Penlogd (Segmentation Fault) Problem

From: Markus Binder (markus.binder@globalways.net)
Date: Wed Sep 04 2002 - 19:31:02 CEST


Hi there, hi Ulric,
I have a little Problem, we are running pen on a big Webserver
and want to do Lofileanalysis put at certain time penlogd says good bye.
Does anyone have the same Problem, I tried to do a work arround by
writing a little Perl Script for Monitoring penlog and automatically
restart it, when it is crashed. The result was only to get aproximately
half of the Logentries it should be ;-(
I also tried mergelogs, but don'z like the handling.

Below is how penlog(d) is configured and the strace output of penlogd.
(It seems to has sth. to do with DNS ??!)

On the PEN Loadbalancer Box:

/www/pen/bin/pen -l penlog.rz.infos.inh:10003
-p /var/run/pen/pen.www.infos.com.80.pid
-h www.infos.com:80
shuttle2.www.infos.com.rz.infos.inh:80
shuttle1.www.infos.com.rz.infos.inh:80

On the Apache Boxes:

CustomLog "|/www/pen/bin/penlog penlog.rz.infos.inh 10003
shuttle1.www.infos.com.rz.infos.inh" combined

CustomLog "|/www/pen/bin/penlog penlog.rz.infos.inh 10003
shuttle2.www.infos.com.rz.infos.inh" combined

On the penlogd Box:

/www/pen/bin/penlogd -l /data/logs/access_log.www.infos.com -p
/www/pen/logs/penlogd.www.infos.com.pid 10003

>>========= STRACE SNIP >>=============

recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33406),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 247
recvfrom(4, "+ 195.154.29.219 192.168.1.51 GE"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 82
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33325),
sin_addr=inet_addr("192.168.1.51")}}, [16]) = 209
write(3, "atible; MSIE 5.0; Windows 98; Di"..., 4096) = 4096
recvfrom(4, "+ 195.154.29.219 192.168.1.51 GE"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 82
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33325),
sin_addr=inet_addr("192.168.1.51")}}, [16]) = 123
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \26\3\0\0a"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 102
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \24\3\0\0\1"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 95
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \27\3\0\0022"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 144
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33406),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 269
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \26\3\0\0a"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 102
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \24\3\0\0\1"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 95
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \27\3\0\1\235"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 204
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33406),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 248
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \26\3\0\0a"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 102
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \24\3\0\0\1"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 95
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \27\3\0\2\4"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 135
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33406),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 223
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \26\3\0\0a"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 102
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \24\3\0\0\1"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 95
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \27\3\0\1n"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 91
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33406),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 201
recvfrom(4, "+ 195.154.29.219 192.168.1.51 GE"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 82
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33325),
sin_addr=inet_addr("192.168.1.51")}}, [16]) = 170
recvfrom(4, "+ 195.154.29.219 192.168.1.51 GE"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 82
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33325),
sin_addr=inet_addr("192.168.1.51")}}, [16]) = 170
recvfrom(4, "+ 195.154.29.219 192.168.1.51 GE"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 82
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33325),
sin_addr=inet_addr("192.168.1.51")}}, [16]) = 168
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \26\3\0\0a"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 102
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \24\3\0\0\1"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 95
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \27\3\0\1g"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 201
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33406),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 197
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \26\3\0\0a"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 55
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \24\3\0\0\1"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 95
recvfrom(4, "+ 80.129.99.1 192.168.1.52 \27\3\0\1$"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 198
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 62
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 193
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33406),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 163
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 72
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 195
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 66
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 189
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 74
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 197
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 69
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 192
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 84
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 65
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 188
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 78
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 201
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 209
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 68
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 191
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 67
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 71
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 194
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 72
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 195
write(3, "MSIE 5.0; Windows 98; DigExt)\"\n1"..., 4096) = 4096
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 65
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 188
recvfrom(4, "+ 212.86.62.170 192.168.1.52 GET"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33327),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 70
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 193
recvfrom(4, "192.168.1.2 - - [04/Sep/2002:19:"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33405),
sin_addr=inet_addr("192.168.1.52")}}, [16]) = 190
recvfrom(4, "+ 216.239.46.204 192.168.1.52 \2005"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 53
recvfrom(4, "+ 216.239.46.204 192.168.1.52 \26\3"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 213
recvfrom(4, "+ 216.239.46.204 192.168.1.52 \27\3"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(33328),
sin_addr=inet_addr("192.168.1.2")}}, [16]) = 136
socket(PF_UNIX, SOCK_STREAM, 0) = 5
connect(5, {sin_family=AF_UNIX, path="/var/run/.nscd_socket"}, 110) = -1
ENOENT (No such file or directory)
close(5) = 0
open("/etc/nsswitch.conf", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=1342, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0) = 0x40017000
read(5, "#\n# /etc/nsswitch.conf\n#\n# An ex"..., 4096) = 1342
read(5, "", 4096) = 0
close(5) = 0
munmap(0x40017000, 4096) = 0
open("/etc/ld.so.cache", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=46759, ...}) = 0
old_mmap(NULL, 46759, PROT_READ, MAP_PRIVATE, 5, 0) = 0x40190000
close(5) = 0
open("/lib/libnss_files.so.2", O_RDONLY) = 5
read(5, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\"\0\000"...,
1024) = 1024
fstat64(5, {st_mode=S_IFREG|0755, st_size=44166, ...}) = 0
old_mmap(NULL, 38492, PROT_READ|PROT_EXEC, MAP_PRIVATE, 5, 0) =
0x40017000
mprotect(0x40020000, 1628, PROT_NONE) = 0
old_mmap(0x40020000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED,
5, 0x8000) = 0x40020000
close(5) = 0
munmap(0x40190000, 46759) = 0
open("/etc/host.conf", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=370, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0) = 0x40021000
read(5, "#\n# /etc/host.conf - resolver co"..., 4096) = 370
read(5, "", 4096) = 0
close(5) = 0
munmap(0x40021000, 4096) = 0
open("/etc/hosts", O_RDONLY) = 5
fcntl64(5, F_GETFD) = 0
fcntl64(5, F_SETFD, FD_CLOEXEC) = 0
fstat64(5, {st_mode=S_IFREG|0644, st_size=736, ...}) = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0) = 0x40021000
read(5, "#\n# hosts This file desc"..., 4096) = 736
read(5, "", 4096) = 0
close(5) = 0
munmap(0x40021000, 4096) = 0
open("/etc/ld.so.cache", O_RDONLY) = 5
fstat64(5, {st_mode=S_IFREG|0644, st_size=46759, ...}) = 0
old_mmap(NULL, 46759, PROT_READ, MAP_PRIVATE, 5, 0) = 0x40190000
close(5) = 0
open("/lib/libnss_dns.so.2", O_RDONLY) = 5
read(5, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\16\0\000"...,
1024) = 1024
fstat64(5, {st_mode=S_IFREG|0755, st_size=16408, ...}) = 0
old_mmap(NULL, 15784, PROT_READ|PROT_EXEC, MAP_PRIVATE, 5, 0) =
0x4019c000
mprotect(0x4019f000, 3496, PROT_NONE) = 0
old_mmap(0x4019f000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED,
5, 0x2000) = 0x4019f000
close(5) = 0
open("/lib/libresolv.so.2", O_RDONLY) = 5
read(5, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\240*\0"...,
1024) = 1024
fstat64(5, {st_mode=S_IFREG|0755, st_size=66718, ...}) = 0
old_mmap(NULL, 69920, PROT_READ|PROT_EXEC, MAP_PRIVATE, 5, 0) =
0x401a0000
mprotect(0x401ae000, 12576, PROT_NONE) = 0
old_mmap(0x401ae000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED,
5, 0xd000) = 0x401ae000
old_mmap(0x401af000, 8480, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x401af000
close(5) = 0
munmap(0x40190000, 46759) = 0
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 5
connect(5, {sin_family=AF_INET, sin_port=htons(53),
sin_addr=inet_addr("192.168.1.3")}}, 28) = 0
send(5, "7J\1\0\0\1\0\0\0\0\0\0\0\0\1\0\1", 17, 0) = 17
gettimeofday({1031159864, 408933}, NULL) = 0
poll([{fd=5, events=POLLIN, revents=POLLIN}], 1, 5000) = 1
recvfrom(5,
"7J\205\200\0\1\0\0\0\1\0\0\0\0\1\0\1\0\0\6\0\1\0\1Q\200"..., 1024, 0,
{sin_family=AF_INET, sin_port=htons(53),
sin_addr=inet_addr("192.168.1.3")}}, [16]) = 92
close(5) = 0
--- SIGSEGV (Segmentation fault) ---

<<============= STRACE SNAP <<======================

Hope you can help me !!!

Mit freundlichen Grüssen

            Markus Binder

--
globalways Internetservice | Sickenhäuser Str. 65 | D-72760 Reutlingen
Tel: +49 (0) 7121 38119-10 | Fax: +49 (0) 7121 38119-12
www.globalways.net | info@globalways.net



This archive was generated by hypermail 2.1.2 : Wed Sep 04 2002 - 19:38:11 CEST