initial import of vlogsplitter

* addresses #1
This commit is contained in:
Jan Dittberner 2008-08-22 20:08:09 +00:00
commit 3818c4eaf7

82
vlogsplitter/vlogsplitter Executable file
View file

@ -0,0 +1,82 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This script is a replacement for split-logfile and rotatelogs from the
Apache webserver distribution suited as a pipe log consumer. It has a
file handle pool like vlogger.
Copyright (c) 2008 Jan Dittberner <jan@dittberner.info>
Version: $Id$
"""
import os
import sys
import time
FHPOOL = {}
FHPOOLSIZE = 100
def getfromfhpool(key, dirname, filename):
"""Get a file handle from the filehandle pool.
The function checks whether a filehandle for the given key is in
the pool and returns it. If there is no such handle the pool size
is checked and when it would be exceeded an LRU algorithm is used
to remove the oldest handle from the cache. When removing the item
from the cache the file is closed. After the pool size check the
file identified by dirname and filename is opened for appending
log date..
"""
now = time.time()
if key in FHPOOL:
FHPOOL[key][0] = now
return FHPOOL[key][1]
if len(FHPOOL) == FHPOOLSIZE:
mintime = now
minkey = None
for (ckey, value) in FHPOOL.iteritems():
if value[0] <= mintime:
mintime = value[0]
minkey = ckey
FHPOOL[minkey][1].close()
del FHPOOL[minkey]
FHPOOL[key] = [now, open(os.path.join(dirname, filename), "a")]
return FHPOOL[key][1]
def handlelogs(basedir, logfilename, inputdata):
"""
This function controls the input parsing process.
It reads one log line at a time and adds it to the appropriate log
file.
"""
while 1:
line = inputdata.readline()
if line.find(" ") == -1:
continue
(vhost, logentry) = line.split(" ", 1)
if vhost and logentry:
logdir = os.path.join(basedir, vhost)
if not os.path.isdir(logdir):
os.makedirs(logdir)
fhandle = getfromfhpool(vhost, logdir, logfilename)
fhandle.write(logentry)
fhandle.flush()
def usage():
"""Usage: %s <logbasedir> <logfilename>
Logs incoming lines in a format starting with a vhost field to
<logbasedir>/<vhost>/<logfilename>. Directories below <logbasedir>
are created automatically.
"""
print usage.__doc__ % (sys.argv[0])
if __name__ == '__main__':
if len(sys.argv) < 3:
usage()
sys.exit(1)
handlelogs(sys.argv[1], sys.argv[2], sys.stdin)