Changeset 86:5f6de6db11ec in livinglogic.python.tipimaid

Show
Ignore:
Timestamp:
02/04/09 18:32:38 (10 years ago)
Author:
Nik Tautenhahn <nik@…>
Branch:
default
Message:

docstrings!

Files:
3 modified

Legend:

Unmodified
Added
Removed
  • liaalh.py

    r85 r86  
    1212 
    1313class LogLine(tuple): 
     14    """ 
     15    Helper Class which overwrites "<" and "<=" to do the right things for liaalh-loglines - it should only be sorted according to the datetime of the logline. 
     16    """ 
    1417    def __lt__(self, other): 
    1518        return self[0] < other[0] 
     
    2023 
    2124class Buffer(object): 
     25    """ 
     26    The main class of liaalh 
     27    """ 
    2228    def __init__(self, pattern='', gzip_logs=None, buffertime=0, stream=sys.stdin, utcrotate=False, symlinkpattern=None, execute=None, progname="liaalh"): # TODO: Name 
    2329        self.pattern = pattern 
     
    4248 
    4349    def openfile(self, filename, server): 
     50        """ 
     51        opens a file filename for server server which may be continuously gzipped. Closes old files, creates directories, if necessary 
     52        """ 
    4453        try: 
    4554            f = open(filename, "a", 1) 
     
    6978 
    7079    def readlines(self): 
     80        """ 
     81        Gets lines from stdin and splits the virtual host from the rest of the line if virtual hosts are used. 
     82        """ 
    7183        while True: 
    7284            try: 
     
    88100 
    89101    def writeline(self, utclogdate, server, line): 
     102        """ 
     103        Writes the logline line for server server which has the date utcdate to the right logfile (i.e. it checks if an already opened logfile is still correct or if it has to be rotated). 
     104        This method also triggers the execution of external scripts after a logfile has been rotated. 
     105        """ 
    90106        if not self.utcrotate: 
    91107            utclogdate += self.localutcoffset 
     
    109125 
    110126    def run_unbuffered(self): 
     127        """ 
     128        Tries to find the date/time of a logline. This is the unbuffered case. 
     129        """ 
    111130        for (server, data) in self.readlines(): 
    112131            try: 
     
    120139 
    121140    def run_buffered(self): 
     141        """ 
     142        Tries to find the date/time of a logline. This is the buffered case. 
     143        """ 
    122144        signal.signal(signal.SIGHUP, self.sighandler) 
    123145        signal.signal(signal.SIGTERM, self.sighandler) 
     
    135157                    continue # ignore it 
    136158                except Exception, exc: 
    137                     print server, data 
     159                    sys.stderr.write("[%s] [%s] Exception encountered: %r\nVhost was: %s\nData was: %r\n" % (datetime.datetime.now().strftime("%a %b %d %H:%M:%S %Y"), self.progname, exc, server, data)) 
     160                    sys.stderr.flush() 
    138161                    raise 
    139162        except Exception, exc: 
     
    142165 
    143166    def do_something(self, filename): 
     167        """ 
     168        Execute the command which was given with the --execute option 
     169        """ 
    144170        try: 
    145171            if subprocessworks: 
     
    154180            sys.stderr.flush() 
    155181 
    156     def add(self, logline): # keep entries in the buffer sorted 
     182    def add(self, logline): 
     183        """ 
     184        Keeps entries in the buffer sorted. 
     185        """ 
    157186        if not self.data or self.data[-1] <= logline: 
    158187            self.data.append(logline) 
     
    162191 
    163192    def flushall(self): 
     193        """ 
     194        Write all loglines content to their files. 
     195        """ 
    164196        for (utclogdate, server, logdata) in self.data: 
    165197            self.writeline(utclogdate, server, logdata) 
     
    167199 
    168200    def flush(self): 
     201        """ 
     202        Write buffered loglines to their files if they have been in the buffer for buffertime seconds. 
     203        """ 
    169204        while self.data: 
    170205            line = self.data[0] 
     
    176211 
    177212    def apachedate2utc(self, d): 
     213        """ 
     214        Converts an "apachedate", i.e. a string like "01/Jan/2009:01:02:03 +0100", to a (UTC) datetime object. 
     215        """ 
    178216        temp = d.split() 
    179217        utcdate = datetime.datetime(*(time.strptime(temp[0], "%d/%b/%Y:%H:%M:%S")[0:6])) # support ancient distributions with python < 2.5 
     
    184222 
    185223    def updateutcoffset(self): 
     224        """ 
     225        Updates the offset of the local system clock to UTC time. (Daylight savings time might have changed... or you managed to move your server across a timezone without switching it off or...) 
     226        """ 
    186227        temp = datetime.datetime.now() - datetime.datetime.utcnow() 
    187228        self.localutcoffset = datetime.timedelta(days=temp.days, seconds=temp.seconds+1, microseconds=0) 
    188229 
    189230    def sighandler(self, signum, frame): 
     231        """ 
     232        Signal handler which specifies what to do if someone wants to quit, term or interrupt us. (If someone wants to kill us, we can't react...) 
     233        """ 
    190234        self.flushall() 
    191235        if signum in (signal.SIGQUIT, signal.SIGTERM, signal.SIGINT): 
  • liaalh_sender.py

    r85 r86  
    55 
    66def intify(s, defaultint=0): 
     7    """ 
     8    Take something, convert it to int, return defaultint if bad things happen. 
     9    """ 
    710    try: 
    811        return int(s) 
     
    1114 
    1215class Sender(object): 
     16    """ 
     17    The main class for the sender... yadda yadda 
     18    """ # TODO: Write more stuff 
    1319    def __init__(self, ip, port, buffertime=0, backuppath=None, stream=sys.stdin, progname="liaalh_sender"): # TODO: Put the right name in here 
    1420        self.ip = ip 
     
    5056 
    5157    def readlines(self): 
     58        """ 
     59        Gets input from stdin. 
     60        """ 
    5261        while True: 
    5362            try: 
     
    6372 
    6473    def create_socket(self): 
     74        """ 
     75        Creates a new socket connection. 
     76        """ 
    6577        if self.s is None: 
    6678            for (af, socktype, proto, canonname, sa) in socket.getaddrinfo(self.ip, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM): 
     
    8092 
    8193    def send_socket(self, line): 
     94        """ 
     95        Sends line over socket, sees if this works. If not buffer it and try again later or write it to a tempfile. 
     96        """ 
    8297        (l, t) = (len(line), 0) 
    8398        while t < l: # it might be that only a part of line gets transmitted 
     
    91106                self.s = None 
    92107                self.last_check_connection = self.startedbuffering = datetime.datetime.now() 
    93                 sys.stderr.write("%s (%s): Socket connection lost, starting to log to buffer and/or recovery file\n" % (self.progname, self.startedbuffering.isoformat())) 
     108                sys.stderr.write("[%s] [%s]: Socket connection lost, starting to log to buffer and/or recovery file\n" % (self.startedbuffering.strftime("%a %b %d %H:%M:%S %Y"), self.progname)) 
     109                sys.stderr.flush() 
    94110                self.send_tempfile(line) 
    95111 
    96112    def send_tempfile(self, line): 
     113        """ 
     114        Puts line into the local buffer if they are not too old for the server 
     115        to be ordered in correctly. Otherwise line is dumped to a local 
     116        tempfile. This method also checks periodically if a new socket 
     117        connection can be established and, if yes and the buffered data is not 
     118        too old, it is transmitted to the server. 
     119        """ 
    97120        if datetime.datetime.now() - self.startedbuffering < self.buffertime: 
    98121            self.buffer.append(line) 
     
    106129                self.f = open(filename, "a", 1) 
    107130            if self.buffer: # if we have buffered entries these must be handled before "line" 
    108                 sys.stderr.write("%s (%s): Dumping buffered data to local recovery file\n" % (self.progname, datetime.datetime.now().isoformat())) 
     131                sys.stderr.write("[%s] [%s]: Dumping buffered data to local recovery file\n" % (self.startedbuffering.strftime("%a %b %d %H:%M:%S %Y"), self.progname)) 
    109132                for oldline in self.buffer: 
    110133                    self.f.write(oldline) 
     
    144167 
    145168    def flush_buffer(self): 
     169        """ 
     170        Flushes all local buffers - either via socket or to a tempfile 
     171        """ 
    146172        if not self.s: 
    147173            self.create_socket() 
     
    189215 
    190216    def send(self): 
     217        """ 
     218        Sends loglines - and decides if they are sent via socket or to a tempfile. 
     219        """ 
    191220        try: 
    192221            for line in self.readlines(): 
     
    203232 
    204233    def sighandler(self, signum, frame): 
     234        """ 
     235        Signal handler which specifies what to do if someone wants to quit, term or interrupt us. (If someone wants to kill us, we can't react...) 
     236        """ 
    205237        self.flush_buffer() 
    206238        if signum in (signal.SIGQUIT, signal.SIGTERM, signal.SIGINT): 
  • liaalh_server.py

    r85 r86  
    55 
    66def intify(s, defaultint=0): 
     7    """ 
     8    Take something, convert it to int, return defaultint if bad things happen. 
     9    """ 
    710    try: 
    811        return int(s) 
     
    1114 
    1215def run(port=40000, netcat_compatible=False): 
     16    """ 
     17    Starts the server at port port. If netcat_compatible is True the server does 
     18    not notify the sender that it received something. Thus a connection loss 
     19    might be noticed later than when liaalh_sender is used which is the 
     20    recommended setup here. Actually this switch only exists for paranoid admins 
     21    who don't speak python and don't trust liaalh_sender (but trust netctat 
     22    instead). 
     23    """ 
    1324    backlog = 5 
    1425    size = 8192 
     
    4859                    if data: 
    4960                        if not netcat_compatible: 
    50                             s.send("1") # notify the sender that we got something DISABLE THIS IF WE USE NETCAT AS SENDER! 
     61                            s.send("1") # notify the sender that we got something 
    5162                        inputs[s] += data 
    5263                        pos = inputs[s].rfind("\n")