Update log parser for new ops, etc

report.py

@@ -1,57 +1,138 @@
 # Analyze milter log to find abusers
+import traceback
+import sys
 
-class Connection(object);
-  def __init__(self,dt,tm,id,ip)
+def parse_addr(a):
+  beg = a.find('<')
+  end = a.find('>')
+  if beg >= 0:
+    if end > beg: return a[beg+1:end]
+  return a
+
+class Connection(object):
+  def __init__(self,dt,tm,id,ip=None,conn=None):
     self.dt = dt
     self.tm = tm
     self.id = id
-    _,self.host,self.ip = ip.split(None,2)
+    if ip:
+      _,self.host,self.ip = ip.split(None,2)
+    elif conn:
+      self.ip = conn.ip
+      self.host = conn.host
+      self.helo = conn.helo
+    self.subject = None
+    self.rcpt = []
+    self.mfrom = None
+    self.helo = None
+    self.innoc = []
+    self.whitelist = False
 
 def connections(fp):
   conndict = {}
+  termdict = {}
   for line in fp:
+    if line.startswith('{'): continue
     a = line.split(None,4)
     if len(a) < 4: continue
     dt,tm,id,op = a[:4]
-    if id,op == 'bms','milter':
-      # FIXME: optionally yield all partial connections
+    if (id,op) == ('bms','milter'):
+      # FIXME: optionally yield all partial connections in conndict
       conndict = {}
-    key = id
+      termdict = {}
+      continue
+    if id[0] == '[' and id[-1] == ']':
+      try:
+	key = int(id[1:-1])
+      except:
+        print >>sys.stderr,'bad id:',line.rstrip()
+	continue
+    else: continue
     if op == 'connect':
       ip = a[4].rstrip()
-      conn = Connection(dt,tm,id,ip)
+      conn = Connection(dt,tm,id,ip=ip)
       conndict[key] = conn
+    elif op in (
+	'DISCARD:','TAG:','CBV:','Large','No',
+	'NOTE:','From:','Sender:','TRAIN:'):
+      continue
     else:
-      conn = conndict[key]
-      if op == 'Subject:':
-        if len(a) > 4: conn.subject = a[4].rstrip()
-      elif op == 'mail':
-        _,conn.mfrom = a[4].split(None,2)
-      elif op == 'rcpt':
-        _,conn.rcpt = a[4].split(None,2)
-      elif op in ('eom','dspam','abort'):
-        del conndict[key]
-	conn.enddt = dt
-	conn.endtm = tm
-	conn.result = op
-	yield conn
-      elif op in ('REJECT:','DSPAM:','SPAM:'):
-	conn.enddt = dt
-	conn.endtm = tm
-        conn.result = op
-	conn.resmsg = a[4].rstrip()
-	yield conn
-      else:
-	print line.rstrip()
-
+      op = op.lower()
+      try:
+	conn = conndict[key]
+      except KeyError:
+        try:
+	  conn = termdict[key]
+	  del termdict[key]
+	  conndict[key] = conn
+	except KeyError:
+	  print >>sys.stderr,'key error:',line.rstrip()
+	  continue
+      try:
+	if op == 'subject:':
+	  if len(a) > 4:
+	    conn.subject = a[4].rstrip()
+	elif op == 'innoc:':
+	  conn.innoc.append(a[4].rstrip())
+	elif op == 'whitelist':
+	  conn.whitelist = True
+	elif op == 'x-mailer:':
+	  if len(a) > 4:
+	    conn.mailer = a[4].rstrip()
+        elif op == 'x-guessed-spf:':
+          conn.spfguess = a[4]
+	elif op == 'received-spf:':
+	  conn.spfres,conn.spfmsg = a[4].rstrip().split(None,1)
+	elif op == 'received:':
+	  conn.received = a[4].rstrip()
+	elif op == 'temp':
+	  _,conn.tempfile = a[4].rstrip().split(None,1)
+	elif op == 'srs':
+	  _,conn.srsrcpt = a[4].rstrip().split(None,1)
+	elif op == 'mail':
+	  _,conn.mfrom = a[4].rstrip().split(None,1)
+	elif op == 'rcpt':
+	  _,rcpt = a[4].rstrip().split(None,1)
+	  conn.rcpt.append(rcpt)
+	elif op == 'hello':
+	  _,conn.helo = a[4].rstrip().split(None,1)
+	elif op in ('eom','dspam','abort'):
+	  del conndict[key]
+	  conn.enddt = dt
+	  conn.endtm = tm
+	  conn.result = op
+	  yield conn
+	  termdict[key] = Connection(conn.dt,conn.tm,conn.id,conn=conn)
+	elif op in ('reject:','dspam:','tempfail:','reject','fail:','honeypot:'):
+	  del conndict[key]
+	  conn.enddt = dt
+	  conn.endtm = tm
+	  conn.result = op
+	  conn.resmsg = a[4].rstrip()
+	  yield conn
+	  termdict[key] = Connection(conn.dt,conn.tm,conn.id,conn=conn)
+	elif op in ('fp:','spam:'):
+	  del conndict[key]
+	  termdict[key] = Connection(conn.dt,conn.tm,conn.id,conn=conn)
+	else:
+	  print >>sys.stderr,'unknown op:',line.rstrip()
+      except Exception:
+	print >>sys.stderr,'error:',line.rstrip()
+        traceback.print_exc()
 
 if __name__ == '__main__':
   import gzip
-  import sys
-  for fn in sys.argv[:1]:
+  for fn in sys.argv[1:]:
     if fn.endswith('.gz'):
       fp = gzip.open(fn)
     else:
       fp = open(fn)
     for conn in connections(fp):
-      print conn.dt,conn.tm,conn.id,conn.subject,conn.mfrom,conn.rcpt
+      if conn.rcpt and conn.mfrom:
+        for r in conn.rcpt:
+	  if r.lower().find('iancarter') > 0: break
+	else:
+	  if conn.mfrom.lower().find('iancarter') < 0: continue
+	print >>sys.stderr,conn.result,conn.dt,conn.tm,conn.id,conn.subject,parse_addr(conn.mfrom),
+	for a in conn.rcpt:
+	  print parse_addr(a),
+	print