Release 0.7.1

MANIFEST.in

@@ -8,8 +8,10 @@ include testsample.py
 include testmime.py
 include testbms.py
 include testdspam.py
+include rejects.py
 include bms.py
 include spf.py
+include spfquery.py
 include test.py
 include sample.py
 include test/*

Milter.py

@@ -8,15 +8,12 @@ import milter
 import thread
 
 from milter import ACCEPT,CONTINUE,REJECT,DISCARD,TEMPFAIL,	\
-  set_flags, setdbg, \
+  set_flags, setdbg, setbacklog, settimeout, \
   ADDHDRS, CHGBODY, ADDRCPT, DELRCPT, CHGHDRS,	\
   V1_ACTS, V2_ACTS, CURR_ACTS
 
-try:
+try: from milter import QUARANTINE
-  from milter import QUARANTINE
+except: pass
-except:
-  #print 'No QUARANTINE support'
-  pass
 
 _seq_lock = thread.allocate_lock()
 _seq = 0
@@ -100,8 +97,10 @@ class Milter:
   def getsymval(self,sym):
     return self.__ctx.getsymval(sym)
 
-  def setreply(self,rcode,xcode,msg):
+  # If sendmail does not support setmlreply, then only the
-    return self.__ctx.setreply(rcode,xcode,msg)
+  # first msg line is used.
+  def setreply(self,rcode,xcode=None,msg=None,*ml):
+    return self.__ctx.setreply(rcode,xcode,msg,*ml)
 
   # Milter methods which can only be called from eom callback.
   def addheader(self,field,value):
@@ -119,6 +118,8 @@ class Milter:
   def replacebody(self,body):
     return self.__ctx.replacebody(body)
 
+  # When quarantined, a message goes into the mailq as if to be delivered,
+  # but delivery is deferred until the message is unquarantined.
   def quarantine(self,reason):
     return self.__ctx.quarantine(reason)
 

NEWS

@@ -1,5 +1,22 @@
 Here is a history of user visible changes to Python milter.
 
+0.7.1	Handle modifying mislabeled multipart messages without an exception
+	Support setbacklog, setmlreply
+	Allow multi-recipient CBV
+	Return TEMPFAIL for SPF softfail
+0.7.0	SPF check hello name
+	Move pythonsock to /var/run/milter
+	Move milter.cfg to /etc/mail/pymilter.cfg
+	Check M$ style XML CID records by converting to SPF
+	Recognize, but never match ip6 - until we properly support it.
+	Option to reject when no PTR and no SPF
+0.6.9	Reject invalid SRS immediately for benefit of callback verifiers
+	Fix include bug in spf.py
+	Fix check_header bug
+	Fix setup.py to work with python < 2.2.3, thanks to Eric S. Johansson
+	Test driver for SPF test suite.  Fix bugs and add features to
+	pass most of test suite.
+	Use best_guess() and get_header() in bms.py for SPF support
 0.6.8	Defang message/rfc822 content_type with boundary 
 	Support SPF delegation
 	Reject neutral SPF result for selected domains
@@ -7,6 +24,7 @@ Here is a history of user visible changes to Python milter.
 	Don't report "spoofed" unless rcpt looks like SRS
 	Check for bounce with multiple rcpts
 	Make dspam see Received-SPF headers
+	Fix sysv init for Redhat 9 and other single ps line per process systems
 0.6.7	Fix failure to remove explicit unix socket thanks to Alexander again.
 	Support SRS forgery detection.
 	Detect thread resource starvation in Milter.py.

TODO

@@ -1,13 +1,25 @@
+Move milter,Milter,mime,spf modules to pymilter
+milter package will have bms.py application
+
+spf.py has no recursion bound on CNAME lookup
+Support SMTP AUTH and disable SPF checks when connection is authorized.
+Web admin interface
+RHSBL
+Check valid domains allowed by internal senders to detect PCs infected
+with spam trojans.
+Do CBV (callback verification) for mail with no published SPF record.
+message log for automated stats and blacklisting
+Skip dspam when SPF pass?
+Report 551 with rcpt on SPF fail?
+check spam keywords with character classes, e.g.
+	{a}=[a@ãä], {i}=[i1í], {e}=[eë], {o}=[o0ö]
+
 Implement RRS - a backdoor for non-SRS forwarders.  User lists non-SRS 
 forwarder accounts, and a util provides a special local alias for the
 user to give to the forwarder.  Alias only works for mail from that
 forwarder.  Milter gets forwarder domain from alias and uses it to
 SPF check forwarder.
 
-adapt init script to work on RH9
-Skip dspam when SPF pass?
-Report 551 with rcpt on SPF fail?
-
 Another special dspam user, 'honeypot', can be listed in innoculations.
 All email to those addresses is treated as known spam.
 

bms.py

@@ -1,6 +1,75 @@
 #!/usr/bin/env python
 # A simple milter.
 # $Log$
+# Revision 1.117  2004/08/23 02:27:53  stuart
+# Allow multi rcpt CBV.  Add some multiline replies.
+#
+# Revision 1.116  2004/08/20 22:27:52  stuart
+# Generate TEMPFAIL for SPF softfail.
+#
+# Revision 1.115  2004/08/19 20:55:49  stuart
+# Always show reversed SRS path.
+# Check if encodings are an ASCII superset.  Some messages were encoded as
+# BIG5 and getting rejected even though chars were all in ascii subset.
+#
+# Revision 1.114  2004/07/27 00:40:12  stuart
+# Make reject on no PTR optional.
+#
+# Revision 1.113  2004/07/23 23:11:14  stuart
+# Log known malformed messages differently than general processing exceptions.
+#
+# Revision 1.112  2004/07/21 19:18:33  stuart
+# Punt on UnicodeDecodeError when decoding headers.
+# Accept a pass with default SPF for missing reverse IP.
+#
+# Revision 1.111  2004/07/18 13:13:31  stuart
+# Reject invalid SRS only for SRS domain (which is the only one we
+# know the key for).
+# Reject senders that have neither reverse IP nor SPF.
+#
+# Revision 1.110  2004/06/12 03:13:18  stuart
+# Block bounces only for SRS domain.  Also treat mail from
+# postmaster or mailer-daemon as DSN for SRS/SES checking purposes.
+#
+# Revision 1.109  2004/05/01 02:56:55  stuart
+# Let multiple screeners share work.
+#
+# Revision 1.108  2004/04/29 20:36:23  stuart
+# Require HELO name
+#
+# Revision 1.107  2004/04/24 22:55:29  stuart
+# Move some files to make the RPM more standard.
+#
+# Revision 1.106  2004/04/21 18:29:08  stuart
+# Validate hello name with SPF.
+#
+# Revision 1.105  2004/04/20 15:16:00  stuart
+# Release 0.6.9
+#
+# Revision 1.104  2004/04/19 21:56:26  stuart
+# Support SPF best_guess and get_header
+#
+# Revision 1.103  2004/04/10 02:31:01  stuart
+# Fix timeout config
+#
+# Revision 1.102  2004/04/08 20:25:11  stuart
+# Make libmilter timeout a config option
+#
+# Revision 1.101  2004/04/08 19:18:16  stuart
+# Preserve case of local part in sender
+#
+# Revision 1.100  2004/04/08 18:41:15  stuart
+# Reject numeric hello names
+#
+# Revision 1.99  2004/04/06 19:46:39  stuart
+# Reject invalid SRS immediately for benefit of CallBack Verifiers.
+#
+# Revision 1.98  2004/04/06 15:28:20  stuart
+# Release 0.6.8-2
+#
+# Revision 1.97  2004/04/06 13:07:43  stuart
+# Pass original header name to check_header
+#
 # Revision 1.96  2004/04/06 03:27:03  stuart
 # bugs from Redhat 9 testing
 #
@@ -154,90 +223,6 @@
 # Revision 1.47  2003/08/26 05:01:38  stuart
 # Release 0.6.0
 #
-# Revision 1.46  2003/08/26 04:45:16  stuart
-# Modest dspam control
-#
-# Revision 1.43  2003/06/25 17:00:02  stuart
-# fix hostaddr test
-#
-# Revision 1.42  2003/06/25 16:45:59  stuart
-# Not using checking hostaddr properly
-#
-# Revision 1.41  2003/06/25 15:57:54  stuart
-# Ready for 5.5 release.
-#
-# Revision 1.40  2003/06/25 15:41:41  stuart
-# recognize internal connections.
-# Give legitimate users a clue about banned subject keywords.
-#
-# Revision 1.39  2002/12/14 00:36:59  stuart
-# Smart alias feature
-#
-# Revision 1.38  2002/11/14 17:52:53  stuart
-# Redirection feature for wiretap
-#
-# Revision 1.37  2002/11/07 23:52:09  stuart
-# config fixes
-#
-# Revision 1.36  2002/10/04 05:27:38  stuart
-# Add get_submsg to allow modifying rfc822 attachment.
-#
-# Revision 1.35  2002/10/03 01:31:18  stuart
-# Test encoded rfc822 attachment
-#
-# Revision 1.34  2002/10/03 00:55:42  stuart
-# Decode rfc822 attachments
-#
-# Revision 1.33  2002/10/02 18:49:02  stuart
-# Save and log messages which cause an exception while parsing attachments.
-#
-# Revision 1.32  2002/09/24 01:38:05  stuart
-# Doc updates.
-#
-# Revision 1.31  2002/09/13 22:14:06  stuart
-# Release 0.5.0 wrapup
-#
-# Revision 1.30  2002/09/13 20:22:37  stuart
-# Additional config items
-#
-# Revision 1.29  2002/08/20 04:40:46  stuart
-# Use config file
-#
-# Revision 1.28  2002/07/12 19:40:38  stuart
-# Update docs, minor bugs.
-#
-# Revision 1.27  2002/06/16 02:06:24  stuart
-# SPAM tweaks
-#
-# Revision 1.26  2002/06/07 22:07:30  stuart
-# Isolate local hacks to configuration data.
-#
-# Revision 1.25  2002/05/02 20:41:00  stuart
-# Top level virus needs top level header change.
-#
-# Revision 1.24  2002/05/02 20:31:43  stuart
-# Handle quoted-printable HTML attachments.
-# Remove entire attachment when HTML can't be parsed by sgmllib.
-#
-# Revision 1.23  2002/05/02 03:42:31  stuart
-# base64 no longer needed
-#
-# Revision 1.22  2002/05/02 03:12:39  stuart
-# Move check_html to mime module.
-#
-# Revision 1.21  2002/05/02 02:48:22  stuart
-# Remove scripts from HTML even with base64 encoding.
-#
-# Revision 1.20  2002/05/02 00:21:01  stuart
-# Test filtering HTML attachments.
-#
-# Revision 1.19  2002/05/01 22:12:41  stuart
-# Remove scripts from HTML attachments.
-#
-# Revision 1.18  2002/03/01 20:29:00  stuart
-# Ready for release.
-#
-
 # Author: Stuart D. Gathman <stuart@bmsi.com>
 # Copyright 2001 Business Management Systems, Inc.
 # This code is under GPL.  See COPYING for details.
@@ -252,17 +237,22 @@ import Milter
 import tempfile
 import ConfigParser
 import time
+import re
+
 from fnmatch import fnmatchcase
 from email.Header import decode_header
 
 # Import pysrs if available
 try:
   import SRS
-  import re
   srsre = re.compile(r'^SRS[01][+-=]',re.IGNORECASE)
 except: SRS = None
+
+# Import spf if available
 try: import spf
 except: spf = None
+
+ip4re = re.compile(r'^[1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*$')
 #import syslog
 #syslog.openlog('milter')
 
@@ -294,13 +284,17 @@ dspam_users = {}
 dspam_userdir = None
 dspam_exempt = {}
 dspam_whitelist = {}
-dspam_screener = None
+dspam_screener = ()
 dspam_internal = True	# True if internal mail should be dspammed
 dspam_reject = ()
-dspam_sizelimit = 80000
+dspam_sizelimit = 180000
 srs = None
 srs_reject_spoofed = False
+srs_fwdomain = None
 spf_reject_neutral = ()
+spf_best_guess = False
+spf_reject_noptr = False
+timeout = 600
 
 class MilterConfigParser(ConfigParser.ConfigParser):
 
@@ -350,7 +344,8 @@ class MilterConfigParser(ConfigParser.ConfigParser):
 def read_config(list):
   cp = MilterConfigParser({
     'tempdir': "/var/log/milter/save",
-    'socket': "/var/log/milter/pythonsock",
+    'socket': "/var/run/milter/pythonsock",
+    'timeout': '600',
     'scan_html': 'no',
     'scan_rfc822': 'yes',
     'block_chinese': 'no',
@@ -358,12 +353,15 @@ def read_config(list):
     'blind_wiretap': 'yes',
     'maxage': '8',
     'hashlength': '8',
-    'reject_spoofed': 'no'
+    'reject_spoofed': 'no',
+    'reject_noptr': 'no',
+    'best_guess': 'no'
   })
   cp.read(list)
   tempfile.tempdir = cp.get('milter','tempdir')
-  global socketname, scan_rfc822, scan_html, block_chinese
+  global socketname, scan_rfc822, scan_html, block_chinese, timeout
   socketname = cp.get('milter','socket')
+  timeout = cp.getint('milter','timeout')
   scan_rfc822 = cp.getboolean('milter','scan_rfc822')
   scan_html = cp.getboolean('milter','scan_html')
   block_chinese = cp.getboolean('milter','block_chinese')
@@ -402,13 +400,13 @@ def read_config(list):
 
   global dspam_dict, dspam_users, dspam_userdir, dspam_exempt
   global dspam_screener,dspam_whitelist,dspam_reject,dspam_sizelimit
-  global spf_reject_neutral,SRS
+  global spf_reject_neutral,spf_best_guess,SRS,spf_reject_noptr
   dspam_dict = cp.getdefault('dspam','dspam_dict')
   dspam_exempt = cp.getaddrset('dspam','dspam_exempt')
   dspam_whitelist = cp.getaddrset('dspam','dspam_whitelist')
   dspam_users = cp.getaddrdict('dspam','dspam_users')
   dspam_userdir = cp.getdefault('dspam','dspam_userdir')
-  dspam_screener = cp.getdefault('dspam','dspam_screener')
+  dspam_screener = cp.getlist('dspam','dspam_screener')
   dspam_reject = cp.getlist('dspam','dspam_reject')
   if cp.has_option('dspam','dspam_sizelimit'):
     dspam_sizelimit = cp.getint('dspam','dspam_sizelimit')
@@ -416,11 +414,13 @@ def read_config(list):
   if spf:
     spf.DELEGATE = cp.getdefault('spf','delegate')
     spf_reject_neutral = cp.getlist('spf','reject_neutral')
+    spf_best_guess = cp.getboolean('spf','best_guess')
+    spf_reject_noptr = cp.getboolean('spf','reject_noptr')
   srs_config = cp.getdefault('srs','config')
   if srs_config: cp.read([srs_config])
   srs_secret = cp.getdefault('srs','secret')
   if SRS and srs_secret:
-    global srs,srs_reject_spoofed
+    global srs,srs_reject_spoofed,srs_fwdomain
     database = cp.getdefault('srs','database')
     srs_reject_spoofed = cp.getboolean('srs','reject_spoofed')
     maxage = cp.getint('srs','maxage')
@@ -433,7 +433,7 @@ def read_config(list):
     else:
       srs = SRS.Guarded.Guarded(secret=srs_secret,
         maxage=maxage,hashlength=hashlength,separator=separator)
-
+    srs_fwdomain = cp.getdefault('srs','fwdomain')
 
 def parse_addr(t):
   if t.startswith('<') and t.endswith('>'): t = t[1:-1]
@@ -446,7 +446,10 @@ def parse_header(val):
     u = []
     for s,enc in h:
       if enc:
+        try:
 	  u.append(unicode(s,enc))
+	except LookupError:
+	  u.append(unicode(s))
       else:
 	u.append(unicode(s))
     u = ''.join(u)
@@ -454,11 +457,14 @@ def parse_header(val):
       try:
 	return u.encode(enc)
       except UnicodeError: continue
-  except LookupError:
+  except UnicodeDecodeError: pass
+  except LookupError: pass
   return val
 
 class bmsMilter(Milter.Milter):
-  "Milter to replace attachments poisonous to Windows with a WARNING message."
+  """Milter to replace attachments poisonous to Windows with a WARNING message,
+     check SPF, and other anti-forgery features, and implement wiretapping
+     and smart alias redirection."""
 
   def log(self,*msg):
     print "%s [%d]" % (time.strftime('%Y%b%d %H:%M:%S'),self.id),
@@ -494,6 +500,7 @@ class bmsMilter(Milter.Milter):
     self.log('%s: %s' % (name,val))
 
   def connect(self,hostname,unused,hostaddr):
+    self.missing_ptr = hostname.startswith('[') and hostname.endswith(']')
     self.internal_connection = False
     self.trusted_relay = False
     self.receiver = self.getsymval('j')
@@ -521,11 +528,17 @@ class bmsMilter(Milter.Milter):
     if self.trusted_relay:
       connecttype += ' TRUSTED'
     self.log("connect from %s at %s %s" % (hostname,hostaddr,connecttype))
+    self.hello_name = None
+    self.connecthost = hostname
     return Milter.CONTINUE
 
   def hello(self,hostname):
     self.hello_name = hostname
     self.log("hello from %s" % hostname)
+    if ip4re.match(hostname):
+      self.log("REJECT: numeric hello name:",hostname)
+      self.setreply('550','5.7.1','hello name cannot be numeric ip')
+      return Milter.REJECT
     if not self.internal_connection and hostname in hello_blacklist:
       self.log("REJECT: spam from self:",hostname)
       self.setreply('550','5.7.1','I hate talking to myself.')
@@ -573,79 +586,78 @@ class bmsMilter(Milter.Milter):
 	self.dspam = False
     else:
       self.rejectvirus = False
+    if not self.hello_name:
+      self.log("REJECT: missing HELO")
+      self.setreply('550','5.7.1',"It's polite to say HELO first.")
+      return Milter.REJECT
     if not (self.internal_connection or self.trusted_relay)	\
     	and self.connectip and spf:
       return self.check_spf()
     return Milter.CONTINUE
 
   def check_spf(self):
-    user,host = spf.split_email(self.canon_from,self.hello_name)
+    t = parse_addr(self.mailfrom)
-    self.sender = '@'.join((user,host))
+    if len(t) == 2: t[1] = t[1].lower()
-    res,code,txt = spf.check(self.connectip,self.canon_from,self.hello_name)
+    q = spf.query(self.connectip,'@'.join(t),self.hello_name)
-    if res in ('deny', 'fail'):
+    q.set_default_explanation('SPF fail: see http://spf.pobox.com/why.html')
-      self.log('REJECT: SPF %s %i %s' % (res,code,txt))
+    res,code,txt = q.check()
-      # improve default explanation, but don't wipe out text from SPF record
+    receiver = self.receiver
-      if txt == 'access denied':	
+    if res == 'none':
-        txt = 'SPF fail: see http://spf.pobox.com/why.html'
+      if self.mailfrom != '<>':
-      self.setreply(str(code),'5.7.1',txt)
+	# check hello name via spf
-      return Milter.REJECT
+	hres,hcode,htxt = spf.check(self.connectip,'',self.hello_name)
-    if res == 'pass':
+	if hres in ('deny','fail','neutral','softfail'):
-#       Received-SPF: pass (mybox.example.org: domain of
+	  self.log('REJECT: hello SPF: %s 550 %s' % (hres,htxt))
-#                           myname@example.com designates 192.0.2.1 as
+	  self.setreply('550','5.7.1',htxt,
-#                           permitted sender);
+	    "The hostname given in your MTA's HELO response is not listed",
-#                           receiver=mybox.example.org;
+	    "as a legitimate MTA in the SPF records for your domain.",
-#                           client_ip=192.0.2.1;
+	    "If you get this bounce, the message was not in fact a forgery,",
-#                           envelope-from=myname@example.com;
+	    "and you should notify your email administrator of the problem."
-      self.add_header('Received-SPF',"""pass (%(receiver)s: domain of
-      %(sender)s designates %(connectip)s as permitted sender);
-      receiver=%(receiver)s; client_ip=%(connectip)s;
-      envelope-from=%(canon_from)s;""" % self.__dict__)
-    elif res == 'none' or res == 'unknown' and txt == 'no SPF record':
-#       Received-SPF: none (mybox.example.org: myname@example.com does
-#                           not designated permitted sender hosts)
-      self.add_header('Received-SPF',"""none (%(receiver)s: %(sender)s does
-      	not designate permitted sender hosts)""" % self.__dict__)
-    elif res == 'softfail':
-#       Received-SPF: softfail (mybox.example.org: domain of transitioning
-#                              myname@example.com does not designate
-#                              192.0.2.1 as permitted sender)
-      self.add_header('Received-SPF',
-      	"""softfail (%(receiver)s: domain of transitioning
-	%(sender)s does not designate
-	%(connectip)s as permitted sender)""" % self.__dict__)
-    elif res == 'neutral':
-      if host in spf_reject_neutral:
-        self.log('REJECT: SPF neutral for',self.sender)
-	self.setreply('550','5.7.1',
-	  'mail from %s must pass SPF: http://spf.pobox.com/why.html' % host
 	  )
 	  return Milter.REJECT
-#       Received-SPF: neutral (mybox.example.org: 192.0.2.1 is neither
+      if spf_best_guess:
-#                             permitted nor denied by domain of
+	#self.log('SPF: no record published, guessing')
-#                             myname@example.com)
+	q.set_default_explanation(
-      self.add_header('Received-SPF',
+		'SPF guess: see http://spf.pobox.com/why.html')
-      	"""neutral (%(receiver)s: %(connectip)s is neither
+	# best_guess should not result in fail
-	permitted nor denied by domain of %(sender)s)""" % self.__dict__)
+	res,code,txt = q.best_guess()
-    elif res == 'unknown':
+	receiver += ': guessing'
-#       Received-SPF: unknown -extension:foo (mybox.example.org: domain
+      if self.missing_ptr and res in ('neutral', 'none') and spf_reject_noptr:
-#                      of myname@example.com uses mechanism
+        self.log('REJECT: no PTR or SPF')
-#			not recognized by this client)
+	self.setreply('550','5.7.1',
-      self.spf_mech = txt
+  'You must have a reverse lookup or publish SPF: http://spf.pobox.com'
-      self.add_header('Received-SPF',
+	)
-      	"""unknown %(spf_mech)s (%(receiver)s: domain
+	return Milter.REJECT
-	of %(sender)s uses mechanism not recognized by this client)"""
+    if res in ('deny', 'fail'):
-	% self.__dict__)
+      self.log('REJECT: SPF %s %i %s' % (res,code,txt))
-    elif res == 'error':
+      self.setreply(str(code),'5.7.1',txt)
-#   	Received-SPF: error (mybox.example.org: error in processing
+      return Milter.REJECT
-#                           during lookup of myname@example.com: DNS
+    if res == 'softfail':
-#                           timeout)
+      self.log('TEMPFAIL: SPF %s 450 %s' % (res,txt))
-      self.add_header('Received-SPF',
+      self.setreply('450','4.3.0',
-      	"""error (%s: error in processing
+	'SPF softfail: will keep trying until your SPF record is fixed.',
-	during lookup of %s: %s)""" % (self.receiver,self.sender,txt))
+	'If you get this Delivery Status Notice, your email was probably',
+	'legitimate.  Your administrator has published SPF records in a',
+	'testing mode.  The SPF record reported your email as a forgery,',
+	'which is a mistake if you are reading this.  Please notify your',
+	'administrator of the problem.'
+      )
+      return Milter.TEMPFAIL
+    if res == 'neutral' and q.o in spf_reject_neutral:
+      self.log('REJECT: SPF neutral for',q.s)
+      self.setreply('550','5.7.1',
+	'mail from %s must pass SPF: http://spf.pobox.com/why.html' % q.o,
+	'The %s domain is one that spammers love to forge.  Due to' % q.o,
+	'the volume of forged mail, we can only accept mail that',
+	'the SPF record for %s explicitly designates as legitimate.' % q.o,
+	'Sending your email through the recommended outgoing SMTP',
+	'servers for %s should accomplish this.' % q.o
+      )
+      return Milter.REJECT
+    if res == 'error':
       self.setreply(str(code),'4.3.0',txt)
       return Milter.TEMPFAIL
-    else:
+    self.add_header('Received-SPF',q.get_header(res,receiver))
-      self.log('SPF: %s %i %s' % (res,code,txt))
     return Milter.CONTINUE
 
   # hide_path causes a copy of the message to be saved - until we
@@ -659,27 +671,31 @@ class bmsMilter(Milter.Milter):
     self.log("rcpt to",to,str)
     t = parse_addr(to.lower())
     if len(t) == 2:
-      if self.mailfrom == '<>':
+      user,domain = t
+      if self.mailfrom == '<>' or self.canon_from.startswith('postmaster@') \
+      	or self.canon_from.startswith('mailer-daemon@'):
         if self.recipients:
-	  self.log('REJECT: Multiple bounce recipients')
+	  self.data_allowed = False
-	  self.setreply('550','5.7.1','Multiple bounce recipients')
+        if srs and domain == srs_fwdomain:
-	  return Milter.REJECT
-        if srs and not (self.internal_connection or self.trusted_relay):
 	  oldaddr = '@'.join(parse_addr(to))
 	  try:
 	    newaddr = srs.reverse(oldaddr)
+	    # Currently, a sendmail map reverses SRS.  We just log it here.
 	    self.log("srs rcpt:",newaddr)
 	  except:
+            if not (self.internal_connection or self.trusted_relay):
 	      if srsre.match(oldaddr):
-	      self.log("srs spoofed:",oldaddr)
+		self.log("REJECT: srs spoofed:",oldaddr)
+		self.setreply('550','5.7.1','Invalid SRS signature')
+		return Milter.REJECT
 	      self.data_allowed = not srs_reject_spoofed
+      # non DSN mail to SRS address will bounce due to invalid local part
       self.recipients.append('@'.join(t))
-      user,domain = t
       users = check_user.get(domain)
       if self.discard:
         self.del_recipient(to)
       if users and not user in users:
-        self.log('REJECT: RCPT TO:',to,str)
+        self.log('REJECT: RCPT TO:',to)
 	return Milter.REJECT
       if user in block_forward.get(domain,()):
         self.forward = False
@@ -708,7 +724,8 @@ class bmsMilter(Milter.Milter):
     return Milter.CONTINUE
 
   # Heuristic checks for spam headers
-  def check_header(self,lname,val):
+  def check_header(self,name,val):
+    lname = name.lower()
     # val is decoded header value
     if lname == 'subject':
 
@@ -743,6 +760,7 @@ class bmsMilter(Milter.Milter):
       if not self.forward:
 	if lval.startswith("fwd:") or lval.startswith("[fw"):
 	  self.log('REJECT: %s: %s' % (name,val))
+	  self.setreply('550','5.7.1','I find unedited forwards annoying')
 	  return Milter.REJECT
 
     # check for invalid message id
@@ -764,8 +782,12 @@ class bmsMilter(Milter.Milter):
 
   def header(self,name,hval):
     if not self.data_allowed:
+      if len(self.recipients) > 1:
+	self.log('REJECT: Multiple bounce recipients')
+	self.setreply('550','5.7.1','Multiple bounce recipients')
+      else:
 	self.log('REJECT: bounce with no SRS encoding')
-      self.setreply('550','5.7.1',"spoofed reply address")
+	self.setreply('550','5.7.1',"I did not send you that message.")
       return Milter.REJECT
     lname = name.lower()
     # decode near ascii text to unobfuscate
@@ -773,11 +795,11 @@ class bmsMilter(Milter.Milter):
     if not self.internal_connection:
       # even if we wanted the Taiwanese spam, we can't read Chinese
       if block_chinese and lname == 'subject':
-	if hval.startswith('=?big5') or hval.startswith('=?ISO-2022-JP'):
+	if val.startswith('=?big5') or val.startswith('=?ISO-2022-JP'):
-	  self.log('REJECT: %s: %s' % (name,hval))
+	  self.log('REJECT: %s: %s' % (name,val))
 	  self.setreply('550','5.7.1',"We don't understand chinese")
 	  return Milter.REJECT
-      rc = self.check_header(lname,val)
+      rc = self.check_header(name,val)
       if rc != Milter.CONTINUE: return rc
     # log selected headers
     if log_headers or lname in ('subject','x-mailer'):
@@ -911,21 +933,22 @@ class bmsMilter(Milter.Milter):
 	    print x
     # screen if no recipients are dspam_users
     if not modified and dspam_screener and not self.internal_connection \
-    	and (self.dspam or self.reject_spam):
+    	and self.dspam:
       self.fp.seek(0)
       txt = self.fp.read()
       if len(txt) > dspam_sizelimit:
 	self.log("Large message:",len(txt))
 	return False
-      if not ds.check_spam(dspam_screener,txt,self.recipients,
+      screener = dspam_screener[self.id % len(dspam_screener)]
+      if not ds.check_spam(screener,txt,self.recipients,
       	classify=True,quarantine=not self.reject_spam):
 	self.fp = None
 	if self.reject_spam:
-	  self.log("DSPAM:",dspam_screener,
+	  self.log("DSPAM:",screener,
 	  	'REJECT: X-DSpam-Score: %f' % ds.probability)
 	  self.setreply('550','5.7.1','Your Message looks spammy')
 	  return True
-	self.log("DSPAM:",dspam_screener,"SCREENED")
+	self.log("DSPAM:",screener,"SCREENED")
     return modified
 
   def eom(self):
@@ -960,16 +983,18 @@ class bmsMilter(Milter.Milter):
       fname = tempfile.mktemp(".fail")	# save message that caused crash
       os.rename(self.tempname,fname)
       self.tempname = None
-      self.log("FAIL: %s" % fname)	# log filename
       if exc_type == email.Errors.BoundaryError:
+	self.log("MALFORMED: %s" % fname)	# log filename
 	self.setreply('554','5.7.7',
 		'Boundary error in your message, are you a spammer?')
         return Milter.REJECT
       if exc_type == email.Errors.HeaderParseError:
+	self.log("MALFORMED: %s" % fname)	# log filename
 	self.setreply('554','5.7.7',
 		'Header parse error in your message, are you a spammer?')
         return Milter.REJECT
       # let default exception handler print traceback and return 451 code
+      self.log("FAIL: %s" % fname)	# log filename
       raise
     if rc == Milter.REJECT: return rc;
     if rc == Milter.DISCARD: return rc;
@@ -1031,6 +1056,7 @@ class bmsMilter(Milter.Milter):
       os.remove(self.tempname)	# remove in case session aborted
     if self.fp:
       self.fp.close()
+    sys.stdout.flush()
     return Milter.CONTINUE
 
   def abort(self):
@@ -1045,13 +1071,13 @@ def main():
   if srs or len(discard_users) > 0 or smart_alias or dspam_userdir:
     flags = flags + Milter.DELRCPT
   Milter.set_flags(flags)
-  print "bms milter startup"
+  print "%s bms milter startup" % time.strftime('%Y%b%d %H:%M:%S')
   sys.stdout.flush()
-  Milter.runmilter("pythonfilter",socketname,600)
+  Milter.runmilter("pythonfilter",socketname,timeout)
-  print "bms milter shutdown"
+  print "%s bms milter shutdown" % time.strftime('%Y%b%d %H:%M:%S')
 
 if __name__ == "__main__":
-  read_config(["milter.cfg"])
+  read_config(["/etc/mail/pymilter.cfg","milter.cfg"])
   if dspam_dict:
     import dspam	# low level spam check
   if dspam_userdir:

cid2spf.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python2.3
+
+# Convert a MS Caller-ID entry (XML) to a SPF entry
+#
+# (c) 2004 by Ernesto Baschny
+# (c) 2004 Python version by Stuart Gathman
+#
+# Date: 2004-02-25
+# Version: 1.0
+#
+# Usage:
+#  ./cid2spf.pl "<ep xmlns='http://ms.net/1'>...</ep>"
+#
+# Note that the 'include' directives will also have to be checked and
+# "translated". Future versions of this script might be able to get a
+# domain name as an argument and "crawl" the DNS for the necessary
+# information.
+#
+# A complete reverse translation (SPF -> CID) might be impossible, since
+# there are no way to handle:
+# - PTR and EXISTS mechanism 
+# - MX mechanism with an different domain as argument
+# - macros
+# 
+# References:
+# http://www.microsoft.com/mscorp/twc/privacy/spam_callerid.mspx
+# http://spf.pobox.com/
+#
+# Known bugs:
+# - Currently it won't handle the exclusions provided in the A and R
+#   tags (prefix '!'). They will show up "as-is" in the SPF record
+# - I really haven't read the MS-CID specs in-depth, so there are probably
+#   other bugs too :)
+#
+# Ernesto Baschny <ernst@baschny.de>
+#
+
+import xml.sax
+import spf
+
+# -------------------------------------------------------------------------
+class CIDParser(xml.sax.ContentHandler):
+  "Convert a MS Caller-ID entry (XML) to a SPF entry"
+
+  def __init__(self,q=None):
+    self.spf = []
+    self.action = '-all'
+    self.has_servers = None
+    self.spf_entry = None
+    if q:
+      self.spf_query = q
+    else:
+      self.spf_query = spf.query(i='127.0.0.1', s='localhost', h='unknown')
+
+  def startElement(self,tag,attr):
+      if tag == 'm':
+	if self.has_servers != None and not self.has_servers:
+	  raise ValueError(
+    "Declared <noMailServers\> and later <m>, this CID entry is not valid."
+	  )
+	self.has_servers = True
+      elif tag == 'noMailServers':
+	if self.has_servers:
+	  raise ValueError(
+    "Declared <m> and later <noMailServers\>, this CID entry is not valid."
+	  )
+	self.has_servers = False
+      elif tag == 'ep':
+	if attr.has_key('testing') and attr.getValue('testing') == 'true':
+	  # A CID with 'testing' found:
+	  # From the MS-specs:
+	  #  "Documents in which such attribute is present with a true
+	  #  value SHOULD be entirely ignored (one should act as if the
+	  #  document were absent)"
+	  # From the SPF-specs:
+	  #  "Neutral (?): The SPF client MUST proceed as if a domain did
+	  #  not publish SPF data."
+	  # So we set SPF action to "neutral":
+	  self.action = '?all'
+      elif tag == 'mx':
+	  # The empty MX-tag, same as SPF's MX-mechanism
+	  self.spf.append('mx')
+      self.tag = tag
+
+  def characters(self,text):
+	tag = self.tag
+	# Remove starting and trailing spaces from text:
+	text = text.strip()
+
+	if tag == 'a' or tag == 'r':
+	    # The A and R tags from MS-CID are both handled by the 
+	    # ipv4/6-mechanisms from SPF:
+	    if text.find(':') < 0:
+	      mechanism = 'ip4'
+	    else:
+	      mechanism = 'ip6'
+	    self.spf.append(mechanism + ':' + text)
+	elif tag == 'indirect':
+	    # MS-CID's indirect is "sort of" the include from SPF:
+	    # Not really true, because the <indirect> tag from MS-CID also 
+	    # provides a fallback in case the included domain doesn't provide
+	    # _ep-records: The inbound MX-servers of the included domains
+	    # are added to the list of allowed outgoing mailservers for the
+	    # domain that declared the _ep-record with the <indirect> tag.
+	    # In SPF you would use the 'mx:domain' to handle this, but this
+	    # wouldn't depend on referred domain having or not SPF-records.
+	    cid_xml = self.cid_txt(text)
+	    if cid_xml:
+	      p = CIDParser()
+	      xml.sax.parseString(cid_xml,p)
+	      if p.has_servers != False:
+		self.spf += p.spf
+	    else:
+	      self.spf.append('mx:' + text)
+
+  def cid_txt(self,domain):
+    q = self.spf_query
+    domain='_ep.' + domain
+    a = q.dns_txt(domain)
+    if not a: return None
+    if a[0].lower().startswith('<ep ') and a[-1].lower().endswith('</ep>'):
+      return ''.join(a)
+    return None
+
+  def endElement(self,tag):
+      if tag == 'ep':
+	# This is the end... assemble what we've got
+	spf_entry = ['v=spf1']
+	if self.has_servers != False:
+	  spf_entry += self.spf
+	spf_entry.append(self.action)
+	self.spf_entry = ' '.join(spf_entry)
+
+  def spf_txt(self,cid_xml):
+    if not cid_xml.startswith('<'):
+      cid_xml = self.cid_txt(cid_xml)
+      if not cid_xml: return None
+    # Parse the beast. Any XML-problem will be reported by xlm.sax
+    self.spf_entry = None
+    xml.sax.parseString(cid_xml,self)
+    return self.spf_entry
+
+if __name__ == '__main__':
+  import sys
+  if len(sys.argv) < 2:
+    print >>sys.stderr, \
+      """Usage: %s "<ep xmlns='http://ms.net/1'>...</ep>" """ % sys.argv[0]
+    sys.exit(1)
+
+  cid_xml = sys.argv[1]
+
+  p = CIDParser()
+  print p.spf_txt(cid_xml)

faq.html

@@ -134,5 +134,26 @@ is a milter declaration for sendmail.cf with all timeouts specified:
 Xpythonfilter, S=local:/var/log/milter/pythonsock, F=T, T=C:5m;S:20s;R:60s;E:5m
 </pre>
 
+<a name="spf">
+<li> Q. So how do I use the SPF support?  The sample.py milter doesn't seem
+        to use it.
+<p>  A. The bms.py milter supports spf.  The RedHat RPMs will set almost
+everything up for you.  For other systems:
+<ol type=i>
+<li> Arrange to run bms.py in the background (as a service perhaps) and
+     redirect output and errors to a logfile.  For instance, on AIX you'll want
+     to use SRC (System Resource Controller).  
+<li> Copy milter.cfg to the directory you run bms.py in, and edit it.  The
+     comments should explain the options. 
+<li> Start bms.py in the background as arranged.
+<li> Add Xpythonfilter to sendmail.cf or add an INPUT_MAIL_FILTER to
+     sendmail.mc.  Regen sendmail.cf if you use sendmail.mc and restart 
+     sendmail.
+<li> Arrange to rotate log files and remove old defang files in 
+     <code>tempdir</code>.  The RedHat RPM uses <code>logrotate</code> for
+     logfiles and a simple cron script using <code>find</code> to clean
+     <code>tempdir</code>.
+</ol>
+
 </ol>
 </html>

milter.cfg

@@ -1,54 +1,79 @@
 # features intended to filter or block incoming mail
 [milter]
+# the socket used to communicate with sendmail.  Must match sendmail.cf
+;socket=/var/run/milter/pythonsock
+# where to save original copies of defanged and failed messages
 tempdir = /var/log/milter/save
+# how long to wait for a response from sendmail before giving up 
+;timeout=600
+
+# do virus scanning on attached messages also
 scan_rfc822 = 1
-# can be CPU intensive
+# Comment out scripts in HTML attachments.  Can be CPU intensive.
 scan_html = 0
+# reject messages with asian fonts because we can't read them
 block_chinese = 1
+# list users who hate forwarded mail
 ;block_forward = egghead@mycorp.com, busybee@mycorp.com
 log_headers = 0
 # Reject mail for domains mentioned unless user is mentioned here also
 ;check_user = joe@mycorp.com, mary@mycorp.com, file:bigcorp.com
-# porn words are case insensitive
+# reject mail with these case insensitive strings in the subject
 porn_words = penis, breast, pussy, horse cock, porn, xenical, diet pill, d1ck,
 	vi*gra, vi-a-gra, viag, tits, p0rn, hunza, horny, sexy, c0ck,
-	p-e-n-i-s, hydrocodone, vicodin, xanax, vicod1n, x@nax
+	p-e-n-i-s, hydrocodone, vicodin, xanax, vicod1n, x@nax, diazepam,
-# spam words are case sensitive
+	v1@gra, xan@x, cialis, ci@lis, frëe, xãnax, valíum, vãlium, via-gra,
+	x@n3x, vicod3n, penís, c0d1n, phentermine, en1arge, dip1oma, v1codin
+# reject mail with these case sensitive strings in the subject
 spam_words = $$$, !!!, XXX, FREE, HGH
 
 # connection ips and hostnames are matched against this glob style list
 # to recognize internal senders
 ;internal_connect = 192.168.*.*
+
 # mail that is not an internal_connect and claims to be from an
-# internal domain is rejected.
+# internal domain is rejected.  You should enable SPF instead if you can.
+# SPF is much more comprehensive and flexible.
 ;internal_domains = mycorp.com
+
 # connections from a trusted relay can trust the first Received header
+# SPF checks are bypassed for internal connections and trusted relays.
 ;trusted_relay = 1.2.3.4, 66.12.34.56
+
 # reject external senders with hello names no legit external sender would use
+# SPF will do this also, but listing your own domain and mailserver here
+# will save some DNS lookups when rejecting certain viruses.
 ;hello_blacklist = mycorp.com, 66.12.34.56
 
+# See http://bmsi.com/python/pysrs.html for details
 [srs]
 config=/etc/mail/pysrs.cfg
+# SRS options can be set here also, but must match the sendmail plugin
 ;secret="shhhh!"
 ;maxage=21
 ;hashlength=4
 ;database=/var/log/milter/srsdata
 ;fwdomain = mydomain.com
-# turn this on after a grace period
+# turn this on after a grace period to reject spoofed DSNs
 reject_spoofed = 0
 
+# See http://spf.pobox.com for more info on SPF.
 [spf]
 # namespace where SPF records can be supplied for domains without one
-# records are search for under _spf.domain.com
+# records are searched for under _spf.domain.com
 ;delegate = domain.com
 # domains where a neutral SPF result should cause mail to be rejected
 ;reject_neutral = aol.com
+# use a default (v=spf1 a/24 mx/24 ptr) when no SPF records are published
+;best_guess = 0
+# reject senders that have neither PTR nor SPF records
+;reject_noptr = 0
 
 # features intended to clean up outgoing mail
 [scrub]
-# domains that stupidly block visible private nodes
+# domains that block visible private nodes
 ;hide_path = jcpenney.com	
-# block, don't just replace with warning, viruses from these domains
+# reject, don't just replace with warning, viruses from these domains
 ;reject_virus_from = mycorp.com
 
 # features intended for spying on users and coworkers
@@ -77,22 +102,26 @@ blind = 1
 ;walter1 = cust@othercorp.com,walter@bigcorp.com,boss@bigcorp.com,
 ;	walter@bigcorp.com
 
+# See http://bmsi.com/python/dspam.html
 [dspam]
-# Select a well moderated dspam dictionary to reject spammy headers
+# Select a well moderated dspam dictionary to reject spammy headers.
-# dspam-python must be installed to use: http://bmsi.com/python/dspam.html
+# To filter on the entire message, use the full setup below.
 # only EXTERNAL messages are dspam filtered
 ;dspam_dict=/var/lib/dspam/moderator.dict
+
 # Opt-opt recipients from dspam screening and header triage
 ;dspam_exempt=getitall@mycorp.com
 # Do not scan mail (ostensibly) from these senders
 ;dspam_whitelist=getitall@sender.com
-# Reject spam to these domains, perhaps because we are a backup MX server
+# Reject spam to these domains instead of quarantining it.
 ;dspam_reject=othercorp.com
 
 # directory for dspam user quarantine, signature db, and dictionaries
 # defining this activates the dspam application
 # dspam and dspam-python must be installed
 ;dspam_userdir=/var/lib/dspam
+# do not dspam messages larger than this
+;dspam_sizelimit=180000
 
 # Map email addresses and aliases to dspam users
 ;dspam_users=david,goliath,spam,falsepositive
@@ -102,8 +131,9 @@ blind = 1
 ;spam=spam@foocorp.com
 # address to forward false positives to.  milter will process and not deliver
 ;falsepositive=ham@foocorp.com
-# the dspam_screener is used to screen mail for all recipients who are
+# the dspam_screener is a list of dspam users who screen mail for all
-# not dspam_users.  Spam goes to the screeners quarantine, and the original
+# recipients who are not dspam_users.  Spam goes to the screeners quarantine,
-# recipients saved so that false positives can be properly delivered.
+# and the original recipients are saved so that false positives can be properly
-
+# delivered.
+;dspam_screener=david,goliath
 # The dspam CGI can also be used: logins must match dspam users

milter.html

@@ -13,8 +13,8 @@ ALT="Viewable With Any Browser" BORDER="0"></A>
 	usemap="#banner_4" alt="Your vote?">
 <map name="banner_4">
   <area shape="rect" coords="330,25,426,59"
-  	href="http://www.sepschool.org/survey/" alt="I Disagree">
+  	href="http://education-survey.org/" alt="I Disagree">
-  <area shape="rect" coords="234,28,304,57" href="http://sepschool.org/" alt="I Agree">
+  <area shape="rect" coords="234,28,304,57" href="http://www.honestEd.com/" alt="I Agree">
 </map>
 
 </P>
@@ -24,12 +24,14 @@ ALT="Viewable With Any Browser" BORDER="0"></A>
   Stuart D. Gathman</a><br>
 This web page is written by Stuart D. Gathman<br>and<br>sponsored by
 <a href="http://www.bmsi.com">Business Management Systems, Inc.</a> <br>
-Last updated Apr 05, 2004</h4>
+Last updated Aug 06, 2004</h4>
 
 See the <a href="faq.html">FAQ</a> | <a href="#download">Download now</a> |
-<a href="/mailman/listinfo/pymilter">Subscribe to mailing list</a>
+<a href="/mailman/listinfo/pymilter">Subscribe to mailing list</a> |
+<a href="#overview">Overview</a>
 <p>
-<img src="python55.gif" align=left alt="A Python">
+<a href="//www.python.org">
+<img src="python55.gif" align=left alt="A Python"></a>
 <a href="//www.sendmail.org/">Sendmail</a> introduced a
 <a href="http://www.milter.org/milter_api/api.html"> new API</a> beginning with version 8.10 -
 libmilter.  The milter module for <a href="//www.python.org">Python</a>
@@ -40,20 +42,34 @@ Version 8.12 seems to be more robust, and includes new privilege
 separation features to enhance security.
 I recommend upgrading.
 
-<h2> <a name=dspam>Bayesian Filtering</a> </h2>
+<h2> Recent Changes </h2>
-
+The RPM for release 0.7.0 moves the config file and socket locations to
+/etc/mail and /var/run/milter respectively.  We now parse Microsoft CID records
+- but only hotmail.com uses them.  They seem to have a patent on the brilliant
+idea of examining the mail headers to see who the message is from.
+We aren't doing that here, so not to worry - but I am not a lawyer, so if you
+are worried, change spf.py around line 626 to return None instead of
+calling CIDParser().  There is a new option to reject mail with no PTR
+and no SPF.
+<p>
+<a href="http://spf.pobox.com">
+<img src="SPF.gif" align=left alt="SPF logo"></a>
+Release 0.6.6 adds support for <a href="http://spf.pobox.com/">SPF</a>,
+a protocol to prevent forging of the envelope from address.  
+SPF support requires <a href="http://pydns.sourceforge.net/">pydns</a>.
+The included spf.py module is an updated version of the original 1.6
+version at <a href="http://www.wayforward.net/spf/">wayforward.net</a>.
+The updated version tracks the draft RFC and test suite.
+<p>
+The FAQ addresses <a href="faq.html#spf">how to get started with SPF</a>.
+<p>
+Release 0.6.1 adds a full milter based dspam application.
+<p>
 I have selected the <a href="http://www.nuclearelephant.com/projects/dspam/">
 dspam bayes filter project</a> and <a href="dspam.html">
 packaged it for python</a>.
 Release 0.6.0 offers a simple application of dspam I call "header triage",
-which rejects messages with spammy headers.  Since sendmail has to
+which rejects messages with spammy headers.  
-read the entire message anyway once we start reading headers, it
-would probably be better to scan the whole message - except that 
-we replace dangerous attachments elsewhere in the milter  - which screws up the
-body statistics for messages with dangerous attachments.
-<p>
-Release 0.6.1 adds a full milter based dspam application.
-<p>
 To use header triage, you must have <a href="dspam.html">DSPAM</a> installed,
 and select a dictionary that is well moderated by someone who gets
 lots of spam.  That dictionary can be used to block spam that is 
@@ -102,7 +118,7 @@ entries.  Be sure to handle both Bcc and file copies, and designating what
 mail should be copied.  How should "outgoing" be defined?  Implementing it is
 easy once the configuration is designed.
 
-<h3>Overview</h3>
+<h3><a name=overview>Overview</a></h3>
 
 This package provides a robust toolkit for Python <a
 href="#milter">milters</a>, and the beginnings of a general purpose mail
@@ -134,20 +150,78 @@ methods that
 do nothing, and also provides wrappers for the libmilter methods to mutate
 the message.
 <p>
+The 'spf' module provides an implementation of <a href="http://spf.pobox.com">
+SPF</a> useful for detecting email forgery.
+<p>
+The 'mime' module provides a wrapper for the Python email package that
+fixes some bugs, and simplifies modifying selected parts of a MIME message.
+<p>
 Finally, the bms.py application is both a sample of how to use the
-Milter module, and the beginnings of a general purpose SPAM filtering,
+Milter and spf modules, and the beginnings of a general purpose SPAM filtering,
-wiretapping, and Win32 virus protection milter.
+wiretapping, SPF checking, and Win32 virus protecting milter.  It can
+make use of the <a href="pysrs.html">pysrs</a> package when available for
+SRS/SES checking and the <a href="dspam.html">pydspam</a> package for Bayesian
+content filtering.  SPF checking
+requires <a href="http://pydns.sourceforge.net/">
+pydns</a>.  Configuration documentation is currently included as comments
+in the <a href="milter.cfg">sample config file</a> for the bms.py milter.
 
 <h3><a name=download>Downloading</a></h3>
 
-The latest stable release is <a href="#stable">0.6.6</a>. A stable
+The latest stable release is <a href="#stable">0.7.0</a>. A stable
 release is one which has been installed (and working correctly) on
 production systems long enough to convince me that it is stable.  As
 the package gains more features and complexity, stable will mean no
 bug reports from outside users either.
 <p>
-The latest version is 0.6.7.  See the <a href=NEWS>Change Log</a>.
+The latest version is 0.7.0-1.  See the <a href=NEWS>Change Log</a>.
-
+<p>
+<a name="stable"><b>Stable</b></a>
+<a href="http://bmsi.com/python/milter-0.7.0.tar.gz">
+milter-0.7.0.tar.gz</a> Move config file and default socket location.
+Parse M$ CID records.
+<br>
+<a href="http://bmsi.com/linux/rh72/milter-0.7.0-1.i386.rpm">
+milter-0.7.0-1.i386.rpm</a> Binary RPM for Redhat 7.x, now requires 
+	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
+	python2.3</a>.
+<br>
+<a href="http://bmsi.com/linux/rh9/milter-0.7.0-1rh9.i386.rpm">
+milter-0.7.0-1rh9.i386.rpm</a> Binary RPM for Redhat 9, requires 
+	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
+	python2.3</a>.
+<br>
+<a href="http://bmsi.com/linux/rh9/milter-0.7.0-1.src.rpm">
+milter-0.7.0-1.src.rpm</a> Source RPM for Redhat 9,7.x.  
+<p>
+<a href="http://bmsi.com/python/milter-0.6.9.tar.gz">
+milter-0.6.9.tar.gz</a> Add SPF test suite driver, and validate
+spf.py against test suite.  Add best_guess and get_header to spf.py.
+Libmilter timeout option in config.
+<br>
+<a href="http://bmsi.com/linux/rh72/milter-0.6.9-1.i386.rpm">
+milter-0.6.9-1.i386.rpm</a> Binary RPM for Redhat 7.x, now requires 
+	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
+	python2.3</a>.
+<br>
+<a href="http://bmsi.com/linux/rh9/milter-0.6.9-1.src.rpm">
+milter-0.6.9-1.src.rpm</a> Source RPM for Redhat 9,7.x.  
+<p>
+<a href="http://bmsi.com/python/milter-0.6.8.tar.gz">
+milter-0.6.8.tar.gz</a> Include Received-SPF headers in Dspam analysis.
+Fix sysv init for Redhat 9 and later.  Reject bounces with multiple
+recipients.
+<br>
+<a href="http://bmsi.com/python/milter-0.6.8.patch">milter-0.6.8.patch</a>
+Last minutes fixes from production testing.
+<p>
+<a href="http://bmsi.com/linux/rh72/milter-0.6.8-3.i386.rpm">
+milter-0.6.8-3.i386.rpm</a> Binary RPM for Redhat 7.x, now requires 
+	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
+	python2.3</a>. 
+<br>
+<a href="http://bmsi.com/linux/rh9/milter-0.6.8-3.src.rpm">
+milter-0.6.8-3.src.rpm</a> Source RPM for Redhat 9,7.x.  
 <p>
 <a href="http://bmsi.com/python/milter-0.6.7.tar.gz">
 milter-0.6.7.tar.gz</a> Explicit local socket bug,
@@ -169,7 +243,6 @@ Release 0.6.7-3 patches:
 <li> Reject neutral SPF result for selected domains
 </ul>
 <p>
-<a name="stable"><b>Stable</b></a>
 <a href="http://bmsi.com/python/milter-0.6.6.tar.gz">
 milter-0.6.6.tar.gz</a> Plug another memory leak, 
 <a href="http://spf.pobox.com/">SPF</a> support, hello blacklist.

milter.spec

@@ -1,10 +1,10 @@
 %define name milter
-%define version 0.6.8
+%define version 0.7.1
 %define release 1
 # Redhat 7.x and earlier (multiple ps lines per thread)
-#%define sysvinit rc7
+%define sysvinit milter.rc7
 # RH9, other systems (single ps line per process)
-%define sysvinit rc
+#define sysvinit milter.rc
 %ifos Linux
 %define python python2.3
 %else
@@ -16,7 +16,7 @@ Name: %{name}
 Version: %{version}
 Release: %{release}
 Source: %{name}-%{version}.tar.gz
-#Patch: %{name}.patch
+#Patch: %{name}-%{version}.patch
 Copyright: GPL
 Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-buildroot
@@ -24,8 +24,8 @@ Prefix: %{_prefix}
 Vendor: Stuart D. Gathman <stuart@bmsi.com>
 Packager: Stuart D. Gathman <stuart@bmsi.com>
 Url: http://www.bmsi.com/python/milter.html
-Requires: %{python} >= 2.2.2, sendmail >= 8.12
+Requires: %{python} >= 2.2.2, sendmail >= 8.12.10
-BuildRequires: %{python}-devel >= 2.2.2, sendmail-devel >= 8.12
+BuildRequires: %{python}-devel >= 2.2.2, sendmail-devel >= 8.12.10
 
 %description
 This is a python extension module to enable python scripts to
@@ -43,8 +43,10 @@ env CFLAGS="$RPM_OPT_FLAGS" %{python} setup.py build
 rm -rf $RPM_BUILD_ROOT
 %{python} setup.py install --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES
 mkdir -p $RPM_BUILD_ROOT/var/log/milter
+mkdir -p $RPM_BUILD_ROOT/etc/mail
 mkdir $RPM_BUILD_ROOT/var/log/milter/save
-cp bms.py milter.cfg $RPM_BUILD_ROOT/var/log/milter
+cp bms.py $RPM_BUILD_ROOT/var/log/milter
+cp milter.cfg $RPM_BUILD_ROOT/etc/mail/pymilter.cfg
 
 # logfile rotation
 mkdir -p $RPM_BUILD_ROOT/etc/logrotate.d
@@ -81,7 +83,7 @@ exec >>milter.log 2>&1
 echo $! >/var/run/milter/milter.pid
 EOF
 mkdir -p $RPM_BUILD_ROOT/etc/rc.d/init.d
-cp milter.%{sysvinit} $RPM_BUILD_ROOT/etc/rc.d/init.d/milter
+cp %{sysvinit} $RPM_BUILD_ROOT/etc/rc.d/init.d/milter
 ed $RPM_BUILD_ROOT/etc/rc.d/init.d/milter <<'EOF'
 /^python=/
 c
@@ -103,6 +105,9 @@ mkssys -s milter -p /var/log/milter/start.sh -u 25 -S -n 15 -f 9 -G mail || :
 if [ $1 = 0 ]; then
   rmssys -s milter || :
 fi
+%else
+%post
+echo "pythonsock has moved to /var/run/milter, update /etc/mail/sendmail.cf"
 %endif
 
 %clean
@@ -124,9 +129,33 @@ rm -rf $RPM_BUILD_ROOT
 %dir /var/log/milter/save
 %config /var/log/milter/start.sh
 %config /var/log/milter/bms.py
-%config /var/log/milter/milter.cfg
+%config(noreplace) /etc/mail/pymilter.cfg
 
 %changelog
+* Sun Aug 22 2004 Stuart Gathman <stuart@bmsi.com> 0.7.1-1
+- Handle modifying mislabeled multipart messages without an exception
+- Support setbacklog, setmlreply
+- allow multi-recipient CBV
+- return TEMPFAIL for SPF softfail
+* Fri Jul 23 2004 Stuart Gathman <stuart@bmsi.com> 0.7.0-1
+- SPF check hello name
+- Move pythonsock to /var/run/milter
+- Move milter.cfg to /etc/mail/pymilter.cfg
+- Check M$ style XML CID records by converting to SPF
+- Recognize, but never match ip6 until we properly support it.
+- Option to reject when no PTR and no SPF
+* Fri Apr 09 2004 Stuart Gathman <stuart@bmsi.com> 0.6.9-1
+- Validate spf.py against test suite, and add Received-SPF support to spf.py
+- Support best_guess for SPF
+- Reject numeric hello names
+- Preserve case of local part in sender
+- Make libmilter timeout a config option
+- Fix setup.py to work with python < 2.2.3
+* Tue Apr 06 2004 Stuart Gathman <stuart@bmsi.com> 0.6.8-3
+- Reject invalid SRS immediately for benefit of callback verifiers
+- Fix include bug in spf.py
+* Tue Apr 06 2004 Stuart Gathman <stuart@bmsi.com> 0.6.8-2
+- Bug in check_header
 * Mon Apr 05 2004 Stuart Gathman <stuart@bmsi.com> 0.6.8-1
 - Don't report spoofed unless rcpt looks like SRS
 - Check for bounce with multiple rcpts

miltermodule.c

@@ -33,6 +33,18 @@ $ python setup.py help
      libraries=["milter","smutil","resolv"]
 
  * $Log$
+ * Revision 2.31  2004/08/23 02:24:36  stuart
+ * Support setbacklog
+ *
+ * Revision 2.30  2004/08/21 20:29:53  stuart
+ * Support option of 11 lines max for mlreply.
+ *
+ * Revision 2.29  2004/08/21 04:14:29  stuart
+ * mlreply support
+ *
+ * Revision 2.28  2004/08/21 02:45:21  stuart
+ * Don't leak int constants if module unloaded.
+ *
  * Revision 2.27  2004/04/06 03:19:59  stuart
  * Release 0.6.8
  *
@@ -127,11 +139,20 @@ $ python setup.py help
  *
  */
 
+#ifndef MAX_ML_REPLY
+#define MAX_ML_REPLY 32
+#endif
+#if MAX_ML_REPLY != 1 && MAX_ML_REPLY != 32 && MAX_ML_REPLY != 11
+#error MAX_ML_REPLY must be 1 or 11 or 32
+#endif
+#define _FFR_MULTILINE (MAX_ML_REPLY > 1)
+
 #include <pthread.h>
 #include <netinet/in.h>
 #include <Python.h>
 #include <libmilter/mfapi.h>
 
+
 /* See if we have IPv4 and/or IPv6 support in this OS and in
  * libmilter.  We need to make several macro tests because some OS's
  * may define some if IPv6 is only partially supported, and we may
@@ -746,6 +767,18 @@ milter_setdbg(PyObject *self, PyObject *args) {
   return _generic_return(smfi_setdbg(val), "cannot set debug value");
 }
 
+static char milter_setbacklog__doc__[] =
+"setbacklog(int) -> None\n\
+Set the TCP connection queue size for the milter socket.";
+
+static PyObject *
+milter_setbacklog(PyObject *self, PyObject *args) {
+   int val;
+
+   if (!PyArg_ParseTuple(args, "i:setbacklog", &val)) return NULL;
+   return _generic_return(smfi_setbacklog(val), "cannot set backlog");
+}
+
 static char milter_settimeout__doc__[] =
 "settimeout(int) -> None\n\
 Set the time (in seconds) that sendmail will wait before\n\
@@ -820,13 +853,54 @@ static PyObject *
 milter_setreply(PyObject *self, PyObject *args) {
   char *rcode;
   char *xcode;
-  char *message;
+  char *message[MAX_ML_REPLY];
+  char fmt[MAX_ML_REPLY + 16];
   SMFICTX *ctx;
-  if (!PyArg_ParseTuple(args, "szz:setreply", &rcode, &xcode, &message))
+  int i;
+  strcpy(fmt,"sz|");
+  for (i = 0; i < MAX_ML_REPLY; ++i) {
+    message[i] = 0;
+    fmt[i+3] = 's';
+  }
+  strcpy(fmt+i+3,":setreply");
+  if (!PyArg_ParseTuple(args, fmt,
+	&rcode, &xcode, message
+#if MAX_ML_REPLY > 1
+	,message+1,message+2,message+3,message+4,message+5,message+6,
+	message+7,message+8,message+9,message+10
+#if MAX_ML_REPLY > 11
+	,message+11,message+12,message+13,message+14,message+15,
+	message+16,message+17,message+18,message+19,message+20,
+	message+21,message+22,message+23,message+24,message+25,
+	message+26,message+27,message+28,message+29,message+30,
+	message+31
+#endif
+#endif
+  ))
     return NULL;
   ctx = _find_context(self);
   if (ctx == NULL) return NULL;
-  return _generic_return(smfi_setreply(ctx, rcode, xcode, message),
+#if MAX_ML_REPLY > 1
+  /*
+   * C varargs might be convenient for some things, but they sure are a pain
+   * when the number of args is not known at compile time.
+   */
+  if (message[0] && message[1])
+    return _generic_return(smfi_setmlreply(ctx, rcode, xcode,
+	  message[0],
+	  message[1],message[2],message[3],message[4],message[5],
+	  message[6],message[7],message[8],message[9],message[10],
+#if MAX_ML_REPLY > 11
+	  message[11],message[12],message[13],message[14],message[15],
+	  message[16],message[17],message[18],message[19],message[20],
+	  message[21],message[22],message[23],message[24],message[25],
+	  message[26],message[27],message[28],message[29],message[30],
+	  message[31],
+#endif
+	  (char *)0
+    ), "cannot set reply");
+#endif
+  return _generic_return(smfi_setreply(ctx, rcode, xcode, message[0]),
 			 "cannot set reply");
 }
 
@@ -986,7 +1060,7 @@ milter_getpriv(PyObject *self, PyObject *args) {
   return o;
 }
 
-#if _FFR_QUARANTINE
+#ifdef SMFIF_QUARANTINE
 static char milter_quarantine__doc__[] =
 "quarantine(string) -> None\n\
 Place the message in quarantine.  A string with a description of the reason\n\
@@ -1035,7 +1109,7 @@ static PyMethodDef context_methods[] = {
   { "replacebody", milter_replacebody, METH_VARARGS, milter_replacebody__doc__},
   { "setpriv",     milter_setpriv,     METH_VARARGS, milter_setpriv__doc__},
   { "getpriv",     milter_getpriv,     METH_VARARGS, milter_getpriv__doc__},
-#if _FFR_QUARANTINE
+#ifdef SMFIF_QUARANTINE
   { "quarantine",  milter_quarantine,  METH_VARARGS, milter_quarantine__doc__},
 #endif
 #if _FFR_SMFI_PROGRESS
@@ -1081,6 +1155,7 @@ static PyMethodDef milter_methods[] = {
    { "main",                 milter_main,                 METH_VARARGS, milter_main__doc__},
    { "setdbg",               milter_setdbg,               METH_VARARGS, milter_setdbg__doc__},
    { "settimeout",           milter_settimeout,           METH_VARARGS, milter_settimeout__doc__},
+   { "setbacklog",           milter_setbacklog,           METH_VARARGS, milter_setbacklog__doc__},
    { "setconn",              milter_setconn,              METH_VARARGS, milter_setconn__doc__},
    { "stop",                 milter_stop,                 METH_VARARGS, milter_stop__doc__},
    { NULL, NULL }
@@ -1116,6 +1191,12 @@ allowing one to write email filters directly in Python.\n\
 Libmilter is currently marked FFR, and needs to be explicitly installed.\n\
 See <sendmailsource>/libmilter/README for details on setting it up.\n";
 
+static void setitem(PyObject *d,const char *name,long val) {
+  PyObject *v = PyInt_FromLong(val);
+  PyDict_SetItemString(d,name,v);
+  Py_DECREF(v);
+}
+
 void
 initmilter(void) {
    PyObject *m, *d;
@@ -1125,24 +1206,24 @@ initmilter(void) {
    d = PyModule_GetDict(m);
    MilterError = PyErr_NewException("milter.error", NULL, NULL);
    PyDict_SetItemString(d,"error", MilterError);
-   PyDict_SetItemString(d,"SUCCESS", PyInt_FromLong((long) MI_SUCCESS));
+   setitem(d,"SUCCESS",  MI_SUCCESS);
-   PyDict_SetItemString(d,"FAILURE", PyInt_FromLong((long) MI_FAILURE));
+   setitem(d,"FAILURE",  MI_FAILURE);
-   PyDict_SetItemString(d,"VERSION", PyInt_FromLong((long) SMFI_VERSION));
+   setitem(d,"VERSION",  SMFI_VERSION);
-   PyDict_SetItemString(d,"ADDHDRS", PyInt_FromLong((long) SMFIF_ADDHDRS));
+   setitem(d,"ADDHDRS",  SMFIF_ADDHDRS);
-   PyDict_SetItemString(d,"CHGBODY", PyInt_FromLong((long) SMFIF_CHGBODY));
+   setitem(d,"CHGBODY",  SMFIF_CHGBODY);
-   PyDict_SetItemString(d,"MODBODY", PyInt_FromLong((long) SMFIF_MODBODY));
+   setitem(d,"MODBODY",  SMFIF_MODBODY);
-   PyDict_SetItemString(d,"ADDRCPT", PyInt_FromLong((long) SMFIF_ADDRCPT));
+   setitem(d,"ADDRCPT",  SMFIF_ADDRCPT);
-   PyDict_SetItemString(d,"DELRCPT", PyInt_FromLong((long) SMFIF_DELRCPT));
+   setitem(d,"DELRCPT",  SMFIF_DELRCPT);
-   PyDict_SetItemString(d,"CHGHDRS", PyInt_FromLong((long) SMFIF_CHGHDRS));
+   setitem(d,"CHGHDRS",  SMFIF_CHGHDRS);
-   PyDict_SetItemString(d,"V1_ACTS", PyInt_FromLong((long) SMFI_V1_ACTS));
+   setitem(d,"V1_ACTS",  SMFI_V1_ACTS);
-   PyDict_SetItemString(d,"V2_ACTS", PyInt_FromLong((long) SMFI_V2_ACTS));
+   setitem(d,"V2_ACTS",  SMFI_V2_ACTS);
-   PyDict_SetItemString(d,"CURR_ACTS", PyInt_FromLong((long) SMFI_CURR_ACTS));
+   setitem(d,"CURR_ACTS",  SMFI_CURR_ACTS);
 #ifdef SMFIF_QUARANTINE
-   PyDict_SetItemString(d,"QUARANTINE",PyInt_FromLong((long)SMFIF_QUARANTINE));
+   setitem(d,"QUARANTINE",SMFIF_QUARANTINE);
 #endif
-   PyDict_SetItemString(d,"CONTINUE", PyInt_FromLong((long) SMFIS_CONTINUE));
+   setitem(d,"CONTINUE",  SMFIS_CONTINUE);
-   PyDict_SetItemString(d,"REJECT", PyInt_FromLong((long) SMFIS_REJECT));
+   setitem(d,"REJECT",  SMFIS_REJECT);
-   PyDict_SetItemString(d,"DISCARD", PyInt_FromLong((long) SMFIS_DISCARD));
+   setitem(d,"DISCARD",  SMFIS_DISCARD);
-   PyDict_SetItemString(d,"ACCEPT", PyInt_FromLong((long) SMFIS_ACCEPT));
+   setitem(d,"ACCEPT",  SMFIS_ACCEPT);
-   PyDict_SetItemString(d,"TEMPFAIL", PyInt_FromLong((long) SMFIS_TEMPFAIL));
+   setitem(d,"TEMPFAIL",  SMFIS_TEMPFAIL);
 }

mime.py

@@ -1,4 +1,13 @@
 # $Log$
+# Revision 1.54  2004/08/18 01:59:46  stuart
+# Handle mislabeled multipart messages
+#
+# Revision 1.53  2004/04/24 22:53:20  stuart
+# Rename some local variables to avoid shadowing builtins
+#
+# Revision 1.52  2004/04/24 22:47:13  stuart
+# Convert header values to str
+#
 # Revision 1.51  2004/03/25 03:19:10  stuart
 # Correctly defang rfc822 attachments when boundary specified with
 # content-type message/rfc822.
@@ -52,6 +61,18 @@ except: from email.Parser import nlcre as NLCRE
 
 from email import Errors
 
+class MimeGenerator(Generator):
+    def _dispatch(self, msg):
+        # Get the Content-Type: for the message, then try to dispatch to
+        # self._handle_<maintype>_<subtype>().  If there's no handler for the
+        # full MIME type, then dispatch to self._handle_<maintype>().  If
+        # that's missing too, then dispatch to self._writeBody().
+        main = msg.get_content_maintype()
+	if msg.is_multipart() and main.lower() != 'multipart':
+	  self._handle_multipart(msg)
+	else:
+	  Generator._dispatch(self,msg)
+
 class MimeParser(Parser):
 
     # This is a copy of _parsebody from email.Parser, with a fix
@@ -192,19 +213,19 @@ class MimeParser(Parser):
                 text = firstbodyline + '\n' + text
             container.set_payload(text)
 
-def unquote(str):
+def unquote(s):
     """Remove quotes from a string."""
-    if len(str) > 1:
+    if len(s) > 1:
-        if str.startswith('"'):
+        if s.startswith('"'):
-	  if str.endswith('"'):
+	  if s.endswith('"'):
-            str = str[1:-1]
+            s = s[1:-1]
 	  else: # remove garbage after trailing quote
-	    try: str = str[1:str[1:].index('"')+1]
+	    try: s = s[1:s[1:].index('"')+1]
-	    except: return str
+	    except: return s
-	  return str.replace('\\\\', '\\').replace('\\"', '"')
+	  return s.replace('\\\\', '\\').replace('\\"', '"')
-        if str.startswith('<') and str.endswith('>'):
+        if s.startswith('<') and s.endswith('>'):
-            return str[1:-1]
+            return s[1:-1]
-    return str
+    return s
 
 from types import TupleType
 
@@ -216,21 +237,21 @@ def _unquotevalue(value):
 
 email.Message._unquotevalue = _unquotevalue
 
-def _parseparam(str):
+def _parseparam(s):
     plist = []
-    while str[:1] == ';':
+    while s[:1] == ';':
-	str = str[1:]
+	s = s[1:]
-	end = str.find(';')
+	end = s.find(';')
-	while end > 0 and (str.count('"',0,end) & 1):
+	while end > 0 and (s.count('"',0,end) & 1):
-	  end = str.find(';',end + 1)
+	  end = s.find(';',end + 1)
-	if end < 0: end = len(str)
+	if end < 0: end = len(s)
-	f = str[:end]
+	f = s[:end]
 	if '=' in f:
 	    i = f.index('=')
 	    f = f[:i].strip().lower() + \
 		    '=' + f[i+1:].strip()
 	plist.append(f.strip())
-	str = str[end:]
+	s = s[end:]
     return plist
 
 # Enhance email.Message 
@@ -343,16 +364,22 @@ class MimeMessage(Message):
 
   def dump(self,file,unixfrom=False):
     "Write this message (and all subparts) to a file"
-    g = Generator(file)
+    g = MimeGenerator(file)
     g.flatten(self,unixfrom=unixfrom)
 
+  def as_string(self, unixfrom=False):
+      "Return the entire formatted message as a string."
+      fp = StringIO.StringIO()
+      self.dump(fp,unixfrom=unixfrom)
+      return fp.getvalue()
+
   def getencoding(self):
     return self.get('content-transfer-encoding',None)
 
   # Decode body to stream according to transfer encoding, return encoding name
-  def decode(self,filter):
+  def decode(self,filt):
     try:
-      filter.write(self.get_payload(decode=True))
+      filt.write(self.get_payload(decode=True))
     except:
       pass
     return self.getencoding()
@@ -363,7 +390,7 @@ class MimeMessage(Message):
   def __setitem__(self, name, value):
     rc = Message.__setitem__(self,name,value)
     self.modified = True
-    if self.headerchange: self.headerchange(self,name,value)
+    if self.headerchange: self.headerchange(self,name,str(value))
     return rc
 
   def __delitem__(self, name):
@@ -423,7 +450,7 @@ See your administrator.
 
 def check_name(msg,savname=None,ckname=check_ext):
   "Replace attachment with a warning if its name is suspicious."
-  for (key,name) in msg.getnames():
+  for key,name in msg.getnames():
     badname = ckname(name)
     if badname:
       hostname = socket.gethostname()
@@ -571,7 +598,6 @@ class HTMLScriptFilter(SGMLFilter):
   def handle_comment(self,comment):
     if not self.ignoring: SGMLFilter.handle_comment(self,comment)
 
-
 def check_html(msg,savname=None):
   "Remove scripts from HTML attachments."
   msgtype = msg.get_content_type().lower()
@@ -582,14 +608,14 @@ def check_html(msg,savname=None):
 	msgtype = 'text/html'
   if msgtype == 'text/html':
     out = StringIO.StringIO()
-    filter = HTMLScriptFilter(out)
+    htmlfilter = HTMLScriptFilter(out)
     try:
-      filter.write(msg.get_payload(decode=True))
+      htmlfilter.write(msg.get_payload(decode=True))
-      filter.close()
+      htmlfilter.close()
     #except sgmllib.SGMLParseError:
     except:
       #mimetools.copyliteral(msg.get_payload(),open('debug.out','w')
-      filter.close()
+      htmlfilter.close()
       hostname = socket.gethostname()
       msg.set_payload(
   "An HTML attachment could not be parsed.  The original is saved as '%s:%s'"
@@ -600,7 +626,7 @@ def check_html(msg,savname=None):
       name = "WARNING.TXT"
       msg["Content-Type"] = "text/plain; name="+name
       return Milter.CONTINUE
-    if filter.modified:
+    if htmlfilter.modified:
       msg.set_payload(out)	# remove embedded scripts
       del msg["content-transfer-encoding"]
       email.Encoders.encode_quopri(msg)

rejects.py

@@ -0,0 +1,38 @@
+# Analyze milter log to find abusers
+
+fp = open('/var/log/milter/milter.log','r')
+subdict = {}
+ipdict = {}
+spamcnt = {}
+for line in fp:
+  a = line.split(None,4)
+  if len(a) < 4: continue
+  dt,tm,id,op = a[:4]
+  if op == 'Subject:':
+    if len(a) > 4: subdict[id] = a[4].rstrip()
+  elif op == 'connect':
+    ipdict[id] = a[4].rstrip()
+  elif op in ('eom','dspam'):
+    if id in subdict: del subdict[id]
+    if id in ipdict: del ipdict[id]
+  elif op in ('REJECT:','DSPAM:','SPAM:','abort'):
+    if id in subdict:
+      if id in ipdict:
+        ip = ipdict[id]
+	del ipdict[id]
+	f,host,raw = ip.split(None,2)
+	if host in spamcnt:
+	  spamcnt[host] += 1
+	else:
+	  spamcnt[host] = 1
+      else: ip = ''
+      print dt,tm,op,a[4].rstrip(),subdict[id]
+      del subdict[id]
+    else:
+      print line.rstrip()
+print len(subdict),'leftover entries'
+
+spamlist = filter(lambda x: x[1] > 1,spamcnt.items())
+spamlist.sort(lambda x,y: x[1] - y[1])
+for ip,cnt in spamlist:
+  print cnt,ip

setup.py

@@ -1,10 +1,18 @@
 import os
+import sys
 from distutils.core import setup, Extension
 
 # FIXME: on some versions of sendmail, smutil is renamed to sm
 libs = ["milter", "smutil"]
 
-setup(name = "milter", version = "0.6.8",
+# patch distutils if it can't cope with the "classifiers" or
+# "download_url" keywords
+if sys.version < '2.2.3':
+  from distutils.dist import DistributionMetadata
+  DistributionMetadata.classifiers = None
+  DistributionMetadata.download_url = None
+
+setup(name = "milter", version = "0.7.1",
 	description="Python interface to sendmail milter API",
 	long_description="""\
 This is a python extension module to enable python scripts to
@@ -20,7 +28,10 @@ querying SPF records.
 	url="http://www.bmsi.com/python/milter.html",
 	py_modules=["Milter","mime","spf"],
 	ext_modules=[
-	  Extension("milter", ["miltermodule.c"],libraries=libs),
+	  Extension("milter", ["miltermodule.c"],
+	    libraries=libs,
+	    define_macros = [ ('MAX_ML_REPLY',32) ]
+	  ),
 	],
 	keywords = ['sendmail','milter'],
 	classifiers = [

spf.py

@@ -40,7 +40,38 @@ For news, bugfixes, etc. visit the home page for this implementation at
 #                      ditch the annoying Python 2.4 FutureWarning
 #   18-dec-2003, v1.6, Failures on Intel hardware: endianness.  Use ! on
 #                      struct.pack(), struct.unpack().
+#
+# Development taken over by Stuart Gathman <stuart@bmsi.com> since
+# Terrence is not responding to email.
+#
 # $Log$
+# Revision 1.14  2004/08/23 02:28:24  stuart
+# Remove Perl usage message.
+#
+# Revision 1.13  2004/07/23 19:23:12  stuart
+# Always fail to match on ip6, until we support it properly.
+#
+# Revision 1.12  2004/07/23 18:48:15  stuart
+# Fold CID parsing into spf
+#
+# Revision 1.11  2004/07/21 21:32:01  stuart
+# Handle CID records (Microsoft XML format).
+#
+# Revision 1.10  2004/04/19 22:12:11  stuart
+# Release 0.6.9
+#
+# Revision 1.9  2004/04/18 03:29:35  stuart
+# Pass most tests except -local and -rcpt-to
+#
+# Revision 1.8  2004/04/17 22:17:55  stuart
+# Header comment method.
+#
+# Revision 1.7  2004/04/17 18:22:48  stuart
+# Support default explanation.
+#
+# Revision 1.6  2004/04/06 20:18:02  stuart
+# Fix bug in include
+#
 # Revision 1.5  2004/04/05 22:29:46  stuart
 # SPF best_guess,
 #
@@ -78,6 +109,135 @@ import struct  # for pack() and unpack()
 import time    # for time()
 
 import DNS	# http://pydns.sourceforge.net
+import xml.sax
+
+# -------------------------------------------------------------------------
+# Convert a MS Caller-ID entry (XML) to a SPF entry
+#
+# (c) 2004 by Ernesto Baschny
+# (c) 2004 Python version by Stuart Gathman
+#
+# Date: 2004-02-25
+#
+# A complete reverse translation (SPF -> CID) might be impossible, since
+# there are no way to handle:
+# - PTR and EXISTS mechanism 
+# - MX mechanism with an different domain as argument
+# - macros
+# 
+# References:
+# http://www.microsoft.com/mscorp/twc/privacy/spam_callerid.mspx
+# http://spf.pobox.com/
+#
+# Known bugs:
+# - Currently it won't handle the exclusions provided in the A and R
+#   tags (prefix '!'). They will show up "as-is" in the SPF record
+# - I really haven't read the MS-CID specs in-depth, so there are probably
+#   other bugs too :)
+#
+# Ernesto Baschny <ernst@baschny.de>
+#
+
+class CIDParser(xml.sax.ContentHandler):
+  "Convert a MS Caller-ID entry (XML) to a SPF entry."
+
+  def __init__(self,q=None):
+    self.spf = []
+    self.action = '-all'
+    self.has_servers = None
+    self.spf_entry = None
+    if q:
+      self.spf_query = q
+    else:
+      self.spf_query = query(i='127.0.0.1', s='localhost', h='unknown')
+
+  def startElement(self,tag,attr):
+      if tag == 'm':
+	if self.has_servers != None and not self.has_servers:
+	  raise ValueError(
+    "Declared <noMailServers\> and later <m>, this CID entry is not valid."
+	  )
+	self.has_servers = True
+      elif tag == 'noMailServers':
+	if self.has_servers:
+	  raise ValueError(
+    "Declared <m> and later <noMailServers\>, this CID entry is not valid."
+	  )
+	self.has_servers = False
+      elif tag == 'ep':
+	if attr.has_key('testing') and attr.getValue('testing') == 'true':
+	  # A CID with 'testing' found:
+	  # From the MS-specs:
+	  #  "Documents in which such attribute is present with a true
+	  #  value SHOULD be entirely ignored (one should act as if the
+	  #  document were absent)"
+	  # From the SPF-specs:
+	  #  "Neutral (?): The SPF client MUST proceed as if a domain did
+	  #  not publish SPF data."
+	  # So we set SPF action to "neutral":
+	  self.action = '?all'
+      elif tag == 'mx':
+	  # The empty MX-tag, same as SPF's MX-mechanism
+	  self.spf.append('mx')
+      self.tag = tag
+
+  def characters(self,text):
+	tag = self.tag
+	# Remove starting and trailing spaces from text:
+	text = text.strip()
+
+	if tag == 'a' or tag == 'r':
+	    # The A and R tags from MS-CID are both handled by the 
+	    # ipv4/6-mechanisms from SPF:
+	    if text.find(':') < 0:
+	      mechanism = 'ip4'
+	    else:
+	      mechanism = 'ip6'
+	    self.spf.append(mechanism + ':' + text)
+	elif tag == 'indirect':
+	    # MS-CID's indirect is "sort of" the include from SPF:
+	    # Not really true, because the <indirect> tag from MS-CID also 
+	    # provides a fallback in case the included domain doesn't provide
+	    # _ep-records: The inbound MX-servers of the included domains
+	    # are added to the list of allowed outgoing mailservers for the
+	    # domain that declared the _ep-record with the <indirect> tag.
+	    # In SPF you would use the 'mx:domain' to handle this, but this
+	    # wouldn't depend on referred domain having or not SPF-records.
+	    cid_xml = self.cid_txt(text)
+	    if cid_xml:
+	      p = CIDParser()
+	      xml.sax.parseString(cid_xml,p)
+	      if p.has_servers != False:
+		self.spf += p.spf
+	    else:
+	      self.spf.append('mx:' + text)
+
+  def cid_txt(self,domain):
+    q = self.spf_query
+    domain='_ep.' + domain
+    a = q.dns_txt(domain)
+    if not a: return None
+    if a[0].lower().startswith('<ep ') and a[-1].lower().endswith('</ep>'):
+      return ''.join(a)
+    return None
+
+  def endElement(self,tag):
+      if tag == 'ep':
+	# This is the end... assemble what we've got
+	spf_entry = ['v=spf1']
+	if self.has_servers != False:
+	  spf_entry += self.spf
+	spf_entry.append(self.action)
+	self.spf_entry = ' '.join(spf_entry)
+
+  def spf_txt(self,cid_xml):
+    if not cid_xml.startswith('<'):
+      cid_xml = self.cid_txt(cid_xml)
+      if not cid_xml: return None
+    # Parse the beast. Any XML-problem will be reported by xlm.sax
+    self.spf_entry = None
+    xml.sax.parseString(cid_xml,self)
+    return self.spf_entry
 
 # 32-bit IPv4 address mask
 MASK = 0xFFFFFFFFL
@@ -99,12 +259,13 @@ JOINERS = {'l': '.', 's': '.'}
 RESULTS = {'+': 'pass', '-': 'fail', '?': 'neutral', '~': 'softfail',
            'pass': 'pass', 'fail': 'fail', 'unknown': 'unknown',
 	   'neutral': 'neutral', 'softfail': 'softfail',
-	   'none': 'none' }
+	   'none': 'none', 'deny': 'fail' }
 
 EXPLANATIONS = {'pass': 'sender SPF verified', 'fail': 'access denied',
-                'unknown': 'SPF unknown', 'softfail': 'domain in transition',
+                'unknown': 'SPF unknown',
+		'softfail': 'domain in transition',
 		'neutral': 'access neither permitted nor denied',
-		'none': 'no SPF records'
+		'none': ''
 		}
 
 # if set to a domain name, search _spf.domain namespace if no SPF record
@@ -123,7 +284,7 @@ except NameError:
 # standard default SPF record
 DEFAULT_SPF = 'v=spf1 a/24 mx/24 ptr'
 
-def check(i, s, h,default=None):
+def check(i, s, h,local=None):
 	"""Test an incoming MAIL FROM:<s>, from a client with ip address i.
 	h is the HELO/EHLO domain name.
 
@@ -137,21 +298,7 @@ def check(i, s, h,default=None):
 	#>>> check(i='61.51.192.42', s='liukebing@bcc.com', h='bmsi.com')
 
 	"""
-	if i.startswith('127.'):
+	return query(i=i, s=s, h=h,local=local).check()
-		return ('pass', 250, 'local connections always pass')
-
-	try:
-		q = query(i=i, s=s, h=h)
-		spf = q.dns_spf(q.d)
-		if not spf and default:
-		  spf = default
-		return q.check(spf)
-	except DNS.DNSError:
-		return ('error', 450, 'SPF DNS Error')
-
-def best_guess(i, s, h,spf=DEFAULT_SPF):
-	q = query(i=i, s=s, h=h)
-	return q.check(spf)
 
 class query(object):
 	"""A query object keeps the relevant information about a single SPF
@@ -172,7 +319,7 @@ class query(object):
 
 	Also keeps cache: DNS cache.
 	"""
-	def __init__(self, i, s, h):
+	def __init__(self, i, s, h,local=None):
 		self.i, self.s, self.h = i, s, h
 		self.l, self.o = split_email(s, h)
 		self.t = str(int(time.time()))
@@ -180,6 +327,13 @@ class query(object):
 		self.d = self.o
 		self.p = None
 		self.cache = {}
+		self.exps = dict(EXPLANATIONS)
+		self.local = local	# local policy
+
+	def set_default_explanation(self,exp):
+		exps = self.exps
+		for i in 'softfail','fail','unknown':
+		  exps[i] = exp
 
 	def getp(self):
 		if not self.p:
@@ -190,17 +344,32 @@ class query(object):
 				self.p = self.i
 		return self.p
 
-	def check(self, spf):
+	def best_guess(self,spf=DEFAULT_SPF):
+		"""Return a best guess based on a default SPF record"""
+		return self.check(spf)
+
+	def check(self, spf=None):
 		"""
 	Returns (result, mta-status-code, explanation) where
-		result in ['fail', 'unknown', 'pass']
+	result in ['fail', 'softfail', 'neutral' 'unknown', 'pass', 'error']
 		"""
+		if self.i.startswith('127.'):
+			return ('pass', 250, 'local connections always pass')
+
+		try:
+			if not spf:
+			    spf = self.dns_spf(self.d)
+			if self.local and spf:
+			    spf += ' ' + self.local
 			return self.check1(spf, self.d, 0)
+		except DNS.DNSError:
+			return ('error', 450, 'SPF DNS Error')
 
 	def check1(self, spf, domain, recursion):
 		# spf rfc: 3.7 Processing Limits
 		#
-		if recursion > 10:
+		if recursion > 20:
+			self.prob =  'Mechanisms used too many DNS lookups'
 			return ('unknown', 250, 'SPF recursion limit exceeded')
 		try:
 			tmp, self.d = self.d, domain
@@ -216,20 +385,21 @@ class query(object):
 		"""
 
 		if not spf:
-			return ('none', 250, 'no SPF records')
+			return ('none', 250, EXPLANATIONS['none'])
 
 		# split string by whitespace, drop the 'v=spf1'
 		#
 		spf = spf.split()[1:]
 
 		# copy of explanations to be modified by exp=
-		exps = dict(EXPLANATIONS)
+		exps = self.exps
 		redirect = None
 
 		# no mechanisms at all cause unknown result, unless
 		# overridden with 'default=' modifier
 		#
 		default = 'neutral'
+		self.mech = []		# unknown mechanisms
 
 		# Look for modifiers
 		#
@@ -268,12 +438,21 @@ class query(object):
 
 			if m == 'include':
 			    if arg != self.d:
-					tmp = self.check1(self.dns_spf(arg),
+				res,code,txt = self.check1(self.dns_spf(arg),
 						  arg, recursion + 1)
-					if tmp[0] == 'pass':
+				if res == 'pass':
 					break
-					if tmp[0] != 'fail':
+				if res in ('fail','neutral','softfail'):
-						return tmp
+					continue
+				if res == 'none':
+				  	self.prob = \
+					  'Could not find a valid SPF record'
+				  	res = 'unknown'
+				return res,code,txt
+			    else:
+			    	self.prob = 'Required option is missing'
+				self.mech.append(mech)
+				return ('unknown', 250, 'missing SPF option')
 
 			elif m == 'all':
 				break
@@ -292,9 +471,14 @@ class query(object):
 				             cidrlength):
 					break
 
-			elif m in ('ip4', 'ipv4') and arg != self.d:
+			elif m in ('ip4', 'ipv4', 'ip') and arg != self.d:
 				if cidrmatch(self.i, [arg], cidrlength):
 					break
+			elif m == 'ip6':
+			# Until we support IPV6, we should never
+			# get an IPv6 connection.  So this mech
+			# will never match.
+				pass
 
 			elif m in ('ptr', 'prt'):
 				if domainmatch(self.validated_ptrs(self.i),
@@ -304,7 +488,9 @@ class query(object):
 			else:
 				# unknown mechanisms cause immediate unknown
 				# abort results
-				return ('unknown', 250, mech)
+				self.mech.append(mech)
+				self.prob = 'Unknown mechanism found'
+				return ('unknown',250,'unknown SPF mechanism')
 
 		else:
 			# no matches
@@ -321,7 +507,10 @@ class query(object):
 
 	def get_explanation(self, spec):
 		"""Expand an explanation."""
+		if spec:
 		  return self.expand(''.join(self.dns_txt(self.expand(spec))))
+		else:
+		  return 'explanation : Required option is missing'
 
 	def expand(self, str):
 		"""Do SPF RFC macro expansion.
@@ -422,18 +611,27 @@ class query(object):
 		is found.
 		"""
 		a = [t for t in self.dns_txt(domain) if t.startswith('v=spf1')]
-		if not a and DELEGATE:
+		if not a:
+		  if DELEGATE:
 		    a = [t
 		      for t in self.dns_txt(domain+'._spf.'+DELEGATE)
 			if t.startswith('v=spf1')
 		    ]
+		  if not a:
+		    # No SPF record: convert and return CID if present
+		    p = CIDParser(q=self)
+		    return p.spf_txt(domain)
+
 		if len(a) == 1:
 			return a[0]
 		else:
 			return None
 
 	def dns_txt(self, domainname):
+		"Get a list of TXT records for a domain name."
+		if domainname:
 		  return [t for a in self.dns(domainname, 'TXT') for t in a]
+		return []
 
 	def dns_mx(self, domainname):
 		"""Get a list of IP addresses for all MX exchanges for a
@@ -490,6 +688,46 @@ class query(object):
 			result = self.dns(cname, qtype)
 		return result
 
+	def get_header(self,res,receiver):
+	  if res in ('pass','fail'):
+	    return '%s (%s: %s) client-ip=%s; envelope-from=%s; helo=%s;' % (
+	  	res,receiver,self.get_header_comment(res),self.i,
+	        self.l + '@' + self.o, self.h)
+	  if res == 'unknown':
+	    return '%s (%s: %s)' % (' '.join([res] + self.mech),
+	      receiver,self.get_header_comment(res))
+	  return '%s (%s: %s)' % (res,receiver,self.get_header_comment(res))
+
+	def get_header_comment(self,res):
+		"""Return comment for Received-SPF header.
+		"""
+		sender = self.o
+		if res == 'pass':
+		  if self.i.startswith('127.'):
+		    return "localhost is always allowed."
+		  else: return \
+		    "domain of %s designates %s as permitted sender" \
+			% (sender,self.i)
+		elif res == 'softfail': return \
+      "transitioning domain of %s does not designate %s as permitted sender" \
+			% (sender,self.i)
+		elif res == 'neutral': return \
+		    "%s is neither permitted nor denied by domain of %s" \
+		    	% (self.i,sender)
+		elif res == 'none': return \
+		    "%s is neither permitted nor denied by domain of %s" \
+		    	% (self.i,sender)
+		    #"%s does not designate permitted sender hosts" % sender
+		elif res == 'unknown': return \
+		    "error in processing during lookup of domain of %s: %s" \
+		    	% (sender, self.prob)
+		elif res == 'error': return \
+		    "error in processing during lookup of %s" % sender
+		elif res == 'fail': return \
+		    "domain of %s does not designate %s as permitted sender" \
+			% (sender,self.i)
+		raise ValueError("invalid SPF result for header comment: "+res)
+
 def split_email(s, h):
 	"""Given a sender email s and a HELO domain h, create a valid tuple
 	(l, d) local-part and domain-part.

spfquery.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python2.3
+# $Log$
+# Revision 2.3  2004/04/19 22:12:11  stuart
+# Release 0.6.9
+#
+# Revision 2.2  2004/04/18 03:29:35  stuart
+# Pass most tests except -local and -rcpt-to
+#
+# Revision 2.1  2004/04/08 18:41:15  stuart
+# Reject numeric hello names
+#
+# Driver for SPF test system
+
+import spf
+import sys
+
+from optparse import OptionParser
+
+class PerlOptionParser(OptionParser):
+    def _process_args (self, largs, rargs, values):
+        """_process_args(largs : [string],
+                         rargs : [string],
+                         values : Values)
+
+        Process command-line arguments and populate 'values', consuming
+        options and arguments from 'rargs'.  If 'allow_interspersed_args' is
+        false, stop at the first non-option argument.  If true, accumulate any
+        interspersed non-option arguments in 'largs'.
+        """
+        while rargs:
+            arg = rargs[0]
+            # We handle bare "--" explicitly, and bare "-" is handled by the
+            # standard arg handler since the short arg case ensures that the
+            # len of the opt string is greater than 1.
+            if arg == "--":
+                del rargs[0]
+                return
+            elif arg[0:2] == "--":
+                # process a single long option (possibly with value(s))
+                self._process_long_opt(rargs, values)
+            elif arg[:1] == "-" and len(arg) > 1:
+                # process a single perl style long option
+		rargs[0] = '-' + arg
+                self._process_long_opt(rargs, values)
+            elif self.allow_interspersed_args:
+                largs.append(arg)
+                del rargs[0]
+            else:
+		return
+
+def format(q):
+  res,code,txt = q.check()
+  print res
+  if res in ('pass','neutral','unknown'): print
+  else: print txt
+  print 'spfquery:',q.get_header_comment(res)
+  print 'Received-SPF:',q.get_header(res,'spfquery')
+
+def main(argv):
+  parser = PerlOptionParser()
+  parser.add_option("--file",dest="file")
+  parser.add_option("--ip",dest="ip")
+  parser.add_option("--sender",dest="sender")
+  parser.add_option("--helo",dest="hello_name")
+  parser.add_option("--local",dest="local_policy")
+  parser.add_option("--rcpt-to",dest="rcpt")
+  parser.add_option("--default-explanation",dest="explanation")
+  parser.add_option("--sanitize",type="int",dest="sanitize")
+  parser.add_option("--debug",type="int",dest="debug")
+  opts,args = parser.parse_args(argv)
+  if opts.ip:
+    q = spf.query(opts.ip,opts.sender,opts.hello_name,local=opts.local_policy)
+    if opts.explanation:
+      q.set_default_explanation(opts.explanation)
+    format(q)
+  if opts.file:
+    if opts.file == '0':
+      fp = sys.stdin
+    else:
+      fp = open(opts.file,'r')
+    for ln in fp:
+      ip,sender,helo,rcpt = ln.split(None,3)
+      q = spf.query(ip,sender,helo,local=opts.local_policy)
+      if opts.explanation:
+	q.set_default_explanation(opts.explanation)
+      format(q)
+    fp.close()
+    
+if __name__ == "__main__":
+  import sys
+  main(sys.argv[1:])