Release 0.7.2

MANIFEST.in

@@ -20,3 +20,4 @@ include start.sh
 include milter.rc
 include milter.rc7
 include milter.cfg
+include rhsbl.m4

Milter.py

@@ -8,15 +8,12 @@ import milter
 import thread
 
 from milter import ACCEPT,CONTINUE,REJECT,DISCARD,TEMPFAIL,	\
-  set_flags, setdbg, \
+  set_flags, setdbg, setbacklog, settimeout, \
   ADDHDRS, CHGBODY, ADDRCPT, DELRCPT, CHGHDRS,	\
   V1_ACTS, V2_ACTS, CURR_ACTS
 
-try:
+try: from milter import QUARANTINE
-  from milter import QUARANTINE
+except: pass
-except:
-  #print 'No QUARANTINE support'
-  pass
 
 _seq_lock = thread.allocate_lock()
 _seq = 0
@@ -100,8 +97,10 @@ class Milter:
   def getsymval(self,sym):
     return self.__ctx.getsymval(sym)
 
-  def setreply(self,rcode,xcode,msg):
+  # If sendmail does not support setmlreply, then only the
-    return self.__ctx.setreply(rcode,xcode,msg)
+  # first msg line is used.
+  def setreply(self,rcode,xcode=None,msg=None,*ml):
+    return self.__ctx.setreply(rcode,xcode,msg,*ml)
 
   # Milter methods which can only be called from eom callback.
   def addheader(self,field,value):
@@ -119,6 +118,8 @@ class Milter:
   def replacebody(self,body):
     return self.__ctx.replacebody(body)
 
+  # When quarantined, a message goes into the mailq as if to be delivered,
+  # but delivery is deferred until the message is unquarantined.
   def quarantine(self,reason):
     return self.__ctx.quarantine(reason)
 

NEWS

@@ -1,5 +1,18 @@
 Here is a history of user visible changes to Python milter.
 
+0.7.2	Return unknown for invalid ip address in mechanism
+	Recognize dynamic PTR names, and don't count them as authentication.
+	Three strikes and yer out rule.
+	Block softfail by default when no PTR or HELO
+	Return unknown for null mechanism
+	Return unknown for invalid ip address in mechanism
+	Try best guess on HELO also
+	Expand setreply for common errors
+	make rhsbl.m4 hack available for sendmail.mc
+0.7.1	Handle modifying mislabeled multipart messages without an exception
+	Support setbacklog, setmlreply
+	Allow multi-recipient CBV
+	Return TEMPFAIL for SPF softfail
 0.7.0	SPF check hello name
 	Move pythonsock to /var/run/milter
 	Move milter.cfg to /etc/mail/pymilter.cfg

TODO

@@ -1,21 +1,30 @@
-Message not saved for following traceback:
+Checking in mime.py;
-Traceback (most recent call last):
+/bms/cvs/milter/mime.py,v  <--  mime.py
-  File "/usr/lib/python2.3/site-packages/Milter.py", line 188, in <lambda>
+new revision: 1.56; previous revision: 1.55
-    milter.set_eom_callback(lambda ctx: ctx.getpriv().eom())
+done
-  File "bms.py", line 935, in eom
+Checking in spf.py;
-    msg.dump(out)
+/bms/cvs/milter/spf.py,v  <--  spf.py
-  File "/usr/lib/python2.3/site-packages/mime.py", line 347, in dump
+new revision: 1.18; previous revision: 1.17
-    g.flatten(self,unixfrom=unixfrom)
+done
-  File "/var/tmp/python2.3-2.3.3-root/usr/lib/python2.3/email/Generator.py", line 102, in flatten
+Checking in testmime.py;
-  File "/var/tmp/python2.3-2.3.3-root/usr/lib/python2.3/email/Generator.py", line 130, in _write
+/bms/cvs/milter/testmime.py,v  <--  testmime.py
-  File "/var/tmp/python2.3-2.3.3-root/usr/lib/python2.3/email/Generator.py", line 156, in _dispatch
+new revision: 1.19; previous revision: 1.18
-  File "/var/tmp/python2.3-2.3.3-root/usr/lib/python2.3/email/Generator.py", line 199, in _handle_text
+
-TypeError: string payload expected: <type 'list'>
+Auto whitelist based on outgoing email - perhaps with magic subject
-------------
+or recipient prefix.
-spf.py has no recursion bound on CNAME lookup
+
+Can't output messages with malformed rfc822 attachments.
+
+Use python exceptions in SPF to cleanly handle unknown and error results.
+
+Example malformed SPF:
+onvunvuvvx.usafisnews.org text "v=spf1 mx ptr ip4:207.44.199.970 -all"
+
+Move milter,Milter,mime,spf modules to pymilter
+milter package will have bms.py application
+
 Support SMTP AUTH and disable SPF checks when connection is authorized.
 Web admin interface
-RHSBL
 Check valid domains allowed by internal senders to detect PCs infected
 with spam trojans.
 Do CBV (callback verification) for mail with no published SPF record.
@@ -63,3 +72,4 @@ Wrap smfi_setbacklog(int) - but it is only available in sendmail >= 8.12.3,
 Need a test module to feed sample messages to a milter though a live 
 sendmail and SMTP.  The mockup currently used is probably not very accurate,
 and doesn't test the threading code.
+

bms.py

@@ -1,6 +1,41 @@
 #!/usr/bin/env python
 # A simple milter.
 # $Log$
+# Revision 1.126  2004/11/24 14:39:38  stuart
+# Also accept softfail if valid PTR or HELO.
+#
+# Revision 1.125  2004/11/19 16:40:14  stuart
+# Block softfail except for listed domains.
+#
+# Revision 1.124  2004/11/19 06:18:04  stuart
+# block softfail for configured domains only
+#
+# Revision 1.123  2004/11/18 20:36:49  stuart
+# Recognize more dynamic hosts.  Ignore dynamic PTR for best_guess.
+#
+# Revision 1.122  2004/11/18 17:16:10  stuart
+# Recognize more dynamic ips.
+#
+# Revision 1.121  2004/11/09 22:37:48  stuart
+# Don't accept helo names which are dynamic IP addresses.
+#
+# Revision 1.120  2004/11/09 20:33:50  stuart
+# Recognize more dynamic PTR variations.
+#
+# Revision 1.118  2004/08/30 21:19:50  stuart
+# Try best guess for HELO, expand setreply for common errors
+#
+# Revision 1.117  2004/08/23 02:27:53  stuart
+# Allow multi rcpt CBV.  Add some multiline replies.
+#
+# Revision 1.116  2004/08/20 22:27:52  stuart
+# Generate TEMPFAIL for SPF softfail.
+#
+# Revision 1.115  2004/08/19 20:55:49  stuart
+# Always show reversed SRS path.
+# Check if encodings are an ASCII superset.  Some messages were encoded as
+# BIG5 and getting rejected even though chars were all in ascii subset.
+#
 # Revision 1.114  2004/07/27 00:40:12  stuart
 # Make reject on no PTR optional.
 #
@@ -281,6 +316,7 @@ srs = None
 srs_reject_spoofed = False
 srs_fwdomain = None
 spf_reject_neutral = ()
+spf_accept_softfail = ()
 spf_best_guess = False
 spf_reject_noptr = False
 timeout = 600
@@ -390,6 +426,7 @@ def read_config(list):
   global dspam_dict, dspam_users, dspam_userdir, dspam_exempt
   global dspam_screener,dspam_whitelist,dspam_reject,dspam_sizelimit
   global spf_reject_neutral,spf_best_guess,SRS,spf_reject_noptr
+  global spf_accept_softfail
   dspam_dict = cp.getdefault('dspam','dspam_dict')
   dspam_exempt = cp.getaddrset('dspam','dspam_exempt')
   dspam_whitelist = cp.getaddrset('dspam','dspam_whitelist')
@@ -403,6 +440,7 @@ def read_config(list):
   if spf:
     spf.DELEGATE = cp.getdefault('spf','delegate')
     spf_reject_neutral = cp.getlist('spf','reject_neutral')
+    spf_accept_softfail = cp.getlist('spf','accept_softfail')
     spf_best_guess = cp.getboolean('spf','best_guess')
     spf_reject_noptr = cp.getboolean('spf','reject_noptr')
   srs_config = cp.getdefault('srs','config')
@@ -435,7 +473,10 @@ def parse_header(val):
     u = []
     for s,enc in h:
       if enc:
+        try:
 	  u.append(unicode(s,enc))
+	except LookupError:
+	  u.append(unicode(s))
       else:
 	u.append(unicode(s))
     u = ''.join(u)
@@ -443,13 +484,51 @@ def parse_header(val):
       try:
 	return u.encode(enc)
       except UnicodeError: continue
-  except UnicodeDecodeError:
+  except UnicodeDecodeError: pass
-    return val
+  except LookupError: pass
-  except LookupError:
   return val
 
+ip3 = re.compile('([0-9]{1,3})[.-]([0-9]{1,3})[.-]([0-9]{1,3})')
+rehmac = re.compile('h[0-9a-f]{12}[.]|pcp[0-9]{6,10}pcs[.]|no-reverse')
+
+def dynip(host,addr):
+  """Return True if hostname is for a dynamic ip.
+  Examples:
+
+  >>> is_dynip('post3.fabulousdealz.com','69.60.99.112')
+  False
+  >>> is_dynip('adsl-69-208-201-177.dsl.emhril.ameritech.net','69.208.201.177')
+  True
+  """
+  if host.startswith('[') and host.endswith(']'):
+    return True
+  if addr:
+    if host.find(addr) >= 0: return True
+    a = addr.split('.')
+    m = ip3.search(host)
+    if m:
+      g = list(m.groups())
+      if g == a[1:] or g == a[:3]: return True
+      g.reverse()
+      if g == a[1:] or g == a[:3]: return True
+    if rehmac.search(host): return True
+    if host.find("-%s." % '-'.join(a[2:])) >= 0: return True
+    if host.find("w%s." % '-'.join(a[:2])) >= 0: return True
+    if host.find(''.join(a[:3])) >= 0: return True
+    if host.find(''.join(a[1:])) >= 0: return True
+    x = "%02x%02x%02x%02x" % tuple(map(int,a))
+    if host.lower().find(x) >= 0: return True
+    z = [n.zfill(3) for n in a]
+    if host.find('-'.join(z)) >= 0: return True
+    if host.find("-%s." % '-'.join(z[2:])) >= 0: return True
+    if host.find("%s." % ''.join(z[2:])) >= 0: return True
+    if host.find(''.join(z)) >= 0: return True
+  return False
+
 class bmsMilter(Milter.Milter):
-  "Milter to replace attachments poisonous to Windows with a WARNING message."
+  """Milter to replace attachments poisonous to Windows with a WARNING message,
+     check SPF, and other anti-forgery features, and implement wiretapping
+     and smart alias redirection."""
 
   def log(self,*msg):
     print "%s [%d]" % (time.strftime('%Y%b%d %H:%M:%S'),self.id),
@@ -485,7 +564,6 @@ class bmsMilter(Milter.Milter):
     self.log('%s: %s' % (name,val))
 
   def connect(self,hostname,unused,hostaddr):
-    self.missing_ptr = hostname.startswith('[') and hostname.endswith(']')
     self.internal_connection = False
     self.trusted_relay = False
     self.receiver = self.getsymval('j')
@@ -502,6 +580,7 @@ class bmsMilter(Milter.Milter):
       self.connectip = ipaddr
     else:
       self.connectip = None
+    self.missing_ptr = dynip(hostname,self.connectip)
     for pat in internal_connect:
       if fnmatchcase(hostname,pat):
 	self.internal_connection = True
@@ -587,38 +666,78 @@ class bmsMilter(Milter.Milter):
     q.set_default_explanation('SPF fail: see http://spf.pobox.com/why.html')
     res,code,txt = q.check()
     receiver = self.receiver
-    if res == 'none':
+    if res in ('none', 'softfail'):
       if self.mailfrom != '<>':
 	# check hello name via spf
-	hres,hcode,htxt = spf.check(self.connectip,'',self.hello_name)
+	h = spf.query(self.connectip,'',self.hello_name)
+	hres,hcode,htxt = h.check()
 	if hres in ('deny','fail','neutral','softfail'):
-	  self.log('REJECT: hello SPF: %s %i %s' % (hres,hcode,htxt))
+	  self.log('REJECT: hello SPF: %s 550 %s' % (hres,htxt))
-	  self.setreply('550','5.7.1',htxt)
+	  self.setreply('550','5.7.1',htxt,
+	    "The hostname given in your MTA's HELO response is not listed",
+	    "as a legitimate MTA in the SPF records for your domain.  If you",
+	    "get this bounce, the message was not in fact a forgery, and you",
+	    "should IMMEDIATELY notify your email administrator of the problem."
+	  )
 	  return Milter.REJECT
-      if spf_best_guess:
+	if hres == 'none' and spf_best_guess \
+	  and not dynip(self.hello_name,self.connectip):
+	  hres,hcode,htxt = h.best_guess()
+      else: hres = res
+      if spf_best_guess and res == 'none':
 	#self.log('SPF: no record published, guessing')
 	q.set_default_explanation(
 		'SPF guess: see http://spf.pobox.com/why.html')
 	# best_guess should not result in fail
+	if self.missing_ptr:
+	  # ignore dynamic PTR for best guess
+	  res,code,txt = q.best_guess('v=spf1 a/24 mx/24')
+	else:
 	  res,code,txt = q.best_guess()
 	receiver += ': guessing'
-      if self.missing_ptr and res in ('neutral', 'none') and spf_reject_noptr:
+      if self.missing_ptr and res in ('neutral', 'none')	\
-        self.log('REJECT: no PTR or SPF')
+      	and spf_reject_noptr and hres != 'pass':
+        self.log('REJECT: no PTR, HELO or SPF')
 	self.setreply('550','5.7.1',
-  'You must have a reverse lookup or publish SPF: http://spf.pobox.com'
+  'You must have a reverse lookup or publish SPF: http://spf.pobox.com',
+  'Contact your mail administrator IMMEDIATELY!  Your mail server is',
+  'severely misconfigured.  It has no PTR record (dynamic PTR records',
+  "that contain your IP don't count), an invalid HELO, and no SPF record."
 	)
 	return Milter.REJECT
     if res in ('deny', 'fail'):
       self.log('REJECT: SPF %s %i %s' % (res,code,txt))
       self.setreply(str(code),'5.7.1',txt)
       return Milter.REJECT
+    if res == 'softfail' and not q.o in spf_accept_softfail:
+      if self.missing_ptr and spf_reject_noptr and hres != 'pass':
+	self.log('TEMPFAIL: SPF %s 450 %s' % (res,txt))
+	self.setreply('450','4.3.0',
+	  'SPF softfail: will keep trying until your SPF record is fixed.',
+	  'If you get this Delivery Status Notice, your email was probably',
+	  'legitimate.  Your administrator has published SPF records in a',
+	  'testing mode.  The SPF record reported your email as a forgery,',
+	  'which is a mistake if you are reading this.  Please notify your',
+	  'administrator of the problem immediately.'
+	)
+	return Milter.TEMPFAIL
     if res == 'neutral' and q.o in spf_reject_neutral:
       self.log('REJECT: SPF neutral for',q.s)
       self.setreply('550','5.7.1',
-	'mail from %s must pass SPF: http://spf.pobox.com/why.html' % q.o
+	'mail from %s must pass SPF: http://spf.pobox.com/why.html' % q.o,
+	'The %s domain is one that spammers love to forge.  Due to' % q.o,
+	'the volume of forged mail, we can only accept mail that',
+	'the SPF record for %s explicitly designates as legitimate.' % q.o,
+	'Sending your email through the recommended outgoing SMTP',
+	'servers for %s should accomplish this.' % q.o
       )
       return Milter.REJECT
     if res == 'error':
+      if code >= 500:
+        self.log('REJECT: SPF %s %i %s' % (res,code,txt))
+	self.setreply(str(code),'5.7.1',txt)
+	return Milter.REJECT
+      self.log('TEMPFAIL: SPF %s %i %s' % (res,code,txt))
       self.setreply(str(code),'4.3.0',txt)
       return Milter.TEMPFAIL
     self.add_header('Received-SPF',q.get_header(res,receiver))
@@ -639,17 +758,15 @@ class bmsMilter(Milter.Milter):
       if self.mailfrom == '<>' or self.canon_from.startswith('postmaster@') \
       	or self.canon_from.startswith('mailer-daemon@'):
         if self.recipients:
-	  self.log('REJECT: Multiple bounce recipients')
+	  self.data_allowed = False
-	  self.setreply('550','5.7.1','Multiple bounce recipients')
+        if srs and domain == srs_fwdomain:
-	  return Milter.REJECT
-        if srs and not (self.internal_connection or self.trusted_relay) \
-		and domain == srs_fwdomain:
 	  oldaddr = '@'.join(parse_addr(to))
 	  try:
 	    newaddr = srs.reverse(oldaddr)
 	    # Currently, a sendmail map reverses SRS.  We just log it here.
 	    self.log("srs rcpt:",newaddr)
 	  except:
+            if not (self.internal_connection or self.trusted_relay):
 	      if srsre.match(oldaddr):
 		self.log("REJECT: srs spoofed:",oldaddr)
 		self.setreply('550','5.7.1','Invalid SRS signature')
@@ -748,8 +865,12 @@ class bmsMilter(Milter.Milter):
 
   def header(self,name,hval):
     if not self.data_allowed:
+      if len(self.recipients) > 1:
+	self.log('REJECT: Multiple bounce recipients')
+	self.setreply('550','5.7.1','Multiple bounce recipients')
+      else:
 	self.log('REJECT: bounce with no SRS encoding')
-      self.setreply('550','5.7.1',"I did not send you this message.")
+	self.setreply('550','5.7.1',"I did not send you that message.")
       return Milter.REJECT
     lname = name.lower()
     # decode near ascii text to unobfuscate
@@ -757,8 +878,8 @@ class bmsMilter(Milter.Milter):
     if not self.internal_connection:
       # even if we wanted the Taiwanese spam, we can't read Chinese
       if block_chinese and lname == 'subject':
-	if hval.startswith('=?big5') or hval.startswith('=?ISO-2022-JP'):
+	if val.startswith('=?big5') or val.startswith('=?ISO-2022-JP'):
-	  self.log('REJECT: %s: %s' % (name,hval))
+	  self.log('REJECT: %s: %s' % (name,val))
 	  self.setreply('550','5.7.1',"We don't understand chinese")
 	  return Milter.REJECT
       rc = self.check_header(name,val)
@@ -947,6 +1068,9 @@ class bmsMilter(Milter.Milter):
       self.tempname = None
       if exc_type == email.Errors.BoundaryError:
 	self.log("MALFORMED: %s" % fname)	# log filename
+        if self.internal_connection:
+	  # accept anyway for now
+	  return Milter.ACCEPT
 	self.setreply('554','5.7.7',
 		'Boundary error in your message, are you a spammer?')
         return Milter.REJECT

milter.cfg

@@ -1,52 +1,67 @@
 # features intended to filter or block incoming mail
 [milter]
+# the socket used to communicate with sendmail.  Must match sendmail.cf
 ;socket=/var/run/milter/pythonsock
+# where to save original copies of defanged and failed messages
 tempdir = /var/log/milter/save
+# how long to wait for a response from sendmail before giving up 
 ;timeout=600
 
+# do virus scanning on attached messages also
 scan_rfc822 = 1
-# can be CPU intensive
+# Comment out scripts in HTML attachments.  Can be CPU intensive.
 scan_html = 0
-# reject asian fonts because we can't read them
+# reject messages with asian fonts because we can't read them
 block_chinese = 1
-# users who hate forwarded mail
+# list users who hate forwarded mail
 ;block_forward = egghead@mycorp.com, busybee@mycorp.com
 log_headers = 0
 # Reject mail for domains mentioned unless user is mentioned here also
 ;check_user = joe@mycorp.com, mary@mycorp.com, file:bigcorp.com
-# porn words are case insensitive
+# reject mail with these case insensitive strings in the subject
 porn_words = penis, breast, pussy, horse cock, porn, xenical, diet pill, d1ck,
-	vi*gra, vi-a-gra, viag, tits, p0rn, hunza, horny, sexy, c0ck,
+	vi*gra, vi-a-gra, viag, tits, p0rn, hunza, horny, sexy, c0ck, xanaax,
 	p-e-n-i-s, hydrocodone, vicodin, xanax, vicod1n, x@nax, diazepam,
 	v1@gra, xan@x, cialis, ci@lis, frëe, xãnax, valíum, vãlium, via-gra,
-	x@n3x, vicod3n, penís, c0d1n, phentermine, en1arge, dip1oma, v1codin
+	x@n3x, vicod3n, penís, c0d1n, phentermine, en1arge, dip1oma, v1codin,
-# spam words are case sensitive
+	valium, rolex
+# reject mail with these case sensitive strings in the subject
 spam_words = $$$, !!!, XXX, FREE, HGH
 
 # connection ips and hostnames are matched against this glob style list
 # to recognize internal senders
 ;internal_connect = 192.168.*.*
+
 # mail that is not an internal_connect and claims to be from an
-# internal domain is rejected.
+# internal domain is rejected.  You should enable SPF instead if you can.
+# SPF is much more comprehensive and flexible.
 ;internal_domains = mycorp.com
+
 # connections from a trusted relay can trust the first Received header
+# SPF checks are bypassed for internal connections and trusted relays.
 ;trusted_relay = 1.2.3.4, 66.12.34.56
+
 # reject external senders with hello names no legit external sender would use
+# SPF will do this also, but listing your own domain and mailserver here
+# will save some DNS lookups when rejecting certain viruses.
 ;hello_blacklist = mycorp.com, 66.12.34.56
 
+# See http://bmsi.com/python/pysrs.html for details
 [srs]
 config=/etc/mail/pysrs.cfg
+# SRS options can be set here also, but must match the sendmail plugin
 ;secret="shhhh!"
 ;maxage=21
 ;hashlength=4
 ;database=/var/log/milter/srsdata
 ;fwdomain = mydomain.com
-# turn this on after a grace period
+# turn this on after a grace period to reject spoofed DSNs
 reject_spoofed = 0
 
+# See http://spf.pobox.com for more info on SPF.
 [spf]
 # namespace where SPF records can be supplied for domains without one
-# records are search for under _spf.domain.com
+# records are searched for under _spf.domain.com
 ;delegate = domain.com
 # domains where a neutral SPF result should cause mail to be rejected
 ;reject_neutral = aol.com
@@ -54,12 +69,14 @@ reject_spoofed = 0
 ;best_guess = 0
 # reject senders that have neither PTR nor SPF records
 ;reject_noptr = 0
+# always accept softfail from these domains
+;accept_softfail = bounces.amazon.com
 
 # features intended to clean up outgoing mail
 [scrub]
-# domains that stupidly block visible private nodes
+# domains that block visible private nodes
 ;hide_path = jcpenney.com	
-# block, don't just replace with warning, viruses from these domains
+# reject, don't just replace with warning, viruses from these domains
 ;reject_virus_from = mycorp.com
 
 # features intended for spying on users and coworkers
@@ -88,16 +105,18 @@ blind = 1
 ;walter1 = cust@othercorp.com,walter@bigcorp.com,boss@bigcorp.com,
 ;	walter@bigcorp.com
 
+# See http://bmsi.com/python/dspam.html
 [dspam]
-# Select a well moderated dspam dictionary to reject spammy headers
+# Select a well moderated dspam dictionary to reject spammy headers.
-# dspam-python must be installed to use: http://bmsi.com/python/dspam.html
+# To filter on the entire message, use the full setup below.
 # only EXTERNAL messages are dspam filtered
 ;dspam_dict=/var/lib/dspam/moderator.dict
+
 # Opt-opt recipients from dspam screening and header triage
 ;dspam_exempt=getitall@mycorp.com
 # Do not scan mail (ostensibly) from these senders
 ;dspam_whitelist=getitall@sender.com
-# Reject spam to these domains, perhaps because we are a backup MX server
+# Reject spam to these domains instead of quarantining it.
 ;dspam_reject=othercorp.com
 
 # directory for dspam user quarantine, signature db, and dictionaries
@@ -115,8 +134,9 @@ blind = 1
 ;spam=spam@foocorp.com
 # address to forward false positives to.  milter will process and not deliver
 ;falsepositive=ham@foocorp.com
-# the dspam_screener is used to screen mail for all recipients who are
+# the dspam_screener is a list of dspam users who screen mail for all
-# not dspam_users.  Spam goes to the screeners quarantine, and the original
+# recipients who are not dspam_users.  Spam goes to the screeners quarantine,
-# recipients saved so that false positives can be properly delivered.
+# and the original recipients are saved so that false positives can be properly
-
+# delivered.
+;dspam_screener=david,goliath
 # The dspam CGI can also be used: logins must match dspam users

milter.html

@@ -13,8 +13,8 @@ ALT="Viewable With Any Browser" BORDER="0"></A>
 	usemap="#banner_4" alt="Your vote?">
 <map name="banner_4">
   <area shape="rect" coords="330,25,426,59"
-  	href="http://www.sepschool.org/survey/" alt="I Disagree">
+  	href="http://education-survey.org/" alt="I Disagree">
-  <area shape="rect" coords="234,28,304,57" href="http://sepschool.org/" alt="I Agree">
+  <area shape="rect" coords="234,28,304,57" href="http://www.honestEd.com/" alt="I Agree">
 </map>
 
 </P>
@@ -24,12 +24,14 @@ ALT="Viewable With Any Browser" BORDER="0"></A>
   Stuart D. Gathman</a><br>
 This web page is written by Stuart D. Gathman<br>and<br>sponsored by
 <a href="http://www.bmsi.com">Business Management Systems, Inc.</a> <br>
-Last updated Jun 08, 2004</h4>
+Last updated Nov 24, 2004</h4>
 
 See the <a href="faq.html">FAQ</a> | <a href="#download">Download now</a> |
-<a href="/mailman/listinfo/pymilter">Subscribe to mailing list</a>
+<a href="/mailman/listinfo/pymilter">Subscribe to mailing list</a> |
+<a href="#overview">Overview</a>
 <p>
-<img src="python55.gif" align=left alt="A Python">
+<a href="//www.python.org">
+<img src="python55.gif" align=left alt="A Python"></a>
 <a href="//www.sendmail.org/">Sendmail</a> introduced a
 <a href="http://www.milter.org/milter_api/api.html"> new API</a> beginning with version 8.10 -
 libmilter.  The milter module for <a href="//www.python.org">Python</a>
@@ -42,6 +44,54 @@ I recommend upgrading.
 
 <h2> Recent Changes </h2>
 
+Release 0.7.2 tightens the authentication screws with a "3 strikes and
+your out" policy.  A sender must have a valid PTR, HELO, or SPF record
+to send email.  Specific senders can be whitelisted using the
+"delegate" option in the spf configuration section by adding a
+default SPF record for them.  The PTR and HELO are required
+by RFC anyway, so this is not an unreasonable requirement.
+There is now a coherent policy for an SPF softfail result.  A softfail
+is accepted if there is a valid PTR or HELO, or if the domain
+is listed in the "accept_softfail" option of the spf configuration section.
+A neutral result is accepted by default if there is a valid PTR or
+HELO, (and the SPF record was not guessed), unless the domain is listed in the
+"reject_neutral" option.  Common forms of PTR records for dynamic IPs are
+recognized, and do not count as a valid PTR.  This does not prevent anyone
+from sending mail from a dynamic IP - they just need to configure a
+valid HELO name or publish an SPF record.
+<p>
+The RPM for release 0.7.0 moves the config file and socket locations to
+/etc/mail and /var/run/milter respectively.  We now parse Microsoft CID records
+- but only hotmail.com uses them.  They seem to have applied for a patent on
+the brilliant idea of examining the mail headers to see who the message is
+from.  We aren't doing that here, so not to worry - but I am not a lawyer, so
+if you are worried, change spf.py around line 626 to return None instead of
+calling CIDParser().  There is a new option to reject mail with no PTR
+and no SPF.
+<p>
+Microsoft is pushing an anti-opensource license for their pending patent
+along with their sender-ID proposal before the IETF.
+It is royalty free - but requires anyone distributing a binary they've
+compiled from source to sign a license agreement.  The Apache Software
+Foundation <a
+href="http://www.apache.org/foundation/docs/sender-id-position.html"> explains
+the problem with sender-ID</a>, and Debian <a
+href="http://www.debian.org/News/2004/20040904">concurs</a>.  Since 
+the <a href="http://download.microsoft.com/download/4/3/9/439b024b-09fd-44ee-8ff0-10e834004c36/senderid_FAQ.PDF">Microsoft license</a> is
+<a href="http://www.circleid.com/article/732_0_1_0_C/">incompatible with free
+software in general</a> and the <a
+href="http://www.imc.org/ietf-mxcomp/mail-archive/msg03678.html">GPL in
+particular</a>, Python milter will not be able to implement sender-ID in its
+current form.  This was, no doubt, Microsoft's intent all along.
+<p>
+Sender-ID attempts to do for RFC2822 headers what SPF does for RFC2821 headers.
+Unlike SPF, it has never been tried, and is encumbered by a stupid patent.  I
+recommend ignoring it and continuing to implement and improve SPF until a
+working and unencumbered proposal for RFC2822 headers surfaces.
+
+<p>
+<a href="http://spf.pobox.com">
+<img src="SPF.gif" align=left alt="SPF logo"></a>
 Release 0.6.6 adds support for <a href="http://spf.pobox.com/">SPF</a>,
 a protocol to prevent forging of the envelope from address.  
 SPF support requires <a href="http://pydns.sourceforge.net/">pydns</a>.
@@ -106,7 +156,7 @@ entries.  Be sure to handle both Bcc and file copies, and designating what
 mail should be copied.  How should "outgoing" be defined?  Implementing it is
 easy once the configuration is designed.
 
-<h3>Overview</h3>
+<h3><a name=overview>Overview</a></h3>
 
 This package provides a robust toolkit for Python <a
 href="#milter">milters</a>, and the beginnings of a general purpose mail
@@ -138,21 +188,80 @@ methods that
 do nothing, and also provides wrappers for the libmilter methods to mutate
 the message.
 <p>
+The 'spf' module provides an implementation of <a href="http://spf.pobox.com">
+SPF</a> useful for detecting email forgery.
+<p>
+The 'mime' module provides a wrapper for the Python email package that
+fixes some bugs, and simplifies modifying selected parts of a MIME message.
+<p>
 Finally, the bms.py application is both a sample of how to use the
-Milter module, and the beginnings of a general purpose SPAM filtering,
+Milter and spf modules, and the beginnings of a general purpose SPAM filtering,
-wiretapping, and Win32 virus protection milter.
+wiretapping, SPF checking, and Win32 virus protecting milter.  It can
+make use of the <a href="pysrs.html">pysrs</a> package when available for
+SRS/SES checking and the <a href="dspam.html">pydspam</a> package for Bayesian
+content filtering.  SPF checking
+requires <a href="http://pydns.sourceforge.net/">
+pydns</a>.  Configuration documentation is currently included as comments
+in the <a href="milter.cfg">sample config file</a> for the bms.py milter.
 
 <h3><a name=download>Downloading</a></h3>
 
-The latest stable release is <a href="#stable">0.6.9</a>. A stable
+The latest stable release is <a href="#stable">0.7.2</a>. A stable
 release is one which has been installed (and working correctly) on
 production systems long enough to convince me that it is stable.  As
 the package gains more features and complexity, stable will mean no
 bug reports from outside users either.
 <p>
-The latest version is 0.6.9-1.  See the <a href=NEWS>Change Log</a>.
+The latest version is 0.7.2-2.  See the <a href=NEWS>Change Log</a>.
+PLEASE NOTE - if you are using the modules, but not the bms milter application,
+then ignore the RPMs and milter.spec.  Use 'python setup.py bdist_rpm' to 
+build source and binary rpms that do not include the milter application.
+<p>
+I want to split the bms milter application to a new project once I figure
+out the renaming.  The current plan is to rename 'milter' to 'pymilter', which
+will have the Python modules.  The bms milter application will still be named
+'milter' and depend on pymilter (so that my installs won't notice anything).
 <p>
 <a name="stable"><b>Stable</b></a>
+<a href="http://bmsi.com/python/milter-0.7.2.tar.gz">
+milter-0.7.2.tar.gz</a> Three strikes and your out policy.  Some SPF fixes.
+Recognizes PTR records for dynamic IPs.
+<p>
+<a href="http://bmsi.com/python/milter-0.7.1.tar.gz">
+milter-0.7.1.tar.gz</a> Support setmlreply, handle some more exceptions
+for malformed spam.  Compiling pymilter with sendmail-8.12.10, requires
+sendmail-devel with _FFR_MULTILINE set.  The binary will work with older
+sendmails.  The _FFR_MULTILINE option only affects libmilter.a.
+<br>
+<a href="http://bmsi.com/linux/rh72/milter-0.7.1-1.i386.rpm">
+milter-0.7.1-1.i386.rpm</a> Binary RPM for Redhat 7.x, now requires 
+	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
+	python2.3</a>.
+<br>
+<a href="http://bmsi.com/linux/rh9/milter-0.7.1-1.src.rpm">
+milter-0.7.1-1.src.rpm</a> Source RPM for Redhat 9,7.x. 
+<p>
+<a name="stable"><b>Stable</b></a>
+<a href="http://bmsi.com/python/milter-0.7.0.tar.gz">
+milter-0.7.0.tar.gz</a> Move config file and default socket location.
+Parse M$ CID records.
+<br>
+<a href="http://bmsi.com/linux/rh72/milter-0.7.0-1.i386.rpm">
+milter-0.7.0-1.i386.rpm</a> Binary RPM for Redhat 7.x, now requires 
+	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
+	python2.3</a>.
+<br>
+<a href="http://bmsi.com/linux/rh9/milter-0.7.0-1rh9.i386.rpm">
+milter-0.7.0-1rh9.i386.rpm</a> Binary RPM for Redhat 9, requires 
+	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
+	python2.3</a>.
+<br>
+<a href="http://bmsi.com/aix/milter-0.7.0-1.ppc.rpm">
+milter-0.7.0-1.ppc.rpm</a> Binary RPM for AIX, requires sendmail-8.13.1.
+<br>
+<a href="http://bmsi.com/linux/rh9/milter-0.7.0-1.src.rpm">
+milter-0.7.0-1.src.rpm</a> Source RPM for Redhat 9,7.x.  
+<p>
 <a href="http://bmsi.com/python/milter-0.6.9.tar.gz">
 milter-0.6.9.tar.gz</a> Add SPF test suite driver, and validate
 spf.py against test suite.  Add best_guess and get_header to spf.py.

milter.spec

@@ -1,6 +1,6 @@
 %define name milter
-%define version 0.7.0
+%define version 0.7.2
-%define release 1
+%define release 2
 # Redhat 7.x and earlier (multiple ps lines per thread)
 %define sysvinit milter.rc7
 # RH9, other systems (single ps line per process)
@@ -24,8 +24,11 @@ Prefix: %{_prefix}
 Vendor: Stuart D. Gathman <stuart@bmsi.com>
 Packager: Stuart D. Gathman <stuart@bmsi.com>
 Url: http://www.bmsi.com/python/milter.html
-Requires: %{python} >= 2.2.2, sendmail >= 8.12
+Requires: %{python} >= 2.2.2, sendmail >= 8.12.10
-BuildRequires: %{python}-devel >= 2.2.2, sendmail-devel >= 8.12
+%ifnos aix4.1
+Requires: chkconfig
+%endif
+BuildRequires: %{python}-devel >= 2.2.2, sendmail-devel >= 8.12.10
 
 %description
 This is a python extension module to enable python scripts to
@@ -59,10 +62,15 @@ EOF
 
 # purge saved defanged message copies
 mkdir -p $RPM_BUILD_ROOT/etc/cron.daily
+%ifos aix4.1
+R=
+%else
+R='-r'
+%endif
 cat >$RPM_BUILD_ROOT/etc/cron.daily/milter <<'EOF'
 #!/bin/sh
 
-find /var/log/milter/save -mtime +7 | xargs -r rm
+find /var/log/milter/save -mtime +7 | xargs $R rm
 EOF
 chmod a+x $RPM_BUILD_ROOT/etc/cron.daily/milter
 
@@ -96,6 +104,8 @@ EOF
 chmod a+x $RPM_BUILD_ROOT/var/log/milter/start.sh
 
 mkdir -p $RPM_BUILD_ROOT/var/run/milter
+mkdir -p $RPM_BUILD_ROOT/usr/share/sendmail-cf/hack
+cp -p rhsbl.m4 $RPM_BUILD_ROOT/usr/share/sendmail-cf/hack
 
 %ifos aix4.1
 %post
@@ -107,7 +117,13 @@ if [ $1 = 0 ]; then
 fi
 %else
 %post
-echo "pythonsock has moved to /var/run/milter, update /etc/mail/sendmail.cf"
+#echo "pythonsock has moved to /var/run/milter, update /etc/mail/sendmail.cf"
+/sbin/chkconfig --add milter
+
+%preun
+if [ $1 = 0 ]; then
+  /sbin/chkconfig --del milter
+fi
 %endif
 
 %clean
@@ -130,8 +146,24 @@ rm -rf $RPM_BUILD_ROOT
 %config /var/log/milter/start.sh
 %config /var/log/milter/bms.py
 %config(noreplace) /etc/mail/pymilter.cfg
+/usr/share/sendmail-cf/hack/rhsbl.m4
 
 %changelog
+* Mon Aug 30 2004 Stuart Gathman <stuart@bmsi.com> 0.7.2-1
+- Fix various SPF bugs
+- Recognize dynamic PTR names, and don't count them as authentication.
+- Three strikes and yer out rule.
+- Block softfail by default unless valid PTR or HELO
+- Return unknown for null mechanism
+- Return unknown for invalid ip address in mechanism
+- Try best guess on HELO also
+- Expand setreply for common errors
+- make rhsbl.m4 hack available for sendmail.mc
+* Sun Aug 22 2004 Stuart Gathman <stuart@bmsi.com> 0.7.1-1
+- Handle modifying mislabeled multipart messages without an exception
+- Support setbacklog, setmlreply
+- allow multi-recipient CBV
+- return TEMPFAIL for SPF softfail
 * Fri Jul 23 2004 Stuart Gathman <stuart@bmsi.com> 0.7.0-1
 - SPF check hello name
 - Move pythonsock to /var/run/milter

miltermodule.c

@@ -33,6 +33,18 @@ $ python setup.py help
      libraries=["milter","smutil","resolv"]
 
  * $Log$
+ * Revision 2.31  2004/08/23 02:24:36  stuart
+ * Support setbacklog
+ *
+ * Revision 2.30  2004/08/21 20:29:53  stuart
+ * Support option of 11 lines max for mlreply.
+ *
+ * Revision 2.29  2004/08/21 04:14:29  stuart
+ * mlreply support
+ *
+ * Revision 2.28  2004/08/21 02:45:21  stuart
+ * Don't leak int constants if module unloaded.
+ *
  * Revision 2.27  2004/04/06 03:19:59  stuart
  * Release 0.6.8
  *
@@ -127,11 +139,20 @@ $ python setup.py help
  *
  */
 
+#ifndef MAX_ML_REPLY
+#define MAX_ML_REPLY 32
+#endif
+#if MAX_ML_REPLY != 1 && MAX_ML_REPLY != 32 && MAX_ML_REPLY != 11
+#error MAX_ML_REPLY must be 1 or 11 or 32
+#endif
+#define _FFR_MULTILINE (MAX_ML_REPLY > 1)
+
 #include <pthread.h>
 #include <netinet/in.h>
 #include <Python.h>
 #include <libmilter/mfapi.h>
 
+
 /* See if we have IPv4 and/or IPv6 support in this OS and in
  * libmilter.  We need to make several macro tests because some OS's
  * may define some if IPv6 is only partially supported, and we may
@@ -746,6 +767,18 @@ milter_setdbg(PyObject *self, PyObject *args) {
   return _generic_return(smfi_setdbg(val), "cannot set debug value");
 }
 
+static char milter_setbacklog__doc__[] =
+"setbacklog(int) -> None\n\
+Set the TCP connection queue size for the milter socket.";
+
+static PyObject *
+milter_setbacklog(PyObject *self, PyObject *args) {
+   int val;
+
+   if (!PyArg_ParseTuple(args, "i:setbacklog", &val)) return NULL;
+   return _generic_return(smfi_setbacklog(val), "cannot set backlog");
+}
+
 static char milter_settimeout__doc__[] =
 "settimeout(int) -> None\n\
 Set the time (in seconds) that sendmail will wait before\n\
@@ -820,13 +853,54 @@ static PyObject *
 milter_setreply(PyObject *self, PyObject *args) {
   char *rcode;
   char *xcode;
-  char *message;
+  char *message[MAX_ML_REPLY];
+  char fmt[MAX_ML_REPLY + 16];
   SMFICTX *ctx;
-  if (!PyArg_ParseTuple(args, "szz:setreply", &rcode, &xcode, &message))
+  int i;
+  strcpy(fmt,"sz|");
+  for (i = 0; i < MAX_ML_REPLY; ++i) {
+    message[i] = 0;
+    fmt[i+3] = 's';
+  }
+  strcpy(fmt+i+3,":setreply");
+  if (!PyArg_ParseTuple(args, fmt,
+	&rcode, &xcode, message
+#if MAX_ML_REPLY > 1
+	,message+1,message+2,message+3,message+4,message+5,message+6,
+	message+7,message+8,message+9,message+10
+#if MAX_ML_REPLY > 11
+	,message+11,message+12,message+13,message+14,message+15,
+	message+16,message+17,message+18,message+19,message+20,
+	message+21,message+22,message+23,message+24,message+25,
+	message+26,message+27,message+28,message+29,message+30,
+	message+31
+#endif
+#endif
+  ))
     return NULL;
   ctx = _find_context(self);
   if (ctx == NULL) return NULL;
-  return _generic_return(smfi_setreply(ctx, rcode, xcode, message),
+#if MAX_ML_REPLY > 1
+  /*
+   * C varargs might be convenient for some things, but they sure are a pain
+   * when the number of args is not known at compile time.
+   */
+  if (message[0] && message[1])
+    return _generic_return(smfi_setmlreply(ctx, rcode, xcode,
+	  message[0],
+	  message[1],message[2],message[3],message[4],message[5],
+	  message[6],message[7],message[8],message[9],message[10],
+#if MAX_ML_REPLY > 11
+	  message[11],message[12],message[13],message[14],message[15],
+	  message[16],message[17],message[18],message[19],message[20],
+	  message[21],message[22],message[23],message[24],message[25],
+	  message[26],message[27],message[28],message[29],message[30],
+	  message[31],
+#endif
+	  (char *)0
+    ), "cannot set reply");
+#endif
+  return _generic_return(smfi_setreply(ctx, rcode, xcode, message[0]),
 			 "cannot set reply");
 }
 
@@ -986,7 +1060,7 @@ milter_getpriv(PyObject *self, PyObject *args) {
   return o;
 }
 
-#if _FFR_QUARANTINE
+#ifdef SMFIF_QUARANTINE
 static char milter_quarantine__doc__[] =
 "quarantine(string) -> None\n\
 Place the message in quarantine.  A string with a description of the reason\n\
@@ -1035,7 +1109,7 @@ static PyMethodDef context_methods[] = {
   { "replacebody", milter_replacebody, METH_VARARGS, milter_replacebody__doc__},
   { "setpriv",     milter_setpriv,     METH_VARARGS, milter_setpriv__doc__},
   { "getpriv",     milter_getpriv,     METH_VARARGS, milter_getpriv__doc__},
-#if _FFR_QUARANTINE
+#ifdef SMFIF_QUARANTINE
   { "quarantine",  milter_quarantine,  METH_VARARGS, milter_quarantine__doc__},
 #endif
 #if _FFR_SMFI_PROGRESS
@@ -1081,6 +1155,7 @@ static PyMethodDef milter_methods[] = {
    { "main",                 milter_main,                 METH_VARARGS, milter_main__doc__},
    { "setdbg",               milter_setdbg,               METH_VARARGS, milter_setdbg__doc__},
    { "settimeout",           milter_settimeout,           METH_VARARGS, milter_settimeout__doc__},
+   { "setbacklog",           milter_setbacklog,           METH_VARARGS, milter_setbacklog__doc__},
    { "setconn",              milter_setconn,              METH_VARARGS, milter_setconn__doc__},
    { "stop",                 milter_stop,                 METH_VARARGS, milter_stop__doc__},
    { NULL, NULL }
@@ -1116,6 +1191,12 @@ allowing one to write email filters directly in Python.\n\
 Libmilter is currently marked FFR, and needs to be explicitly installed.\n\
 See <sendmailsource>/libmilter/README for details on setting it up.\n";
 
+static void setitem(PyObject *d,const char *name,long val) {
+  PyObject *v = PyInt_FromLong(val);
+  PyDict_SetItemString(d,name,v);
+  Py_DECREF(v);
+}
+
 void
 initmilter(void) {
    PyObject *m, *d;
@@ -1125,24 +1206,24 @@ initmilter(void) {
    d = PyModule_GetDict(m);
    MilterError = PyErr_NewException("milter.error", NULL, NULL);
    PyDict_SetItemString(d,"error", MilterError);
-   PyDict_SetItemString(d,"SUCCESS", PyInt_FromLong((long) MI_SUCCESS));
+   setitem(d,"SUCCESS",  MI_SUCCESS);
-   PyDict_SetItemString(d,"FAILURE", PyInt_FromLong((long) MI_FAILURE));
+   setitem(d,"FAILURE",  MI_FAILURE);
-   PyDict_SetItemString(d,"VERSION", PyInt_FromLong((long) SMFI_VERSION));
+   setitem(d,"VERSION",  SMFI_VERSION);
-   PyDict_SetItemString(d,"ADDHDRS", PyInt_FromLong((long) SMFIF_ADDHDRS));
+   setitem(d,"ADDHDRS",  SMFIF_ADDHDRS);
-   PyDict_SetItemString(d,"CHGBODY", PyInt_FromLong((long) SMFIF_CHGBODY));
+   setitem(d,"CHGBODY",  SMFIF_CHGBODY);
-   PyDict_SetItemString(d,"MODBODY", PyInt_FromLong((long) SMFIF_MODBODY));
+   setitem(d,"MODBODY",  SMFIF_MODBODY);
-   PyDict_SetItemString(d,"ADDRCPT", PyInt_FromLong((long) SMFIF_ADDRCPT));
+   setitem(d,"ADDRCPT",  SMFIF_ADDRCPT);
-   PyDict_SetItemString(d,"DELRCPT", PyInt_FromLong((long) SMFIF_DELRCPT));
+   setitem(d,"DELRCPT",  SMFIF_DELRCPT);
-   PyDict_SetItemString(d,"CHGHDRS", PyInt_FromLong((long) SMFIF_CHGHDRS));
+   setitem(d,"CHGHDRS",  SMFIF_CHGHDRS);
-   PyDict_SetItemString(d,"V1_ACTS", PyInt_FromLong((long) SMFI_V1_ACTS));
+   setitem(d,"V1_ACTS",  SMFI_V1_ACTS);
-   PyDict_SetItemString(d,"V2_ACTS", PyInt_FromLong((long) SMFI_V2_ACTS));
+   setitem(d,"V2_ACTS",  SMFI_V2_ACTS);
-   PyDict_SetItemString(d,"CURR_ACTS", PyInt_FromLong((long) SMFI_CURR_ACTS));
+   setitem(d,"CURR_ACTS",  SMFI_CURR_ACTS);
 #ifdef SMFIF_QUARANTINE
-   PyDict_SetItemString(d,"QUARANTINE",PyInt_FromLong((long)SMFIF_QUARANTINE));
+   setitem(d,"QUARANTINE",SMFIF_QUARANTINE);
 #endif
-   PyDict_SetItemString(d,"CONTINUE", PyInt_FromLong((long) SMFIS_CONTINUE));
+   setitem(d,"CONTINUE",  SMFIS_CONTINUE);
-   PyDict_SetItemString(d,"REJECT", PyInt_FromLong((long) SMFIS_REJECT));
+   setitem(d,"REJECT",  SMFIS_REJECT);
-   PyDict_SetItemString(d,"DISCARD", PyInt_FromLong((long) SMFIS_DISCARD));
+   setitem(d,"DISCARD",  SMFIS_DISCARD);
-   PyDict_SetItemString(d,"ACCEPT", PyInt_FromLong((long) SMFIS_ACCEPT));
+   setitem(d,"ACCEPT",  SMFIS_ACCEPT);
-   PyDict_SetItemString(d,"TEMPFAIL", PyInt_FromLong((long) SMFIS_TEMPFAIL));
+   setitem(d,"TEMPFAIL",  SMFIS_TEMPFAIL);
 }

mime.py

@@ -1,4 +1,7 @@
 # $Log$
+# Revision 1.54  2004/08/18 01:59:46  stuart
+# Handle mislabeled multipart messages
+#
 # Revision 1.53  2004/04/24 22:53:20  stuart
 # Rename some local variables to avoid shadowing builtins
 #
@@ -58,6 +61,18 @@ except: from email.Parser import nlcre as NLCRE
 
 from email import Errors
 
+class MimeGenerator(Generator):
+    def _dispatch(self, msg):
+        # Get the Content-Type: for the message, then try to dispatch to
+        # self._handle_<maintype>_<subtype>().  If there's no handler for the
+        # full MIME type, then dispatch to self._handle_<maintype>().  If
+        # that's missing too, then dispatch to self._writeBody().
+        main = msg.get_content_maintype()
+	if msg.is_multipart() and main.lower() != 'multipart':
+	  self._handle_multipart(msg)
+	else:
+	  Generator._dispatch(self,msg)
+
 class MimeParser(Parser):
 
     # This is a copy of _parsebody from email.Parser, with a fix
@@ -349,9 +364,15 @@ class MimeMessage(Message):
 
   def dump(self,file,unixfrom=False):
     "Write this message (and all subparts) to a file"
-    g = Generator(file)
+    g = MimeGenerator(file)
     g.flatten(self,unixfrom=unixfrom)
 
+  def as_string(self, unixfrom=False):
+      "Return the entire formatted message as a string."
+      fp = StringIO.StringIO()
+      self.dump(fp,unixfrom=unixfrom)
+      return fp.getvalue()
+
   def getencoding(self):
     return self.get('content-transfer-encoding',None)
 
@@ -577,7 +598,6 @@ class HTMLScriptFilter(SGMLFilter):
   def handle_comment(self,comment):
     if not self.ignoring: SGMLFilter.handle_comment(self,comment)
 
-
 def check_html(msg,savname=None):
   "Remove scripts from HTML attachments."
   msgtype = msg.get_content_type().lower()

rhsbl.m4

@@ -0,0 +1,44 @@
+divert(-1)
+#
+# Copyright (c) 2002 Derek J. Balling
+#	All rights reserved.
+#
+# Permission to use granted for all purposes. If modifications are made
+# they are requested to be sent to <dredd@megacity.org> for inclusion in future
+# versions 
+#
+# Allows (hopefully) for checking of access.db whitelisting now. This ONLY
+# works on sendmail-8.12.x ... use on any other version may require tinkering
+# by you the downloader.
+#
+# Incorporates many changes by Sergey S. Mokryshev <mokr@mokr.net>
+#
+#
+
+divert(0)
+ifdef(`_RHSBL_R_',`dnl',`dnl
+VERSIONID(`$Id$')
+define(`_RHSBL_R_',`')
+ifdef(`_DNSBL_R_',`dnl',`dnl
+LOCAL_CONFIG
+# map for DNS based blacklist lookups based on the sender RHS
+Kdnsbl host -T<TMP>')')
+divert(-1)
+define(`_RHSBL_SRV_', `_ARG_')dnl
+define(`_RHSBL_MSG_', `ifelse(len(X`'_ARG2_),`1',`"550 Mail from " $`'&{RHS} " refused by blackhole site '_RHSBL_SRV_`"',`_ARG2_')')dnl
+define(`_RHSBL_MSG_TMP_', `ifelse(_ARG3_,`t',`"451 Temporary lookup failure of " $`'&{RHS} " at '_RHSBL_SRV_`"',`_ARG3_')')dnl
+
+MAILER_DEFINITIONS
+
+SLocal_check_mail
+# DNS based RHS spam list blackholes.bmsi.com
+R$*			$: <?> $>CanonAddr $1
+R<?> $*<@$+.>		$: <?> $1<@$2.> $| $>SearchList <+ rhs> $| <F:$1@$2> <D:$2> <>
+R<?> $* $| <$={Accept}>	$: OKSOFAR
+R<?> $*<@$+.> $| $*	$: <?> $(dnsbl $2._RHSBL_SRV_. $: OK $) $(macro {RHS} $@ $2 $)
+R<?> OK			$: OKSOFAR
+R<?> $*<@$*>		$: OKSOFAR
+ifelse(len(X`'_ARG3_),`1',
+`R<?>$+<TMP>		$: TMPOK',
+`R<?>$+<TMP>		$#error $@ 4.7.1 $: _RHSBL_MSG_TMP_')
+R<?>$+			$#error $@ 5.7.1 $: _RHSBL_MSG_

setup.py

@@ -12,7 +12,7 @@ if sys.version < '2.2.3':
   DistributionMetadata.classifiers = None
   DistributionMetadata.download_url = None
 
-setup(name = "milter", version = "0.7.0",
+setup(name = "milter", version = "0.7.2",
 	description="Python interface to sendmail milter API",
 	long_description="""\
 This is a python extension module to enable python scripts to
@@ -28,7 +28,10 @@ querying SPF records.
 	url="http://www.bmsi.com/python/milter.html",
 	py_modules=["Milter","mime","spf"],
 	ext_modules=[
-	  Extension("milter", ["miltermodule.c"],libraries=libs),
+	  Extension("milter", ["miltermodule.c"],
+	    libraries=libs,
+	    define_macros = [ ('MAX_ML_REPLY',32) ]
+	  ),
 	],
 	keywords = ['sendmail','milter'],
 	classifiers = [

spf.py

@@ -45,6 +45,28 @@ For news, bugfixes, etc. visit the home page for this implementation at
 # Terrence is not responding to email.
 #
 # $Log$
+# Revision 1.21  2004/11/20 16:37:03  stuart
+# Handle multi-segment TXT records.
+#
+# Revision 1.20  2004/11/19 06:10:30  stuart
+# Use PermError exception instead of reporting unknown.
+#
+# Revision 1.19  2004/11/09 23:00:18  stuart
+# Limit recursion and DNS lookups separately.
+#
+#
+# Revision 1.17  2004/09/10 18:08:26  stuart
+# Return unknown for null mechanism
+#
+# Revision 1.16  2004/09/04 23:27:06  stuart
+# More mechanism aliases.
+#
+# Revision 1.15  2004/08/30 21:19:05  stuart
+# Return unknown for invalid ip syntax in mechanism
+#
+# Revision 1.14  2004/08/23 02:28:24  stuart
+# Remove Perl usage message.
+#
 # Revision 1.13  2004/07/23 19:23:12  stuart
 # Always fail to match on ip6, until we support it properly.
 #
@@ -115,18 +137,9 @@ import xml.sax
 # (c) 2004 Python version by Stuart Gathman
 #
 # Date: 2004-02-25
-# Version: 1.0
-#
-# Usage:
-#  ./cid2spf.pl "<ep xmlns='http://ms.net/1'>...</ep>"
-#
-# Note that the 'include' directives will also have to be checked and
-# "translated". Future versions of this script might be able to get a
-# domain name as an argument and "crawl" the DNS for the necessary
-# information.
 #
 # A complete reverse translation (SPF -> CID) might be impossible, since
-# there are no way to handle:
+# there are no ways to handle:
 # - PTR and EXISTS mechanism 
 # - MX mechanism with an different domain as argument
 # - macros
@@ -290,6 +303,16 @@ except NameError:
 # standard default SPF record
 DEFAULT_SPF = 'v=spf1 a/24 mx/24 ptr'
 
+# maximum DNS lookups allowed
+MAX_LOOKUP = 50
+MAX_RECURSION = 20
+
+class TempError(Exception):
+	"Temporary SPF error"
+
+class PermError(Exception):
+	"Permanent SPF error"
+
 def check(i, s, h,local=None):
 	"""Test an incoming MAIL FROM:<s>, from a client with ip address i.
 	h is the HELO/EHLO domain name.
@@ -335,6 +358,7 @@ class query(object):
 		self.cache = {}
 		self.exps = dict(EXPLANATIONS)
 		self.local = local	# local policy
+    		self.lookups = 0
 
 	def set_default_explanation(self,exp):
 		exps = self.exps
@@ -363,27 +387,32 @@ class query(object):
 			return ('pass', 250, 'local connections always pass')
 
 		try:
+			self.lookups = 0
 			if not spf:
 			    spf = self.dns_spf(self.d)
 			if self.local and spf:
 			    spf += ' ' + self.local
 			return self.check1(spf, self.d, 0)
-		except DNS.DNSError:
+		except DNS.DNSError,x:
-			return ('error', 450, 'SPF DNS Error')
+			return ('error', 450, 'SPF DNS Error: ' + str(x))
+		except TempError,x:
+			return ('error', 450, 'SPF Temporary Error: ' + str(x))
+		except PermError,x:
+			return ('error', 550, 'SPF Permanent Error: ' + str(x))
 
 	def check1(self, spf, domain, recursion):
 		# spf rfc: 3.7 Processing Limits
 		#
-		if recursion > 20:
+		if recursion > MAX_RECURSION:
-			self.prob =  'Mechanisms used too many DNS lookups'
+			self.prob =  'Too many levels of recursion'
 			return ('unknown', 250, 'SPF recursion limit exceeded')
 		try:
 			tmp, self.d = self.d, domain
-			return self.check0(spf, recursion)
+			return self.check0(spf,recursion)
 		finally:
 			self.d = tmp
 
-	def check0(self, spf, recursion):
+	def check0(self, spf,recursion):
 		"""Test this query information against SPF text.
 
 		Returns (result, mta-status-code, explanation) where
@@ -431,6 +460,7 @@ class query(object):
 			m, arg, cidrlength = parse_mechanism(mech, self.d)
 
 			# map '?' '+' or '-' to 'unknown' 'pass' or 'fail'
+			if m:
 			  result = RESULTS.get(m[0])
 			  if result:
 				# eat '?' '+' or '-'
@@ -439,7 +469,7 @@ class query(object):
 				# default pass
 				result = 'pass'
 
-			if m in ['a', 'mx', 'ptr', 'exists', 'include']:
+			if m in ['a', 'mx', 'ptr', 'prt', 'exists', 'include']:
 				arg = self.expand(arg)
 
 			if m == 'include':
@@ -448,18 +478,12 @@ class query(object):
 					      arg, recursion + 1)
 			    if res == 'pass':
 			      break
-				if res in ('fail','neutral','softfail'):
-					continue
 			    if res == 'none':
-				  	self.prob = \
+			      raise PermError(
-					  'Could not find a valid SPF record'
+			      	'No valid SPF record for included domain')
-				  	res = 'unknown'
+			    continue
-				return res,code,txt
 			  else:
-			    	self.prob = 'Required option is missing'
+			    raise PermError('include mechanism missing domain')
-				self.mech.append(mech)
-				return ('unknown', 250, 'missing SPF option')
-
 			elif m == 'all':
 				break
 
@@ -478,9 +502,15 @@ class query(object):
 					break
 
 			elif m in ('ip4', 'ipv4', 'ip') and arg != self.d:
+			    try:
 				if cidrmatch(self.i, [arg], cidrlength):
 				    break
-			elif m == 'ip6':
+			    except socket.error:
+				self.mech.append(mech)
+				self.prob = 'Bad mechanism syntax found'
+				return ('unknown',250,'SPF mechanism syntax error')
+			        
+			elif m in ('ip6', 'ipv6'):
 			# Until we support IPV6, we should never
 			# get an IPv6 connection.  So this mech
 			# will never match.
@@ -494,15 +524,12 @@ class query(object):
 			else:
 			  # unknown mechanisms cause immediate unknown
 			  # abort results
-				self.mech.append(mech)
+			  raise PermError('Unknown mechanism found: ' + mech)
-				self.prob = 'Unknown mechanism found'
-				return ('unknown',250,'unknown SPF mechanism')
-
 		else:
 			# no matches
 			if redirect:
 				return self.check1(self.dns_spf(redirect),
-				                   redirect, recursion+1)
+				                   redirect, recursion + 1)
 			else:
 				result = default
 
@@ -636,7 +663,7 @@ class query(object):
 	def dns_txt(self, domainname):
 		"Get a list of TXT records for a domain name."
 		if domainname:
-		  return [t for a in self.dns(domainname, 'TXT') for t in a]
+		  return [''.join(a) for a in self.dns(domainname, 'TXT')]
 		return []
 
 	def dns_mx(self, domainname):
@@ -678,6 +705,9 @@ class query(object):
 		pre: qtype in ['A', 'AAAA', 'MX', 'PTR', 'TXT', 'SPF']
 		post: isinstance(__return__, types.ListType)
 		"""
+		self.lookups += 1
+		if self.lookups > MAX_LOOKUP:
+			raise PermError('Too many DNS lookups')
 		result = self.cache.get( (name, qtype) )
 		cname = None
 		if not result:

test/missingboundary

@@ -0,0 +1,128 @@
+From leec@windowsshop.com Fri Sep 10 11:48:25 2004
+Message-ID: <4141CDD4.7040305@windowsshop.com>
+Date: Fri, 10 Sep 2004 11:52:52 -0400
+From: Lee Connor <leec@windowsshop.com>
+User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)
+X-Accept-Language: en-us, en
+MIME-Version: 1.0
+To: Cleo Matthews-Conley <cleom@windowsshop.com>, 
+ Tony Collini <tonyc@windowsshop.com>,
+ John Higinbothom <johnh@windowsshop.com>
+CC: Rich Higgins <richh@windowsshop.com>
+Subject: [Fwd: [Fwd: Customer Concerns]]
+Content-Type: multipart/mixed;
+ boundary="------------020209070802060007090105"
+
+This is a multi-part message in MIME format.
+--------------020209070802060007090105
+Content-Type: text/plain; charset=us-ascii; format=flowed
+Content-Transfer-Encoding: 7bit
+
+Cleo - please review attached feedback from Sales team.......I recall at 
+an early meeting after we moved in you and Tony (and maybe 1 or 2 
+others) were going to develop a voice mail procedure or instruction 
+sheet for all staff. It looks like we really need this to get what we 
+are looking for from the system. Please let me know when you can produce 
+this and give a draft to the managers here for review.
+Thanks,
+Lee
+
+
+--------------020209070802060007090105
+Content-Type: message/rfc822;
+ name="[Fwd: Customer Concerns]"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: inline;
+ filename="[Fwd: Customer Concerns]"
+
+Return-Path: <richh@windowsshop.com>
+Received: from windowsshop.com (pc147.windowsshop.com [192.168.100.147] (may be forged))
+	by lord.windowsshop.com (8.12.10/8.12.10) with ESMTP id i89KCClX003425
+	for <leec@windowsshop.com>; Thu, 9 Sep 2004 16:12:12 -0400
+Message-ID: <4140B851.3020501@windowsshop.com>
+Date: Thu, 09 Sep 2004 16:08:49 -0400
+From: Rich <richh@windowsshop.com>
+User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.0.2) Gecko/20021120 Netscape/7.01
+X-Accept-Language: en-us, en
+MIME-Version: 1.0
+To: Lee Connor <leec@windowsshop.com>
+Subject: [Fwd: Customer Concerns]
+Content-Type: multipart/mixed;
+ boundary="------------030301030706020401010801"
+X-DSpam-Score: 0.000000
+
+This is a multi-part message in MIME format.
+--------------030301030706020401010801
+Content-Type: text/plain; charset=us-ascii; format=flowed
+Content-Transfer-Encoding: 7bit
+
+Lee - do you want me to do anything else with this?
+
+Rich
+
+<!DSPAM:FEE4D3278234264874834386>
+
+
+--------------030301030706020401010801
+Content-Type: message/rfc822; name="Customer Concerns";
+	boundary="===============0045392615=="
+Content-Transfer-Encoding: 7bit
+Content-Disposition: inline;
+ filename="Customer Concerns"
+
+
+Return-Path: <joes@windowsshop.com>
+Received: from joes (pc148.windowsshop.com [192.168.100.148] (may be forged))
+	by lord.windowsshop.com (8.12.10/8.12.10) with SMTP id i89K9BlX003262
+	for <richh@windowsshop.com>; Thu, 9 Sep 2004 16:09:11 -0400
+From: "Joe Schmuck" <joes@windowsshop.com>
+To: <richh@windowsshop.com>
+Subject: Customer Concerns
+Date: Thu, 9 Sep 2004 16:08:26 -0400
+Message-ID: <OFEPKHCCLPIECLFBLDHBAEAECAAA.joes@windowsshop.com>
+MIME-Version: 1.0
+Content-Type: text/plain;
+	charset="iso-8859-1"
+Content-Transfer-Encoding: 7bit
+X-Priority: 3 (Normal)
+X-MSMail-Priority: Normal
+X-Mailer: Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0)
+Importance: Normal
+X-MimeOLE: Produced By Microsoft MimeOLE V6.00.2800.1106
+X-DSpam-Score: 0.000000
+
+Rich:
+
+Following is a summary of concerns from customers regarding internal
+communications within WS:
+
+	- Not all employees have activated their voice mail - when this is the
+case, the system will automatically cut you off
+	- When employees are out of the office, phones are not forwarded to a back
+up, ie manager
+	- Reception has no record of employee attendance, and therefore will
+forward call to individual requested - see point 2
+ 	- Reception directs calls to incorrect individuals
+ 	- When entering voice mail, if you press '0', system does not default to
+operator, but puts you back into individual 	voice mail
+	- Reception phone demeanor has no 'pep'
+
+Thanks
+Joe
+
+
+
+
+---
+Outgoing mail is certified Virus Free.
+Checked by AVG anti-virus system (http://www.grisoft.com).
+Version: 6.0.752 / Virus Database: 503 - Release Date: 9/3/2004
+
+
+<!DSPAM:FEE4D05F1332634871908793>
+
+--===============0045392615==--
+--------------030301030706020401010801--
+
+--------------020209070802060007090105--
+