This commit was manufactured by cvs2svn to create tag 'milter-0_8_0a1'.

Sprout from bmsi 2005-05-31 18:23:49 UTC Stuart Gathman <stuart@gathman.org> 'Development changes since 0.7.2'
Cherrypick from master 2005-05-31 20:39:16 UTC Stuart Gathman <stuart@gathman.org> 'Create Milter package.':
    MANIFEST.in
    milter.html
    setup.py
    spf.py

MANIFEST.in

@@ -8,14 +8,17 @@ include testsample.py
 include testmime.py
 include testbms.py
 include testdspam.py
+include rejects.py
 include bms.py
 include spf.py
 include spfquery.py
 include test.py
 include sample.py
 include test/*
+include Milter/*.py
 include *.spec
 include start.sh
 include milter.rc
 include milter.rc7
 include milter.cfg
+include rhsbl.m4

Milter.py

@@ -8,15 +8,12 @@ import milter
 import thread
 
 from milter import ACCEPT,CONTINUE,REJECT,DISCARD,TEMPFAIL,	\
-  set_flags, setdbg, \
+  set_flags, setdbg, setbacklog, settimeout, \
   ADDHDRS, CHGBODY, ADDRCPT, DELRCPT, CHGHDRS,	\
   V1_ACTS, V2_ACTS, CURR_ACTS
 
-try:
+try: from milter import QUARANTINE
-  from milter import QUARANTINE
+except: pass
-except:
-  #print 'No QUARANTINE support'
-  pass
 
 _seq_lock = thread.allocate_lock()
 _seq = 0
@@ -100,8 +97,10 @@ class Milter:
   def getsymval(self,sym):
     return self.__ctx.getsymval(sym)
 
-  def setreply(self,rcode,xcode,msg):
+  # If sendmail does not support setmlreply, then only the
-    return self.__ctx.setreply(rcode,xcode,msg)
+  # first msg line is used.
+  def setreply(self,rcode,xcode=None,msg=None,*ml):
+    return self.__ctx.setreply(rcode,xcode,msg,*ml)
 
   # Milter methods which can only be called from eom callback.
   def addheader(self,field,value):
@@ -119,6 +118,8 @@ class Milter:
   def replacebody(self,body):
     return self.__ctx.replacebody(body)
 
+  # When quarantined, a message goes into the mailq as if to be delivered,
+  # but delivery is deferred until the message is unquarantined.
   def quarantine(self,reason):
     return self.__ctx.quarantine(reason)
 

Milter/__init__.py

@@ -0,0 +1,208 @@
+
+# Author: Stuart D. Gathman <stuart@bmsi.com>
+# Copyright 2001 Business Management Systems, Inc.
+# This code is under GPL.  See COPYING for details.
+
+import os
+import milter
+import thread
+
+from milter import ACCEPT,CONTINUE,REJECT,DISCARD,TEMPFAIL,	\
+  set_flags, setdbg, setbacklog, settimeout, \
+  ADDHDRS, CHGBODY, ADDRCPT, DELRCPT, CHGHDRS,	\
+  V1_ACTS, V2_ACTS, CURR_ACTS
+
+try: from milter import QUARANTINE
+except: pass
+
+_seq_lock = thread.allocate_lock()
+_seq = 0
+
+def uniqueID():
+  """Return a sequence number unique to this process.
+  """
+  global _seq
+  _seq_lock.acquire()
+  seqno = _seq = _seq + 1
+  _seq_lock.release()
+  return seqno
+  
+class Milter:
+  """A simple class interface to the milter module.
+  """
+  def _setctx(self,ctx):
+    self.__ctx = ctx
+    if ctx:
+      ctx.setpriv(self)
+
+  # user replaceable callbacks
+  def log(self,*msg):
+    print 'Milter:',
+    for i in msg: print i,
+    print
+
+  def connect(self,hostname,unused,hostaddr):
+    "Called for each connection to sendmail."
+    self.log("connect from %s at %s" % (hostname,hostaddr))
+    return CONTINUE
+
+  def hello(self,hostname):
+    "Called after the HELO command."
+    self.log("hello from %s" % hostname)
+    return CONTINUE
+
+  def envfrom(self,f,*str):
+    """Called to begin each message.
+    f -> string		message sender
+    str -> tuple	additional ESMTP parameters
+    """
+    self.log("mail from",f,str)
+    return CONTINUE
+
+  def envrcpt(self,to,*str):
+    "Called for each message recipient."
+    self.log("rcpt to",to,str)
+    return CONTINUE
+
+  def header(self,field,value):
+    "Called for each message header."
+    self.log("%s: %s" % (field,value))
+    return CONTINUE
+
+  def eoh(self):
+    "Called after all headers are processed."
+    self.log("eoh")
+    return CONTINUE
+
+  def body(self,unused):
+    "Called to transfer the message body."
+    return CONTINUE
+
+  def eom(self):
+    "Called at the end of message."
+    self.log("eom")
+    return CONTINUE
+
+  def abort(self):
+    "Called if the connection is terminated abnormally."
+    self.log("abort")
+    return CONTINUE
+
+  def close(self):
+    "Called at the end of connection, even if aborted."
+    self.log("close")
+    return CONTINUE
+
+  # Milter methods which can be invoked from callbacks
+  def getsymval(self,sym):
+    return self.__ctx.getsymval(sym)
+
+  # If sendmail does not support setmlreply, then only the
+  # first msg line is used.
+  def setreply(self,rcode,xcode=None,msg=None,*ml):
+    return self.__ctx.setreply(rcode,xcode,msg,*ml)
+
+  # Milter methods which can only be called from eom callback.
+  def addheader(self,field,value):
+    return self.__ctx.addheader(field,value)
+
+  def chgheader(self,field,idx,value):
+    return self.__ctx.chgheader(field,idx,value)
+
+  def addrcpt(self,rcpt):
+    return self.__ctx.addrcpt(rcpt)
+
+  def delrcpt(self,rcpt):
+    return self.__ctx.delrcpt(rcpt)
+
+  def replacebody(self,body):
+    return self.__ctx.replacebody(body)
+
+  # When quarantined, a message goes into the mailq as if to be delivered,
+  # but delivery is deferred until the message is unquarantined.
+  def quarantine(self,reason):
+    return self.__ctx.quarantine(reason)
+
+  def progress(self):
+    return self.__ctx.progress()
+
+factory = Milter
+
+def connectcallback(ctx,hostname,family,hostaddr):
+  m = factory()
+  m._setctx(ctx)
+  return m.connect(hostname,family,hostaddr)
+
+def closecallback(ctx):
+  m = ctx.getpriv()
+  if not m: return CONTINUE
+  rc = m.close()
+  m._setctx(None)	# release milterContext
+  return rc
+
+def envcallback(c,args):
+  """Convert ESMTP parms to keyword parameters.
+  Can be used in the envfrom and/or envrcpt callbacks to process
+  ESMTP parameters as python keyword parameters."""
+  kw = {}
+  for s in args[1:]:
+    pos = s.find('=')
+    if pos > 0:
+      kw[s[:pos]] = s[pos+1:]
+  return apply(c,args,kw)
+
+def runmilter(name,socketname,timeout = 0):
+  # This bit is here on the assumption that you will be starting this filter
+  # before sendmail.  If sendmail is not running and the socket already exists,
+  # libmilter will throw a warning.  If sendmail is running, this is still
+  # safe if there are no messages currently being processed.  It's safer to
+  # shutdown sendmail, kill the filter process, restart the filter, and then
+  # restart sendmail.
+  pos = socketname.find(':')
+  if pos > 1:
+    s = socketname[:pos]
+    fname = socketname[pos+1:]
+  else:
+    s = "unix"
+    fname = socketname
+  if s == "unix" or s == "local":
+    print "Removing %s" % fname
+    try:
+      os.unlink(fname)
+    except:
+      pass
+
+  # The default flags set include everything
+  # milter.set_flags(milter.ADDHDRS)
+  milter.set_connect_callback(connectcallback)
+  milter.set_helo_callback(lambda ctx, host: ctx.getpriv().hello(host))
+  milter.set_envfrom_callback(lambda ctx,*str:
+  	ctx.getpriv().envfrom(*str))
+#  	envcallback(ctx.getpriv().envfrom,str))
+  milter.set_envrcpt_callback(lambda ctx,*str:
+  	ctx.getpriv().envrcpt(*str))
+#  	envcallback(ctx.getpriv().envrcpt,str))
+  milter.set_header_callback(lambda ctx,fld,val:
+  	ctx.getpriv().header(fld,val))
+  milter.set_eoh_callback(lambda ctx: ctx.getpriv().eoh())
+  milter.set_body_callback(lambda ctx,chunk: ctx.getpriv().body(chunk))
+  milter.set_eom_callback(lambda ctx: ctx.getpriv().eom())
+  milter.set_abort_callback(lambda ctx: ctx.getpriv().abort())
+  milter.set_close_callback(closecallback)
+
+  milter.setconn(socketname)
+  if timeout > 0: milter.settimeout(timeout)
+  # The name *must* match the X line in sendmail.cf (supposedly)
+  milter.register(name)
+  start_seq = _seq
+  try:
+    milter.main()
+  except milter.error:
+    if start_seq == _seq: raise	# couldn't start
+    # milter has been running for a while, but now it can't start new threads
+    raise milter.error("out of thread resources")
+
+__all__ = globals().copy()
+for priv in ('os','milter','thread','factory','_seq','_seq_lock'):
+  del __all__[priv]
+__all__ = __all__.keys()

Milter/dsn.py

@@ -0,0 +1,168 @@
+import smtplib
+import spf
+import socket
+from email.Message import Message
+
+nospf_msg = """This is an automatically generated Delivery Status Notification.
+
+THIS IS A WARNING MESSAGE ONLY.
+
+YOU DO *NOT* NEED TO RESEND YOUR MESSAGE.
+
+Delivery to the following recipients has been delayed.
+
+	%(rcpt)s
+
+Subject: %(subject)s 
+
+Someone at IP address %(connectip)s sent an email claiming
+to be from %(sender)s.  
+
+If that wasn't you, then your domain, %(sender_domain)s,
+was forged - i.e. used without your knowlege or authorization by
+someone attempting to steal your mail identity.  This is a very
+serious problem, and you need to provide authentication for your
+SMTP (email) servers to prevent criminals from forging your
+domain.  The simplest step is usually to publish an SPF record
+with your Sender Policy.  
+
+For more information, see: http://spfhelp.net
+
+I hate to annoy you with a DSN (Delivery Status
+Notification) from a possibly forged email, but since you
+have not published a sender policy, there is no other way
+of bringing this to your attention.
+
+If it *was* you that sent the email, then your email domain
+or configuration is in error.  If you don't know anything
+about mail servers, then pass this on to your SMTP (mail)
+server administrator.  We have accepted the email anyway, in
+case it is important, but we couldn't find anything about
+the mail submitter at %(connectip)s to distinguish it from a
+zombie (compromised/infected computer - usually a Windows
+PC).  There was no PTR record for its IP address (PTR names
+that contain the IP address don't count).  RFC2821 requires
+that your hello name be a FQN (Fully Qualified domain Name,
+i.e. at least one dot) that resolves to the IP address of
+the mail sender.  In addition, just like for PTR, we don't
+accept a helo name that contains the IP, since this doesn't
+help to identify you.  The hello name you used,
+%(heloname)s, was invalid.
+
+Furthermore, there was no SPF record for the sending domain
+%(sender_domain)s.  We even tried to find its IP in any A or
+MX records for your domain, but that failed also.  We really
+should reject mail from anonymous mail clients, but in case
+it is important, we are accepting it anyway.
+
+We are sending you this message to alert you to the fact that
+
+Either - Someone is forging your domain.
+Or - You have problems with your email configuration.
+Or - Possibly both.
+
+If you need further assistance, please do not hesitate to
+contact me again.
+
+Kind regards,
+Stuart D. Gathman
+postmaster@%(receiver)s
+"""
+
+softfail_msg = """
+This is an automatically generated Delivery Status Notification.
+
+THIS IS A WARNING MESSAGE ONLY.
+
+YOU DO *NOT* NEED TO RESEND YOUR MESSAGE.
+
+Delivery to the following recipients has been delayed.
+
+       %(rcpt)s
+
+Subject: %(subject)s
+Received-SPF: %(spf_result)s
+"""
+
+def send_dsn(mailfrom,receiver,msg=None):
+  "Send DSN.  If msg is None, do callback verification."
+  user,domain = mailfrom.split('@')
+  q = spf.query(None,None,None)
+  mxlist = q.dns(domain,'MX')
+  if not mxlist:
+    mxlist = (0,domain),
+  else:
+    mxlist.sort()
+  smtp = smtplib.SMTP()
+  for prior,host in mxlist:
+    try:
+      smtp.connect(host)
+      code,resp = smtp.helo(receiver)
+      # some wiley spammers have MX records that resolve to 127.0.0.1
+      if resp.split()[0] == receiver:
+        return (553,'Fraudulent MX for %s' % domain)
+      if not (200 <= code <= 299):
+	raise SMTPHeloError(code, resp)
+      if msg:
+        try:
+	  smtp.sendmail('<>',mailfrom,msg)
+	except smtplib.SMTPSenderRefused:
+	  # does not accept DSN, try postmaster (at the risk of mail loops)
+	  smtp.sendmail('<postmaster@%s>'%receiver,mailfrom,msg)
+      else:	# CBV
+	code,resp = smtp.docmd('MAIL FROM: <>')
+	if code != 250:
+	  raise SMTPSenderRefused(code, resp, '<>')
+	code,resp = smtp.rcpt(mailfrom)
+	if code not in (250,251):
+	  return (code,resp)		# permanent error
+	smtp.quit()
+      return None			# success
+    except smtplib.SMTPRecipientsRefused,x:
+      return x.recipients[mailfrom]	# permanent error
+    except smtplib.SMTPSenderRefused,x:
+      return x		# does not accept DSN
+    except smtplib.SMTPDataError,x:
+      return x				# permanent error
+    except smtplib.SMTPException:
+      pass		# any other error, try next MX
+    except socket.error:
+      pass		# MX didn't accept connections, try next one
+    smtp.close()
+  return (450,'No MX servers available')	# temp error
+
+def create_msg(q,rcptlist,origmsg):
+  heloname = q.h
+  sender = q.s
+  connectip = q.i
+  receiver = q.r
+  sender_domain = q.o
+  rcpt = '\n\t'.join(rcptlist)
+  try: subject = origmsg['Subject']
+  except: subject = '(none)'
+  try:
+    spf_result = origmsg['Received-SPF']
+    if not spf_result.startswith('softfail'):
+      spf_result = None
+  except: spf_result = None
+  msg = Message()
+  msg.add_header('To',sender)
+  msg.add_header('From','postmaster@%s'%receiver)
+  msg.add_header('Auto-Submitted','auto-generated (configuration error)')
+  msg.set_type('text/plain')
+  if spf_result:
+    msg.add_header('Subject','SPF softfail (POSSIBLE FORGERY)')
+    msg.set_payload(softfail_msg % locals())
+  else:
+    msg.add_header('Subject','Critical mail server configuration error')
+    msg.set_payload(nospf_msg % locals())
+  return msg
+
+if __name__ == '__main__':
+  q = spf.query('192.168.9.50',
+  'SRS0=pmeHL=RH=bmsi.com=stuart@bmsi.com',
+  'bmsred.bmsi.com',receiver='mail.bmsi.com')
+  msg = create_msg(q,'charlie@jsconnor.com')
+  #print msg.as_string()
+  # print send_dsn(f,msg.as_string())
+  print send_dsn(q.s,'mail.bmsi.com',msg.as_string())

Milter/dynip.py

@@ -0,0 +1,87 @@
+# examples we don't yet recognize:
+#
+# wiley-268-8196.roadrunner.nf.net at ('205.251.174.46', 4810)
+# cbl-sd-02-79.aster.com.do at ('200.88.62.79', 4153)
+
+import re
+
+ip3 = re.compile('[0-9]{1,3}')
+hpats = (
+ 'h[0-9a-f]{12}[.]',
+ 'h\d*n\d*c\d*o\d*\.',
+ 'pcp\d{6,10}pcs[.]',
+ 'no-reverse',
+ 'S[0-9a-f]{16}[.][a-z]{2}[.]',
+ 'user<3>\.',
+ '[Cc]ust<3>\.',
+ '^<3>\.',
+ 'ppp[^.]*<3>\.',
+ '-ppp\d*\.',
+ '\d*-<3>\.',
+ '[0-9a-f]{1,3}-<3>\.',
+ 'p<3>\.pool',
+ 'h<3>\.',
+ 'xdsl-\d*\.',
+ '-\d*-\d*\.',
+ '\.adsl\.',
+ '\.cable\.'
+)
+rehmac = re.compile('|'.join(hpats))
+
+def is_dynip(host,addr):
+  """Return True if hostname is for a dynamic ip.
+  Examples:
+
+  >>> is_dynip('post3.fabulousdealz.com','69.60.99.112')
+  False
+  >>> is_dynip('adsl-69-208-201-177.dsl.emhril.ameritech.net','69.208.201.177')
+  True
+  >>> is_dynip('[1.2.3.4]','1.2.3.4')
+  True
+  """
+  if host.startswith('[') and host.endswith(']'):
+    return True
+  if addr:
+    if host.find(addr) >= 0: return True
+    a = addr.split('.')
+    ia = map(int,a)
+    h = host
+    m = ip3.findall(host)
+    if m:
+      g = map(int,m)
+      ia3 = (ia[1:],ia[:3])
+      if g[-3:] in ia3: return True
+      if g[0] == ia[3] and g[1:3] == ia[:2]: return True
+      if g[-2:] == ia[2:]: return True
+      g.reverse()
+      if g[:3] in ia3: return True
+      if g[:2] == ia[2:]: return True
+      if ia[2:] in (g[:2],g[-2:]): return True
+      for m in ip3.finditer(host):
+        if int(m.group()) == ia[3]:
+	  h = host[:m.start()] + '<3>' + host[m.end():]
+	  break
+    if rehmac.search(h): return True
+    if host.find(''.join(a[:3])) >= 0: return True
+    if host.find(''.join(a[1:])) >= 0: return True
+    x = "%02x%02x%02x%02x" % tuple(ia)
+    if host.lower().find(x) >= 0: return True
+  return False
+
+if __name__ == '__main__':
+  import fileinput
+  import sets
+  seen = sets.Set()
+  for ln in fileinput.input():
+    a = ln.split()
+    if a[3:5] == ['connect','from']:
+      host = a[5]
+      if host.startswith('[') and host.endswith(']'):
+	continue	# no PTR
+      ip = a[7][2:-2]
+      if ip in seen: continue
+      seen.add(ip)
+      if is_dynip(host,ip):
+        print '%s\t%s DYN' % (ip,host)
+      else:
+        print '%s\t%s' % (ip,host)

NEWS

@@ -1,5 +1,23 @@
 Here is a history of user visible changes to Python milter.
 
+0.7.2	Return unknown for invalid ip address in mechanism
+	Recognize dynamic PTR names, and don't count them as authentication.
+	Three strikes and yer out rule.
+	Block softfail by default when no PTR or HELO
+	Return unknown for null mechanism
+	Try best guess on HELO also
+	Expand setreply for common errors
+	make rhsbl.m4 hack available for sendmail.mc
+0.7.1	Handle modifying mislabeled multipart messages without an exception
+	Support setbacklog, setmlreply
+	Allow multi-recipient CBV
+	Return TEMPFAIL for SPF softfail
+0.7.0	SPF check hello name
+	Move pythonsock to /var/run/milter
+	Move milter.cfg to /etc/mail/pymilter.cfg
+	Check M$ style XML CID records by converting to SPF
+	Recognize, but never match ip6 - until we properly support it.
+	Option to reject when no PTR and no SPF
 0.6.9	Reject invalid SRS immediately for benefit of callback verifiers
 	Fix include bug in spf.py
 	Fix check_header bug

README

@@ -92,7 +92,6 @@ milter.  This milter's socket is a unix-domain socket in the filesystem.
 See libmilter/README for the definitive list of options.
 NB: The name is specified in two places: here, in sendmail's cf file, and
 in the milter itself.  Make sure the two match.
-NB: OpenBSD must use an inet socket.  See the web page for details.
 NB: The above lines can be added in your .mc file with this line:
 
 INPUT_MAIL_FILTER(`pythonfilter', `S=local:/home/username/pythonsock')
@@ -124,16 +123,6 @@ and headers at
 
 http://www.bmsi.com/linux/sendmail-rh72.spec
 
-OpenBSD Notes
--------------
-
-Sendmail is broken on OpenBSD for unix domain sockets.  You must use an
-inet socket for milter.  The sendmail.cf 'X' config line would look like:
-
-Xpythonfilter,        S=inet:1234@localhost
-
-and the sample milter needs to be modified accordingly.
-
 IPv6 Notes
 ----------
 

TODO

@@ -1,10 +1,46 @@
+Defer TEMPERROR in SPF evaluation - give precedence to security
+(only defer for PASS mechanisms).
+
+Allow multiple recipients for MAIL FROM: <> by default.
+
+Option to add Received-SPF header, but never reject on SPF.
+
+Option to configure banned extension list for mime.py.  Default to empty.
+
+Create null config that does nothing - except maybe add Received-SPF
+headers.  Many admins would like to turn features on one at a time.
+
+Checking in mime.py;
+/bms/cvs/milter/mime.py,v  <--  mime.py
+new revision: 1.56; previous revision: 1.55
+done
+Checking in spf.py;
+/bms/cvs/milter/spf.py,v  <--  spf.py
+new revision: 1.18; previous revision: 1.17
+done
+Checking in testmime.py;
+/bms/cvs/milter/testmime.py,v  <--  testmime.py
+new revision: 1.19; previous revision: 1.18
+
+Auto whitelist based on outgoing email - perhaps with magic subject
+or recipient prefix.
+
+Can't output messages with malformed rfc822 attachments.
+
+Use python exceptions in SPF to cleanly handle unknown and error results.
+
+Example malformed SPF:
+onvunvuvvx.usafisnews.org text "v=spf1 mx ptr ip4:207.44.199.970 -all"
+
+Move milter,Milter,mime,spf modules to pymilter
+milter package will have bms.py application
+
+Support SMTP AUTH and disable SPF checks when connection is authorized.
 Web admin interface
-RHBL
 Check valid domains allowed by internal senders to detect PCs infected
 with spam trojans.
 Do CBV (callback verification) for mail with no published SPF record.
 message log for automated stats and blacklisting
-adapt init script to work on RH9
 Skip dspam when SPF pass?
 Report 551 with rcpt on SPF fail?
 check spam keywords with character classes, e.g.
@@ -48,3 +84,4 @@ Wrap smfi_setbacklog(int) - but it is only available in sendmail >= 8.12.3,
 Need a test module to feed sample messages to a milter though a live 
 sendmail and SMTP.  The mockup currently used is probably not very accurate,
 and doesn't test the threading code.
+

bms.py

@@ -1,6 +1,99 @@
 #!/usr/bin/env python
 # A simple milter.
 # $Log$
+# Revision 1.134  2005/05/25 15:36:43  stuart
+# Use dynip module.
+# Support smart aliasing of wiretap destination.
+# Always send DSN for SOFTFAIL.
+# Close forged bounce loophole when there are no headers.
+#
+# Revision 1.133  2005/03/16 21:58:04  stuart
+# Auto DSN feature.
+#
+# Revision 1.132  2005/02/12 02:11:10  stuart
+# Pass unit tests with python2.4.
+#
+# Revision 1.131  2005/02/11 18:34:13  stuart
+# Handle garbage after quote in boundary.
+#
+# Revision 1.130  2005/02/10 01:10:58  stuart
+# Fixed MimeMessage.ismodified()
+#
+# Revision 1.129  2005/02/10 00:56:48  stuart
+# Runs with python2.4.  Defang not working correctly - more work needed.
+#
+# Revision 1.128  2005/02/09 17:53:34  stuart
+# Optionally run dspam on internal mail.
+#
+# Revision 1.127  2004/12/03 14:26:21  stuart
+# Mark DYN PTR, REJECT softfail, log Received-SPF from trusted MTA.
+#
+# Revision 1.126  2004/11/24 14:39:38  stuart
+# Also accept softfail if valid PTR or HELO.
+#
+# Revision 1.125  2004/11/19 16:40:14  stuart
+# Block softfail except for listed domains.
+#
+# Revision 1.124  2004/11/19 06:18:04  stuart
+# block softfail for configured domains only
+#
+# Revision 1.123  2004/11/18 20:36:49  stuart
+# Recognize more dynamic hosts.  Ignore dynamic PTR for best_guess.
+#
+# Revision 1.122  2004/11/18 17:16:10  stuart
+# Recognize more dynamic ips.
+#
+# Revision 1.121  2004/11/09 22:37:48  stuart
+# Don't accept helo names which are dynamic IP addresses.
+#
+# Revision 1.120  2004/11/09 20:33:50  stuart
+# Recognize more dynamic PTR variations.
+#
+# Revision 1.118  2004/08/30 21:19:50  stuart
+# Try best guess for HELO, expand setreply for common errors
+#
+# Revision 1.117  2004/08/23 02:27:53  stuart
+# Allow multi rcpt CBV.  Add some multiline replies.
+#
+# Revision 1.116  2004/08/20 22:27:52  stuart
+# Generate TEMPFAIL for SPF softfail.
+#
+# Revision 1.115  2004/08/19 20:55:49  stuart
+# Always show reversed SRS path.
+# Check if encodings are an ASCII superset.  Some messages were encoded as
+# BIG5 and getting rejected even though chars were all in ascii subset.
+#
+# Revision 1.114  2004/07/27 00:40:12  stuart
+# Make reject on no PTR optional.
+#
+# Revision 1.113  2004/07/23 23:11:14  stuart
+# Log known malformed messages differently than general processing exceptions.
+#
+# Revision 1.112  2004/07/21 19:18:33  stuart
+# Punt on UnicodeDecodeError when decoding headers.
+# Accept a pass with default SPF for missing reverse IP.
+#
+# Revision 1.111  2004/07/18 13:13:31  stuart
+# Reject invalid SRS only for SRS domain (which is the only one we
+# know the key for).
+# Reject senders that have neither reverse IP nor SPF.
+#
+# Revision 1.110  2004/06/12 03:13:18  stuart
+# Block bounces only for SRS domain.  Also treat mail from
+# postmaster or mailer-daemon as DSN for SRS/SES checking purposes.
+#
+# Revision 1.109  2004/05/01 02:56:55  stuart
+# Let multiple screeners share work.
+#
+# Revision 1.108  2004/04/29 20:36:23  stuart
+# Require HELO name
+#
+# Revision 1.107  2004/04/24 22:55:29  stuart
+# Move some files to make the RPM more standard.
+#
+# Revision 1.106  2004/04/21 18:29:08  stuart
+# Validate hello name with SPF.
+#
 # Revision 1.105  2004/04/20 15:16:00  stuart
 # Release 0.6.9
 #
@@ -85,102 +178,6 @@
 # Revision 1.79  2003/12/04 23:46:06  stuart
 # Release 0.6.4
 #
-# Revision 1.78  2003/12/04 23:20:24  stuart
-# Make headerChange handle deleting absent header
-#
-# Revision 1.77  2003/12/04 22:01:40  stuart
-# Limit size of messages which will be dspammed.  This works around a bug
-# in dspam-2.6.5.2 where it scans large binary attachments.  I've never
-# seen really big spam anyway.
-#
-# Revision 1.76  2003/12/04 21:44:33  stuart
-# Pass header changes from Dspam to sendmail
-#
-# Revision 1.75  2003/11/25 17:43:07  stuart
-# Update FAQ.
-#
-# Revision 1.74  2003/11/25 17:36:58  stuart
-# dspam_reject
-#
-# Revision 1.73  2003/11/24 15:46:00  stuart
-# Missing global for dspam_whitelist
-#
-# Revision 1.72  2003/11/22 02:52:07  stuart
-# Handle multiple x-dspam-recipients properly on false positive
-#
-# Revision 1.71  2003/11/22 02:49:57  stuart
-# dspam whitelist
-#
-# Revision 1.70  2003/11/09 03:53:34  stuart
-# Don't block delivery of defanged false positives.
-#
-# Revision 1.69  2003/11/08 22:47:04  stuart
-# Exempt entire domains with '@domain.com'
-#
-# Revision 1.68  2003/11/02 03:06:16  stuart
-# Adjust error codes again.
-#
-# Revision 1.67  2003/11/02 03:01:46  stuart
-# Adjust SMTP error codes after careful reading of standard.
-#
-# Revision 1.66  2003/11/02 01:56:43  stuart
-# Use busy SMTP code.
-#
-# Revision 1.65  2003/11/02 01:44:11  stuart
-# Suppress traceback for Dspam lock timeouts
-#
-# Revision 1.64  2003/10/28 01:00:19  stuart
-# Dspam internal mail for dspam users
-#
-# Revision 1.63  2003/10/25 02:10:34  stuart
-# Match hostname for internal connection test, even if no ipaddr.
-#
-# Revision 1.62  2003/10/24 04:34:52  stuart
-# Fix for not saving defang of false positive triggered rejecting it
-# as a virus from self.
-#
-# Revision 1.61  2003/10/22 22:03:14  stuart
-# Apply dspam_exempt to screening
-#
-# Revision 1.60  2003/10/22 21:58:42  stuart
-# Don't save false positives as defang file.
-#
-# Revision 1.59  2003/10/22 05:02:27  stuart
-# Add support for dspam screeners
-#
-# Revision 1.58  2003/10/16 22:19:24  stuart
-# Redirect Dspam logging to bms milter
-#
-# Revision 1.57  2003/10/10 00:15:04  stuart
-# DISCARD message if quarrantined for any recipient.
-#
-# Revision 1.56  2003/10/06 19:30:27  stuart
-# REJECT messages with boundard errors
-#
-# Revision 1.55  2003/10/03 18:20:31  stuart
-# Opt-out feature to exempt certain recipients from header filtering.
-#
-# Revision 1.54  2003/09/22 13:36:04  stuart
-# Release 0.6.1
-#
-# Revision 1.53  2003/09/06 07:08:36  stuart
-# dspam support improvements.
-#
-# Revision 1.51  2003/09/02 00:27:27  stuart
-# Should have full milter based dspam support working
-#
-# Revision 1.50  2003/08/26 06:08:17  stuart
-# Use new python boolean since we now require 2.2.2
-#
-# Revision 1.49  2003/08/26 05:45:51  stuart
-# Fix conditional import of dspam.  Update web page.
-#
-# Revision 1.48  2003/08/26 05:10:43  stuart
-# Readability tweaks
-#
-# Revision 1.47  2003/08/26 05:01:38  stuart
-# Release 0.6.0
-#
 # Author: Stuart D. Gathman <stuart@bmsi.com>
 # Copyright 2001 Business Management Systems, Inc.
 # This code is under GPL.  See COPYING for details.
@@ -196,6 +193,8 @@ import tempfile
 import ConfigParser
 import time
 import re
+import Milter.dsn as dsn
+from Milter.dynip import is_dynip as dynip
 
 from fnmatch import fnmatchcase
 from email.Header import decode_header
@@ -242,15 +241,23 @@ dspam_users = {}
 dspam_userdir = None
 dspam_exempt = {}
 dspam_whitelist = {}
-dspam_screener = None
+dspam_screener = ()
 dspam_internal = True	# True if internal mail should be dspammed
 dspam_reject = ()
 dspam_sizelimit = 180000
 srs = None
 srs_reject_spoofed = False
+srs_fwdomain = None
 spf_reject_neutral = ()
+spf_accept_softfail = ()
 spf_best_guess = False
+spf_reject_noptr = False
 timeout = 600
+cbv_cache = {}
+try:
+  for rcpt in open('send_dsn.log'):
+    cbv_cache[rcpt.strip()] = None
+except IOError: pass
 
 class MilterConfigParser(ConfigParser.ConfigParser):
 
@@ -300,7 +307,7 @@ class MilterConfigParser(ConfigParser.ConfigParser):
 def read_config(list):
   cp = MilterConfigParser({
     'tempdir': "/var/log/milter/save",
-    'socket': "/var/log/milter/pythonsock",
+    'socket': "/var/run/milter/pythonsock",
     'timeout': '600',
     'scan_html': 'no',
     'scan_rfc822': 'yes',
@@ -310,7 +317,9 @@ def read_config(list):
     'maxage': '8',
     'hashlength': '8',
     'reject_spoofed': 'no',
-    'best_guess': 'no'
+    'reject_noptr': 'no',
+    'best_guess': 'no',
+    'dspam_internal': 'yes'
   })
   cp.read(list)
   tempfile.tempdir = cp.get('milter','tempdir')
@@ -353,28 +362,32 @@ def read_config(list):
     key = (sm[0],sm[1])
     smart_alias[key] = sm[2:]
 
-  global dspam_dict, dspam_users, dspam_userdir, dspam_exempt
+  global dspam_dict, dspam_users, dspam_userdir, dspam_exempt, dspam_internal
   global dspam_screener,dspam_whitelist,dspam_reject,dspam_sizelimit
-  global spf_reject_neutral,spf_best_guess,SRS
+  global spf_reject_neutral,spf_best_guess,SRS,spf_reject_noptr
+  global spf_accept_softfail
   dspam_dict = cp.getdefault('dspam','dspam_dict')
   dspam_exempt = cp.getaddrset('dspam','dspam_exempt')
   dspam_whitelist = cp.getaddrset('dspam','dspam_whitelist')
   dspam_users = cp.getaddrdict('dspam','dspam_users')
   dspam_userdir = cp.getdefault('dspam','dspam_userdir')
-  dspam_screener = cp.getdefault('dspam','dspam_screener')
+  dspam_screener = cp.getlist('dspam','dspam_screener')
   dspam_reject = cp.getlist('dspam','dspam_reject')
+  dspam_internal = cp.getboolean('dspam','dspam_internal')
   if cp.has_option('dspam','dspam_sizelimit'):
     dspam_sizelimit = cp.getint('dspam','dspam_sizelimit')
 
   if spf:
     spf.DELEGATE = cp.getdefault('spf','delegate')
     spf_reject_neutral = cp.getlist('spf','reject_neutral')
+    spf_accept_softfail = cp.getlist('spf','accept_softfail')
     spf_best_guess = cp.getboolean('spf','best_guess')
+    spf_reject_noptr = cp.getboolean('spf','reject_noptr')
   srs_config = cp.getdefault('srs','config')
   if srs_config: cp.read([srs_config])
   srs_secret = cp.getdefault('srs','secret')
   if SRS and srs_secret:
-    global srs,srs_reject_spoofed
+    global srs,srs_reject_spoofed,srs_fwdomain
     database = cp.getdefault('srs','database')
     srs_reject_spoofed = cp.getboolean('srs','reject_spoofed')
     maxage = cp.getint('srs','maxage')
@@ -387,7 +400,7 @@ def read_config(list):
     else:
       srs = SRS.Guarded.Guarded(secret=srs_secret,
         maxage=maxage,hashlength=hashlength,separator=separator)
-
+    srs_fwdomain = cp.getdefault('srs','fwdomain')
 
 def parse_addr(t):
   if t.startswith('<') and t.endswith('>'): t = t[1:-1]
@@ -400,7 +413,10 @@ def parse_header(val):
     u = []
     for s,enc in h:
       if enc:
-	u.append(unicode(s,enc))
+        try:
+	  u.append(unicode(s,enc))
+	except LookupError:
+	  u.append(unicode(s))
       else:
 	u.append(unicode(s))
     u = ''.join(u)
@@ -408,11 +424,14 @@ def parse_header(val):
       try:
 	return u.encode(enc)
       except UnicodeError: continue
-  except LookupError:
+  except UnicodeDecodeError: pass
-    return val
+  except LookupError: pass
+  return val
 
 class bmsMilter(Milter.Milter):
-  "Milter to replace attachments poisonous to Windows with a WARNING message."
+  """Milter to replace attachments poisonous to Windows with a WARNING message,
+     check SPF, and other anti-forgery features, and implement wiretapping
+     and smart alias redirection."""
 
   def log(self,*msg):
     print "%s [%d]" % (time.strftime('%Y%b%d %H:%M:%S'),self.id),
@@ -461,9 +480,9 @@ class bmsMilter(Milter.Milter):
 	if fnmatchcase(ipaddr,pat):
 	  self.trusted_relay = True
 	  break
-      self.connectip = ipaddr
+    else: ipaddr = ''
-    else:
+    self.connectip = ipaddr
-      self.connectip = None
+    self.missing_ptr = dynip(hostname,self.connectip)
     for pat in internal_connect:
       if fnmatchcase(hostname,pat):
 	self.internal_connection = True
@@ -474,7 +493,16 @@ class bmsMilter(Milter.Milter):
       connecttype = 'EXTERNAL'
     if self.trusted_relay:
       connecttype += ' TRUSTED'
+    if self.missing_ptr:
+      connecttype += ' DYN'
     self.log("connect from %s at %s %s" % (hostname,hostaddr,connecttype))
+    self.hello_name = None
+    self.connecthost = hostname
+    if hostname == 'localhost' and not ipaddr.startswith('127.') \
+    or hostname == '.':
+      self.log("REJECT: PTR is",hostname)
+      self.setreply('550','5.7.1', '"%s" is not a reasonable PTR name'%hostname)
+      return Milter.REJECT
     return Milter.CONTINUE
 
   def hello(self,hostname):
@@ -490,6 +518,25 @@ class bmsMilter(Milter.Milter):
       return Milter.REJECT
     return Milter.CONTINUE
 
+  def smart_alias(self,to):
+    if smart_alias:
+      t = parse_addr(to.lower())
+      if len(t) == 2:
+	ct = '@'.join(t)
+      else:
+	ct = t[0]
+      cf = self.canon_from
+      cf0 = cf.split('@',1)
+      if len(cf0) == 2:
+	cf0 = '@' + cf0[1]
+      else:
+	cf0 = cf
+      for key in ((cf,ct),(cf0,ct)):
+	if smart_alias.has_key(key):
+	  self.del_recipient(to)
+	  for t in smart_alias[key]:
+	    self.add_recipient('<%s>'%t)
+
   # multiple messages can be received on a single connection
   # envfrom (MAIL FROM in the SMTP protocol) seems to mark the start
   # of each message.
@@ -506,10 +553,12 @@ class bmsMilter(Milter.Milter):
     self.reject_spam = True
     self.data_allowed = True
     self.trust_received = self.trusted_relay
+    self.trust_spf = self.trusted_relay
     self.redirect_list = []
     self.discard_list = []
     self.new_headers = []
     self.recipients = []
+    self.cbv_needed = None
     t = parse_addr(f.lower())
     self.canon_from = '@'.join(t)
     self.fp.write('From %s %s\n' % (self.canon_from,time.ctime()))
@@ -524,6 +573,7 @@ class bmsMilter(Milter.Milter):
       self.rejectvirus = domain in reject_virus_from
       if user in wiretap_users.get(domain,()):
         self.add_recipient(wiretap_dest)
+	self.smart_alias(wiretap_dest)
       if user in discard_users.get(domain,()):
 	self.discard = True
       exempt_users = dspam_whitelist.get(domain,())
@@ -531,6 +581,10 @@ class bmsMilter(Milter.Milter):
 	self.dspam = False
     else:
       self.rejectvirus = False
+    if not self.hello_name:
+      self.log("REJECT: missing HELO")
+      self.setreply('550','5.7.1',"It's polite to say HELO first.")
+      return Milter.REJECT
     if not (self.internal_connection or self.trusted_relay)	\
     	and self.connectip and spf:
       return self.check_spf()
@@ -539,27 +593,89 @@ class bmsMilter(Milter.Milter):
   def check_spf(self):
     t = parse_addr(self.mailfrom)
     if len(t) == 2: t[1] = t[1].lower()
-    q = spf.query(self.connectip,'@'.join(t),self.hello_name)
+    receiver = self.receiver
+    q = spf.query(self.connectip,'@'.join(t),self.hello_name,receiver=receiver)
     q.set_default_explanation('SPF fail: see http://spf.pobox.com/why.html')
     res,code,txt = q.check()
-    receiver = self.receiver
+    if res in ('none', 'softfail'):
-    if res == 'none' and spf_best_guess:
+      if self.mailfrom != '<>':
-      #self.log('SPF: no record published, guessing')
+	# check hello name via spf
-      q.set_default_explanation('SPF guess: see http://spf.pobox.com/why.html')
+	h = spf.query(self.connectip,'',self.hello_name,receiver=receiver)
-      # best_guess should not result in fail
+	hres,hcode,htxt = h.check()
-      res,code,txt = q.best_guess()
+	if hres in ('deny','fail','neutral','softfail'):
-      receiver += ': guessing'
+	  self.log('REJECT: hello SPF: %s 550 %s' % (hres,htxt))
+	  self.setreply('550','5.7.1',htxt,
+	    "The hostname given in your MTA's HELO response is not listed",
+	    "as a legitimate MTA in the SPF records for your domain.  If you",
+	    "get this bounce, the message was not in fact a forgery, and you",
+	    "should IMMEDIATELY notify your email administrator of the problem."
+	  )
+	  return Milter.REJECT
+	if hres == 'none' and spf_best_guess \
+	  and not dynip(self.hello_name,self.connectip):
+	  hres,hcode,htxt = h.best_guess()
+      else: hres = res
+      if spf_best_guess and res == 'none':
+	#self.log('SPF: no record published, guessing')
+	q.set_default_explanation(
+		'SPF guess: see http://spf.pobox.com/why.html')
+	# best_guess should not result in fail
+	if self.missing_ptr:
+	  # ignore dynamic PTR for best guess
+	  res,code,txt = q.best_guess('v=spf1 a/24 mx/24')
+	else:
+	  res,code,txt = q.best_guess()
+	receiver += ': guessing'
+      if self.missing_ptr and res in ('neutral', 'none') and hres != 'pass':
+	if spf_reject_noptr:
+	  self.log('REJECT: no PTR, HELO or SPF')
+	  self.setreply('550','5.7.1',
+    'You must have a reverse lookup or publish SPF: http://spf.pobox.com',
+    'Contact your mail administrator IMMEDIATELY!  Your mail server is',
+    'severely misconfigured.  It has no PTR record (dynamic PTR records',
+    "that contain your IP don't count), an invalid HELO, and no SPF record."
+	  )
+	  return Milter.REJECT
+        if self.mailfrom != '<>':
+	  self.cbv_needed = q
     if res in ('deny', 'fail'):
       self.log('REJECT: SPF %s %i %s' % (res,code,txt))
       self.setreply(str(code),'5.7.1',txt)
+      # A proper SPF fail error message would read:
+      # forger.biz [1.2.3.4] is not allowed to send mail with the domain
+      # "forged.org" in the sender address.  Contact <postmaster@forged.org>.
       return Milter.REJECT
+    if res == 'softfail' and not q.o in spf_accept_softfail:
+      if self.missing_ptr and hres != 'pass':
+        if spf_reject_noptr or q.o in spf_reject_neutral:
+	  self.log('REJECT: SPF %s %i %s' % (res,code,txt))
+	  self.setreply('550','5.7.1',
+	    'SPF softfail: If you get this Delivery Status Notice, your email',
+	    'was probably legitimate.  Your administrator has published SPF',
+	    'records in a testing mode.  The SPF record reported your email as',
+	    'a forgery, which is a mistake if you are reading this.  Please',
+	    'notify your administrator of the problem immediately.'
+	  )
+	  return Milter.REJECT
+      if self.mailfrom != '<>':
+	self.cbv_needed = q
     if res == 'neutral' and q.o in spf_reject_neutral:
       self.log('REJECT: SPF neutral for',q.s)
       self.setreply('550','5.7.1',
-	'mail from %s must pass SPF: http://spf.pobox.com/why.html' % q.o
+	'mail from %s must pass SPF: http://spf.pobox.com/why.html' % q.o,
+	'The %s domain is one that spammers love to forge.  Due to' % q.o,
+	'the volume of forged mail, we can only accept mail that',
+	'the SPF record for %s explicitly designates as legitimate.' % q.o,
+	'Sending your email through the recommended outgoing SMTP',
+	'servers for %s should accomplish this.' % q.o
       )
       return Milter.REJECT
     if res == 'error':
+      if code >= 500:
+        self.log('REJECT: SPF %s %i %s' % (res,code,txt))
+	self.setreply(str(code),'5.7.1',txt)
+	return Milter.REJECT
+      self.log('TEMPFAIL: SPF %s %i %s' % (res,code,txt))
       self.setreply(str(code),'4.3.0',txt)
       return Milter.TEMPFAIL
     self.add_header('Received-SPF',q.get_header(res,receiver))
@@ -576,29 +692,31 @@ class bmsMilter(Milter.Milter):
     self.log("rcpt to",to,str)
     t = parse_addr(to.lower())
     if len(t) == 2:
-      if self.mailfrom == '<>':
+      user,domain = t
+      if self.mailfrom == '<>' or self.canon_from.startswith('postmaster@') \
+      	or self.canon_from.startswith('mailer-daemon@'):
         if self.recipients:
-	  self.log('REJECT: Multiple bounce recipients')
+	  self.data_allowed = False
-	  self.setreply('550','5.7.1','Multiple bounce recipients')
+        if srs and domain == srs_fwdomain:
-	  return Milter.REJECT
-        if srs and not (self.internal_connection or self.trusted_relay):
 	  oldaddr = '@'.join(parse_addr(to))
 	  try:
 	    newaddr = srs.reverse(oldaddr)
+	    # Currently, a sendmail map reverses SRS.  We just log it here.
 	    self.log("srs rcpt:",newaddr)
 	  except:
-	    if srsre.match(oldaddr):
+            if not (self.internal_connection or self.trusted_relay):
-	      self.log("REJECT: srs spoofed:",oldaddr)
+	      if srsre.match(oldaddr):
-	      self.setreply('550','5.7.1','Invalid SRS signature')
+		self.log("REJECT: srs spoofed:",oldaddr)
-	      return Milter.REJECT
+		self.setreply('550','5.7.1','Invalid SRS signature')
-	    self.data_allowed = not srs_reject_spoofed
+		return Milter.REJECT
+	      self.data_allowed = not srs_reject_spoofed
+      # non DSN mail to SRS address will bounce due to invalid local part
       self.recipients.append('@'.join(t))
-      user,domain = t
       users = check_user.get(domain)
       if self.discard:
         self.del_recipient(to)
       if users and not user in users:
-        self.log('REJECT: RCPT TO:',to,str)
+        self.log('REJECT: RCPT TO:',to)
 	return Milter.REJECT
       if user in block_forward.get(domain,()):
         self.forward = False
@@ -609,19 +727,7 @@ class bmsMilter(Milter.Milter):
         self.hidepath = True
       if not domain in dspam_reject:
         self.reject_spam = False
-    if smart_alias:
+    self.smart_alias(to)
-      cf = self.canon_from
-      cf0 = cf.split('@',1)
-      if len(cf0) == 2:
-	cf0 = '@' + cf0[1]
-      else:
-	cf0 = cf
-      ct = '@'.join(t)
-      for key in ((cf,ct),(cf0,ct)):
-	if smart_alias.has_key(key):
-	  self.del_recipient(to)
-	  for t in smart_alias[key]:
-	    self.add_recipient('<%s>'%t)
     #rcpt = self.getsymval("{rcpt_addr}")
     #self.log("rcpt-addr",rcpt);
     return Milter.CONTINUE
@@ -681,21 +787,36 @@ class bmsMilter(Milter.Milter):
     elif self.trust_received and lname == 'received':
       self.trust_received = False
       self.log('%s: %s' % (name,val.splitlines()[0]))
+    elif self.trust_spf and lname == 'received-spf':
+      self.trust_spf = False
+      self.log('%s: %s' % (name,val.splitlines()[0]))
     return Milter.CONTINUE
 
+  def forged_bounce(self):
+    if len(self.recipients) > 1:
+      self.log('REJECT: Multiple bounce recipients')
+      self.setreply('550','5.7.1','Multiple bounce recipients')
+    else:
+      self.log('REJECT: bounce with no SRS encoding')
+      self.setreply('550','5.7.1',
+      "I did not send you that message. Please consider implementing SPF",
+      "(http://spf.pobox.com) to avoid bouncing mail to spoofed senders.",
+      "Thank you."
+      )
+    return Milter.REJECT
+    
   def header(self,name,hval):
     if not self.data_allowed:
-      self.log('REJECT: bounce with no SRS encoding')
+      return self.forged_bounce()
-      self.setreply('550','5.7.1',"spoofed reply address")
+	  
-      return Milter.REJECT
     lname = name.lower()
     # decode near ascii text to unobfuscate
     val = parse_header(hval)
     if not self.internal_connection:
       # even if we wanted the Taiwanese spam, we can't read Chinese
       if block_chinese and lname == 'subject':
-	if hval.startswith('=?big5') or hval.startswith('=?ISO-2022-JP'):
+	if val.startswith('=?big5') or val.startswith('=?ISO-2022-JP'):
-	  self.log('REJECT: %s: %s' % (name,hval))
+	  self.log('REJECT: %s: %s' % (name,val))
 	  self.setreply('550','5.7.1',"We don't understand chinese")
 	  return Milter.REJECT
       rc = self.check_header(name,val)
@@ -707,12 +828,14 @@ class bmsMilter(Milter.Milter):
       try:
         val = val.encode('us-ascii')
       except:
-        val = hval
+	val = hval
       self.fp.write("%s: %s\n" % (name,val))	# add header to buffer
     return Milter.CONTINUE
 
   def eoh(self):
     if not self.fp: return Milter.TEMPFAIL	# not seen by envfrom
+    if not self.data_allowed:
+      return self.forged_bounce()
     for name,val in self.new_headers:
       self.fp.write("%s: %s\n" % (name,val))	# add new headers to buffer
     self.fp.write("\n")				# terminate headers
@@ -761,7 +884,8 @@ class bmsMilter(Milter.Milter):
     # don't let a tricky virus slip one past us
     if scan_rfc822:
       msg = msg.get_submsg()
-      if msg: return mime.check_attachments(msg,self._chk_attach)
+      if isinstance(msg,email.Message.Message):
+	return mime.check_attachments(msg,self._chk_attach)
     return Milter.CONTINUE
 
   def alter_recipients(self,discard_list,redirect_list):
@@ -832,21 +956,22 @@ class bmsMilter(Milter.Milter):
 	    print x
     # screen if no recipients are dspam_users
     if not modified and dspam_screener and not self.internal_connection \
-    	and (self.dspam or self.reject_spam):
+    	and self.dspam:
       self.fp.seek(0)
       txt = self.fp.read()
       if len(txt) > dspam_sizelimit:
 	self.log("Large message:",len(txt))
 	return False
-      if not ds.check_spam(dspam_screener,txt,self.recipients,
+      screener = dspam_screener[self.id % len(dspam_screener)]
+      if not ds.check_spam(screener,txt,self.recipients,
       	classify=True,quarantine=not self.reject_spam):
 	self.fp = None
 	if self.reject_spam:
-	  self.log("DSPAM:",dspam_screener,
+	  self.log("DSPAM:",screener,
 	  	'REJECT: X-DSpam-Score: %f' % ds.probability)
 	  self.setreply('550','5.7.1','Your Message looks spammy')
 	  return True
-	self.log("DSPAM:",dspam_screener,"SCREENED")
+	self.log("DSPAM:",screener,"SCREENED")
     return modified
 
   def eom(self):
@@ -862,11 +987,12 @@ class bmsMilter(Milter.Milter):
 
       # analyze all mail for dangerous attachments and scripts
       self.fp.seek(0)
-      msg = mime.MimeMessage(self.fp)
+      msg = mime.message_from_file(self.fp)
       # pass header changes in top level message to sendmail
       msg.headerchange = self._headerChange
 
       # filter leaf attachments through _chk_attach
+      assert not msg.ismodified()
       rc = mime.check_attachments(msg,self._chk_attach)
     except:	# milter crashed trying to analyze mail
       exc_type,exc_value = sys.exc_info()[0:2]
@@ -881,16 +1007,21 @@ class bmsMilter(Milter.Milter):
       fname = tempfile.mktemp(".fail")	# save message that caused crash
       os.rename(self.tempname,fname)
       self.tempname = None
-      self.log("FAIL: %s" % fname)	# log filename
       if exc_type == email.Errors.BoundaryError:
+	self.log("MALFORMED: %s" % fname)	# log filename
+        if self.internal_connection:
+	  # accept anyway for now
+	  return Milter.ACCEPT
 	self.setreply('554','5.7.7',
 		'Boundary error in your message, are you a spammer?')
         return Milter.REJECT
       if exc_type == email.Errors.HeaderParseError:
+	self.log("MALFORMED: %s" % fname)	# log filename
 	self.setreply('554','5.7.7',
 		'Header parse error in your message, are you a spammer?')
         return Milter.REJECT
       # let default exception handler print traceback and return 451 code
+      self.log("FAIL: %s" % fname)	# log filename
       raise
     if rc == Milter.REJECT: return rc;
     if rc == Milter.DISCARD: return rc;
@@ -916,6 +1047,33 @@ class bmsMilter(Milter.Milter):
     for name,val in self.new_headers:
       self.addheader(name,val)
 
+    if self.cbv_needed:
+      sender = self.cbv_needed.s
+      cached = cbv_cache.has_key(sender)
+      if cached:
+	self.log('CBV:',sender,'(cached)')
+        res = cbv_cache[sender]
+      else:
+	self.log('CBV:',sender)
+	m = dsn.create_msg(self.cbv_needed,self.recipients,msg)
+        m = m.as_string()
+        print >>open('last_dsn','w'),m
+	res = dsn.send_dsn(sender,self.receiver,m)
+      if res:
+        desc = "CBV: %d %s" % res[:2]
+        if 400 <= res[0] < 500:
+	  self.log('TEMPFAIL:',desc)
+          self.setreply('450','4.2.0',*desc.splitlines())
+	  return Milter.TEMPFAIL
+	cbv_cache[sender] = res
+	self.log('REJECT:',desc)
+        self.setreply('550','5.7.1',*desc.splitlines())
+	return Milter.REJECT
+      cbv_cache[sender] = res
+      if not cached:
+	print >>open('send_dsn.log','a'),sender # log who we sent DSNs to
+      self.cbv_needed = None
+
     if not defanged and not spam_checked:
       os.remove(self.tempname)
       self.tempname = None	# prevent re-removal
@@ -967,13 +1125,13 @@ def main():
   if srs or len(discard_users) > 0 or smart_alias or dspam_userdir:
     flags = flags + Milter.DELRCPT
   Milter.set_flags(flags)
-  print "bms milter startup"
+  print "%s bms milter startup" % time.strftime('%Y%b%d %H:%M:%S')
   sys.stdout.flush()
   Milter.runmilter("pythonfilter",socketname,timeout)
-  print "bms milter shutdown"
+  print "%s bms milter shutdown" % time.strftime('%Y%b%d %H:%M:%S')
 
 if __name__ == "__main__":
-  read_config(["milter.cfg"])
+  read_config(["/etc/mail/pymilter.cfg","milter.cfg"])
   if dspam_dict:
     import dspam	# low level spam check
   if dspam_userdir:

cid2spf.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python2.3
+
+# Convert a MS Caller-ID entry (XML) to a SPF entry
+#
+# (c) 2004 by Ernesto Baschny
+# (c) 2004 Python version by Stuart Gathman
+#
+# Date: 2004-02-25
+# Version: 1.0
+#
+# Usage:
+#  ./cid2spf.pl "<ep xmlns='http://ms.net/1'>...</ep>"
+#
+# Note that the 'include' directives will also have to be checked and
+# "translated". Future versions of this script might be able to get a
+# domain name as an argument and "crawl" the DNS for the necessary
+# information.
+#
+# A complete reverse translation (SPF -> CID) might be impossible, since
+# there are no way to handle:
+# - PTR and EXISTS mechanism 
+# - MX mechanism with an different domain as argument
+# - macros
+# 
+# References:
+# http://www.microsoft.com/mscorp/twc/privacy/spam_callerid.mspx
+# http://spf.pobox.com/
+#
+# Known bugs:
+# - Currently it won't handle the exclusions provided in the A and R
+#   tags (prefix '!'). They will show up "as-is" in the SPF record
+# - I really haven't read the MS-CID specs in-depth, so there are probably
+#   other bugs too :)
+#
+# Ernesto Baschny <ernst@baschny.de>
+#
+
+import xml.sax
+import spf
+
+# -------------------------------------------------------------------------
+class CIDParser(xml.sax.ContentHandler):
+  "Convert a MS Caller-ID entry (XML) to a SPF entry"
+
+  def __init__(self,q=None):
+    self.spf = []
+    self.action = '-all'
+    self.has_servers = None
+    self.spf_entry = None
+    if q:
+      self.spf_query = q
+    else:
+      self.spf_query = spf.query(i='127.0.0.1', s='localhost', h='unknown')
+
+  def startElement(self,tag,attr):
+      if tag == 'm':
+	if self.has_servers != None and not self.has_servers:
+	  raise ValueError(
+    "Declared <noMailServers\> and later <m>, this CID entry is not valid."
+	  )
+	self.has_servers = True
+      elif tag == 'noMailServers':
+	if self.has_servers:
+	  raise ValueError(
+    "Declared <m> and later <noMailServers\>, this CID entry is not valid."
+	  )
+	self.has_servers = False
+      elif tag == 'ep':
+	if attr.has_key('testing') and attr.getValue('testing') == 'true':
+	  # A CID with 'testing' found:
+	  # From the MS-specs:
+	  #  "Documents in which such attribute is present with a true
+	  #  value SHOULD be entirely ignored (one should act as if the
+	  #  document were absent)"
+	  # From the SPF-specs:
+	  #  "Neutral (?): The SPF client MUST proceed as if a domain did
+	  #  not publish SPF data."
+	  # So we set SPF action to "neutral":
+	  self.action = '?all'
+      elif tag == 'mx':
+	  # The empty MX-tag, same as SPF's MX-mechanism
+	  self.spf.append('mx')
+      self.tag = tag
+
+  def characters(self,text):
+	tag = self.tag
+	# Remove starting and trailing spaces from text:
+	text = text.strip()
+
+	if tag == 'a' or tag == 'r':
+	    # The A and R tags from MS-CID are both handled by the 
+	    # ipv4/6-mechanisms from SPF:
+	    if text.find(':') < 0:
+	      mechanism = 'ip4'
+	    else:
+	      mechanism = 'ip6'
+	    self.spf.append(mechanism + ':' + text)
+	elif tag == 'indirect':
+	    # MS-CID's indirect is "sort of" the include from SPF:
+	    # Not really true, because the <indirect> tag from MS-CID also 
+	    # provides a fallback in case the included domain doesn't provide
+	    # _ep-records: The inbound MX-servers of the included domains
+	    # are added to the list of allowed outgoing mailservers for the
+	    # domain that declared the _ep-record with the <indirect> tag.
+	    # In SPF you would use the 'mx:domain' to handle this, but this
+	    # wouldn't depend on referred domain having or not SPF-records.
+	    cid_xml = self.cid_txt(text)
+	    if cid_xml:
+	      p = CIDParser()
+	      xml.sax.parseString(cid_xml,p)
+	      if p.has_servers != False:
+		self.spf += p.spf
+	    else:
+	      self.spf.append('mx:' + text)
+
+  def cid_txt(self,domain):
+    q = self.spf_query
+    domain='_ep.' + domain
+    a = q.dns_txt(domain)
+    if not a: return None
+    if a[0].lower().startswith('<ep ') and a[-1].lower().endswith('</ep>'):
+      return ''.join(a)
+    return None
+
+  def endElement(self,tag):
+      if tag == 'ep':
+	# This is the end... assemble what we've got
+	spf_entry = ['v=spf1']
+	if self.has_servers != False:
+	  spf_entry += self.spf
+	spf_entry.append(self.action)
+	self.spf_entry = ' '.join(spf_entry)
+
+  def spf_txt(self,cid_xml):
+    if not cid_xml.startswith('<'):
+      cid_xml = self.cid_txt(cid_xml)
+      if not cid_xml: return None
+    # Parse the beast. Any XML-problem will be reported by xlm.sax
+    self.spf_entry = None
+    xml.sax.parseString(cid_xml,self)
+    return self.spf_entry
+
+if __name__ == '__main__':
+  import sys
+  if len(sys.argv) < 2:
+    print >>sys.stderr, \
+      """Usage: %s "<ep xmlns='http://ms.net/1'>...</ep>" """ % sys.argv[0]
+    sys.exit(1)
+
+  cid_xml = sys.argv[1]
+
+  p = CIDParser()
+  print p.spf_txt(cid_xml)

faq.html

@@ -134,5 +134,26 @@ is a milter declaration for sendmail.cf with all timeouts specified:
 Xpythonfilter, S=local:/var/log/milter/pythonsock, F=T, T=C:5m;S:20s;R:60s;E:5m
 </pre>
 
+<a name="spf">
+<li> Q. So how do I use the SPF support?  The sample.py milter doesn't seem
+        to use it.
+<p>  A. The bms.py milter supports spf.  The RedHat RPMs will set almost
+everything up for you.  For other systems:
+<ol type=i>
+<li> Arrange to run bms.py in the background (as a service perhaps) and
+     redirect output and errors to a logfile.  For instance, on AIX you'll want
+     to use SRC (System Resource Controller).  
+<li> Copy milter.cfg to the directory you run bms.py in, and edit it.  The
+     comments should explain the options. 
+<li> Start bms.py in the background as arranged.
+<li> Add Xpythonfilter to sendmail.cf or add an INPUT_MAIL_FILTER to
+     sendmail.mc.  Regen sendmail.cf if you use sendmail.mc and restart 
+     sendmail.
+<li> Arrange to rotate log files and remove old defang files in 
+     <code>tempdir</code>.  The RedHat RPM uses <code>logrotate</code> for
+     logfiles and a simple cron script using <code>find</code> to clean
+     <code>tempdir</code>.
+</ol>
+
 </ol>
 </html>

milter.cfg

@@ -1,63 +1,82 @@
-# features intended to filter or block incoming mail
 [milter]
-;socket=/var/log/milter/pythonsock
+# the socket used to communicate with sendmail.  Must match sendmail.cf
+;socket=/var/run/milter/pythonsock
+# where to save original copies of defanged and failed messages
 tempdir = /var/log/milter/save
+# how long to wait for a response from sendmail before giving up 
 ;timeout=600
-
-scan_rfc822 = 1
-# can be CPU intensive
-scan_html = 0
-# reject asian fonts because we can't read them
-block_chinese = 1
-# users who hate forwarded mail
-;block_forward = egghead@mycorp.com, busybee@mycorp.com
 log_headers = 0
-# Reject mail for domains mentioned unless user is mentioned here also
-;check_user = joe@mycorp.com, mary@mycorp.com, file:bigcorp.com
-# porn words are case insensitive
-porn_words = penis, breast, pussy, horse cock, porn, xenical, diet pill, d1ck,
-	vi*gra, vi-a-gra, viag, tits, p0rn, hunza, horny, sexy, c0ck,
-	p-e-n-i-s, hydrocodone, vicodin, xanax, vicod1n, x@nax, diazepam,
-	v1@gra, xan@x, cialis, ci@lis, frëe, xãnax, valíum, vãlium, via-gra,
-	x@n3x, vicod3n, penís, v|c0d1n, phentermine, en1arge, dip1oma, v1codin
-# spam words are case sensitive
-spam_words = $$$, !!!, XXX, FREE, HGH
-
 # connection ips and hostnames are matched against this glob style list
 # to recognize internal senders
 ;internal_connect = 192.168.*.*
+
 # mail that is not an internal_connect and claims to be from an
-# internal domain is rejected.
+# internal domain is rejected.  You should enable SPF instead if you can.
+# SPF is much more comprehensive and flexible.
 ;internal_domains = mycorp.com
+
 # connections from a trusted relay can trust the first Received header
+# SPF checks are bypassed for internal connections and trusted relays.
 ;trusted_relay = 1.2.3.4, 66.12.34.56
+
 # reject external senders with hello names no legit external sender would use
+# SPF will do this also, but listing your own domain and mailserver here
+# will save some DNS lookups when rejecting certain viruses.
 ;hello_blacklist = mycorp.com, 66.12.34.56
 
+# features intended to filter or block incoming mail
+;[defang]
+# do virus scanning on attached messages also
+scan_rfc822 = 1
+# Comment out scripts in HTML attachments.  Can be CPU intensive.
+scan_html = 0
+# reject messages with asian fonts because we can't read them
+block_chinese = 1
+# list users who hate forwarded mail
+;block_forward = egghead@mycorp.com, busybee@mycorp.com
+# Reject mail for domains mentioned unless user is mentioned here also
+;check_user = joe@mycorp.com, mary@mycorp.com, file:bigcorp.com
+# reject mail with these case insensitive strings in the subject
+porn_words = penis, breast, pussy, horse cock, porn, xenical, diet pill, d1ck,
+	vi*gra, vi-a-gra, viag, tits, p0rn, hunza, horny, sexy, c0ck, xanaax,
+	p-e-n-i-s, hydrocodone, vicodin, xanax, vicod1n, x@nax, diazepam,
+	v1@gra, xan@x, cialis, ci@lis, frëe, xãnax, valíum, vãlium, via-gra,
+	x@n3x, vicod3n, penís, c0d1n, phentermine, en1arge, dip1oma, v1codin,
+	valium, rolex, sexual
+# reject mail with these case sensitive strings in the subject
+spam_words = $$$, !!!, XXX, FREE, HGH
+
+# See http://bmsi.com/python/pysrs.html for details
 [srs]
 config=/etc/mail/pysrs.cfg
+# SRS options can be set here also, but must match the sendmail plugin
 ;secret="shhhh!"
 ;maxage=21
 ;hashlength=4
 ;database=/var/log/milter/srsdata
 ;fwdomain = mydomain.com
-# turn this on after a grace period
+# turn this on after a grace period to reject spoofed DSNs
 reject_spoofed = 0
 
+# See http://spf.pobox.com for more info on SPF.
 [spf]
 # namespace where SPF records can be supplied for domains without one
-# records are search for under _spf.domain.com
+# records are searched for under _spf.domain.com
 ;delegate = domain.com
 # domains where a neutral SPF result should cause mail to be rejected
 ;reject_neutral = aol.com
 # use a default (v=spf1 a/24 mx/24 ptr) when no SPF records are published
 ;best_guess = 0
+# reject senders that have neither PTR nor SPF records
+;reject_noptr = 0
+# always accept softfail from these domains
+;accept_softfail = bounces.amazon.com
 
 # features intended to clean up outgoing mail
 [scrub]
-# domains that stupidly block visible private nodes
+# domains that block visible private nodes
 ;hide_path = jcpenney.com	
-# block, don't just replace with warning, viruses from these domains
+# reject, don't just replace with warning, viruses from these domains
 ;reject_virus_from = mycorp.com
 
 # features intended for spying on users and coworkers
@@ -86,17 +105,21 @@ blind = 1
 ;walter1 = cust@othercorp.com,walter@bigcorp.com,boss@bigcorp.com,
 ;	walter@bigcorp.com
 
+# See http://bmsi.com/python/dspam.html
 [dspam]
-# Select a well moderated dspam dictionary to reject spammy headers
+# Select a well moderated dspam dictionary to reject spammy headers.
-# dspam-python must be installed to use: http://bmsi.com/python/dspam.html
+# To filter on the entire message, use the full setup below.
 # only EXTERNAL messages are dspam filtered
 ;dspam_dict=/var/lib/dspam/moderator.dict
+
 # Opt-opt recipients from dspam screening and header triage
 ;dspam_exempt=getitall@mycorp.com
 # Do not scan mail (ostensibly) from these senders
 ;dspam_whitelist=getitall@sender.com
-# Reject spam to these domains, perhaps because we are a backup MX server
+# Reject spam to these domains instead of quarantining it.
 ;dspam_reject=othercorp.com
+# Scan internal mail - often a good source of stats on legit mail.
+;dspam_internal=1
 
 # directory for dspam user quarantine, signature db, and dictionaries
 # defining this activates the dspam application
@@ -113,8 +136,9 @@ blind = 1
 ;spam=spam@foocorp.com
 # address to forward false positives to.  milter will process and not deliver
 ;falsepositive=ham@foocorp.com
-# the dspam_screener is used to screen mail for all recipients who are
+# the dspam_screener is a list of dspam users who screen mail for all
-# not dspam_users.  Spam goes to the screeners quarantine, and the original
+# recipients who are not dspam_users.  Spam goes to the screeners quarantine,
-# recipients saved so that false positives can be properly delivered.
+# and the original recipients are saved so that false positives can be properly
-
+# delivered.
+;dspam_screener=david,goliath
 # The dspam CGI can also be used: logins must match dspam users

milter.html

@@ -13,8 +13,8 @@ ALT="Viewable With Any Browser" BORDER="0"></A>
 	usemap="#banner_4" alt="Your vote?">
 <map name="banner_4">
   <area shape="rect" coords="330,25,426,59"
-  	href="http://www.sepschool.org/survey/" alt="I Disagree">
+  	href="http://education-survey.org/" alt="I Disagree">
-  <area shape="rect" coords="234,28,304,57" href="http://sepschool.org/" alt="I Agree">
+  <area shape="rect" coords="234,28,304,57" href="http://www.honestEd.com/" alt="I Agree">
 </map>
 
 </P>
@@ -24,12 +24,14 @@ ALT="Viewable With Any Browser" BORDER="0"></A>
   Stuart D. Gathman</a><br>
 This web page is written by Stuart D. Gathman<br>and<br>sponsored by
 <a href="http://www.bmsi.com">Business Management Systems, Inc.</a> <br>
-Last updated Apr 21, 2004</h4>
+Last updated May 31, 2005</h4>
 
-See the <a href="faq.html">FAQ</a> | <a href="#download">Download now</a> |
+See the <a href="faq.html">FAQ</a> | <a href="http://sourceforge.net/project/showfiles.php?group_id=139894">Download now</a> |
-<a href="/mailman/listinfo/pymilter">Subscribe to mailing list</a>
+<a href="/mailman/listinfo/pymilter">Subscribe to mailing list</a> |
+<a href="#overview">Overview</a>
 <p>
-<img src="python55.gif" align=left alt="A Python">
+<a href="//www.python.org">
+<img src="python55.gif" align=left alt="A Python"></a>
 <a href="//www.sendmail.org/">Sendmail</a> introduced a
 <a href="http://www.milter.org/milter_api/api.html"> new API</a> beginning with version 8.10 -
 libmilter.  The milter module for <a href="//www.python.org">Python</a>
@@ -40,11 +42,79 @@ Version 8.12 seems to be more robust, and includes new privilege
 separation features to enhance security.
 I recommend upgrading.
 
-<h2> <a name=dspam>Bayesian Filtering</a> </h2>
+<h2> Recent Changes </h2>
 
-I have selected the <a href="http://www.nuclearelephant.com/projects/dspam/">
+Python milter is being moved to
-dspam bayes filter project</a> and <a href="dspam.html">
+<a href="http://sourceforge.net/projects/pymilter/">Sourceforge</a> for
-packaged it for python</a>.
+development.
+<p>
+Release 0.7.2 tightens the authentication screws with a "3 strikes and
+your out" policy.  A sender must have a valid PTR, HELO, or SPF record
+to send email.  Specific senders can be whitelisted using the
+"delegate" option in the spf configuration section by adding a
+default SPF record for them.  The PTR and HELO are required
+by RFC anyway, so this is not an unreasonable requirement.
+There is now a coherent policy for an SPF softfail result.  A softfail
+is accepted if there is a valid PTR or HELO, or if the domain
+is listed in the "accept_softfail" option of the spf configuration section.
+A neutral result is accepted by default if there is a valid PTR or
+HELO, (and the SPF record was not guessed), unless the domain is listed in the
+"reject_neutral" option.  Common forms of PTR records for dynamic IPs are
+recognized, and do not count as a valid PTR.  This does not prevent anyone
+from sending mail from a dynamic IP - they just need to configure a
+valid HELO name or publish an SPF record.
+<p>
+As SPF adoption continues to rise, forged spam is not getting through.  So
+spammers are publishing their SPF records as predicted.  The 0.7.2 RPM
+now provides the <code>rhsbl</code> sendmail hack so that spammer domains
+can be blacklisted.  With the RPM installed, add a line like the following
+to your <code>sendmail.mc</code>.
+<pre>
+HACK(rhsbl,`blackholes.example.com',"550 Rejected: " $&{RHS} " has been spamming our customers.")dnl
+</pre>
+<p>
+Of course, spammers are now starting to register
+throwaway domains.  The next thing we need is a custom DNS server,
+in Python, that
+can recognize patterns.  For instance, one spammer registers ded304.com,
+ded305.com, ded306.com, etc.  We also need the custom DNS server to
+let SPF classic clients check SES (which will be part of pysrs).  
+The <a href="http://twistedmatrix.com/products/twisted">Twisted Python</a>
+framework provides a custom DNS server - but I
+would like a smaller implementation for our use.
+<p>
+The RPM for release 0.7.0 moves the config file and socket locations to
+/etc/mail and /var/run/milter respectively.  We now parse Microsoft CID records
+- but only hotmail.com uses them.  They seem to have applied for a patent on
+the brilliant idea of examining the mail headers to see who the message is
+from.  We aren't doing that here, so not to worry - but I am not a lawyer, so
+if you are worried, change spf.py around line 626 to return None instead of
+calling CIDParser().  There is a new option to reject mail with no PTR
+and no SPF.
+<p>
+Microsoft is pushing an anti-opensource license for their pending patent
+along with their sender-ID proposal before the IETF.
+It is royalty free - but requires anyone distributing a binary they've
+compiled from source to sign a license agreement.  The Apache Software
+Foundation <a
+href="http://www.apache.org/foundation/docs/sender-id-position.html"> explains
+the problem with sender-ID</a>, and Debian <a
+href="http://www.debian.org/News/2004/20040904">concurs</a>.  Since 
+the <a href="http://download.microsoft.com/download/4/3/9/439b024b-09fd-44ee-8ff0-10e834004c36/senderid_FAQ.PDF">Microsoft license</a> is
+<a href="http://www.circleid.com/article/732_0_1_0_C/">incompatible with free
+software in general</a> and the <a
+href="http://www.imc.org/ietf-mxcomp/mail-archive/msg03678.html">GPL in
+particular</a>, Python milter will not be able to implement sender-ID in its
+current form.  This was, no doubt, Microsoft's intent all along.
+<p>
+Sender-ID attempts to do for RFC2822 headers what SPF does for RFC2821 headers.
+Unlike SPF, it has never been tried, and is encumbered by a stupid patent.  I
+recommend ignoring it and continuing to implement and improve SPF until a
+working and unencumbered proposal for RFC2822 headers surfaces.
+
+<p>
+<a href="http://spf.pobox.com">
+<img src="SPF.gif" align=left alt="SPF logo"></a>
 Release 0.6.6 adds support for <a href="http://spf.pobox.com/">SPF</a>,
 a protocol to prevent forging of the envelope from address.  
 SPF support requires <a href="http://pydns.sourceforge.net/">pydns</a>.
@@ -52,15 +122,15 @@ The included spf.py module is an updated version of the original 1.6
 version at <a href="http://www.wayforward.net/spf/">wayforward.net</a>.
 The updated version tracks the draft RFC and test suite.
 <p>
-Release 0.6.0 offers a simple application of dspam I call "header triage",
+The FAQ addresses <a href="faq.html#spf">how to get started with SPF</a>.
-which rejects messages with spammy headers.  Since sendmail has to
-read the entire message anyway once we start reading headers, it
-would probably be better to scan the whole message - except that 
-we replace dangerous attachments elsewhere in the milter  - which screws up the
-body statistics for messages with dangerous attachments.
 <p>
 Release 0.6.1 adds a full milter based dspam application.
 <p>
+I have selected the <a href="http://www.nuclearelephant.com/projects/dspam/">
+dspam bayes filter project</a> and <a href="dspam.html">
+packaged it for python</a>.
+Release 0.6.0 offers a simple application of dspam I call "header triage",
+which rejects messages with spammy headers.  
 To use header triage, you must have <a href="dspam.html">DSPAM</a> installed,
 and select a dictionary that is well moderated by someone who gets
 lots of spam.  That dictionary can be used to block spam that is 
@@ -109,7 +179,7 @@ entries.  Be sure to handle both Bcc and file copies, and designating what
 mail should be copied.  How should "outgoing" be defined?  Implementing it is
 easy once the configuration is designed.
 
-<h3>Overview</h3>
+<h3><a name=overview>Overview</a></h3>
 
 This package provides a robust toolkit for Python <a
 href="#milter">milters</a>, and the beginnings of a general purpose mail
@@ -141,244 +211,24 @@ methods that
 do nothing, and also provides wrappers for the libmilter methods to mutate
 the message.
 <p>
+The 'spf' module provides an implementation of <a href="http://spf.pobox.com">
+SPF</a> useful for detecting email forgery.
+<p>
+The 'mime' module provides a wrapper for the Python email package that
+fixes some bugs, and simplifies modifying selected parts of a MIME message.
+<p>
 Finally, the bms.py application is both a sample of how to use the
-Milter module, and the beginnings of a general purpose SPAM filtering,
+Milter and spf modules, and the beginnings of a general purpose SPAM filtering,
-wiretapping, and Win32 virus protection milter.
+wiretapping, SPF checking, and Win32 virus protecting milter.  It can
-
+make use of the <a href="pysrs.html">pysrs</a> package when available for
-<h3><a name=download>Downloading</a></h3>
+SRS/SES checking and the <a href="dspam.html">pydspam</a> package for Bayesian
-
+content filtering.  SPF checking
-The latest stable release is <a href="#stable">0.6.9</a>. A stable
+requires <a href="http://pydns.sourceforge.net/">
-release is one which has been installed (and working correctly) on
+pydns</a>.  Configuration documentation is currently included as comments
-production systems long enough to convince me that it is stable.  As
+in the <a href="milter.cfg">sample config file</a> for the bms.py milter.
-the package gains more features and complexity, stable will mean no
-bug reports from outside users either.
-<p>
-The latest version is 0.6.9-1.  See the <a href=NEWS>Change Log</a>.
-<p>
-<a name="stable"><b>Stable</b></a>
-<a href="http://bmsi.com/python/milter-0.6.9.tar.gz">
-milter-0.6.9.tar.gz</a> Add SPF test suite driver, and validate
-spf.py against test suite.  Add best_guess and get_header to spf.py.
-Libmilter timeout option in config.
-<br>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.9-1.i386.rpm">
-milter-0.6.9-1.i386.rpm</a> Binary RPM for Redhat 7.x, now requires 
-	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
-	python2.3</a>.
-<br>
-<a href="http://bmsi.com/linux/rh9/milter-0.6.9-1.src.rpm">
-milter-0.6.9-1.src.rpm</a> Source RPM for Redhat 9,7.x.  
-<p>
-<a href="http://bmsi.com/python/milter-0.6.8.tar.gz">
-milter-0.6.8.tar.gz</a> Include Received-SPF headers in Dspam analysis.
-Fix sysv init for Redhat 9 and later.  Reject bounces with multiple
-recipients.
-<br>
-<a href="http://bmsi.com/python/milter-0.6.8.patch">milter-0.6.8.patch</a>
-Last minutes fixes from production testing.
-<p>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.8-3.i386.rpm">
-milter-0.6.8-3.i386.rpm</a> Binary RPM for Redhat 7.x, now requires 
-	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
-	python2.3</a>. 
-<br>
-<a href="http://bmsi.com/linux/rh9/milter-0.6.8-3.src.rpm">
-milter-0.6.8-3.src.rpm</a> Source RPM for Redhat 9,7.x.  
-<p>
-<a href="http://bmsi.com/python/milter-0.6.7.tar.gz">
-milter-0.6.7.tar.gz</a> Explicit local socket bug,
-<a href="http://spf.pobox.com/srs.html">SRS</a> forgery detection,
-thread resource starvation detection.
-SRS support requires <a href="http://bmsi.com/python/pysrs.html">pysrs</a>.
-<p>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.7-3.i386.rpm">
-milter-0.6.7-3.i386.rpm</a> Binary RPM for Redhat 7.x, now requires 
-	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
-	python2.3</a>. 
-<br>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.7-3.src.rpm">
-milter-0.6.7-3.src.rpm</a> Source RPM for Redhat 7.x. 
-Release 0.6.7-3 patches:
-<ul>
-<li> Defang message/rfc822 content_type with boundary 
-<li> Support SPF delegation
-<li> Reject neutral SPF result for selected domains
-</ul>
-<p>
-<a href="http://bmsi.com/python/milter-0.6.6.tar.gz">
-milter-0.6.6.tar.gz</a> Plug another memory leak, 
-<a href="http://spf.pobox.com/">SPF</a> support, hello blacklist.
-SPF support requires <a href="http://pydns.sourceforge.net/">pydns</a>.
-NOTE - the spf.py module included is modified from the official 1.6
-version at <a href="http://www.wayforward.net/spf/">wayforward.net</a>.
-I neglected to add the CVS log.  The changes are expanded result codes
-and tolerating common method misspellings in SPF records.  I have notified the
-author, but haven't heard back.  At some point, the RPM will
-include the official pyspf tarball and apply patches.
-<p>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.6-2.i386.rpm">
-milter-0.6.6-2.i386.rpm</a> Binary RPM for Redhat 7.x, now requires 
-	sendmail-8.12 and <a href="http://www.python.org/2.3.3/rpms.html">
-	python2.3</a>.  Release 2 fixes sysv init script bug for python2.3.
-<br>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.6-2.src.rpm">
-milter-0.6.6-2.src.rpm</a> Source RPM for Redhat 7.x
-<p>
-<a href="http://bmsi.com/python/milter-0.6.5.tar.gz">
-milter-0.6.5.tar.gz</a> Plug memory leak, progress reporting, trusted relay.
-Redhat RPM now requires sendmail-8.12.
-<p>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.5-2.i386.rpm">
-milter-0.6.5-2.i386.rpm</a> Binary RPM for Redhat 7.x
-<br>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.5-2.src.rpm">
-milter-0.6.5-2.src.rpm</a> Source RPM for Redhat 7.x
-<p>
-<a href="http://bmsi.com/python/milter-0.6.4.tar.gz">
-milter-0.6.4.tar.gz</a> Numerous Dspam fixes.  Requires
-<a href="dspam.html">pydspam-1.1.5</a> and 
-<a href="/libdspam/dspam.html">dspam-2.6.5.2</a>
-for Dspam features.  The dspam-python RPM has been replaced by pydspam.
-<p>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.4-1.i386.rpm">
-milter-0.6.4-1.i386.rpm</a> Binary RPM for Redhat 7.x
-<p>
-<a href="http://bmsi.com/python/milter-0.6.3.1.tar.gz">
-milter-0.6.3.1.tar.gz</a> New dspam SCREENER feature with pydspam-1.1.4.
-Don't save a defang copy of false positives.  Fixed an oops from last fix,
-rejecting false positives.  BUG: sendmail-8.11 doesn't invoke milter
-when sending mail via sendmail from command line (8.12 works).  Therefore,
-the supplied falsepositive script for milter based dspam doesn't work
-with stock RedHat 7.x.  I am writing a HOWTO for configuring milter
-based dspam that will address this (and a fix in the next version).
-<p>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.3-1.i386.rpm">
-milter-0.6.3-1.i386.rpm</a> Binary RPM for Redhat 7.x
-<p>
-<a href="http://bmsi.com/python/milter-0.6.2.tar.gz">
-milter-0.6.2.tar.gz</a> work around email.Message.get_filename bug,
-dspam_exempt list, REJECT messages with missing MIME boundaries (which
-are almost always spam),
-DISCARD messages which any dspam user flags as spam,
-start.sh was calling python instead of python2 on Linux.
-<p>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.2-1.src.rpm">
-milter-0.6.2-1.src.rpm</a> Source RPM for Redhat 7.x (and likely
-higher versions)
-<p>
-<a href="http://bmsi.com/python/milter-0.6.1.tar.gz">
-milter-0.6.1.tar.gz</a> dspam milter application, python-2.2.3 support.
-<p>
-You must have <a href=dspam.html>dspam and dspam-python</a> loaded for
-the dspam feature to work.  Brief instructions for configuring are
-in the default config file.  This is working at a customer, but I'm
-sure a few more iterations will be required to make setup as smooth
-as possible.
-<p>
-NOTE: Outlook destroys dspam tags when forwarding mail (while converting
-HTML to text).  Perhaps some config option will turn this abominable
-"feature" off.  Working around this by making dspam tags visble on
-HTML mail is ugly.  My suggestion is to not use Outlook, for this and
-many other reasons - especially security.  Any other suggestions for
-those married to Microsoft are welcome.  The DSPAM LDA works around this
-by making the tags visible in HTML attachments.  This is ugly, and
-occasionally corrupts attachments.
-<p>
-We have to supply workarounds for bugs in the email module (reported
-to sourceforge).  The workarounds reference some internal variables
-which change with python versions.
-<p>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.1-1.i386.rpm">
-milter-0.6.1-1.i386.rpm</a> Binary RPM for Redhat 7.x
-<p>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.1-1.src.rpm">
-milter-0.6.1-1.src.rpm</a> Source RPM for Redhat 7.x (and likely
-higher versions)
-<p>
-<a href="http://bmsi.com/python/milter-0.6.0.tar.gz">
-milter-0.6.0.tar.gz</a> simple dspam pre-filtering, use email module,
-requires python &gt;= 2.2.2.  
-<ul>
-<li> The milter.so module from 0.5.4
-is needed to run this release on AIX.  Haven't tracked this down yet.
-<li> The patches to fix the email packages in mime.py don't work
-on python-2.2.3.  The email package is still broken in 2.3, and patches
-required for that will likely be different still.
-</ul>
-
-<p>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.0-1.i386.rpm">
-milter-0.6.0-1.i386.rpm</a> Binary RPM for Redhat 7.x
-<p>
-<a href="http://bmsi.com/linux/rh72/milter-0.6.0-1.src.rpm">
-milter-0.6.0-1.src.rpm</a> Source RPM for Redhat 7.x (and likely
-higher versions)
-<p>
-<a href="http://www.bmsi.com/python/milter-0.5.5.tar.gz">
-milter-0.5.5.tar.gz</a> IPV6 support, passing None to set_XXX_callback,
-set_reply, chg_header, detect internal connections.  Note, this release
-did not work on AIX4.1.5, probably due to IPV6 support breaking something.
-The milter.so module from 0.5.4 can be installed to use this release 
-with AIX.
-<p>
-<a href="http://www.bmsi.com/python/milter-0.5.4.tar.gz">
-milter-0.5.4.tar.gz</a> wiretap, smart alias features, quarantine support.
-<p>
-The name of the production "sample" milter "bms.py" now
-stands for "Basic Milter System" until someone suggests a better name.
-The test coverage is rather
-  sparse at present.  
-  Please <a href="mailto:%73%74%75%61%72%74%40%62%6D%73%69%2E%63%6F%6D">email</a> with proposals for what
-  to name the milter application.
-<h4>NOTES</h4>
-<ul>
-<li>
-  Quarantine support requires that you define _FFR_QUARANTINE 
-  when compiling miltermodule.c.  I am not sure how to make setup.py
-  do that for you iff sendmail was actually compiled with _FFR_QUARANTINE.
-<li>
-  While 0.6.0 will use the new email package in Python-2.2, that
-  package seems to be buggy in Python-2.2.1.  The list example in the docs
-  doesn't find all MIME parts.  Update: Python-2.2.2 has fixed the email
-  package.  It can now parse my test cases.
-<li>
-  Preliminary testing with python-2.2 shows that most things work after
-  adding <code>self.readahead = ""</code> to <code>mimepart.seek</code>.
-  Python-2.2 <code>multifile</code> reads one less newline per section than
-  2.1.  I'm not not sure which is correct.  After adding some calls to
-  <code>rstrip()</code> in testmime.py, all milter modules pass unit testing
-  with python-2.2.  Python-2.2 patches have been released since 0.5.3.
-<li>
-  sgmlop-1.1a3 has a memory leak (at least Python milter has a
-  memory leak when using sgmlop instead of sgmllib).  Do not make Python
-  milter use sgmlop-1.1a2 or a3 in a production
-  system unless you can restart your milter periodically.  The amount
-  of memory leaked seems roughly proportional to the amount of HTML
-  parsed.
-<li>
-  There are a number of ways that malformed MIME attachments
-  can cause a python traceback.  Uncaught exceptions cause a 415
-  error to be returned to sendmail.  So far, all the malformed messages
-  I've investigated have been SPAM - so good riddance.  I would prefer,
-  however, that the mime handling libraries were more precise.  Beginning
-  with 0.5.1, bms.py will save messages that cause a traceback during
-  scanning in the tempfile directory with a ".fail" extension.  This
-  makes it easier to get samples of mail that causes parsing problems
-  for incorporation into the unit tests.
-</ul>
-<p>
-<a href="http://www.bmsi.com/python/milter-0.5.2.tar.gz">
-milter-0.5.2.tar.gz</a> Fix and unittest another HTML parsing bug.<br>
-<a href="http://www.bmsi.com/python/milter-0.5.1.tar.gz">
-milter-0.5.1.tar.gz</a> Handle encoded rfc822 attachments.<br>
-<a href="http://www.bmsi.com/python/milter-0.5.0.tar.gz">
-milter-0.5.0.tar.gz</a> Use a config file so users don't have to
-keep syncing with bms.py.  <br>
-<a href="http://www.bmsi.com/python/milter-0.4.5.tar.gz">
-milter-0.4.5.tar.gz</a> Work with sgmlop. Reduce local hacks to config variables.
 <p>
 Python milter is under GPL.  The authors can probably be convinced to
-change this to LGPL.
+change this to LGPL if needed.
 
 <h3>What is a <a name="milter">milter</a>?</h3>
 
@@ -429,7 +279,7 @@ The "defang" function of the sample milter was inspired by
 a Perl milter with flexible attachment processing options.  The latest
 version of MIMEDefang uses an apache style process pool to avoid reloading
 the Perl interpreter for each message.  This makes it fast enough for
-production and does not use Perl threading.
+production without using Perl threading.
 <p>
 <a href="http://sourceforge.net/projects/mailchecker">mailchecker</a> is
 a Python project to provide flexible attachment processing for mail.  I
@@ -503,18 +353,10 @@ me if you successfully install milter on a system not mentioned below.
 <td>0.5.4</td><tr>
 <td>RedHat 7.1</td><td>gcc-2.96</td><td>?</td><td>8.12.1</td>
 <td>0.3.5</td><tr>
-<td>RedHat 7.2</td><td>gcc-2.96</td><td>2.1.1</td><td>8.11.6</td>
-<td>0.4.1</td><tr>
-<td>RedHat 7.2</td><td>gcc-2.96</td><td>2.2.1</td><td>8.11.6</td>
-<td>0.4.5</td><tr>
-<td>RedHat 7.2</td><td>gcc-2.96</td><td>2.2.2</td><td>8.11.6</td>
-<td>0.5.5</td><tr>
-<td>RedHat 7.2</td><td>gcc-2.96</td><td>2.3.3</td><td>8.12.10</td>
-<td>0.6.6</td><tr>
 <td>RedHat 7.3</td><td>gcc-2.96</td><td>2.2.2</td><td>8.11.6</td>
 <td>0.5.5</td><tr>
-<td>RedHat 7.3</td><td>gcc-2.96</td><td>2.3.3</td><td>8.12.10</td>
+<td>RedHat 7.3</td><td>gcc-2.96</td><td>2.3.3</td><td>8.13.1</td>
-<td>0.6.6</td><tr>
+<td>0.7.2</td><tr>
 <td>RedHat 8.0</td><td>gcc-3.2</td><td>2.2.1</td><td>8.12.6</td>
 <td>0.5.2</td><tr>
 <td>Debian Linux</td><td>gcc-2.95.2</td><td>2.1.1</td><td>8.12.0</td>
@@ -527,14 +369,14 @@ me if you successfully install milter on a system not mentioned below.
 <td>0.3.4</td><tr>
 <td>AIX-4.1.5</td><td>gcc-2.95.2</td><td>2.1.3</td><td>8.12.3</td>
 <td>0.4.2</td><tr>
-<td>AIX-4.1.5</td><td>gcc-2.95.2</td><td>2.2.2</td><td>8.12.6</td>
+<td>AIX-4.1.5</td><td>gcc-2.95.2</td><td>2.2.3</td><td>8.13.1</td>
-<td>0.5.4</td><tr>
+<td>0.7.1</td><tr>
 <td>Slackware 7.1</td><td>?</td><td>?</td><td>8.12.1</td>
 <td>0.3.8</td><tr>
 <td>Slackware 9.0</td><td>gcc-3.2.2</td><td>2.2.3</td><td>8.12.9</td>
 <td>0.5.4</td><tr>
-<td>OpenBSD</td><td>?</td><td>2.1.1</td><td>8.11.6</td>
+<td>OpenBSD</td><td>?</td><td>2.3.3?</td><td>8.13.1?</td>
-<td>0.3.9</td><tr>
+<td>0.7.2</td><tr>
 <td>SuSE 7.3</td><td>gcc-2.95.3</td><td>2.1.1</td><td>8.12.2</td>
 <td>0.3.9</td><tr>
 <td>FreeBSD</td><td>gcc-2.95.3</td><td>2.2.1</td><td>8.12.3</td>

milter.spec

@@ -1,12 +1,12 @@
 %define name milter
-%define version 0.6.9
+%define version 0.8.0
-%define release 1
+%define release 2.EL3
 # Redhat 7.x and earlier (multiple ps lines per thread)
-%define sysvinit milter.rc7
+#define sysvinit milter.rc7
 # RH9, other systems (single ps line per process)
-#define sysvinit milter.rc
+%define sysvinit milter.rc
 %ifos Linux
-%define python python2.3
+%define python python2.4
 %else
 %define python python
 %endif
@@ -24,8 +24,11 @@ Prefix: %{_prefix}
 Vendor: Stuart D. Gathman <stuart@bmsi.com>
 Packager: Stuart D. Gathman <stuart@bmsi.com>
 Url: http://www.bmsi.com/python/milter.html
-Requires: %{python} >= 2.2.2, sendmail >= 8.12
+Requires: %{python} >= 2.4, sendmail >= 8.12.10
-BuildRequires: %{python}-devel >= 2.2.2, sendmail-devel >= 8.12
+%ifnos aix4.1
+Requires: chkconfig
+%endif
+BuildRequires: %{python}-devel >= 2.2.2, sendmail-devel >= 8.12.10
 
 %description
 This is a python extension module to enable python scripts to
@@ -43,8 +46,10 @@ env CFLAGS="$RPM_OPT_FLAGS" %{python} setup.py build
 rm -rf $RPM_BUILD_ROOT
 %{python} setup.py install --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES
 mkdir -p $RPM_BUILD_ROOT/var/log/milter
+mkdir -p $RPM_BUILD_ROOT/etc/mail
 mkdir $RPM_BUILD_ROOT/var/log/milter/save
-cp bms.py milter.cfg $RPM_BUILD_ROOT/var/log/milter
+cp bms.py $RPM_BUILD_ROOT/var/log/milter
+cp milter.cfg $RPM_BUILD_ROOT/etc/mail/pymilter.cfg
 
 # logfile rotation
 mkdir -p $RPM_BUILD_ROOT/etc/logrotate.d
@@ -57,10 +62,15 @@ EOF
 
 # purge saved defanged message copies
 mkdir -p $RPM_BUILD_ROOT/etc/cron.daily
+%ifos aix4.1
+R=
+%else
+R='-r'
+%endif
 cat >$RPM_BUILD_ROOT/etc/cron.daily/milter <<'EOF'
 #!/bin/sh
 
-find /var/log/milter/save -mtime +7 | xargs -r rm
+find /var/log/milter/save -mtime +7 | xargs $R rm
 EOF
 chmod a+x $RPM_BUILD_ROOT/etc/cron.daily/milter
 
@@ -94,6 +104,8 @@ EOF
 chmod a+x $RPM_BUILD_ROOT/var/log/milter/start.sh
 
 mkdir -p $RPM_BUILD_ROOT/var/run/milter
+mkdir -p $RPM_BUILD_ROOT/usr/share/sendmail-cf/hack
+cp -p rhsbl.m4 $RPM_BUILD_ROOT/usr/share/sendmail-cf/hack
 
 %ifos aix4.1
 %post
@@ -103,6 +115,15 @@ mkssys -s milter -p /var/log/milter/start.sh -u 25 -S -n 15 -f 9 -G mail || :
 if [ $1 = 0 ]; then
   rmssys -s milter || :
 fi
+%else
+%post
+#echo "pythonsock has moved to /var/run/milter, update /etc/mail/sendmail.cf"
+/sbin/chkconfig --add milter
+
+%preun
+if [ $1 = 0 ]; then
+  /sbin/chkconfig --del milter
+fi
 %endif
 
 %clean
@@ -124,9 +145,34 @@ rm -rf $RPM_BUILD_ROOT
 %dir /var/log/milter/save
 %config /var/log/milter/start.sh
 %config /var/log/milter/bms.py
-%config /var/log/milter/milter.cfg
+%config(noreplace) /etc/mail/pymilter.cfg
+/usr/share/sendmail-cf/hack/rhsbl.m4
 
 %changelog
+* Tue Feb 08 2005 Stuart Gathman <stuart@bmsi.com> 0.7.3-1.EL3
+- Compile for EL3 and Python4
+* Mon Aug 30 2004 Stuart Gathman <stuart@bmsi.com> 0.7.2-1
+- Fix various SPF bugs
+- Recognize dynamic PTR names, and don't count them as authentication.
+- Three strikes and yer out rule.
+- Block softfail by default unless valid PTR or HELO
+- Return unknown for null mechanism
+- Return unknown for invalid ip address in mechanism
+- Try best guess on HELO also
+- Expand setreply for common errors
+- make rhsbl.m4 hack available for sendmail.mc
+* Sun Aug 22 2004 Stuart Gathman <stuart@bmsi.com> 0.7.1-1
+- Handle modifying mislabeled multipart messages without an exception
+- Support setbacklog, setmlreply
+- allow multi-recipient CBV
+- return TEMPFAIL for SPF softfail
+* Fri Jul 23 2004 Stuart Gathman <stuart@bmsi.com> 0.7.0-1
+- SPF check hello name
+- Move pythonsock to /var/run/milter
+- Move milter.cfg to /etc/mail/pymilter.cfg
+- Check M$ style XML CID records by converting to SPF
+- Recognize, but never match ip6 until we properly support it.
+- Option to reject when no PTR and no SPF
 * Fri Apr 09 2004 Stuart Gathman <stuart@bmsi.com> 0.6.9-1
 - Validate spf.py against test suite, and add Received-SPF support to spf.py
 - Support best_guess for SPF

miltermodule.c

@@ -33,6 +33,18 @@ $ python setup.py help
      libraries=["milter","smutil","resolv"]
 
  * $Log$
+ * Revision 2.31  2004/08/23 02:24:36  stuart
+ * Support setbacklog
+ *
+ * Revision 2.30  2004/08/21 20:29:53  stuart
+ * Support option of 11 lines max for mlreply.
+ *
+ * Revision 2.29  2004/08/21 04:14:29  stuart
+ * mlreply support
+ *
+ * Revision 2.28  2004/08/21 02:45:21  stuart
+ * Don't leak int constants if module unloaded.
+ *
  * Revision 2.27  2004/04/06 03:19:59  stuart
  * Release 0.6.8
  *
@@ -127,11 +139,20 @@ $ python setup.py help
  *
  */
 
+#ifndef MAX_ML_REPLY
+#define MAX_ML_REPLY 32
+#endif
+#if MAX_ML_REPLY != 1 && MAX_ML_REPLY != 32 && MAX_ML_REPLY != 11
+#error MAX_ML_REPLY must be 1 or 11 or 32
+#endif
+#define _FFR_MULTILINE (MAX_ML_REPLY > 1)
+
 #include <pthread.h>
 #include <netinet/in.h>
 #include <Python.h>
 #include <libmilter/mfapi.h>
 
+
 /* See if we have IPv4 and/or IPv6 support in this OS and in
  * libmilter.  We need to make several macro tests because some OS's
  * may define some if IPv6 is only partially supported, and we may
@@ -746,6 +767,18 @@ milter_setdbg(PyObject *self, PyObject *args) {
   return _generic_return(smfi_setdbg(val), "cannot set debug value");
 }
 
+static char milter_setbacklog__doc__[] =
+"setbacklog(int) -> None\n\
+Set the TCP connection queue size for the milter socket.";
+
+static PyObject *
+milter_setbacklog(PyObject *self, PyObject *args) {
+   int val;
+
+   if (!PyArg_ParseTuple(args, "i:setbacklog", &val)) return NULL;
+   return _generic_return(smfi_setbacklog(val), "cannot set backlog");
+}
+
 static char milter_settimeout__doc__[] =
 "settimeout(int) -> None\n\
 Set the time (in seconds) that sendmail will wait before\n\
@@ -820,13 +853,54 @@ static PyObject *
 milter_setreply(PyObject *self, PyObject *args) {
   char *rcode;
   char *xcode;
-  char *message;
+  char *message[MAX_ML_REPLY];
+  char fmt[MAX_ML_REPLY + 16];
   SMFICTX *ctx;
-  if (!PyArg_ParseTuple(args, "szz:setreply", &rcode, &xcode, &message))
+  int i;
+  strcpy(fmt,"sz|");
+  for (i = 0; i < MAX_ML_REPLY; ++i) {
+    message[i] = 0;
+    fmt[i+3] = 's';
+  }
+  strcpy(fmt+i+3,":setreply");
+  if (!PyArg_ParseTuple(args, fmt,
+	&rcode, &xcode, message
+#if MAX_ML_REPLY > 1
+	,message+1,message+2,message+3,message+4,message+5,message+6,
+	message+7,message+8,message+9,message+10
+#if MAX_ML_REPLY > 11
+	,message+11,message+12,message+13,message+14,message+15,
+	message+16,message+17,message+18,message+19,message+20,
+	message+21,message+22,message+23,message+24,message+25,
+	message+26,message+27,message+28,message+29,message+30,
+	message+31
+#endif
+#endif
+  ))
     return NULL;
   ctx = _find_context(self);
   if (ctx == NULL) return NULL;
-  return _generic_return(smfi_setreply(ctx, rcode, xcode, message),
+#if MAX_ML_REPLY > 1
+  /*
+   * C varargs might be convenient for some things, but they sure are a pain
+   * when the number of args is not known at compile time.
+   */
+  if (message[0] && message[1])
+    return _generic_return(smfi_setmlreply(ctx, rcode, xcode,
+	  message[0],
+	  message[1],message[2],message[3],message[4],message[5],
+	  message[6],message[7],message[8],message[9],message[10],
+#if MAX_ML_REPLY > 11
+	  message[11],message[12],message[13],message[14],message[15],
+	  message[16],message[17],message[18],message[19],message[20],
+	  message[21],message[22],message[23],message[24],message[25],
+	  message[26],message[27],message[28],message[29],message[30],
+	  message[31],
+#endif
+	  (char *)0
+    ), "cannot set reply");
+#endif
+  return _generic_return(smfi_setreply(ctx, rcode, xcode, message[0]),
 			 "cannot set reply");
 }
 
@@ -986,7 +1060,7 @@ milter_getpriv(PyObject *self, PyObject *args) {
   return o;
 }
 
-#if _FFR_QUARANTINE
+#ifdef SMFIF_QUARANTINE
 static char milter_quarantine__doc__[] =
 "quarantine(string) -> None\n\
 Place the message in quarantine.  A string with a description of the reason\n\
@@ -1035,7 +1109,7 @@ static PyMethodDef context_methods[] = {
   { "replacebody", milter_replacebody, METH_VARARGS, milter_replacebody__doc__},
   { "setpriv",     milter_setpriv,     METH_VARARGS, milter_setpriv__doc__},
   { "getpriv",     milter_getpriv,     METH_VARARGS, milter_getpriv__doc__},
-#if _FFR_QUARANTINE
+#ifdef SMFIF_QUARANTINE
   { "quarantine",  milter_quarantine,  METH_VARARGS, milter_quarantine__doc__},
 #endif
 #if _FFR_SMFI_PROGRESS
@@ -1081,6 +1155,7 @@ static PyMethodDef milter_methods[] = {
    { "main",                 milter_main,                 METH_VARARGS, milter_main__doc__},
    { "setdbg",               milter_setdbg,               METH_VARARGS, milter_setdbg__doc__},
    { "settimeout",           milter_settimeout,           METH_VARARGS, milter_settimeout__doc__},
+   { "setbacklog",           milter_setbacklog,           METH_VARARGS, milter_setbacklog__doc__},
    { "setconn",              milter_setconn,              METH_VARARGS, milter_setconn__doc__},
    { "stop",                 milter_stop,                 METH_VARARGS, milter_stop__doc__},
    { NULL, NULL }
@@ -1116,6 +1191,12 @@ allowing one to write email filters directly in Python.\n\
 Libmilter is currently marked FFR, and needs to be explicitly installed.\n\
 See <sendmailsource>/libmilter/README for details on setting it up.\n";
 
+static void setitem(PyObject *d,const char *name,long val) {
+  PyObject *v = PyInt_FromLong(val);
+  PyDict_SetItemString(d,name,v);
+  Py_DECREF(v);
+}
+
 void
 initmilter(void) {
    PyObject *m, *d;
@@ -1125,24 +1206,24 @@ initmilter(void) {
    d = PyModule_GetDict(m);
    MilterError = PyErr_NewException("milter.error", NULL, NULL);
    PyDict_SetItemString(d,"error", MilterError);
-   PyDict_SetItemString(d,"SUCCESS", PyInt_FromLong((long) MI_SUCCESS));
+   setitem(d,"SUCCESS",  MI_SUCCESS);
-   PyDict_SetItemString(d,"FAILURE", PyInt_FromLong((long) MI_FAILURE));
+   setitem(d,"FAILURE",  MI_FAILURE);
-   PyDict_SetItemString(d,"VERSION", PyInt_FromLong((long) SMFI_VERSION));
+   setitem(d,"VERSION",  SMFI_VERSION);
-   PyDict_SetItemString(d,"ADDHDRS", PyInt_FromLong((long) SMFIF_ADDHDRS));
+   setitem(d,"ADDHDRS",  SMFIF_ADDHDRS);
-   PyDict_SetItemString(d,"CHGBODY", PyInt_FromLong((long) SMFIF_CHGBODY));
+   setitem(d,"CHGBODY",  SMFIF_CHGBODY);
-   PyDict_SetItemString(d,"MODBODY", PyInt_FromLong((long) SMFIF_MODBODY));
+   setitem(d,"MODBODY",  SMFIF_MODBODY);
-   PyDict_SetItemString(d,"ADDRCPT", PyInt_FromLong((long) SMFIF_ADDRCPT));
+   setitem(d,"ADDRCPT",  SMFIF_ADDRCPT);
-   PyDict_SetItemString(d,"DELRCPT", PyInt_FromLong((long) SMFIF_DELRCPT));
+   setitem(d,"DELRCPT",  SMFIF_DELRCPT);
-   PyDict_SetItemString(d,"CHGHDRS", PyInt_FromLong((long) SMFIF_CHGHDRS));
+   setitem(d,"CHGHDRS",  SMFIF_CHGHDRS);
-   PyDict_SetItemString(d,"V1_ACTS", PyInt_FromLong((long) SMFI_V1_ACTS));
+   setitem(d,"V1_ACTS",  SMFI_V1_ACTS);
-   PyDict_SetItemString(d,"V2_ACTS", PyInt_FromLong((long) SMFI_V2_ACTS));
+   setitem(d,"V2_ACTS",  SMFI_V2_ACTS);
-   PyDict_SetItemString(d,"CURR_ACTS", PyInt_FromLong((long) SMFI_CURR_ACTS));
+   setitem(d,"CURR_ACTS",  SMFI_CURR_ACTS);
 #ifdef SMFIF_QUARANTINE
-   PyDict_SetItemString(d,"QUARANTINE",PyInt_FromLong((long)SMFIF_QUARANTINE));
+   setitem(d,"QUARANTINE",SMFIF_QUARANTINE);
 #endif
-   PyDict_SetItemString(d,"CONTINUE", PyInt_FromLong((long) SMFIS_CONTINUE));
+   setitem(d,"CONTINUE",  SMFIS_CONTINUE);
-   PyDict_SetItemString(d,"REJECT", PyInt_FromLong((long) SMFIS_REJECT));
+   setitem(d,"REJECT",  SMFIS_REJECT);
-   PyDict_SetItemString(d,"DISCARD", PyInt_FromLong((long) SMFIS_DISCARD));
+   setitem(d,"DISCARD",  SMFIS_DISCARD);
-   PyDict_SetItemString(d,"ACCEPT", PyInt_FromLong((long) SMFIS_ACCEPT));
+   setitem(d,"ACCEPT",  SMFIS_ACCEPT);
-   PyDict_SetItemString(d,"TEMPFAIL", PyInt_FromLong((long) SMFIS_TEMPFAIL));
+   setitem(d,"TEMPFAIL",  SMFIS_TEMPFAIL);
 }

mime.py

@@ -1,4 +1,38 @@
 # $Log$
+# Revision 1.62  2005/02/14 22:31:17  stuart
+# _parseparam replacement not needed for python2.4
+#
+# Revision 1.61  2005/02/12 02:11:11  stuart
+# Pass unit tests with python2.4.
+#
+# Revision 1.60  2005/02/11 18:34:14  stuart
+# Handle garbage after quote in boundary.
+#
+# Revision 1.59  2005/02/10 01:10:59  stuart
+# Fixed MimeMessage.ismodified()
+#
+# Revision 1.58  2005/02/10 00:56:49  stuart
+# Runs with python2.4.  Defang not working correctly - more work needed.
+#
+# Revision 1.57  2004/11/20 16:37:52  stuart
+# fix regex for splitting header and body
+#
+# Revision 1.56  2004/11/09 20:33:51  stuart
+# Recognize more dynamic PTR variations.
+#
+# Revision 1.55  2004/10/06 21:39:20  stuart
+# Handle message attachments with boundary errors by not parsing them
+# until needed.
+#
+# Revision 1.54  2004/08/18 01:59:46  stuart
+# Handle mislabeled multipart messages
+#
+# Revision 1.53  2004/04/24 22:53:20  stuart
+# Rename some local variables to avoid shadowing builtins
+#
+# Revision 1.52  2004/04/24 22:47:13  stuart
+# Convert header values to str
+#
 # Revision 1.51  2004/03/25 03:19:10  stuart
 # Correctly defang rfc822 attachments when boundary specified with
 # content-type message/rfc822.
@@ -34,177 +68,44 @@
 import StringIO
 import socket
 import Milter
+
 import email
 import email.Message
 from email.Message import Message
 from email.Generator import Generator
 from email.Utils import quote
 from email import Utils
+from email.Parser import Parser
+from email import Errors
 
 from types import ListType,StringType
 
-# Enhance email.Parser
+class MimeGenerator(Generator):
-# - Fix _parsebody to decode message attachments before parsing
+    def _dispatch(self, msg):
+        # Get the Content-Type: for the message, then try to dispatch to
+        # self._handle_<maintype>_<subtype>().  If there's no handler for the
+        # full MIME type, then dispatch to self._handle_<maintype>().  If
+        # that's missing too, then dispatch to self._writeBody().
+        main = msg.get_content_maintype()
+	if msg.is_multipart() and main.lower() != 'multipart':
+	  self._handle_multipart(msg)
+	else:
+	  Generator._dispatch(self,msg)
 
-from email.Parser import Parser
+def unquote(s):
-try: from email.Parser import NLCRE
-except: from email.Parser import nlcre as NLCRE
-
-from email import Errors
-
-class MimeParser(Parser):
-
-    # This is a copy of _parsebody from email.Parser, with a fix
-    # for message attachments.  I couldn't find a smaller way to patch it
-    # in a subclass.
-
-    def _parsebody(self, container, fp, firstbodyline=None):
-        # Parse the body, but first split the payload on the content-type
-        # boundary if present.
-        boundary = container.get_boundary()
-        isdigest = (container.get_content_type() == 'multipart/digest')
-        # If there's a boundary, split the payload text into its constituent
-        # parts and parse each separately.  Otherwise, just parse the rest of
-        # the body as a single message.  Note: any exceptions raised in the
-        # recursive parse need to have their line numbers coerced.
-        if boundary:
-            preamble = epilogue = None
-            # Split into subparts.  The first boundary we're looking for won't
-            # always have a leading newline since we're at the start of the
-            # body text, and there's not always a preamble before the first
-            # boundary.
-            separator = '--' + boundary
-            payload = fp.read()
-            if firstbodyline is not None:
-                payload = firstbodyline + '\n' + payload
-            # We use an RE here because boundaries can have trailing
-            # whitespace.
-            mo = re.search(
-                r'(?P<sep>' + re.escape(separator) + r')(?P<ws>[ \t]*)',
-                payload)
-            if not mo:
-                if self._strict:
-                    raise Errors.BoundaryError(
-                        "Couldn't find starting boundary: %s" % boundary)
-                container.set_payload(payload)
-                return
-            start = mo.start()
-            if start > 0:
-                # there's some pre-MIME boundary preamble
-                preamble = payload[0:start]
-            # Find out what kind of line endings we're using
-            start += len(mo.group('sep')) + len(mo.group('ws'))
-            mo = NLCRE.search(payload, start)
-            if mo:
-                start += len(mo.group(0))
-            # We create a compiled regexp first because we need to be able to
-            # specify the start position, and the module function doesn't
-            # support this signature. :(
-            cre = re.compile('(?P<sep>\r\n|\r|\n)' +
-                             re.escape(separator) + '--')
-            mo = cre.search(payload, start)
-            if mo:
-                terminator = mo.start()
-                linesep = mo.group('sep')
-                if mo.end() < len(payload):
-                    # There's some post-MIME boundary epilogue
-                    epilogue = payload[mo.end():]
-            elif self._strict:
-                raise Errors.BoundaryError(
-                        "Couldn't find terminating boundary: %s" % boundary)
-            else:
-                # Handle the case of no trailing boundary.  Check that it ends
-                # in a blank line.  Some cases (spamspamspam) don't even have
-                # that!
-                mo = re.search('(?P<sep>\r\n|\r|\n){2}$', payload)
-                if not mo:
-                    mo = re.search('(?P<sep>\r\n|\r|\n)$', payload)
-                    if not mo:
-                        raise Errors.BoundaryError(
-                          'No terminating boundary and no trailing empty line')
-                linesep = mo.group('sep')
-                terminator = len(payload)
-            # We split the textual payload on the boundary separator, which
-            # includes the trailing newline. If the container is a
-            # multipart/digest then the subparts are by default message/rfc822
-            # instead of text/plain.  In that case, they'll have a optional
-            # block of MIME headers, then an empty line followed by the
-            # message headers.
-            parts = re.split(
-                linesep + re.escape(separator) + r'[ \t]*' + linesep,
-                payload[start:terminator])
-            for part in parts:
-                if isdigest:
-                    if part.startswith(linesep):
-                        # There's no header block so create an empty message
-                        # object as the container, and lop off the newline so
-                        # we can parse the sub-subobject
-                        msgobj = self._class()
-                        part = part[len(linesep):]
-                    else:
-                        parthdrs, part = part.split(linesep+linesep, 1)
-                        # msgobj in this case is the "message/rfc822" container
-                        msgobj = self.parsestr(parthdrs, headersonly=1)
-                    # while submsgobj is the message itself
-                    msgobj.set_default_type('message/rfc822')
-                    maintype = msgobj.get_content_maintype()
-                    if maintype in ('message', 'multipart'):
-                        submsgobj = self.parsestr(part)
-                        msgobj.attach(submsgobj)
-                    else:
-                        msgobj.set_payload(part)
-                else:
-                    msgobj = self.parsestr(part)
-                container.preamble = preamble
-                container.epilogue = epilogue
-                container.attach(msgobj)
-        elif container.get_main_type() == 'multipart':
-            # Very bad.  A message is a multipart with no boundary!
-            raise Errors.BoundaryError(
-                'multipart message with no defined boundary')
-        elif container.get_type() == 'message/delivery-status':
-            # This special kind of type contains blocks of headers separated
-            # by a blank line.  We'll represent each header block as a
-            # separate Message object
-            blocks = []
-            while True:
-                blockmsg = self._class()
-                self._parseheaders(blockmsg, fp)
-                if not len(blockmsg):
-                    # No more header blocks left
-                    break
-                blocks.append(blockmsg)
-            container.set_payload(blocks)
-        elif container.get_main_type() == 'message':
-            # Create a container for the payload, but watch out for there not
-            # being any headers left
-            container.set_payload(fp.read())
-	    fp = StringIO.StringIO(container.get_payload(decode=True))
-            try:
-                msg = self.parse(fp)
-            except Errors.HeaderParseError:
-                msg = self._class()
-                self._parsebody(msg, fp)
-            container.set_payload([msg])
-        else:
-            text = fp.read()
-            if firstbodyline is not None:
-                text = firstbodyline + '\n' + text
-            container.set_payload(text)
-
-def unquote(str):
     """Remove quotes from a string."""
-    if len(str) > 1:
+    if len(s) > 1:
-        if str.startswith('"'):
+        if s.startswith('"'):
-	  if str.endswith('"'):
+	  if s.endswith('"'):
-            str = str[1:-1]
+            s = s[1:-1]
 	  else: # remove garbage after trailing quote
-	    try: str = str[1:str[1:].index('"')+1]
+	    try: s = s[1:s[1:].index('"')+1]
-	    except: return str
+	    except:
-	  return str.replace('\\\\', '\\').replace('\\"', '"')
+	      return s
-        if str.startswith('<') and str.endswith('>'):
+	  return s.replace('\\\\', '\\').replace('\\"', '"')
-            return str[1:-1]
+        if s.startswith('<') and s.endswith('>'):
-    return str
+	  return s[1:-1]
+    return s
 
 from types import TupleType
 
@@ -214,27 +115,11 @@ def _unquotevalue(value):
   else:
       return unquote(value)
 
-email.Message._unquotevalue = _unquotevalue
+#email.Message._unquotevalue = _unquotevalue
 
-def _parseparam(str):
+from email.Message import _parseparam
-    plist = []
-    while str[:1] == ';':
-	str = str[1:]
-	end = str.find(';')
-	while end > 0 and (str.count('"',0,end) & 1):
-	  end = str.find(';',end + 1)
-	if end < 0: end = len(str)
-	f = str[:end]
-	if '=' in f:
-	    i = f.index('=')
-	    f = f[:i].strip().lower() + \
-		    '=' + f[i+1:].strip()
-	plist.append(f.strip())
-	str = str[end:]
-    return plist
 
 # Enhance email.Message 
-# - Fix getparam to parse attributes IE style
 # - Provide a headerchange event for integration with Milter
 #   Headerchange attribute can be assigned a function to be called when
 #   changing headers.  The signature is:
@@ -245,64 +130,19 @@ class MimeMessage(Message):
   """Version of email.Message.Message compatible with old mime module
   """
   def __init__(self,fp=None,seekable=1):
+    Message.__init__(self)
     self.headerchange = None
     self.submsg = None
-    Message.__init__(self)
+    self.modified = False
-    self.fp = fp
-    if fp:
-      parser = MimeParser(MimeMessage)
-      self.startofheaders = fp.tell()
-      parser._parseheaders(self,fp)
-      self.startofbody = fp.tell()
-      parser._parsebody(self,fp)
-    for part in self.walk():
-      part.modified = False
 
-  def rewindbody(self):
+  def get_param(self, param, failobj=None, header='content-type', unquote=True):
-    return self.fp.seek(self.startofbody)
+    val = Message.get_param(self,param,failobj,header,unquote)
+    if val != failobj and param == 'boundary' and unquote:
+      # unquote boundaries an extra time, test case testDefang5
+      return _unquotevalue(val)
+    return val
 
-  # override param parsing to handle quotes
+  getfilename = Message.get_filename
-  def _get_params_preserve(self,failobj=None,header='content-type'):
-    "Return all parameter names and values. Use parser that handles quotes."
-    missing = []
-    value = self.get(header, missing)
-    if value is missing:
-	return failobj
-    params = []
-    for p in _parseparam(';' + value):
-	  try:
-	      name, val = p.split('=', 1)
-	      name = name.strip()
-	      val = val.strip()
-	  except ValueError:
-	      # Must have been a bare attribute
-	      name = p.strip()
-	      val = ''
-	  params.append((name, val))
-    params = Utils.decode_params(params)
-    return params
-
-  def get_filename(self, failobj=None):
-      """Return the filename associated with the payload if present.
-
-      The filename is extracted from the Content-Disposition header's
-      `filename' parameter, and it is unquoted.
-      """
-      missing = []
-      filename = self.get_param('filename', missing, 'content-disposition')
-      if filename is missing:
-	  return failobj
-      if isinstance(filename, TupleType):
-	  # It's an RFC 2231 encoded parameter
-	  newvalue = _unquotevalue(filename)
-	  if newvalue[0]:
-	    return unicode(newvalue[2], newvalue[0])
-	  return unicode(newvalue[2])
-      else:
-	  newvalue = _unquotevalue(filename.strip())
-	  return newvalue
-
-  getfilename = get_filename
   ismultipart = Message.is_multipart
   getheaders = Message.get_all
   gettype = Message.get_content_type
@@ -317,7 +157,7 @@ class MimeMessage(Message):
     """Return a list of (attr,name) pairs of attributes that IE might
        interpret as a name - and hence decide to execute this message."""
     names = []
-    for attr,val in self.get_params([]):
+    for attr,val in self._get_params_preserve([],'content-type'):
       if isinstance(val, TupleType):
 	  # It's an RFC 2231 encoded parameter
 	  newvalue = _unquotevalue(val)
@@ -333,7 +173,7 @@ class MimeMessage(Message):
   def ismodified(self):
     "True if this message or a subpart has been modified."
     if not self.is_multipart():
-      if self.submsg:
+      if isinstance(self.submsg,Message):
         return self.submsg.ismodified()
       return self.modified
     if self.modified: return True
@@ -343,16 +183,22 @@ class MimeMessage(Message):
 
   def dump(self,file,unixfrom=False):
     "Write this message (and all subparts) to a file"
-    g = Generator(file)
+    g = MimeGenerator(file)
     g.flatten(self,unixfrom=unixfrom)
 
+  def as_string(self, unixfrom=False):
+      "Return the entire formatted message as a string."
+      fp = StringIO.StringIO()
+      self.dump(fp,unixfrom=unixfrom)
+      return fp.getvalue()
+
   def getencoding(self):
     return self.get('content-transfer-encoding',None)
 
   # Decode body to stream according to transfer encoding, return encoding name
-  def decode(self,filter):
+  def decode(self,filt):
     try:
-      filter.write(self.get_payload(decode=True))
+      filt.write(self.get_payload(decode=True))
     except:
       pass
     return self.getencoding()
@@ -363,7 +209,7 @@ class MimeMessage(Message):
   def __setitem__(self, name, value):
     rc = Message.__setitem__(self,name,value)
     self.modified = True
-    if self.headerchange: self.headerchange(self,name,value)
+    if self.headerchange: self.headerchange(self,name,str(value))
     return rc
 
   def __delitem__(self, name):
@@ -374,7 +220,7 @@ class MimeMessage(Message):
 
   def get_payload(self,i=None,decode=False):
     msg = self.submsg
-    if msg and msg.ismodified():
+    if isinstance(msg,Message) and msg.ismodified():
       self.set_payload([msg])
     return Message.get_payload(self,i,decode)
 
@@ -388,18 +234,27 @@ class MimeMessage(Message):
     self.submsg = None
 
   def get_submsg(self):
-    if self.get_content_type().lower() == 'message/rfc822':
+    t = self.get_content_type().lower()
+    if t == 'message/rfc822' or t.startswith('multipart/'):
       if not self.submsg:
         txt = self.get_payload()
 	if type(txt) == str:
 	  txt = self.get_payload(decode=True)
-	  parser = MimeParser(MimeMessage)
+	  self.submsg = email.message_from_string(txt,MimeMessage)
-	  self.submsg = parser.parsestr(txt)
+	  for part in self.submsg.walk():
+	    part.modified = False
 	else:
 	  self.submsg = txt[0]
       return self.submsg
     return None
 
+def message_from_file(fp):
+  msg = email.message_from_file(fp,MimeMessage)
+  for part in msg.walk():
+    part.modified = False
+  assert not msg.ismodified()
+  return msg
+
 extlist = ''.join("""
 ade,adp,asd,asx,asp,bas,bat,chm,cmd,com,cpl,crt,dll,exe,hlp,hta,inf,ins,isp,js,
 jse,lnk,mdb,mde,msc,msi,msp,mst,ocx,pcd,pif,reg,scr,sct,shs,url,vb,vbe,vbs,wsc,
@@ -423,7 +278,7 @@ See your administrator.
 
 def check_name(msg,savname=None,ckname=check_ext):
   "Replace attachment with a warning if its name is suspicious."
-  for (key,name) in msg.getnames():
+  for key,name in msg.getnames():
     badname = ckname(name)
     if badname:
       hostname = socket.gethostname()
@@ -444,7 +299,7 @@ msg	MimeMessage
 check	function(MimeMessage): int
 	Return CONTINUE, REJECT, ACCEPT
   """
-  if msg.ismultipart() and not msg.get_content_type() == 'message/rfc822':
+  if msg.is_multipart():
     for i in msg.get_payload():
       rc = check_attachments(i,check)
       if rc != Milter.CONTINUE: return rc
@@ -453,28 +308,33 @@ check	function(MimeMessage): int
 
 # save call context for Python without nested_scopes
 class _defang:
-  def __init__(self,savname,check):
+
-    self._savname = savname
+  def __init__(self):
-    self._check = check
-    self.scan_rfc822 = True
     self.scan_html = True
+
   def _chk_name(self,msg):
     rc = check_name(msg,self._savname,self._check)
     if self.scan_html:
       check_html(msg,self._savname)	# remove scripts from HTML
     if self.scan_rfc822:
       msg = msg.get_submsg()
-      if msg: return check_attachments(msg,self._chk_name)
+      if isinstance(msg,Message):
+        return check_attachments(msg,self._chk_name)
     return rc
 
+  def __call__(self,msg,savname=None,check=check_ext,scan_rfc822=True):
+    """Compatible entry point.
+    Replace all attachments with dangerous names."""
+    self._savname = savname
+    self._check = check
+    self.scan_rfc822 = scan_rfc822
+    check_attachments(msg,self._chk_name)
+    if msg.ismodified():
+      return True
+    return False
+
 # emulate old defang function
-def defang(msg,savname=None,check=check_ext):
+defang = _defang()
-  """Compatible entry point.
-Replace all attachments with dangerous names."""
-  check_attachments(msg,_defang(savname,check)._chk_name)
-  if msg.ismodified():
-    return 1;
-  return 0
 
 import sgmllib
 
@@ -571,7 +431,6 @@ class HTMLScriptFilter(SGMLFilter):
   def handle_comment(self,comment):
     if not self.ignoring: SGMLFilter.handle_comment(self,comment)
 
-
 def check_html(msg,savname=None):
   "Remove scripts from HTML attachments."
   msgtype = msg.get_content_type().lower()
@@ -582,14 +441,14 @@ def check_html(msg,savname=None):
 	msgtype = 'text/html'
   if msgtype == 'text/html':
     out = StringIO.StringIO()
-    filter = HTMLScriptFilter(out)
+    htmlfilter = HTMLScriptFilter(out)
     try:
-      filter.write(msg.get_payload(decode=True))
+      htmlfilter.write(msg.get_payload(decode=True))
-      filter.close()
+      htmlfilter.close()
     #except sgmllib.SGMLParseError:
     except:
       #mimetools.copyliteral(msg.get_payload(),open('debug.out','w')
-      filter.close()
+      htmlfilter.close()
       hostname = socket.gethostname()
       msg.set_payload(
   "An HTML attachment could not be parsed.  The original is saved as '%s:%s'"
@@ -600,8 +459,25 @@ def check_html(msg,savname=None):
       name = "WARNING.TXT"
       msg["Content-Type"] = "text/plain; name="+name
       return Milter.CONTINUE
-    if filter.modified:
+    if htmlfilter.modified:
       msg.set_payload(out)	# remove embedded scripts
       del msg["content-transfer-encoding"]
       email.Encoders.encode_quopri(msg)
   return Milter.CONTINUE
+
+if __name__ == '__main__':
+  import sys
+  def _list_attach(msg):
+    t = msg.get_content_type()
+    p = msg.get_payload(decode=True)
+    print msg.get_filename(),msg.get_content_type(),type(p)
+    msg = msg.get_submsg()
+    if isinstance(msg,Message):
+      return check_attachments(msg,_list_attach)
+    return Milter.CONTINUE
+
+  for fname in sys.argv[1:]:
+    fp = open(fname)
+    msg = message_from_file(fp)
+    email.Iterators._structure(msg)
+    check_attachments(msg,_list_attach)

rejects.py

@@ -0,0 +1,38 @@
+# Analyze milter log to find abusers
+
+fp = open('/var/log/milter/milter.log','r')
+subdict = {}
+ipdict = {}
+spamcnt = {}
+for line in fp:
+  a = line.split(None,4)
+  if len(a) < 4: continue
+  dt,tm,id,op = a[:4]
+  if op == 'Subject:':
+    if len(a) > 4: subdict[id] = a[4].rstrip()
+  elif op == 'connect':
+    ipdict[id] = a[4].rstrip()
+  elif op in ('eom','dspam'):
+    if id in subdict: del subdict[id]
+    if id in ipdict: del ipdict[id]
+  elif op in ('REJECT:','DSPAM:','SPAM:','abort'):
+    if id in subdict:
+      if id in ipdict:
+        ip = ipdict[id]
+	del ipdict[id]
+	f,host,raw = ip.split(None,2)
+	if host in spamcnt:
+	  spamcnt[host] += 1
+	else:
+	  spamcnt[host] = 1
+      else: ip = ''
+      print dt,tm,op,a[4].rstrip(),subdict[id]
+      del subdict[id]
+    else:
+      print line.rstrip()
+print len(subdict),'leftover entries'
+
+spamlist = filter(lambda x: x[1] > 1,spamcnt.items())
+spamlist.sort(lambda x,y: x[1] - y[1])
+for ip,cnt in spamlist:
+  print cnt,ip

rhsbl.m4

@@ -0,0 +1,44 @@
+divert(-1)
+#
+# Copyright (c) 2002 Derek J. Balling
+#	All rights reserved.
+#
+# Permission to use granted for all purposes. If modifications are made
+# they are requested to be sent to <dredd@megacity.org> for inclusion in future
+# versions 
+#
+# Allows (hopefully) for checking of access.db whitelisting now. This ONLY
+# works on sendmail-8.12.x ... use on any other version may require tinkering
+# by you the downloader.
+#
+# Incorporates many changes by Sergey S. Mokryshev <mokr@mokr.net>
+#
+#
+
+divert(0)
+ifdef(`_RHSBL_R_',`dnl',`dnl
+VERSIONID(`$Id$')
+define(`_RHSBL_R_',`')
+ifdef(`_DNSBL_R_',`dnl',`dnl
+LOCAL_CONFIG
+# map for DNS based blacklist lookups based on the sender RHS
+Kdnsbl host -T<TMP>')')
+divert(-1)
+define(`_RHSBL_SRV_', `_ARG_')dnl
+define(`_RHSBL_MSG_', `ifelse(len(X`'_ARG2_),`1',`"550 Mail from " $`'&{RHS} " refused by blackhole site '_RHSBL_SRV_`"',`_ARG2_')')dnl
+define(`_RHSBL_MSG_TMP_', `ifelse(_ARG3_,`t',`"451 Temporary lookup failure of " $`'&{RHS} " at '_RHSBL_SRV_`"',`_ARG3_')')dnl
+
+MAILER_DEFINITIONS
+
+SLocal_check_mail
+# DNS based RHS spam list blackholes.bmsi.com
+R$*			$: <?> $>CanonAddr $1
+R<?> $*<@$+.>		$: <?> $1<@$2.> $| $>SearchList <+ rhs> $| <F:$1@$2> <D:$2> <>
+R<?> $* $| <$={Accept}>	$: OKSOFAR
+R<?> $*<@$+.> $| $*	$: <?> $(dnsbl $2._RHSBL_SRV_. $: OK $) $(macro {RHS} $@ $2 $)
+R<?> OK			$: OKSOFAR
+R<?> $*<@$*>		$: OKSOFAR
+ifelse(len(X`'_ARG3_),`1',
+`R<?>$+<TMP>		$: TMPOK',
+`R<?>$+<TMP>		$#error $@ 4.7.1 $: _RHSBL_MSG_TMP_')
+R<?>$+			$#error $@ 5.7.1 $: _RHSBL_MSG_

sample.py

@@ -126,7 +126,7 @@ class sampleMilter(Milter.Milter):
   def eom(self):
     if not self.fp: return Milter.ACCEPT
     self.fp.seek(0)
-    msg = mime.MimeMessage(self.fp)
+    msg = mime.message_from_file(self.fp)
     msg.headerchange = self._headerChange
     if not mime.defang(msg,self.tempname):
       os.remove(self.tempname)

setup.py

@@ -12,7 +12,7 @@ if sys.version < '2.2.3':
   DistributionMetadata.classifiers = None
   DistributionMetadata.download_url = None
 
-setup(name = "milter", version = "0.6.9",
+setup(name = "milter", version = "0.8.0",
 	description="Python interface to sendmail milter API",
 	long_description="""\
 This is a python extension module to enable python scripts to
@@ -26,9 +26,13 @@ querying SPF records.
 	maintainer_email="stuart@bmsi.com",
 	license="GPL",
 	url="http://www.bmsi.com/python/milter.html",
-	py_modules=["Milter","mime","spf"],
+	py_modules=["mime","spf"],
+	packages = ['Milter'],
 	ext_modules=[
-	  Extension("milter", ["miltermodule.c"],libraries=libs),
+	  Extension("milter", ["miltermodule.c"],
+	    libraries=libs,
+	    define_macros = [ ('MAX_ML_REPLY',32) ]
+	  ),
 	],
 	keywords = ['sendmail','milter'],
 	classifiers = [
@@ -39,6 +43,7 @@ querying SPF records.
 	  'Natural Language :: English',
 	  'Operating System :: POSIX',
 	  'Programming Language :: Python',
-	  'Topic :: Communications :: Email :: Mail Transport Agents'
+	  'Topic :: Communications :: Email :: Mail Transport Agents',
+	  'Topic :: Communications :: Email :: Filters'
 	]
 )

spf.py

@@ -45,6 +45,43 @@ For news, bugfixes, etc. visit the home page for this implementation at
 # Terrence is not responding to email.
 #
 # $Log$
+# Revision 1.24  2005/03/16 21:58:39  stuart
+# Change Milter module to package.
+#
+# Revision 1.22  2005/02/09 17:52:59  stuart
+# Report DNS errors as PermError rather than unknown.
+#
+# Revision 1.21  2004/11/20 16:37:03  stuart
+# Handle multi-segment TXT records.
+#
+# Revision 1.20  2004/11/19 06:10:30  stuart
+# Use PermError exception instead of reporting unknown.
+#
+# Revision 1.19  2004/11/09 23:00:18  stuart
+# Limit recursion and DNS lookups separately.
+#
+#
+# Revision 1.17  2004/09/10 18:08:26  stuart
+# Return unknown for null mechanism
+#
+# Revision 1.16  2004/09/04 23:27:06  stuart
+# More mechanism aliases.
+#
+# Revision 1.15  2004/08/30 21:19:05  stuart
+# Return unknown for invalid ip syntax in mechanism
+#
+# Revision 1.14  2004/08/23 02:28:24  stuart
+# Remove Perl usage message.
+#
+# Revision 1.13  2004/07/23 19:23:12  stuart
+# Always fail to match on ip6, until we support it properly.
+#
+# Revision 1.12  2004/07/23 18:48:15  stuart
+# Fold CID parsing into spf
+#
+# Revision 1.11  2004/07/21 21:32:01  stuart
+# Handle CID records (Microsoft XML format).
+#
 # Revision 1.10  2004/04/19 22:12:11  stuart
 # Release 0.6.9
 #
@@ -97,6 +134,135 @@ import struct  # for pack() and unpack()
 import time    # for time()
 
 import DNS	# http://pydns.sourceforge.net
+import xml.sax
+
+# -------------------------------------------------------------------------
+# Convert a MS Caller-ID entry (XML) to a SPF entry
+#
+# (c) 2004 by Ernesto Baschny
+# (c) 2004 Python version by Stuart Gathman
+#
+# Date: 2004-02-25
+#
+# A complete reverse translation (SPF -> CID) might be impossible, since
+# there are no ways to handle:
+# - PTR and EXISTS mechanism 
+# - MX mechanism with an different domain as argument
+# - macros
+# 
+# References:
+# http://www.microsoft.com/mscorp/twc/privacy/spam_callerid.mspx
+# http://spf.pobox.com/
+#
+# Known bugs:
+# - Currently it won't handle the exclusions provided in the A and R
+#   tags (prefix '!'). They will show up "as-is" in the SPF record
+# - I really haven't read the MS-CID specs in-depth, so there are probably
+#   other bugs too :)
+#
+# Ernesto Baschny <ernst@baschny.de>
+#
+
+class CIDParser(xml.sax.ContentHandler):
+  "Convert a MS Caller-ID entry (XML) to a SPF entry."
+
+  def __init__(self,q=None):
+    self.spf = []
+    self.action = '-all'
+    self.has_servers = None
+    self.spf_entry = None
+    if q:
+      self.spf_query = q
+    else:
+      self.spf_query = query(i='127.0.0.1', s='localhost', h='unknown')
+
+  def startElement(self,tag,attr):
+      if tag == 'm':
+	if self.has_servers != None and not self.has_servers:
+	  raise ValueError(
+    "Declared <noMailServers\> and later <m>, this CID entry is not valid."
+	  )
+	self.has_servers = True
+      elif tag == 'noMailServers':
+	if self.has_servers:
+	  raise ValueError(
+    "Declared <m> and later <noMailServers\>, this CID entry is not valid."
+	  )
+	self.has_servers = False
+      elif tag == 'ep':
+	if attr.has_key('testing') and attr.getValue('testing') == 'true':
+	  # A CID with 'testing' found:
+	  # From the MS-specs:
+	  #  "Documents in which such attribute is present with a true
+	  #  value SHOULD be entirely ignored (one should act as if the
+	  #  document were absent)"
+	  # From the SPF-specs:
+	  #  "Neutral (?): The SPF client MUST proceed as if a domain did
+	  #  not publish SPF data."
+	  # So we set SPF action to "neutral":
+	  self.action = '?all'
+      elif tag == 'mx':
+	  # The empty MX-tag, same as SPF's MX-mechanism
+	  self.spf.append('mx')
+      self.tag = tag
+
+  def characters(self,text):
+	tag = self.tag
+	# Remove starting and trailing spaces from text:
+	text = text.strip()
+
+	if tag == 'a' or tag == 'r':
+	    # The A and R tags from MS-CID are both handled by the 
+	    # ipv4/6-mechanisms from SPF:
+	    if text.find(':') < 0:
+	      mechanism = 'ip4'
+	    else:
+	      mechanism = 'ip6'
+	    self.spf.append(mechanism + ':' + text)
+	elif tag == 'indirect':
+	    # MS-CID's indirect is "sort of" the include from SPF:
+	    # Not really true, because the <indirect> tag from MS-CID also 
+	    # provides a fallback in case the included domain doesn't provide
+	    # _ep-records: The inbound MX-servers of the included domains
+	    # are added to the list of allowed outgoing mailservers for the
+	    # domain that declared the _ep-record with the <indirect> tag.
+	    # In SPF you would use the 'mx:domain' to handle this, but this
+	    # wouldn't depend on referred domain having or not SPF-records.
+	    cid_xml = self.cid_txt(text)
+	    if cid_xml:
+	      p = CIDParser()
+	      xml.sax.parseString(cid_xml,p)
+	      if p.has_servers != False:
+		self.spf += p.spf
+	    else:
+	      self.spf.append('mx:' + text)
+
+  def cid_txt(self,domain):
+    q = self.spf_query
+    domain='_ep.' + domain
+    a = q.dns_txt(domain)
+    if not a: return None
+    if a[0].lower().startswith('<ep ') and a[-1].lower().endswith('</ep>'):
+      return ''.join(a)
+    return None
+
+  def endElement(self,tag):
+      if tag == 'ep':
+	# This is the end... assemble what we've got
+	spf_entry = ['v=spf1']
+	if self.has_servers != False:
+	  spf_entry += self.spf
+	spf_entry.append(self.action)
+	self.spf_entry = ' '.join(spf_entry)
+
+  def spf_txt(self,cid_xml):
+    if not cid_xml.startswith('<'):
+      cid_xml = self.cid_txt(cid_xml)
+      if not cid_xml: return None
+    # Parse the beast. Any XML-problem will be reported by xlm.sax
+    self.spf_entry = None
+    xml.sax.parseString(cid_xml,self)
+    return self.spf_entry
 
 # 32-bit IPv4 address mask
 MASK = 0xFFFFFFFFL
@@ -143,7 +309,25 @@ except NameError:
 # standard default SPF record
 DEFAULT_SPF = 'v=spf1 a/24 mx/24 ptr'
 
-def check(i, s, h,local=None):
+# maximum DNS lookups allowed
+MAX_LOOKUP = 100
+MAX_RECURSION = 20
+
+class TempError(Exception):
+	"Temporary SPF error"
+
+class PermError(Exception):
+	"Permanent SPF error"
+	def __init__(self,msg,mech=None):
+	  Exception.__init__(self,msg,mech)
+	  self.msg = msg
+	  self.mech = mech
+	def __str__(self):
+	  if self.mech:
+	    return '%s: %s'%(self.msg,self.mech)
+	  return self.msg
+
+def check(i, s, h,local=None,receiver=None):
 	"""Test an incoming MAIL FROM:<s>, from a client with ip address i.
 	h is the HELO/EHLO domain name.
 
@@ -157,7 +341,7 @@ def check(i, s, h,local=None):
 	#>>> check(i='61.51.192.42', s='liukebing@bcc.com', h='bmsi.com')
 
 	"""
-	return query(i=i, s=s, h=h,local=local).check()
+	return query(i=i, s=s, h=h,local=local,receiver=receiver).check()
 
 class query(object):
 	"""A query object keeps the relevant information about a single SPF
@@ -178,16 +362,21 @@ class query(object):
 
 	Also keeps cache: DNS cache.
 	"""
-	def __init__(self, i, s, h,local=None):
+	def __init__(self, i, s, h,local=None,receiver=None):
 		self.i, self.s, self.h = i, s, h
+		if not s and h:
+		  self.s = 'postmaster@' + h
 		self.l, self.o = split_email(s, h)
 		self.t = str(int(time.time()))
 		self.v = 'in-addr'
 		self.d = self.o
 		self.p = None
+		if receiver:
+		  self.r = receiver
 		self.cache = {}
 		self.exps = dict(EXPLANATIONS)
 		self.local = local	# local policy
+    		self.lookups = 0
 
 	def set_default_explanation(self,exp):
 		exps = self.exps
@@ -212,31 +401,41 @@ class query(object):
 	Returns (result, mta-status-code, explanation) where
 	result in ['fail', 'softfail', 'neutral' 'unknown', 'pass', 'error']
 		"""
+		self.mech = []		# unknown mechanisms
 		if self.i.startswith('127.'):
 			return ('pass', 250, 'local connections always pass')
 
 		try:
+			self.lookups = 0
 			if not spf:
 			    spf = self.dns_spf(self.d)
 			if self.local and spf:
 			    spf += ' ' + self.local
 			return self.check1(spf, self.d, 0)
-		except DNS.DNSError:
+		except DNS.DNSError,x:
-			return ('error', 450, 'SPF DNS Error')
+			return ('error', 450, 'SPF DNS Error: ' + str(x))
+		except TempError,x:
+			return ('error', 450, 'SPF Temporary Error: ' + str(x))
+		except PermError,x:
+			# Pre-Lentczner draft treats this as an unknown result
+			# and equivalent to no SPF record.
+			self.prob = x.msg
+			self.mech.append(x.mech)
+			return ('unknown', 550, 'SPF Permanent Error: ' + str(x))
 
 	def check1(self, spf, domain, recursion):
 		# spf rfc: 3.7 Processing Limits
 		#
-		if recursion > 20:
+		if recursion > MAX_RECURSION:
-			self.prob =  'Mechanisms used too many DNS lookups'
+			self.prob =  'Too many levels of recursion'
 			return ('unknown', 250, 'SPF recursion limit exceeded')
 		try:
 			tmp, self.d = self.d, domain
-			return self.check0(spf, recursion)
+			return self.check0(spf,recursion)
 		finally:
 			self.d = tmp
 
-	def check0(self, spf, recursion):
+	def check0(self, spf,recursion):
 		"""Test this query information against SPF text.
 
 		Returns (result, mta-status-code, explanation) where
@@ -258,7 +457,6 @@ class query(object):
 		# overridden with 'default=' modifier
 		#
 		default = 'neutral'
-		self.mech = []		# unknown mechanisms
 
 		# Look for modifiers
 		#
@@ -280,84 +478,86 @@ class query(object):
 		# Look for mechanisms
 		#
 		for mech in spf:
-			if RE_MODIFIER.match(mech): continue
+		    if RE_MODIFIER.match(mech): continue
-			m, arg, cidrlength = parse_mechanism(mech, self.d)
+		    m, arg, cidrlength = parse_mechanism(mech, self.d)
 
-			# map '?' '+' or '-' to 'unknown' 'pass' or 'fail'
+		    # map '?' '+' or '-' to 'unknown' 'pass' or 'fail'
-			result = RESULTS.get(m[0])
+		    if m:
-			if result:
+		      result = RESULTS.get(m[0])
-				# eat '?' '+' or '-'
+		      if result:
-				m = m[1:]
+			    # eat '?' '+' or '-'
-			else:
+			    m = m[1:]
-				# default pass
+		      else:
-				result = 'pass'
+			    # default pass
+			    result = 'pass'
 
-			if m in ['a', 'mx', 'ptr', 'exists', 'include']:
+		    if m in ['a', 'mx', 'ptr', 'prt', 'exists', 'include']:
-				arg = self.expand(arg)
+			    arg = self.expand(arg)
 
-			if m == 'include':
+		    if m == 'include':
-			    if arg != self.d:
+		      if arg != self.d:
-				res,code,txt = self.check1(self.dns_spf(arg),
+			res,code,txt = self.check1(self.dns_spf(arg),
-						  arg, recursion + 1)
+					  arg, recursion + 1)
-				if res == 'pass':
+			if res == 'pass':
-					break
+			  break
-				if res in ('fail','neutral','softfail'):
+			if res == 'none':
-					continue
+			  raise PermError(
-				if res == 'none':
+			    'No valid SPF record for included domain: %s'%arg,
-				  	self.prob = \
+			    mech)
-					  'Could not find a valid SPF record'
+			continue
-				  	res = 'unknown'
+		      else:
-				return res,code,txt
+			raise PermError('include mechanism missing domain',mech)
-			    else:
+		    elif m == 'all':
-			    	self.prob = 'Required option is missing'
+			    break
-				self.mech.append(mech)
-				return ('unknown', 250, 'missing SPF option')
 
-			elif m == 'all':
+		    elif m == 'exists':
+			    if len(self.dns_a(arg)) > 0:
+				    break
+
+		    elif m == 'a':
+			    if cidrmatch(self.i, self.dns_a(arg),
+					 cidrlength):
+				    break
+
+		    elif m == 'mx':
+			    if cidrmatch(self.i, self.dns_mx(arg),
+					 cidrlength):
+				    break
+
+		    elif m in ('ip4', 'ipv4', 'ip') and arg != self.d:
+			try:
+			    if cidrmatch(self.i, [arg], cidrlength):
 				break
+			except socket.error:
+			    raise PermError('syntax error',mech)
 			    
-			elif m == 'exists':
+		    elif m in ('ip6', 'ipv6'):
-				if len(self.dns_a(arg)) > 0:
+		    # Until we support IPV6, we should never
-					break
+		    # get an IPv6 connection.  So this mech
+		    # will never match.
+			    pass
 
-			elif m == 'a':
+		    elif m in ('ptr', 'prt'):
-				if cidrmatch(self.i, self.dns_a(arg),
+			    if domainmatch(self.validated_ptrs(self.i),
-				             cidrlength):
+					   arg):
-					break
+				    break
-
-			elif m == 'mx':
-				if cidrmatch(self.i, self.dns_mx(arg),
-				             cidrlength):
-					break
-
-			elif m in ('ip4', 'ipv4') and arg != self.d:
-				if cidrmatch(self.i, [arg], cidrlength):
-					break
-
-			elif m in ('ptr', 'prt'):
-				if domainmatch(self.validated_ptrs(self.i),
-				               arg):
-					break
-
-			else:
-				# unknown mechanisms cause immediate unknown
-				# abort results
-				self.mech.append(mech)
-				self.prob = 'Unknown mechanism found'
-				return ('unknown',250,'unknown SPF mechanism')
 
+		    else:
+		      # unknown mechanisms cause immediate unknown
+		      # abort results
+		      raise PermError('Unknown mechanism found',mech)
 		else:
-			# no matches
+		    # no matches
-			if redirect:
+		    if redirect:
-				return self.check1(self.dns_spf(redirect),
+			return self.check1(self.dns_spf(redirect),
-				                   redirect, recursion+1)
+					       redirect, recursion + 1)
-			else:
+		    else:
-				result = default
+			result = default
 
 		if result == 'fail':
-			return (result, 550, exps[result])
+		    return (result, 550, exps[result])
 		else:
-			return (result, 250, exps[result])
+		    return (result, 250, exps[result])
 
 	def get_explanation(self, spec):
 		"""Expand an explanation."""
@@ -465,19 +665,29 @@ class query(object):
 		is found.
 		"""
 		a = [t for t in self.dns_txt(domain) if t.startswith('v=spf1')]
-		if not a and DELEGATE:
+		if not a:
-		  a = [t
+		  if DELEGATE:
-		    for t in self.dns_txt(domain+'._spf.'+DELEGATE)
+		    a = [t
-		      if t.startswith('v=spf1')
+		      for t in self.dns_txt(domain+'._spf.'+DELEGATE)
-		  ]
+			if t.startswith('v=spf1')
+		    ]
+		  if not a:
+		    # No SPF record: convert and return CID if present
+		    p = CIDParser(q=self)
+		    try:
+		      return p.spf_txt(domain)
+		    except xml.sax._exceptions.SAXParseException,x:
+		      raise PermError("Caller-ID parse error",domain)
+
 		if len(a) == 1:
 			return a[0]
 		else:
 			return None
 
 	def dns_txt(self, domainname):
+		"Get a list of TXT records for a domain name."
 		if domainname:
-		  return [t for a in self.dns(domainname, 'TXT') for t in a]
+		  return [''.join(a) for a in self.dns(domainname, 'TXT')]
 		return []
 
 	def dns_mx(self, domainname):
@@ -519,6 +729,9 @@ class query(object):
 		pre: qtype in ['A', 'AAAA', 'MX', 'PTR', 'TXT', 'SPF']
 		post: isinstance(__return__, types.ListType)
 		"""
+		self.lookups += 1
+		if self.lookups > MAX_LOOKUP:
+			raise PermError('Too many DNS lookups')
 		result = self.cache.get( (name, qtype) )
 		cname = None
 		if not result:
@@ -536,7 +749,7 @@ class query(object):
 		return result
 
 	def get_header(self,res,receiver):
-	  if res in ('pass','fail'):
+	  if res in ('pass','fail','softfail'):
 	    return '%s (%s: %s) client-ip=%s; envelope-from=%s; helo=%s;' % (
 	  	res,receiver,self.get_header_comment(res),self.i,
 	        self.l + '@' + self.o, self.h)
@@ -802,13 +1015,15 @@ if __name__ == '__main__':
 		print USAGE
 		_test()
 	elif len(sys.argv) == 2:
-		q = query(i='127.0.0.1', s='localhost', h='unknown')
+		q = query(i='127.0.0.1', s='localhost', h='unknown',
+			receiver=socket.gethostname())
 		print q.dns_spf(sys.argv[1])
 	elif len(sys.argv) == 4:
-		print check(i=sys.argv[1], s=sys.argv[2], h=sys.argv[3])
+		print check(i=sys.argv[1], s=sys.argv[2], h=sys.argv[3],
+			receiver=socket.gethostname())
 	elif len(sys.argv) == 5:
 		i, s, h = sys.argv[2:]
-		q = query(i=i, s=s, h=h)
+		q = query(i=i, s=s, h=h, receiver=socket.gethostname())
 		print q.check(sys.argv[1])
 	else:
 		print USAGE

test/missingboundary

@@ -0,0 +1,128 @@
+From leec@windowsshop.com Fri Sep 10 11:48:25 2004
+Message-ID: <4141CDD4.7040305@windowsshop.com>
+Date: Fri, 10 Sep 2004 11:52:52 -0400
+From: Lee Connor <leec@windowsshop.com>
+User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)
+X-Accept-Language: en-us, en
+MIME-Version: 1.0
+To: Cleo Matthews-Conley <cleom@windowsshop.com>, 
+ Tony Collini <tonyc@windowsshop.com>,
+ John Higinbothom <johnh@windowsshop.com>
+CC: Rich Higgins <richh@windowsshop.com>
+Subject: [Fwd: [Fwd: Customer Concerns]]
+Content-Type: multipart/mixed;
+ boundary="------------020209070802060007090105"
+
+This is a multi-part message in MIME format.
+--------------020209070802060007090105
+Content-Type: text/plain; charset=us-ascii; format=flowed
+Content-Transfer-Encoding: 7bit
+
+Cleo - please review attached feedback from Sales team.......I recall at 
+an early meeting after we moved in you and Tony (and maybe 1 or 2 
+others) were going to develop a voice mail procedure or instruction 
+sheet for all staff. It looks like we really need this to get what we 
+are looking for from the system. Please let me know when you can produce 
+this and give a draft to the managers here for review.
+Thanks,
+Lee
+
+
+--------------020209070802060007090105
+Content-Type: message/rfc822;
+ name="[Fwd: Customer Concerns]"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: inline;
+ filename="[Fwd: Customer Concerns]"
+
+Return-Path: <richh@windowsshop.com>
+Received: from windowsshop.com (pc147.windowsshop.com [192.168.100.147] (may be forged))
+	by lord.windowsshop.com (8.12.10/8.12.10) with ESMTP id i89KCClX003425
+	for <leec@windowsshop.com>; Thu, 9 Sep 2004 16:12:12 -0400
+Message-ID: <4140B851.3020501@windowsshop.com>
+Date: Thu, 09 Sep 2004 16:08:49 -0400
+From: Rich <richh@windowsshop.com>
+User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.0.2) Gecko/20021120 Netscape/7.01
+X-Accept-Language: en-us, en
+MIME-Version: 1.0
+To: Lee Connor <leec@windowsshop.com>
+Subject: [Fwd: Customer Concerns]
+Content-Type: multipart/mixed;
+ boundary="------------030301030706020401010801"
+X-DSpam-Score: 0.000000
+
+This is a multi-part message in MIME format.
+--------------030301030706020401010801
+Content-Type: text/plain; charset=us-ascii; format=flowed
+Content-Transfer-Encoding: 7bit
+
+Lee - do you want me to do anything else with this?
+
+Rich
+
+<!DSPAM:FEE4D3278234264874834386>
+
+
+--------------030301030706020401010801
+Content-Type: message/rfc822; name="Customer Concerns";
+	boundary="===============0045392615=="
+Content-Transfer-Encoding: 7bit
+Content-Disposition: inline;
+ filename="Customer Concerns"
+
+
+Return-Path: <joes@windowsshop.com>
+Received: from joes (pc148.windowsshop.com [192.168.100.148] (may be forged))
+	by lord.windowsshop.com (8.12.10/8.12.10) with SMTP id i89K9BlX003262
+	for <richh@windowsshop.com>; Thu, 9 Sep 2004 16:09:11 -0400
+From: "Joe Schmuck" <joes@windowsshop.com>
+To: <richh@windowsshop.com>
+Subject: Customer Concerns
+Date: Thu, 9 Sep 2004 16:08:26 -0400
+Message-ID: <OFEPKHCCLPIECLFBLDHBAEAECAAA.joes@windowsshop.com>
+MIME-Version: 1.0
+Content-Type: text/plain;
+	charset="iso-8859-1"
+Content-Transfer-Encoding: 7bit
+X-Priority: 3 (Normal)
+X-MSMail-Priority: Normal
+X-Mailer: Microsoft Outlook IMO, Build 9.0.2416 (9.0.2910.0)
+Importance: Normal
+X-MimeOLE: Produced By Microsoft MimeOLE V6.00.2800.1106
+X-DSpam-Score: 0.000000
+
+Rich:
+
+Following is a summary of concerns from customers regarding internal
+communications within WS:
+
+	- Not all employees have activated their voice mail - when this is the
+case, the system will automatically cut you off
+	- When employees are out of the office, phones are not forwarded to a back
+up, ie manager
+	- Reception has no record of employee attendance, and therefore will
+forward call to individual requested - see point 2
+ 	- Reception directs calls to incorrect individuals
+ 	- When entering voice mail, if you press '0', system does not default to
+operator, but puts you back into individual 	voice mail
+	- Reception phone demeanor has no 'pep'
+
+Thanks
+Joe
+
+
+
+
+---
+Outgoing mail is certified Virus Free.
+Checked by AVG anti-virus system (http://www.grisoft.com).
+Version: 6.0.752 / Virus Database: 503 - Release Date: 9/3/2004
+
+
+<!DSPAM:FEE4D05F1332634871908793>
+
+--===============0045392615==--
+--------------030301030706020401010801--
+
+--------------020209070802060007090105--
+

testbms.py

@@ -4,6 +4,8 @@ import bms
 import mime
 import rfc822
 import StringIO
+import email
+import sys
 #import pdb
 
 class TestMilter(bms.bmsMilter):
@@ -25,7 +27,7 @@ class TestMilter(bms.bmsMilter):
   def replacebody(self,chunk):
     if self._body:
       self._body.write(chunk)
-      self.bodyreplaced = 1
+      self.bodyreplaced = True
     else:
       raise IOError,"replacebody not called from eom()"
 
@@ -39,14 +41,14 @@ class TestMilter(bms.bmsMilter):
       del self._msg[field]
     else:
       self._msg[field] = value
-    self.headerschanged = 1
+    self.headerschanged = True
 
   def addheader(self,field,value):
     if not self._body:
       raise IOError,"addheader not called from eom()"
     self.log('addheader: %s=%s' % (field,value))
     self._msg[field] = value
-    self.headerschanged = 1
+    self.headerschanged = True
 
   def delrcpt(self,rcpt):
     if not self._body:
@@ -63,8 +65,8 @@ class TestMilter(bms.bmsMilter):
 
   def feedFile(self,fp,sender="spam@adv.com",rcpt="victim@lamb.com"):
     self._body = None
-    self.bodyreplaced = 0
+    self.bodyreplaced = False
-    self.headerschanged = 0
+    self.headerschanged = False
     self.reply = None
     msg = rfc822.Message(fp)
     rc = self.envfrom('<%s>'%sender)
@@ -118,7 +120,7 @@ class TestMilter(bms.bmsMilter):
 
   def connect(self,host='localhost'):
     self._body = None
-    self.bodyreplaced = 0
+    self.bodyreplaced = False
     rc =  bms.bmsMilter.connect(self,host,1,('1.2.3.4',1234)) 
     if rc != Milter.CONTINUE and rc != Milter.ACCEPT:
       self.close()
@@ -141,7 +143,7 @@ class BMSMilterTestCase(unittest.TestCase):
     open('test/'+fname+".tstout","w").write(fp.getvalue())
     #self.failUnless(fp.getvalue() == open("test/virus1.out","r").read())
     fp.seek(0)
-    msg = mime.MimeMessage(fp)
+    msg = mime.message_from_file(fp)
     str = msg.get_payload(1).get_payload()
     milter.log(str)
     milter.close()
@@ -218,7 +220,9 @@ class BMSMilterTestCase(unittest.TestCase):
     #pdb.set_trace()
     rc = milter.feedMsg('test8')
     self.assertEqual(rc,Milter.ACCEPT)
-    self.failUnless(milter.bodyreplaced,"Message body not replaced")
+    # python2.4 doesn't scan encoded message attachments
+    if sys.hexversion < 0x02040000:
+      self.failUnless(milter.bodyreplaced,"Message body not replaced")
     #self.failIf(milter.bodyreplaced,"Message body replaced")
     fp = milter._body
     open("test/test8.tstout","w").write(fp.getvalue())
@@ -237,9 +241,12 @@ class BMSMilterTestCase(unittest.TestCase):
     bms.smart_alias[key] = ['ham@eggs.com']
     rc = milter.feedMsg('test8',key[0],key[1])
     self.assertEqual(rc,Milter.ACCEPT)
-    self.failUnless(milter.bodyreplaced,"Message body not replaced")
     self.failUnless(milter._delrcpt == ['<baz@bat.com>'])
     self.failUnless(milter._addrcpt == ['<ham@eggs.com>'])
+    # python2.4 email does not decode message attachments, so script
+    # is not replaced
+    if sys.hexversion < 0x02040000:
+      self.failUnless(milter.bodyreplaced,"Message body not replaced")
 
   def testBadBoundary(self):
     milter = TestMilter()
@@ -247,8 +254,11 @@ class BMSMilterTestCase(unittest.TestCase):
     # test rfc822 attachment with invalid boundaries
     #pdb.set_trace()
     rc = milter.feedMsg('bound')
-    self.assertEqual(rc,Milter.REJECT)
+    if sys.hexversion < 0x02040000:
-    self.assertEqual(milter.reply[0],'554')
+      # python2.4 adds invalid boundaries to decects list and makes
+      # payload a str
+      self.assertEqual(rc,Milter.REJECT)
+      self.assertEqual(milter.reply[0],'554')
     #self.failUnless(milter.bodyreplaced,"Message body not replaced")
     self.failIf(milter.bodyreplaced,"Message body replaced")
     fp = milter._body
@@ -277,7 +287,6 @@ class BMSMilterTestCase(unittest.TestCase):
 def suite(): return unittest.makeSuite(BMSMilterTestCase,'test')
 
 if __name__ == '__main__':
-  import sys
   if len(sys.argv) > 1:
     for fname in sys.argv[1:]:
       milter = TestMilter()

testmime.py

@@ -1,8 +1,23 @@
+# $Log$
+# Revision 1.23  2005/02/11 18:34:14  stuart
+# Handle garbage after quote in boundary.
+#
+# Revision 1.22  2005/02/10 01:10:59  stuart
+# Fixed MimeMessage.ismodified()
+#
+# Revision 1.21  2005/02/10 00:56:49  stuart
+# Runs with python2.4.  Defang not working correctly - more work needed.
+#
+# Revision 1.20  2004/11/20 16:38:17  stuart
+# Add rcs log
+#
 import unittest
 import mime
 import socket
 import StringIO
 import email
+import sys
+from email import Errors
 
 samp1_txt1 = """Dear Agent 1
 I hope you can read this.  Whenever you write label it  P.B.S kids.
@@ -24,22 +39,38 @@ class MimeTestCase(unittest.TestCase):
     self.failUnless(plist[0] == 'name="Jim&Girlz.jpg"')
 
   def testParse(self,fname='samp1'):
-    msg = mime.MimeMessage(open('test/'+fname,"r"))
+    msg = mime.message_from_file(open('test/'+fname,"r"))
     self.failUnless(msg.ismultipart())
     parts = msg.get_payload()
     self.failUnless(len(parts) == 2)
     txt1 = parts[0].get_payload()
     self.failUnless(txt1.rstrip() == samp1_txt1,txt1)
+    msg = mime.message_from_file(open('test/missingboundary',"r"))
+    # should get no exception as long as we don't try to parse
+    # message attachments
+    mime.defang(msg,scan_rfc822=False)
+    msg.dump(open('test/missingboundary.out','w'))
+    msg = mime.message_from_file(open('test/missingboundary',"r"))
+    try:
+      mime.defang(msg)
+      # python 2.4 doesn't get exceptions on missing boundaries, and
+      # if message is modified, output is readable by mail clients
+      if sys.hexversion < 0x02040000:
+	self.fail('should get boundary error parsing bad rfc822 attachment')
+    except Errors.BoundaryError:
+      pass
   
   def testDefang(self,vname='virus1',part=1,
   	fname='LOVE-LETTER-FOR-YOU.TXT.vbs'):
-    msg = mime.MimeMessage(open('test/'+vname,"r"))
+    msg = mime.message_from_file(open('test/'+vname,"r"))
     mime.defang(msg)
+    self.failUnless(msg.ismodified(),"virus not removed")
     oname = vname + '.out'
     msg.dump(open('test/'+oname,"w"))
-    msg = mime.MimeMessage(open('test/'+oname,"r"))
+    msg = mime.message_from_file(open('test/'+oname,"r"))
-    parts = msg.get_payload()
+    txt2 = msg.get_payload()
-    txt2 = parts[part].get_payload()
+    if type(txt2) == list:
+      txt2 = txt2[part].get_payload()
     self.failUnless(txt2.rstrip()+'\n' == mime.virus_msg % (fname,hostname,None),txt2)
 
   def testDefang3(self):
@@ -55,11 +86,11 @@ class MimeTestCase(unittest.TestCase):
 
   # virus6 has no parts - the virus is directly inline
   def testDefang6(self,vname="virus6",fname='FAX20.exe'):
-    msg = mime.MimeMessage(open('test/'+vname,"r"))
+    msg = mime.message_from_file(open('test/'+vname,"r"))
     mime.defang(msg)
     oname = vname + '.out'
     msg.dump(open('test/'+oname,"w"))
-    msg = mime.MimeMessage(open('test/'+oname,"r"))
+    msg = mime.message_from_file(open('test/'+oname,"r"))
     self.failIf(msg.ismultipart())
     txt2 = msg.get_payload()
     self.failUnless(txt2 == mime.virus_msg % \
@@ -68,11 +99,11 @@ class MimeTestCase(unittest.TestCase):
   # honey virus has a sneaky ASP payload which is parsed correctly
   # by email package in python-2.2.2, but not by mime.MimeMessage or 2.2.1
   def testDefang7(self,vname="honey",fname='story[1].scr'):
-    msg = mime.MimeMessage(open('test/'+vname,"r"))
+    msg = mime.message_from_file(open('test/'+vname,"r"))
     mime.defang(msg)
     oname = vname + '.out'
     msg.dump(open('test/'+oname,"w"))
-    msg = mime.MimeMessage(open('test/'+oname,"r"))
+    msg = mime.message_from_file(open('test/'+oname,"r"))
     parts = msg.get_payload()
     txt2 = parts[1].get_payload()
     txt3 = parts[2].get_payload()
@@ -83,7 +114,7 @@ class MimeTestCase(unittest.TestCase):
 	  ('story[1].asp',hostname,None),txt3)
 
   def testParse2(self,fname="spam7"):
-    msg = mime.MimeMessage(open('test/'+fname,"r"))
+    msg = mime.message_from_file(open('test/'+fname,"r"))
     self.failUnless(msg.ismultipart())
     parts = msg.get_payload()
     self.failUnless(len(parts) == 2)
@@ -106,10 +137,9 @@ class MimeTestCase(unittest.TestCase):
 def suite(): return unittest.makeSuite(MimeTestCase,'test')
 
 if __name__ == '__main__':
-  import sys
   if len(sys.argv) < 2:
     unittest.main()
   else:
     for fname in sys.argv[1:]:
       fp = open(fname,'r')
-      msg = mime.MimeMessage(fp)
+      msg = mime.message_from_file(fp)

testsample.py

@@ -17,7 +17,7 @@ class TestMilter(sample.sampleMilter):
   def replacebody(self,chunk):
     if self._body:
       self._body.write(chunk)
-      self.bodyreplaced = 1
+      self.bodyreplaced = True
     else:
       raise IOError,"replacebody not called from eom()"
 
@@ -29,16 +29,16 @@ class TestMilter(sample.sampleMilter):
       del self._msg[field]
     else:
       self._msg[field] = value
-    self.headerschanged = 1
+    self.headerschanged = True
 
   def addheader(self,field,value):
     self.log('addheader: %s=%s' % (field,value))
     self._msg[field] = value
-    self.headerschanged = 1
+    self.headerschanged = True
 
   def feedMsg(self,fname):
     self._body = None
-    self.bodyreplaced = 0
+    self.bodyreplaced = False
     self.headerschanged = 0
     fp = open('test/'+fname,'r')
     msg = rfc822.Message(fp)
@@ -85,7 +85,7 @@ class TestMilter(sample.sampleMilter):
 
   def connect(self,host='localhost'):
     self._body = None
-    self.bodyreplaced = 0
+    self.bodyreplaced = False
     rc =  sample.sampleMilter.connect(self,host,1,0) 
     if rc != Milter.CONTINUE and rc != Milter.ACCEPT:
       self.close()
@@ -108,7 +108,7 @@ class BMSMilterTestCase(unittest.TestCase):
     open('test/'+fname+".tstout","w").write(fp.getvalue())
     #self.failUnless(fp.getvalue() == open("test/virus1.out","r").read())
     fp.seek(0)
-    msg = mime.MimeMessage(fp)
+    msg = mime.message_from_file(fp)
     s = msg.get_payload(1).get_payload()
     milter.log(s)
     milter.close()