This commit was manufactured by cvs2svn to create tag 'milter-0_8_8'.

Sprout from master 2007-07-25 19:04:44 UTC Stuart Gathman <stuart@gathman.org> 'Fixes from test on EL5.'
Cherrypick from bmsi 2005-05-31 18:23:49 UTC Stuart Gathman <stuart@gathman.org> 'Development changes since 0.7.2':
    rejects.py
    rhsbl.m4
    sample.py
    test/amazon
    test/big5
    test/bounce
    test/bounce1
    test/bound
    test/honey
    test/missingboundary
    test/samp1
    test/spam44
    test/spam7
    test/spam8
    test/test1
    test/test8
    test/virus1
    test/virus13
    test/virus2
    test/virus3
    test/virus4
    test/virus5
    test/virus6
    test/virus7
    testsample.py

HOWTO

@@ -1,3 +1,10 @@
+On Sun, 11 Feb 2007, Rick Saul wrote:
+
+> Stuart I was planning to move to centos4.4 in a couple of weeks anyway... 
+> Your advice of where to go from here. 
+
+Oh - you are asking for a howto.
+
 	Step one.  Which DSPAM is right for you?
 
 The DSPAM project makes dspam part of the LDA (Local Delivery Agent).
@@ -28,39 +35,42 @@ wish to install pydspam.
 For basic pymilter you'll need:
 
 python-2.4
-milter-0.8.2	(the RH9 rpm should work on Fedora Core - let me know)
+milter-0.8.7
 sendmail-8.13.x	(with milter support enabled)
 
 and for SPF you'll need:
 
 pydns-2.3.0-2.4
+pyspf-2.0.3-2.py24
 
 and for SRS you'll need:
 
-pysrs-0.30.9-1.py24
+pysrs-0.30.11-1.py24
 
 I'm pretty sure you will want to have SPF and SRS available.
 
 	Step three.  Activate basic milter.
 
-Activate the basic milter by editing /etc/mail/sendmail.mc and adding:
+Activate the basic milter and pysrs by editing /etc/mail/sendmail.mc and adding:
 
+define(`NO_SRS_FILE',`/etc/mail/no-srs-mailers')dnl
+dnl define(`NO_SRS_FROM_LOCAL')dnl
+HACK(`pysrs',`/var/run/milter/pysrs')dnl
 INPUT_MAIL_FILTER(`pythonfilter', `S=local:/var/run/milter/pythonsock, F=T, T=C:5m;S:20s;R:5m;E:5m')
 
 You can then "make sendmail.cf" and restart sendmail.
 
+Start milter and pysrs with "service milter start", "service pysrs start".
+
 Tail /var/log/milter/milter.log while SMTP clients connect to your
 sendmail instance.  This should show you what the milter is doing.
 
-By default, milter-0.8.2 rejects on SPF fail, except for listed domains
-(that are known to be broken).  Some admins don't like that, and 0.8.3 will use
-the /etc/mail/access database to configure SPF responses.  For now,
-if you don't like SPF, you can disable spf by replacing "import spf"
-with "spf = None" around line 285 in /var/log/milter/bms.py.
+By default, milter-0.8.7 rejects on SPF fail.
 
 	Step four.  Tweaking the basic config.
 
-Most pymilter configuration is in /etc/mail/pymilter.cfg.
+Most pymilter configuration is in /etc/mail/pymilter.cfg.  To activate
+changes, "service milter restart".
 
 By default, milter scans attachments for executable extensions.  You can
 turn this off by setting banned_exts to the empty list.  There are options
@@ -76,7 +86,9 @@ should also run pymilter with similar policies.  (But this isn't
 needed for initial testing.)
 
 Configure internal_connect with subnets of your internal SMTP clients.
-Internal connections skip SPF testing and other policies.
+Internal connections skip SPF testing and other policies.  You will
+likely need to set this to allow outgoing mail if you have
+an SPF policy already.
 
 Configure internal_domains with domains used by your internal SMTP clients.
 If they attempt to use any other domain, the attempt is blocked and the
@@ -134,3 +146,9 @@ SRS config
 
 pydspam config
 wiretap config
+
+-- 
+	      Stuart D. Gathman <stuart@bmsi.com>
+    Business Management Systems Inc.  Phone: 703 591-0911 Fax: 703 591-6154
+"Confutatis maledictis, flammis acribus addictis" - background song for
+a Microsoft sponsored "Where do you want to go from here?" commercial.

MANIFEST.in

@@ -8,8 +8,8 @@ include ChangeLog
 include MANIFEST.in
 include testsample.py
 include testmime.py
+include testutils.py
 include testbms.py
-include testdspam.py
 include rejects.py
 include report.py
 include bms.py
@@ -18,6 +18,10 @@ include cid2spf.py
 include spfquery.py
 include test.py
 include sample.py
+include milter-template.py
+include spfmilter.py
+include spfmilter.rc
+include spfmilter.cfg
 include test/*
 include doc/*
 include Milter/*.py

Milter/__init__.py

@@ -9,7 +9,7 @@ import milter
 import thread
 
 from milter import ACCEPT,CONTINUE,REJECT,DISCARD,TEMPFAIL,	\
-  set_flags, setdbg, setbacklog, settimeout, \
+  set_flags, setdbg, setbacklog, settimeout, error,	\
   ADDHDRS, CHGBODY, ADDRCPT, DELRCPT, CHGHDRS,	\
   V1_ACTS, V2_ACTS, CURR_ACTS
 

Milter/cache.py

@@ -0,0 +1,143 @@
+# Email address list with expiration
+#
+# This class acts like a map.  Entries with a value of None are persistent,
+# but disappear after a time limit.  This is useful for automatic whitelists
+# and blacklists with expiration.  The persistent store is a simple ascii
+# file with sender and timestamp on each line.  Entries can be appended
+# to the store, and will be picked up the next time it is loaded.
+#
+# Entries with other values are not persistent.  This is used to hold failed
+# CBV results.
+#
+# $Log$
+# Revision 1.6  2007/01/19 23:31:38  customdesigned
+# Move parse_header to Milter.utils.
+# Test case for delayed DSN parsing.
+# Fix plock when source missing or cannot set owner/group.
+#
+# Revision 1.5  2007/01/11 19:59:40  customdesigned
+# Purge old entries in auto_whitelist and send_dsn logs.
+#
+# Revision 1.4  2007/01/11 04:31:26  customdesigned
+# Negative feedback for bad headers.  Purge cache logs on startup.
+#
+# Revision 1.3  2007/01/08 23:20:54  customdesigned
+# Get user feedback.
+#
+# Revision 1.2  2007/01/05 23:33:55  customdesigned
+# Make blacklist an AddrCache
+#
+# Revision 1.1  2007/01/05 21:25:40  customdesigned
+# Move AddrCache to Milter package.
+#
+
+# Author: Stuart D. Gathman <stuart@bmsi.com>
+# Copyright 2001,2002,2003,2004,2005 Business Management Systems, Inc.
+# This code is under the GNU General Public License.  See COPYING for details.
+
+import time
+from plock import PLock
+
+class AddrCache(object):
+  time_format = '%Y%b%d %H:%M:%S %Z'
+
+  def __init__(self,renew=7,fname=None):
+    self.age = renew
+    self.cache = {}
+    self.fname = fname
+
+  def load(self,fname,age=0):
+    "Load address cache from persistent store."
+    if not age:
+      age = self.age
+    self.fname = fname
+    cache = {}
+    self.cache = cache
+    now = time.time()
+    lock = PLock(self.fname)
+    wfp = lock.lock()
+    changed = False
+    try:
+      too_old = now - age*24*60*60	# max age in days
+      try:
+        fp = open(self.fname)
+      except OSError:
+        fp = ()
+      for ln in fp:
+	try:
+	  rcpt,ts = ln.strip().split(None,1)
+	  l = time.strptime(ts,AddrCache.time_format)
+	  t = time.mktime(l)
+	  if t < too_old:
+	    changed = True
+	    continue
+	  cache[rcpt.lower()] = (t,None)
+	except:
+	  cache[ln.strip().lower()] = (now,None)
+	wfp.write(ln)
+      if changed:
+	lock.commit(self.fname+'.old')
+      else:
+        lock.unlock()
+    except IOError:
+      lock.unlock()
+
+  def has_key(self,sender):
+    "True if sender is cached and has not expired."
+    try:
+      lsender = sender and sender.lower()
+      ts,res = self.cache[lsender]
+      too_old = time.time() - self.age*24*60*60	# max age in days
+      if not ts or ts > too_old:
+        return True
+      del self.cache[lsender]
+      try:
+	user,host = sender.split('@',1)
+	return self.has_key(host)
+      except ValueError:
+        pass
+    except KeyError:
+      try:
+	user,host = sender.split('@',1)
+	return self.has_key(host)
+      except: pass
+    return False
+
+  __contains__ = has_key
+
+  def __getitem__(self,sender):
+    try:
+      lsender = sender.lower()
+      ts,res = self.cache[lsender]
+      too_old = time.time() - self.age*24*60*60	# max age in days
+      if not ts or ts > too_old:
+	return res
+      del self.cache[lsender]
+      raise KeyError, sender
+    except KeyError,x:
+      try:
+	user,host = sender.split('@',1)
+	return self.__getitem__(host)
+      except ValueError:
+        raise x
+
+  def addperm(self,sender,res=None):
+    "Add a permanent sender."
+    lsender = sender.lower()
+    if self.has_key(lsender):
+      ts,res = self.cache[lsender]
+      if not ts: return		# already permanent
+    self.cache[lsender] = (None,res)
+    if not res:
+      print >>open(self.fname,'a'),sender
+    
+  def __setitem__(self,sender,res):
+    lsender = sender.lower()
+    now = time.time()
+    self.cache[lsender] = (now,res)
+    if not res and self.fname:
+      s = time.strftime(AddrCache.time_format,time.localtime(now))
+      print >>open(self.fname,'a'),sender,s # log refreshed senders
+
+  def __len__(self):
+    return len(self.cache)

Milter/config.py

@@ -0,0 +1,59 @@
+from ConfigParser import ConfigParser
+
+class MilterConfigParser(ConfigParser):
+
+  def __init__(self,defaults={}):
+    ConfigParser.__init__(self)
+    self.defaults = defaults
+
+  # The defaults provided by ConfigParser show up in all sections,
+  # which screws up iterating over all options in a section.
+  # Worse, passing "defaults" with vars= overrides the config file!
+  # So we roll our own defaults.
+  def get(self,sect,opt):
+    if not self.has_option(sect,opt) and opt in self.defaults:
+      return self.defaults[opt]
+    return ConfigParser.get(self,sect,opt)
+    
+  def getlist(self,sect,opt):
+    if self.has_option(sect,opt):
+      return [q.strip() for q in self.get(sect,opt).split(',')]
+    return []
+
+  def getaddrset(self,sect,opt):
+    if not self.has_option(sect,opt):
+      return {}
+    s = self.get(sect,opt)
+    d = {}
+    for q in s.split(','):
+      q = q.strip()
+      if q.startswith('file:'):
+        domain = q[5:].lower()
+	d[domain] = d.setdefault(domain,[]) + open(domain,'r').read().split()
+      else:
+	user,domain = q.split('@')
+	d.setdefault(domain.lower(),[]).append(user)
+    return d
+  
+  def getaddrdict(self,sect,opt):
+    if not self.has_option(sect,opt):
+      return {}
+    d = {}
+    for q in self.get(sect,opt).split(','):
+      q = q.strip()
+      if self.has_option(sect,q):
+        l = self.get(sect,q)
+	for addr in l.split(','):
+	  addr = addr.strip()
+	  if addr.startswith('file:'):
+	    fname = addr[5:]
+	    for a in open(fname,'r').read().split():
+	      d[a] = q
+	  else:
+	    d[addr] = q
+    return d
+
+  def getdefault(self,sect,opt,default=None):
+    if self.has_option(sect,opt):
+      return self.get(sect,opt)
+    return default

Milter/dsn.py

@@ -5,6 +5,9 @@
 # Send DSNs, do call back verification,
 # and generate DSN messages from a template
 # $Log$
+# Revision 1.13  2007/01/04 18:01:11  customdesigned
+# Do plain CBV when template missing.
+#
 # Revision 1.12  2006/07/26 16:37:35  customdesigned
 # Support timeout.
 #
@@ -28,8 +31,11 @@ def send_dsn(mailfrom,receiver,msg=None,timeout=600):
      Receiver is the MTA sending the DSN.
      Return None for success or (code,msg) for failure."""
   user,domain = mailfrom.split('@')
+  try:
     q = spf.query(None,None,None)
     mxlist = q.dns(domain,'MX')
+  except spf.TempError:
+    return (450,'DNS Timeout: %s MX'%domain)	# temp error
   if not mxlist:
     mxlist = (0,domain),	# fallback to A record when no MX
   else:

Milter/dynip.py

@@ -44,6 +44,8 @@ def is_dynip(host,addr):
   True
   >>> is_dynip('[1.2.3.4]','1.2.3.4')
   True
+  >>> is_dynip('c-71-63-151-151.hsd1.mn.comcast.net','71.63.151.151')
+  True
   """
   if host.startswith('[') and host.endswith(']'):
     return True
@@ -54,7 +56,7 @@ def is_dynip(host,addr):
     h = host
     m = ip3.findall(host)
     if m:
-      g = map(int,m)
+      g = map(int,m)[:4]
       ia3 = (ia[1:],ia[:3])
       if g[-3:] in ia3: return True
       if g[0] == ia[3] and g[1:3] == ia[:2]: return True

Milter/plock.py

@@ -0,0 +1,66 @@
+# Author: Stuart D. Gathman <stuart@bmsi.com>
+# Copyright 2001 Business Management Systems, Inc.
+# This code is under the GNU General Public License.  See COPYING for details.
+
+import os
+from time import sleep
+
+class PLock(object):
+  "A simple /etc/passwd style lock,update,rename protocol for updating files."
+  def __init__(self,basename):
+    self.basename = basename
+    self.fp = None
+
+  def lock(self,lockname=None,mode=0660,strict_perms=False):
+    "Start an update transaction.  Return FILE to write new version."
+    self.unlock()
+    if not lockname:
+      lockname = self.basename + '.lock'
+    self.lockname = lockname
+    try:
+      st = os.stat(self.basename)
+      mode |= st.st_mode
+    except OSError: pass
+    u = os.umask(0002)
+    try:
+      fd = os.open(lockname,os.O_WRONLY+os.O_CREAT+os.O_EXCL,mode)
+    finally:
+      os.umask(u)
+    self.fp = os.fdopen(fd,'w')
+    try:
+      os.chown(self.lockname,-1,st.st_gid)
+    except:
+      if strict_perms:
+	self.unlock()
+	raise
+    return self.fp
+
+  def wlock(self,lockname=None):
+    "Wait until lock is free, then start an update transaction."
+    while True:
+      try:
+        return self.lock(lockname)
+      except OSError:
+        sleep(2)
+
+  def commit(self,backname=None):
+    "Commit update transaction with optional backup file."
+    if not self.fp:
+      raise IOError,"File not locked"
+    self.fp.close()
+    self.fp = None
+    if backname:
+      try:
+	os.remove(backname)
+      except OSError: pass
+      os.link(self.basename,backname)
+    os.rename(self.lockname,self.basename)
+
+  def unlock(self):
+    "Cancel update transaction."
+    if self.fp:
+      try:
+        self.fp.close()
+      except: pass
+      self.fp = None
+      os.remove(self.lockname)

Milter/unsign.py

@@ -0,0 +1,17 @@
+# Author: Stuart D. Gathman <stuart@bmsi.com>
+# Copyright 2005 Business Management Systems, Inc.
+# This code is under the GNU General Public License.  See COPYING for details.
+
+# The localpart of SMTP return addresses is often signed.  The format
+# of the signing is application specific and doesn't concern us -
+# except that we wish to extract some sort of fixed string from
+# the variable signature which represents the "source" of the message.
+
+def unsign(s):
+  """Attempt to unsign localpart and return original email.
+  No attempt is made to verify the signature.
+  >>> unsign('SRS0=8Y3CZ=3U=jsconnor.com=bills@bmsi.com')
+  'bills@jsconnor.com'
+  """
+  # not implemented yet
+  return s

Milter/utils.py

@@ -0,0 +1,85 @@
+import re
+import struct
+import socket
+import email.Errors
+from fnmatch import fnmatchcase
+from email.Header import decode_header
+
+ip4re = re.compile(r'^[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*$')
+
+# from spf.py
+def addr2bin(str):
+  "Convert a string IPv4 address into an unsigned integer."
+  return struct.unpack("!L", socket.inet_aton(str))[0]
+
+MASK = 0xFFFFFFFFL
+
+def cidr(i,n):
+  return ~(MASK >> n) & MASK & i
+
+def iniplist(ipaddr,iplist):
+  """Return whether ip is in cidr list
+  >>> iniplist('66.179.26.146',['127.0.0.1','66.179.26.128/26'])
+  True
+  >>> iniplist('127.0.0.1',['127.0.0.1','66.179.26.128/26'])
+  True
+  >>> iniplist('192.168.0.45',['192.168.0.*'])
+  True
+  """
+  ipnum = addr2bin(ipaddr)
+  for pat in iplist:
+    p = pat.split('/',1)
+    if ip4re.match(p[0]):
+      if len(p) > 1:
+	n = int(p[1])
+      else:
+        n = 32
+      if cidr(addr2bin(p[0]),n) == cidr(ipnum,n):
+        return True
+    elif fnmatchcase(ipaddr,pat):
+      return True
+  return False
+
+def parse_addr(t):
+  """Split email into user,domain.
+
+  >>> parse_addr('user@example.com')
+  ['user', 'example.com']
+  >>> parse_addr('"user@example.com"')
+  ['user@example.com']
+  >>> parse_addr('"user@bar"@example.com')
+  ['user@bar', 'example.com']
+  >>> parse_addr('foo')
+  ['foo']
+  """
+  if t.startswith('<') and t.endswith('>'): t = t[1:-1]
+  if t.startswith('"'):
+    if t.endswith('"'): return [t[1:-1]]
+    pos = t.find('"@')
+    if pos > 0: return [t[1:pos],t[pos+2:]]
+  return t.split('@')
+
+def parse_header(val):
+  """Decode headers gratuitously encoded to hide the content.
+  """
+  try:
+    h = decode_header(val)
+    if not len(h) or (not h[0][1] and len(h) == 1): return val
+    u = []
+    for s,enc in h:
+      if enc:
+        try:
+	  u.append(unicode(s,enc))
+	except LookupError:
+	  u.append(unicode(s))
+      else:
+	u.append(unicode(s))
+    u = ''.join(u)
+    for enc in ('us-ascii','iso-8859-1','utf8'):
+      try:
+	return u.encode(enc)
+      except UnicodeError: continue
+  except UnicodeDecodeError: pass
+  except LookupError: pass
+  except email.Errors.HeaderParseError: pass
+  return val

NEWS

@@ -1,4 +1,8 @@
 Here is a history of user visible changes to Python milter.
+0.8.8	move AddrCache, parse_addr, iniplist, parse_header to Milter package
+	fix plock for missing source and can't change owner/group
+	add sample spfmilter.py milter
+	private_relay config option
 0.8.7	Move spf module to pyspf
 	Prevent PTR cache poisoning
 	More lame bounce heuristics

README

@@ -42,7 +42,7 @@ Quick Installation
 1. Build and install Sendmail, enabling libmilter (see libmilter/README).
 2. Build and install Python, enabling threading.
 3. Install this module: python setup.py --help
-4. Add these two lines to sendmail.cf:
+4. Add these two lines to sendmail.cf[*]:
 
 O InputMailFilters=pythonfilter
 Xpythonfilter,        S=local:/home/username/pythonsock
@@ -51,9 +51,17 @@ Xpythonfilter,        S=local:/home/username/pythonsock
 Note that milters should almost certainly not run as root.
 
 That's it.  Incoming mail will cause the milter to print some things, and
-some email will be rejected (see the "header" method).  Edit and play.  See
-bms.py for an example milter used in production.
+some email will be rejected (see the "header" method).  Edit and play.  
+See spfmilter.py for a functional SPF milter, or see bms.py for an complex
+milter used in production.
 
+[*] This is for a quick test.  Your sendmail.cf in most distros will get
+overwritten whenever sendmail.mc is updated.  To make a milter permanent,
+add something like:
+
+INPUT_MAIL_FILTER(`pythonfilter', `S=local:/home/username/pythonsock, F=T, T=C:5m;S:20s;R:5m;E:5m')
+
+to sendmail.mc instead.
 
 Not-so-quick Installation
 -------------------------
@@ -90,8 +98,10 @@ some options associated with it.  In this case, we have the "S" option, which
 names the socket that sendmail will use to communicate with this particular
 milter.  This milter's socket is a unix-domain socket in the filesystem.
 See libmilter/README for the definitive list of options.
+
 NB: The name is specified in two places: here, in sendmail's cf file, and
 in the milter itself.  Make sure the two match.
+
 NB: The above lines can be added in your .mc file with this line:
 
 INPUT_MAIL_FILTER(`pythonfilter', `S=local:/home/username/pythonsock')

TODO

@@ -1,66 +1,87 @@
-When bms.py can't find templates, it passes None to dsn.create_msg(),
-which uses local variable as backup, which no longer exist.
+Don't match dynamic ptr in bestguess.
 
-Purge old GOSSiP records nightly.
+When content filtering is not installed, reject BLACKLISTed MFROM
+immediately.  There is no use waiting until EOM.
 
-Find and use X-GOSSiP: header for SPAM: and FP: submissions.  Would need to 
-keep tags longer.
+Configuration is problematic when handling incoming, but not outgoing mail.
+The problem comes when alice@example.com sends mail to bill@example.com,
+and we are the MX for example.com, but alice is sending from some other
+MTA.  The mail is flagged external, so we don't list example.com in
+internal_domains (or we would get "spam from self").  But, if we try to do a
+CBV, we get "fraudulent MX", because the MX is ourself!  So we need to 
+avoid doing CBV on such domains.  Currently, we try to make sure the SPF
+policies don't do CBV.
+
+We now don't check internal domains for incoming mail if there is an
+SPF record.
+
+On the other hand, if alice is sending internally, or with SMTP AUTH, she
+*does* need the domain to be in internal_domains.  The solution to that 
+is to use the new SMTP AUTH access configuration to specify which domains
+can be used by smtp AUTH (by user if desired).
+
+It would be cleaner if CBV would know which domains we have agreed to
+be MX for. Some ideas for external connections:
+
+a) check access file for To:example.com	RELAY
+b) check mailertable
+c) check mx_domains config list
+d) if there is an SPF record, don't check internal_domains
+   (let SPF block unauthorized machines)
+
+But that still doesn't handle the roaming user, who won't use SMTP
+AUTH, but sends through some hotel MTA.  Maybe we don't want to support
+him?
+
+When setting up pydspam, both sender and rcpt must resolve to dspam users
+for falsepositive recognition.  Usually, this means adding 
+honeypot@mail.example.com to alias list for honeypot in pymilter.cfg.
+This needs to be documented.  I was caught by it setting up a new site.
+
+Add signature (x-sig=AB7485f=TS) to Received-SPF, so it can be used
+to blacklist sources of delayed DSNs.
+
+rcpt-addr may let us know when a recipient is unknown.  That should count
+against reputation.
+
+Need to use wildcards in blacklist.log: *.madcowsrecord.net
+Need to exclude emails like !*-admin@example.com in whitelist_sender.
+Need to exclude robot users from autowhitelist.  Don't want to have to
+list all users, so implement something like !*-admin@bmsi.com,@bmsi.com.
+
+GOSSiP feedback from user training is ignored because UMIS has already been
+removed from queue.  Maybe keep UMIS in queue, and add method to 
+alter last feedback for ID.
 
 Generate DSNs according to RFC 3464
 
-Parse incoming 3464 DSNs for "Action: failed" to recognize delayed
-failures.  This works regardless of Subject.
-
 Get temperror policy from access file.
 
-When training with spam, REJECT after data so that mistakenly blacklisted
-senders at least get an error.
-
 Reporting explanation for failure should show source if sender
 provided explanation.
 
-Reports PROBATION even when rejecting message (works, but confusing in log).
-
 Bug in Auto-whitelist.  Recent Auto-whitelist doesn't override expired entry.
 
-DONE Delayed_failure detection needs to handle multi-line header fields.  
-Also, delayed_failure should be recognized when addressed to
-postmaster@helodomain 
-
-Need to use wildcards in blacklist.log: *.madcowsrecord.net
-Need to exclude emails like !*-admin@example.com in whitelist_sender.
-
 SPF permerror diagnostics should include corrected mechanism.
 
 Delay SPF check until RCPT TO.  Cache result to avoid repeating
 for multiple RCPT.  This avoids overhead for invalid RCPT, and
 allows for per RCPT local policy.
 
-Add auto-blacklisted senders to blacklist.log with timestamp.
-
-Received-SPF header field should show identity that was checked.
-
 Check SPF for outgoing mail (including local policy for internal addresses).
 This could also solve the second part of the mail from relay problem below.
 
-Whitelisted sender from trusted relay get PROBATION.  Need to extracted
+Whitelisted senders from trusted relay get PROBATION.  Need to extracted
 SPF result from headers - and in the case of mail internal to relay
 (e.g. bmsi.com), supply 'pass' result.  
 
-FIXME: DSN for Permerror shows 'None' for error under some condition.
-
-Another metaDSN format:
-Subject: Delivery Report
-...
-Original-Envelope-ID: SRS0...@...
-
 For selected domains, check rcpts via CBV before accepting mail.  Cache
 results.  This will kick out dictonary attacks against a mail domain
 behind a gateway sooner.
 
-Allow blacklisted emails as well as domains in blacklist.log.  Use same
-data structure as autowhitelist.log.  Add emails blacklisted via CBV
-so that they are remembered across milter restarts.
+Add auto-blacklisted senders to blacklist.log with timestamp.
+Add emails blacklisted via CBV so that they are remembered across milter
+restarts.
 
 Make all dictionaries work like honeypot.  Do not train as ham unless
 whitelisted.  Train on blacklisted messages, or spam feedback.  This
@@ -71,9 +92,6 @@ to train on error to minimize labor.
 Allow unsigned DSNs from selected domains (that don't accept signed MFROM,
 e.g. verizon.net).
 
-Added Message-ID header to DSN with SRS signed sender.  When seen on incoming
-rfc ignorant failure message, blacklist sender.
-
 Allow verified hostnames for trusted_relay.  E.g. HELO name that
 passes SPF.
 
@@ -98,11 +116,9 @@ wildcard (e.g. empty localpart).
 Quarantined mail is missing headers modified/added by milter after
 checking dspam.
 
-Require signed MFROM for all incoming bounces when signing all outgoing mail -
-except from trusted relays.
-
 Send DSN for permerror before processing extended result.  An additional
-DSN may be sent based on extended result.
+DSN may be sent based on extended result.  Send permerror DSN to
+postmaster@sending_domain.
 
 Rescind whitelist for banned extensions, in case sender is infected.
 
@@ -116,9 +132,6 @@ SPF-Neutral:aol.com	ERROR:"550 AOL mail must get SPF PASS"
 Defer TEMPERROR in SPF evaluation - give precedence to security
 (only defer for PASS mechanisms).
 
-Option to add Received-SPF header, but never reject on SPF.
-  I think the above will handle this.
-
 Create null config that does nothing - except maybe add Received-SPF
 headers.  Many admins would like to turn features on one at a time.
 
@@ -147,8 +160,6 @@ a separate process.  However, a significant amount of memory is wasted
 for each additional Python VM, and communication between milters
 is cumbersome (e.g., adding mail headers, writing external files).
 
-Backup copies for outgoing/incoming mail.
-
 Copy incoming wiretap mail, even though sendmail alias works perfectly
 for the purpose, to avoid having to change two configs for a wiretap.
 
@@ -166,3 +177,50 @@ embarrass yourself), and also removing Received headers with hidepath.
 Need a test module to feed sample messages to a milter though a live 
 sendmail and SMTP.  The mockup currently used is probably not very accurate,
 and doesn't test the threading code.
+
+DONE Require signed MFROM for all incoming bounces when signing all outgoing
+mail - except from trusted relays.
+
+DONE Added Message-ID header to DSN with SRS signed sender.  When seen on
+incoming rfc ignorant failure message, blacklist sender.
+
+DONE Option to add Received-SPF header, but never reject on SPF.
+  I think the above will handle this.
+
+DONE Received-SPF header field should show identity that was checked.
+
+DONE When training with spam, REJECT after data so that mistakenly blacklisted
+senders at least get an error.
+
+DONE Milter won't start when it can't change permissions on *.lock to match
+*.log.  Should maybe ignore that error - the effect will be to set
+the permissions to default.
+
+DONE Milter won't start when a whitelist/blacklist file is missing.
+
+DONE Delayed failure detection should parse From header to find email address.
+
+DONE When bms.py can't find templates, it passes None to dsn.create_msg(),
+which uses local variable as backup, which no longer exist.  Do plain
+CBV in that case instead.
+
+DONE Find and use X-GOSSiP: header for SPAM: and FP: submissions.  Would need
+to keep tags longer.
+
+DONE Parse incoming 3464 DSNs for "Action: failed" to recognize delayed
+failures.  This works regardless of Subject.
+
+DONE Reports PROBATION even when rejecting message (works, but confusing in
+log).
+
+DONE Delayed_failure detection needs to handle multi-line header fields.  
+Also, delayed_failure should be recognized when addressed to
+postmaster@helodomain 
+
+DONE DSN for Permerror shows 'None' for error under some condition.
+
+DONE Allow blacklisted emails as well as domains in blacklist.log.  Use same
+data structure as autowhitelist.log.  
+
+DONE Backup copies for outgoing/incoming mail.
+

doc/changes.ht

@@ -2,6 +2,14 @@ Title: Recent Changes
 
 <h2> Recent Changes </h2>
 
+<h3> 0.8.7 </h3>
+
+The spf module has been moved to the 
+<a href="http://cheeseshop.python.org/pypi/pyspf">pyspf</a> package.
+Download <a href="http://sourceforge.net/project/showfiles.php?group_id=139894&package_id=191419">here</a>.
+
+<h3> 0.8.6 </h3>
+
 Python milter has been moved to 
 <a href="http://sourceforge.net/projects/pymilter/">pymilter Sourceforge
 project</a> for development and release downloads.

doc/faq.ht

@@ -2,8 +2,24 @@ Title: Python Milter FAQ
 
 <h1> Python Milter <a name=faq>FAQ</a> </h1>
 
+<menu>
+<li> <a href="#compiling">Compiling Python Milter</a>
+<li> <a href="#running">Running Python Milter</a>
+<li> <a href="#spf">Using SPF</a>
+<li> <a href="#srs">Using SRS</a>
+</menu>
+
 <ol>
-<h3> Compiling Python Milter </h3>
+
+<h3> <a name="compiling">Compiling Python Milter </a> </h3>
+
+<li> Q. I have tried to download the current milter code and my virus scan
+traps several viruses in the download.
+<p>  A. The milter source includes a number of deactivated viruses in
+the test directory.  All but the first and last lines of the base64
+encoded virus data has been removed.  I suppose I should randomize
+the first and last lines as well, since pymilter just deletes executables,
+and doesn't look for signatures.
 <li> Q. I have installed sendmail from source, but Python milter won't
 compile.
 <p>  A. Even though libmilter is officially supported in sendmail-8.12, 
@@ -36,7 +52,7 @@ in setup.py to
             define_macros = [ ('MAX_ML_REPLY',1) ]
 </pre>
 
-<h3> Running Python Milter </h3>
+<h3> <a name="running">Running Python Milter </a></h3>
 
 <li> Q. The sample.py milter prints a message, then just sits there.
 <pre>
@@ -186,10 +202,16 @@ The <code>internal_domains</code> option is simplistic, it assumes all
 valid senders of the domains are internal.  SPF provides a much more general
 check of IP and MAIL FROM for external email.  Pymilter should soon
 have a local policy feature for more general checking of internal mail.
+<li> Q. <code>mail_archive</code> isn't working.  Or I don't understand how
+	it's suppose to work.  I have
+	<code>mail_archive = /var/mail/mail_archive</code>
+	in <code>pymilter.cfg</code> but nothing ever gets dumped into
+	<code>/var/mail/mail_archive</code>.
+<p>  A. The 'mail' user needs to have write access.  Permission failures
+	should be logged as a traceback in milter.log if it doesn't.
 
-<h3> Using SPF </h3>
+<h3> <a name="spf">Using SPF </a></h3>
 
-<a name="spf">
 <li> Q. So how do I use the SPF support?  The sample.py milter doesn't seem
         to use it.
 <p>  A. The bms.py milter supports spf.  The RedHat RPMs will set almost
@@ -209,5 +231,63 @@ everything up for you.  For other systems:
      logfiles and a simple cron script using <code>find</code> to clean
      <code>tempdir</code>.
 </ol>
+	In CVS, there is <code>spfmilter.py</code>.  Run that as a service,
+	and it does just SPF.  It uses the sendmail <code>access</code>
+	file to configure SPF responses just like <code>bms.py</code>, but
+	supports only REJECT and OK.
+<li> Q. The SPF DSN is sent at least once for domains that don't publish a SPF.
+	How do I stop this behavior?
+<p>  A. The SPF response is controlled by <code>/etc/mail/access</code>
+	(actually the file you specify with <code>access_file</code> in
+	the <code>[spf]</code> section of <code>pymilter.cfg</code>).  
+	Responses are OK, CBV, and REJECT.  CBV sends the DSN.
+<p>
+You can change the defaults.  For instance, I have:
+<pre>
+SPF-None:	REJECT
+SPF-Neutral:	CBV
+SPF-Softfail:	CBV
+SPF-Permerror:	CBV
+</pre>
+I have best_guess = 1, so SPF none is converted to PASS/NEUTRAL for policy
+lookup, and 3 strikes (no PTR, no HELO, no SPF) becomes "SPF NONE" for local
+policy purposes (the Received-SPF header always shows the official SPF
+result.)
+<p>
+You can change the default for specific domains:
+<pre>
+# these guys aren't going to pay attention to CBVs anyway...
+SPF-None:cia.gov	REJECT
+SPF-None:fbi.gov	REJECT
+SPF-Neutral:aol.com	REJECT
+SPF-Softfail:ebay.com	REJECT
+</pre>
+
+<h3> <a name="srs">Using SRS </a></h3>
+
+<li> Q. The SRS part doesn't seem to work as whenever I try to start
+	<code>/etc/init.d/pysrs</code>, I get this in
+	<code>/var/log/milter/pysrs.log</code>:
+<pre>
+ConfigParser.NoOptionError: No option 'fwdomain' in section: 'srs'
+</pre>
+<p>  A. You need to specify the forward domain - i.e. the domain you want
+	SRS to rewrite stuff too.
+<p>
+For instance, I have:
+<pre>
+# sample SRS configuration
+[srs]
+secret = don't you wish
+maxage = 8
+hashlength = 5
+;database=/var/log/milter/srs.db
+fwdomain = bmsi.com
+sign=bmsi.com,mail.bmsi.com,gathman.org
+srs=bmsaix.bmsi.com,bmsred.bmsi.com,stl.gathman.org,bampa.gathman.org
+</pre>
+The <code>sign</code> is for local domains which are signed.
+The <code>srs</code> list is for other domains which you are relaying,
+and which need to have SRS checked/undone for bounces.
 
 </ol>

doc/links.h

@@ -9,6 +9,7 @@
 <li><a href="logmsgs.html">Log&nbsp;Messages</a>
 <li><a href="http://bmsi.com/mailman/listinfo/pymilter">Mailing&nbsp;List</a>
 <li><a href="credits.html">CREDITS</a>
+<li><a href="http://sourceforge.net"><img src="http://sflogo.sourceforge.net/sflogo.php?group_id=139894&amp;type=1" width="88" height="31" border="0" alt="SourceForge.net Logo" /></a>
 <h3>Links</h3>
 <li><a href="http://www.milter.org/milter_api/api.html">C&nbsp;API</a>
 <li><a href="http://www.milter.org/">Milter.Org</a>
@@ -17,5 +18,6 @@
 <li><a href="http://www.openspf.org/">SPF</a>
 <li><a href="pysrs.html">pysrs</a>
 <li><a href="http://cheeseshop.python.org/pypi/pyspf">pyspf</a>
+<li><a href="http://bmsi.com/python/pygossip.html">pygossip</a>
 <li><a href="http://bmsi.com/python/dspam.html">pydspam</a>
 <li><a href="http://bmsi.com/libdspam/dspam.html">libdspam</a>

doc/milter.ht

@@ -20,7 +20,7 @@ ALT="Viewable With Any Browser" BORDER="0"></A>
   Stuart D. Gathman</a><br>
 This web page is written by Stuart D. Gathman<br>and<br>sponsored by
 <a href="http://www.bmsi.com">Business Management Systems, Inc.</a> <br>
-Last updated Dec 29, 2005</h4>
+Last updated Mar 30, 2007</h4>
 
 See the <a href="faq.html">FAQ</a> | <a href="http://sourceforge.net/project/showfiles.php?group_id=139894">Download now</a> |
 <a href="http://bmsi.com/mailman/listinfo/pymilter">Subscribe to mailing list</a> |

milter-template.py

@@ -0,0 +1,155 @@
+## To roll your own milter, create a class that extends Milter.  
+#  See the pymilter project at http://bmsi.com/python/milter.html
+#  based on Sendmail's milter API http://www.milter.org/milter_api/api.html
+#  This code is open-source on the same terms as Python.
+
+## Milter calls methods of your class at milter events.
+## Return REJECT,TEMPFAIL,ACCEPT to short circuit processing for a message.
+## You can also add/del recipients, replacebody, add/del headers, etc.
+
+import Milter
+import StringIO
+import time
+import email
+from socket import AF_INET, AF_INET6
+
+def parse_addr(t):
+  """Split email into user,domain.
+
+  >>> parse_addr('user@example.com')
+  ['user', 'example.com']
+  >>> parse_addr('"user@example.com"')
+  ['user@example.com']
+  >>> parse_addr('"user@bar"@example.com')
+  ['user@bar', 'example.com']
+  >>> parse_addr('foo')
+  ['foo']
+  """
+  if t.startswith('<') and t.endswith('>'): t = t[1:-1]
+  if t.startswith('"'):
+    if t.endswith('"'): return [t[1:-1]]
+    pos = t.find('"@')
+    if pos > 0: return [t[1:pos],t[pos+2:]]
+  return t.split('@')
+
+class myMilter(Milter.Milter):
+
+  def __init__(self):  # A new instance with each new connection.
+    self.id = Milter.uniqueID()  # Integer incremented with each call.
+
+  # each connection runs in its own thread and has its own myMilter
+  # instance.  Python code must be thread safe.  This is trivial if only stuff
+  # in myMilter instances is referenced.
+  def connect(self, IPname, family, hostaddr):
+    # (self, 'ip068.subnet71.example.com', AF_INET, ('215.183.71.68', 4720) )
+    # (self, 'ip6.mxout.example.com', AF_INET6,
+    #	('3ffe:80e8:d8::1', 4720, 1, 0) )
+    self.IP = hostaddr[0]
+    self.port = hostaddr[1]
+    if family == AF_INET6:
+      self.flow = hostaddr[2]
+      self.scope = hostaddr[3]
+    else:
+      self.flow = None
+      self.scope = None
+    self.IPname = IPname  # Name from a reverse IP lookup
+    self.H = None
+    self.fp = None
+    self.receiver = self.getsymval('j')
+    self.log("connect from %s at %s" % (IPname, hostaddr) )
+    
+    return Milter.CONTINUE
+
+
+  ##  def hello(self,hostname):
+  def hello(self, heloname):
+    # (self, 'mailout17.dallas.texas.example.com')
+    self.H = heloname
+    self.log("HELO %s" % heloname)
+    if heloname.find('.') < 0:	# illegal helo name
+      # NOTE: example only - too many real braindead clients to reject on this
+      self.setreply('550','5.7.1','Sheesh people!  Use a proper helo name!')
+      return Milter.REJECT
+      
+    return Milter.CONTINUE
+
+  ##  def envfrom(self,f,*str):
+  def envfrom(self, mailfrom, *str):
+    self.F = mailfrom
+    self.R = []  # list of recipients
+    self.fromparms = Milter.dictfromlist(str)	# ESMTP parms
+    self.user = self.getsymval('{auth_authen}')	# authenticated user
+    self.log("mail from:", mailfrom, *str)
+    self.fp = StringIO.StringIO()
+    self.canon_from = '@'.join(parse_addr(mailfrom))
+    self.fp.write('From %s %s\n' % (self.canon_from,time.ctime()))
+    return Milter.CONTINUE
+
+
+  ##  def envrcpt(self, to, *str):
+  def envrcpt(self, recipient, *str):
+    rcptinfo = to,Milter.dictfromlist(str)
+    self.R.append(rcptinfo)
+    
+    return Milter.CONTINUE
+
+
+  def header(self, name, hval):
+    self.fp.write("%s: %s\n" % (name,hval))	# add header to buffer
+    return Milter.CONTINUE
+
+
+  def eoh(self):
+    self.fp.write("\n")				# terminate headers
+    return Milter.CONTINUE
+
+
+  def body(self, chunk):
+    self.fp.write(chunk)
+    return Milter.CONTINUE
+
+
+  def eom(self):
+    self.fp.seek(0)
+    msg = email.message_from_file(self.fp)
+    self.setreply('250','2.5.1','Grokked by pymilter')
+    # many milter functions can only be called from eom()
+    # example of adding a Bcc:
+    self.addrcpt('<%s>' % 'spy@example.com')
+    return Milter.ACCEPT
+
+
+  def close(self):
+    # always called, even when abort is called.  Clean up
+    # any external resources here.
+    return Milter.CONTINUE
+
+  def abort(self):
+    # client disconnected prematurely
+    return Milter.CONTINUE
+
+  ## === Support Functions ===
+
+  def log(self,*msg):
+    print "%s [%d]" % (time.strftime('%Y%b%d %H:%M:%S'),self.id),
+    # 2005Oct13 02:34:11 [1] msg1 msg2 msg3 ...
+    for i in msg: print i,
+    print
+
+
+## ===
+    
+def main():
+  # Register to have the Milter factory create instances of your class:
+  Milter.factory = myMilter
+  flags = Milter.CHGBODY + Milter.CHGHDRS + Milter.ADDHDRS
+  flags += Milter.ADDRCPT
+  flags += Milter.DELRCPT
+  Milter.set_flags(flags)       # tell Sendmail which features we use
+  print "%s milter startup" % time.strftime('%Y%b%d %H:%M:%S')
+  sys.stdout.flush()
+  Milter.runmilter("pythonfilter",socketname,timeout)
+  print "%s bms milter shutdown" % time.strftime('%Y%b%d %H:%M:%S')
+
+if __name__ == "__main__":
+  main()

milter.cfg

@@ -6,9 +6,10 @@ tempdir = /var/log/milter/save
 # how long to wait for a response from sendmail before giving up 
 ;timeout=600
 log_headers = 0
-# connection ips and hostnames are matched against this glob style list
-# to recognize internal senders.
-;internal_connect = 192.168.*.*,127.*
+# Connection ips and hostnames are matched against this glob style list
+# to recognize internal senders.  You probably need to change this.
+# The default is a good guess to try and prevent newbie frustration.
+internal_connect = 192.168.0.0/16,127.*
 
 # mail that is not an internal_connect and claims to be from an
 # internal domain is rejected.  Furthermore, internal mail that
@@ -23,6 +24,9 @@ log_headers = 0
 # SPF checks are bypassed for internal connections and trusted relays.
 ;trusted_relay = 1.2.3.4, 66.12.34.56
 
+# Relaying to these domains is allowed from internal connections only.
+;private_relay = mycorp.com
+
 # Reject external senders with hello names no legit external sender would use.
 # SPF will do this also, but listing your own domain and mailserver here
 # will save some DNS lookups when rejecting certain viruses.
@@ -78,7 +82,7 @@ reject_spoofed = 0
 # refuses mail from user names commonly abused in that way.
 ;banned_users = postmaster, mailer-daemon, clamav
 
-# See http://spf.pobox.com for more info on SPF.
+# See http://www.openspf.com for more info on SPF.
 [spf]
 # namespace where SPF records can be supplied for domains without one
 # records are searched for under _spf.domain.com
@@ -196,3 +200,18 @@ blind = 1
 # delivered.
 ;dspam_screener=david,goliath
 # The dspam CGI can also be used: logins must match dspam users
+
+# Optional pygossip interface
+#
+# GOSSiP tracks reputation of domain:qualifier pairs.  For instance,
+# the reputation of example.com:SPF is tracked separately from
+# example.com:neutral.  Currently qualifiers are
+# SPF,neutral,softfail,fail,permerror,GUESS,HELO
+[gossip]
+# Use a dedicated GOSSiP server.  If not specified, a local database
+# will be used.
+;server=host:11900
+# If a local database is used, also consult these GOSSiP servers about 
+# domains.  Peer reputation is also tracked as to how often they
+# agree with us, and weighted accordingly.
+;peers=host1:port,host2

milter.rc

@@ -8,7 +8,7 @@
 # config: /etc/mail/pymilter.cfg
 # pidfile: /var/run/milter/milter.pid
 
-python="python2.3"
+python="python2.4"
 
 pidof() {
 	set - ""
@@ -32,7 +32,11 @@ start() {
 	# Start daemons.
 
 	echo -n "Starting $prog: "
-	daemon --check milter --user mail /var/log/milter/start.sh
+        if ! test -d /var/run/milter; then
+		mkdir -p /var/run/milter
+		chown mail:mail /var/run/milter
+	fi
+	daemon --check milter --user mail /var/log/milter/start.sh milter bms
 	RETVAL=$?
 	echo
 	[ $RETVAL -eq 0 ] && touch /var/lock/subsys/milter

milter.rc7

@@ -8,7 +8,7 @@
 # config: /etc/mail/pymilter.cfg
 # pidfile: /var/run/milter/milter.pid
 
-python="python2.3"
+python="python2.4"
 
 pidof() {
 	set - ""
@@ -32,7 +32,11 @@ start() {
 	# Start daemons.
 
 	echo -n "Starting $prog: "
-	daemon --check milter --user mail /var/log/milter/start.sh
+        if ! test -d /var/run/milter; then
+		mkdir -p /var/run/milter
+		chown mail:mail /var/run/milter
+	fi
+	daemon --check milter --user mail /var/log/milter/start.sh milter bms
 	RETVAL=$?
 	echo
 	[ $RETVAL -eq 0 ] && touch /var/lock/subsys/milter

milter.spec

@@ -1,5 +1,5 @@
-%define name milter
-%define version 0.8.7
+%define name pymilter
+%define version 0.8.8
 %define release 1
 # what version of RH are we building for?
 %define redhat7 0
@@ -11,17 +11,23 @@
 # some systems dont have initrddir defined
 %{?_initrddir:%define _initrddir /etc/rc.d/init.d}
 
-%if %{redhat7} # Redhat 7.x and earlier (multiple ps lines per thread)
+%if %{redhat7} 
+# Redhat 7.x and earlier (multiple ps lines per thread)
 %define sysvinit milter.rc7
 %else	
 %define sysvinit milter.rc
 %endif
 # RH9, other systems (single ps line per process)
 %ifos Linux
-%define python python2.4
+%define python python
 %else
 %define python python
 %endif
+%ifos aix4.1
+%define libdir /var/log/milter
+%else
+%define libdir /usr/lib/pymilter
+%endif
 
 Summary: Python interface to sendmail milter API
 Name: %{name}
@@ -29,7 +35,7 @@ Version: %{version}
 Release: %{release}
 Source: %{name}-%{version}.tar.gz
 #Patch: %{name}-%{version}.patch
-Copyright: GPL
+License: GPL
 Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-buildroot
 Prefix: %{_prefix}
@@ -48,6 +54,24 @@ attach to sendmail's libmilter functionality.  Additional python
 modules provide for navigating and modifying MIME parts, sending
 DSNs, and doing CBV.
 
+%package -n milter
+Group: Applications/System
+Summary:  BMS spam and reputation milter
+Requires: pyspf >= 2.0.4
+
+%description -n milter
+A complex but effective spam filtering, SPF checking, and reputation tracking
+mail application.  It uses pydspam if installed for bayesian filtering.
+
+%package spf
+Group: Applications/System
+Summary:  BMS spam and reputation milter
+Requires: pyspf >= 2.0.4
+
+%description spf
+A simple mail filter to add Received-SPF headers and reject forged mail.
+Rejection policy is configured via sendmail access file.
+
 %prep
 %setup
 #patch -p0 -b .bms
@@ -63,11 +87,15 @@ env CFLAGS="$RPM_OPT_FLAGS" LDFLAGS="$LDFLAGS" %{python} setup.py build
 %install
 rm -rf $RPM_BUILD_ROOT
 %{python} setup.py install --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES
+grep '.pyc$' INSTALLED_FILES | sed -e 's/c$/o/' >>INSTALLED_FILES
 mkdir -p $RPM_BUILD_ROOT/var/log/milter
 mkdir -p $RPM_BUILD_ROOT/etc/mail
 mkdir $RPM_BUILD_ROOT/var/log/milter/save
-cp bms.py *.txt $RPM_BUILD_ROOT/var/log/milter
+mkdir -p $RPM_BUILD_ROOT%{libdir}
+cp *.txt $RPM_BUILD_ROOT/var/log/milter
+cp bms.py spfmilter.py $RPM_BUILD_ROOT%{libdir}
 cp milter.cfg $RPM_BUILD_ROOT/etc/mail/pymilter.cfg
+cp spfmilter.cfg $RPM_BUILD_ROOT/etc/mail
 
 # logfile rotation
 mkdir -p $RPM_BUILD_ROOT/etc/logrotate.d
@@ -76,6 +104,11 @@ cat >$RPM_BUILD_ROOT/etc/logrotate.d/milter <<'EOF'
   copytruncate
   compress
 }
+/var/log/milter/banned_ips {
+  rotate 3
+  daily
+  copytruncate
+}
 EOF
 
 # purge saved defanged message copies
@@ -95,7 +128,7 @@ EOF
 chmod a+x $RPM_BUILD_ROOT/etc/cron.daily/milter
 
 %ifos aix4.1
-cat >$RPM_BUILD_ROOT/var/log/milter/start.sh <<'EOF'
+cat >$RPM_BUILD_ROOT%{libdir}/start.sh <<'EOF'
 #!/bin/sh
 cd /var/log/milter
 # uncomment to enable sgmlop if installed
@@ -103,15 +136,10 @@ cd /var/log/milter
 exec /usr/local/bin/python bms.py >>milter.log 2>&1
 EOF
 %else
-cat >$RPM_BUILD_ROOT/var/log/milter/start.sh <<'EOF'
-#!/bin/sh
-cd /var/log/milter
-exec >>milter.log 2>&1
-%{python} bms.py &
-echo $! >/var/run/milter/milter.pid
-EOF
+cp start.sh $RPM_BUILD_ROOT%{libdir}
 mkdir -p $RPM_BUILD_ROOT/etc/rc.d/init.d
 cp %{sysvinit} $RPM_BUILD_ROOT/etc/rc.d/init.d/milter
+cp spfmilter.rc $RPM_BUILD_ROOT/etc/rc.d/init.d/spfmilter
 ed $RPM_BUILD_ROOT/etc/rc.d/init.d/milter <<'EOF'
 /^python=/
 c
@@ -120,8 +148,24 @@ python="%{python}"
 w
 q
 EOF
+ed $RPM_BUILD_ROOT/etc/rc.d/init.d/spfmilter <<'EOF'
+/^python=/
+c
+python="%{python}"
+.
+w
+q
+EOF
+ed $RPM_BUILD_ROOT%{libdir}/start.sh <<'EOF'
+/^python=/
+c
+python="%{python}"
+.
+w
+q
+EOF
 %endif
-chmod a+x $RPM_BUILD_ROOT/var/log/milter/start.sh
+chmod a+x $RPM_BUILD_ROOT%{libdir}/start.sh
 
 mkdir -p $RPM_BUILD_ROOT/var/run/milter
 mkdir -p $RPM_BUILD_ROOT/usr/share/sendmail-cf/hack
@@ -129,21 +173,29 @@ cp -p rhsbl.m4 $RPM_BUILD_ROOT/usr/share/sendmail-cf/hack
 
 %ifos aix4.1
 %post
-mkssys -s milter -p /var/log/milter/start.sh -u 25 -S -n 15 -f 9 -G mail || :
+mkssys -s milter -p %{libdir}/start.sh -u 25 -S -n 15 -f 9 -G mail || :
 
 %preun
 if [ $1 = 0 ]; then
   rmssys -s milter || :
 fi
 %else
-%post
+%post -n milter
 #echo "pythonsock has moved to /var/run/milter, update /etc/mail/sendmail.cf"
 /sbin/chkconfig --add milter
 
-%preun
+%preun -n milter
 if [ $1 = 0 ]; then
   /sbin/chkconfig --del milter
 fi
+%post spf
+#echo "pythonsock has moved to /var/run/milter, update /etc/mail/sendmail.cf"
+/sbin/chkconfig --add spfmilter
+
+%preun spf
+if [ $1 = 0 ]; then
+  /sbin/chkconfig --del spfmilter
+fi
 %endif
 
 %clean
@@ -151,7 +203,11 @@ rm -rf $RPM_BUILD_ROOT
 
 %files -f INSTALLED_FILES
 %defattr(-,root,root)
-%doc README HOWTO ChangeLog NEWS TODO CREDITS sample.py
+%doc README HOWTO ChangeLog NEWS TODO CREDITS sample.py milter-template.py
+%config %{libdir}/start.sh
+
+%files -n milter
+%defattr(-,root,root)
 /etc/logrotate.d/milter
 /etc/cron.daily/milter
 %ifos aix4.1
@@ -161,19 +217,36 @@ rm -rf $RPM_BUILD_ROOT
 %defattr(-,mail,mail)
 %endif
 %dir /var/log/milter
-%dir /var/run/milter
 %dir /var/log/milter/save
-%config /var/log/milter/start.sh
-%config /var/log/milter/bms.py
+%config %{libdir}/bms.py
+%{libdir}/bms.py?
 %config(noreplace) /var/log/milter/strike3.txt
 %config(noreplace) /var/log/milter/softfail.txt
+%config(noreplace) /var/log/milter/fail.txt
 %config(noreplace) /var/log/milter/neutral.txt
 %config(noreplace) /var/log/milter/quarantine.txt
 %config(noreplace) /var/log/milter/permerror.txt
 %config(noreplace) /etc/mail/pymilter.cfg
 /usr/share/sendmail-cf/hack/rhsbl.m4
 
+%files spf
+%defattr(-,root,root)
+%dir /var/log/milter
+%{libdir}/spfmilter.py*
+%config(noreplace) /etc/mail/spfmilter.cfg
+/etc/rc.d/init.d/spfmilter
+
 %changelog
+* Fri Jan 05 2007 Stuart Gathman <stuart@bmsi.com> 0.8.8-1
+- move AddrCache, parse_addr, iniplist to Milter package
+- move parse_header to Milter.utils
+- fix plock for missing source and can't change owner/group
+- add sample spfmilter.py milter
+- private_relay config option
+- persist delayed DSN blacklisting
+- handle gossip server restart without disabling gossip
+- split out pymilter and pymilter-spf packages
+- move milter apps to /usr/lib/pymilter
 * Sat Nov 04 2006 Stuart Gathman <stuart@bmsi.com> 0.8.7-1
 - More lame bounce heuristics
 - SPF moved to pyspf RPM

neutral.txt

@@ -30,6 +30,8 @@ If you never sent the above message, then your domain has been forged.
 Your mail admin needs to publish a strict SPF record so that I can reject
 those forgeries instead of bugging you about them.  
 
+See http://openspf.org for details.
+
 If you need further assistance, please do not hesitate to contact me.
 
 Kind regards,

permerror.txt

@@ -14,6 +14,7 @@ Delivery to the following recipients has been delayed.
 	%(rcpt)s
 
 Subject: %(subject)s 
+Received-SPF: %(spf_result)s
 
 Your spf record has a permanent error.  The error was:
 

setup.py

@@ -15,7 +15,7 @@ if sys.version < '2.2.3':
   DistributionMetadata.download_url = None
 
 # NOTE: importing Milter to obtain version fails when milter.so not built
-setup(name = "milter", version = '0.8.7',
+setup(name = "pymilter", version = '0.8.8',
 	description="Python interface to sendmail milter API",
 	long_description="""\
 This is a python extension module to enable python scripts to

spfmilter.cfg

@@ -0,0 +1,20 @@
+[milter]
+# The socket used to communicate with sendmail
+socketname = /var/run/milter/spfmiltersock
+# Name of the milter given to sendmail
+name = pyspffilter
+# Trusted relays such as secondary MXes that should not have SPF checked.
+;trusted_relay =
+# Internal networks that should not have SPF checked.
+internal_connect = 127.0.0.1,192.168.0.0/16,10.0.0.0/8
+
+# See http://www.openspf.com for more info on SPF.
+[spf]
+# Use sendmail access map or similar format for detailed spf policy.
+# SPF entries in the access map will override defaults.
+access_file = /etc/mail/access.db
+# Connections that get an SPF pass for a pretend MAIL FROM of 
+# postmaster@sometrustedforwarder.com skip SPF checks for the real MAIL FROM.
+# This is for non-SRS forwarders.  It is a simple implementation that
+# is inefficient for more than a few entries.
+;trusted_forwarder = careerbuilder.com

spfmilter.py

@@ -0,0 +1,253 @@
+# A simple SPF milter.
+# You must install pyspf for this to work.
+
+# http://www.sendmail.org/doc/sendmail-current/libmilter/docs/installation.html
+
+# Author: Stuart D. Gathman <stuart@bmsi.com>
+# Copyright 2007 Business Management Systems, Inc.
+# This code is under GPL.  See COPYING for details.
+
+import sys
+import Milter
+import spf
+import syslog
+import anydbm
+from Milter.config import MilterConfigParser
+from Milter.utils import iniplist,parse_addr
+
+syslog.openlog('spfmilter',0,syslog.LOG_MAIL)
+
+class Config(object):
+  "Hold configuration options."
+  pass
+
+def read_config(list):
+  "Return new config object."
+  cp = MilterConfigParser()
+  cp.read(list)
+  if cp.has_option('milter','datadir'):
+        os.chdir(cp.get('milter','datadir'))
+  conf = Config()
+  conf.socketname = cp.getdefault('milter','socketname', '/tmp/spfmiltersock')
+  conf.miltername = cp.getdefault('milter','name','pyspffilter')
+  conf.trusted_relay = cp.getlist('milter','trusted_relay')
+  conf.internal_connect = cp.getlist('milter','internal_connect')
+  conf.trusted_forwarder = cp.getlist('spf','trusted_relay')
+  conf.access_file = cp.getdefault('spf','access_file',None)
+  return conf
+
+class SPFPolicy(object):
+  "Get SPF policy by result from sendmail style access file."
+  def __init__(self,sender,access_file=None):
+    self.sender = sender
+    self.domain = sender.split('@')[-1].lower()
+    if access_file:
+      try: acf = anydbm.open(access_file,'r')
+      except: acf = None
+    else: acf = None
+    self.acf = acf
+
+  def getPolicy(self,pfx):
+    acf = self.acf
+    if not acf: return None
+    try:
+      return acf[pfx + self.sender]
+    except KeyError:
+      try:
+	return acf[pfx + self.domain]
+      except KeyError:
+	try:
+	  return acf[pfx]
+	except KeyError:
+	  return None
+  
+class spfMilter(Milter.Milter):
+  "Milter to check SPF.  Each connection gets its own instance."
+
+  def log(self,*msg):
+    syslog.syslog('[%d] %s' % (self.id,' '.join([str(m) for m in msg])))
+
+  def __init__(self):
+    self.mailfrom = None
+    self.id = Milter.uniqueID()
+    # we don't want config used to change during a connection
+    self.conf = config
+
+  # addheader can only be called from eom().  This accumulates added headers
+  # which can then be applied by alter_headers()
+  def add_header(self,name,val,idx=-1):
+    self.new_headers.append((name,val,idx))
+    self.log('%s: %s' % (name,val))
+
+  def connect(self,hostname,unused,hostaddr):
+    self.internal_connection = False
+    self.trusted_relay = False
+    self.hello_name = None
+    # sometimes people put extra space in sendmail config, so we strip
+    self.receiver = self.getsymval('j').strip()
+    if hostaddr and len(hostaddr) > 0:
+      ipaddr = hostaddr[0]
+      if iniplist(ipaddr,self.conf.internal_connect):
+	self.internal_connection = True
+      if iniplist(ipaddr,self.conf.trusted_relay):
+        self.trusted_relay = True
+    else: ipaddr = ''
+    self.connectip = ipaddr
+    if self.internal_connection:
+      connecttype = 'INTERNAL'
+    else:
+      connecttype = 'EXTERNAL'
+    if self.trusted_relay:
+      connecttype += ' TRUSTED'
+    self.log("connect from %s at %s %s" % (hostname,hostaddr,connecttype))
+    return Milter.CONTINUE
+
+  def hello(self,hostname):
+    self.hello_name = hostname
+    self.log("hello from %s" % hostname)
+    return Milter.CONTINUE
+
+  # multiple messages can be received on a single connection
+  # envfrom (MAIL FROM in the SMTP protocol) seems to mark the start
+  # of each message.
+  def envfrom(self,f,*str):
+    self.log("mail from",f,str)
+    if not self.hello_name:
+      self.log('REJECT: missing HELO')
+      self.setreply('550','5.7.1',"It's polite to say helo first.")
+      return Milter.REJECT
+    self.mailfrom = f
+    self.new_headers = []
+    t = parse_addr(f)
+    if len(t) == 2: t[1] = t[1].lower()
+    self.canon_from = '@'.join(t)
+    if not (self.internal_connection or self.trusted_relay) and self.connectip:
+      rc = self.check_spf()
+      if rc != Milter.CONTINUE: return rc
+    return Milter.CONTINUE
+
+  def envrcpt(self,f,*str):
+    return Milter.CONTINUE
+
+  def header(self,name,hval):
+    return Milter.CONTINUE
+
+  def eoh(self):
+    return Milter.CONTINUE
+
+  def eom(self):
+    for name,val,idx in self.new_headers:
+      try:
+	self.addheader(name,val,idx)
+      except:
+	self.addheader(name,val)	# older sendmail can't insheader
+    return Milter.CONTINUE
+
+  def close(self):
+    return Milter.CONTINUE
+
+  def check_spf(self):
+    receiver = self.receiver
+    for tf in self.conf.trusted_forwarder:
+      q = spf.query(self.connectip,'',tf,receiver=receiver,strict=False)
+      res,code,txt = q.check()
+      if res == 'pass':
+        self.log("TRUSTED_FORWARDER:",tf)
+        break
+    else:
+      q = spf.query(self.connectip,self.canon_from,self.hello_name,
+	  receiver=receiver,strict=False)
+      q.set_default_explanation(
+	'SPF fail: see http://openspf.org/why.html?sender=%s&ip=%s' % (q.s,q.i))
+      res,code,txt = q.check()
+    if res not in ('pass','temperror'):
+      if self.mailfrom != '<>':
+	# check hello name via spf unless spf pass
+	h = spf.query(self.connectip,'',self.hello_name,receiver=receiver)
+	hres,hcode,htxt = h.check()
+	if hres in ('deny','fail','neutral','softfail'):
+	  self.log('REJECT: hello SPF: %s 550 %s' % (hres,htxt))
+	  self.setreply('550','5.7.1',htxt,
+	    "The hostname given in your MTA's HELO response is not listed",
+	    "as a legitimate MTA in the SPF records for your domain.  If you",
+	    "get this bounce, the message was not in fact a forgery, and you",
+	    "should IMMEDIATELY notify your email administrator of the problem."
+	  )
+	  return Milter.REJECT
+      else:
+        hres,hcode,htxt = res,code,txt
+    else: hres = None
+
+    p = SPFPolicy(q.s,self.conf.access_file)
+
+    if res == 'fail':
+      policy = p.getPolicy('spf-fail:')
+      if not policy or policy == 'REJECT':
+	self.log('REJECT: SPF %s %i %s' % (res,code,txt))
+	self.setreply(str(code),'5.7.1',txt)
+	# A proper SPF fail error message would read:
+	# forger.biz [1.2.3.4] is not allowed to send mail with the domain
+	# "forged.org" in the sender address.  Contact <postmaster@forged.org>.
+	return Milter.REJECT
+    if res == 'softfail':
+      policy = p.getPolicy('spf-softfail:')
+      if policy and policy == 'REJECT':
+	self.log('REJECT: SPF %s %i %s' % (res,code,txt))
+	self.setreply(str(code),'5.7.1',txt)
+	# A proper SPF fail error message would read:
+	# forger.biz [1.2.3.4] is not allowed to send mail with the domain
+	# "forged.org" in the sender address.  Contact <postmaster@forged.org>.
+	return Milter.REJECT
+    elif res == 'permerror':
+      policy = p.getPolicy('spf-permerror:')
+      if not policy or policy == 'REJECT':
+	self.log('REJECT: SPF %s %i %s' % (res,code,txt))
+	# latest SPF draft recommends 5.5.2 instead of 5.7.1
+	self.setreply(str(code),'5.5.2',txt,
+	  'There is a fatal syntax error in the SPF record for %s' % q.o,
+	  'We cannot accept mail from %s until this is corrected.' % q.o
+	)
+	return Milter.REJECT
+    elif res == 'temperror':
+      policy = p.getPolicy('spf-temperror:')
+      if not policy or policy == 'REJECT':
+	self.log('TEMPFAIL: SPF %s %i %s' % (res,code,txt))
+	self.setreply(str(code),'4.3.0',txt)
+	return Milter.TEMPFAIL
+    elif res == 'neutral' or res == 'none':
+      policy = p.getPolicy('spf-neutral:')
+      if policy and policy == 'REJECT':
+        self.log('REJECT NEUTRAL:',q.s)
+	self.setreply('550','5.7.1',
+  "%s requires and SPF PASS to accept mail from %s. [http://openspf.org]"
+	  % (receiver,q.s))
+	return Milter.REJECT
+    elif res == 'pass':
+      policy = p.getPolicy('spf-pass:')
+      if policy and policy == 'REJECT':
+        self.log('REJECT PASS:',q.s)
+	self.setreply('550','5.7.1',
+		"%s has been blacklisted by %s." % (q.s,receiver))
+	return Milter.REJECT
+    self.add_header('Received-SPF',q.get_header(res,receiver),0)
+    if hres and q.h != q.o:
+      self.add_header('X-Hello-SPF',hres,0)
+    return Milter.CONTINUE
+
+if __name__ == "__main__":
+  Milter.factory = spfMilter
+  Milter.set_flags(Milter.CHGHDRS + Milter.ADDHDRS)
+  global config
+  config = read_config(['spfmilter.cfg','/etc/mail/spfmilter.cfg'])
+  miltername = config.miltername
+  socketname = config.socketname
+  print """To use this with sendmail, add the following to sendmail.cf:
+
+O InputMailFilters=%s
+X%s,        S=local:%s
+
+See the sendmail README for libmilter.
+sample spfmilter startup""" % (miltername,miltername,socketname)
+  sys.stdout.flush()
+  Milter.runmilter("pyspffilter",socketname,240)
+  print "sample spfmilter shutdown"

spfmilter.rc

@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# spfmilter	This shell script takes care of starting and stopping spfmilter.
+#
+# chkconfig: 2345 80 30
+# description: a process that checks SPF for messages sent through sendmail.
+# processname: spfmilter
+# config: /etc/mail/spfmilter.cfg
+# pidfile: /var/run/milter/spfmilter.pid
+
+python="python2.4"
+
+pidof() {
+	set - ""
+	if set - `ps -e -o pid,cmd | grep "${python} spfmilter.py"` &&
+	  [ "$2" != "grep" ]; then
+	  echo $1
+	  return 0
+	fi
+	return 1
+}
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+[ -x /usr/lib/pymilter/start.sh ] || exit 0
+
+RETVAL=0
+prog="spfmilter"
+
+start() {
+	# Start daemons.
+
+	echo -n "Starting $prog: "
+        if ! test -d /var/run/milter; then
+		mkdir -p /var/run/milter
+		chown mail:mail /var/run/milter
+	fi
+	daemon --check milter --user mail /usr/lib/pymilter/start.sh spfmilter
+	RETVAL=$?
+	echo
+	[ $RETVAL -eq 0 ] && touch /var/lock/subsys/spfmilter
+	return $RETVAL
+}
+
+stop() {
+	# Stop daemons.
+	echo -n "Shutting down $prog: "
+	killproc milter
+	RETVAL=$?
+	echo
+	[ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/spfmilter
+	return $RETVAL
+}
+
+# See how we were called.
+case "$1" in
+  start)
+	start
+	;;
+  stop)
+	stop
+	;;
+  restart|reload)
+	stop
+	start
+	RETVAL=$?
+	;;
+  condrestart)
+	if [ -f /var/lock/subsys/spfmilter ]; then
+	    stop
+	    start
+	    RETVAL=$?
+	fi
+	;;
+  status)
+	status spfmilter
+	RETVAL=$?
+	;;
+  *)
+	echo "Usage: $0 {start|stop|restart|condrestart|status}"
+	exit 1
+esac
+
+exit $RETVAL

start.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+appname="$1"
+script="${2:-${appname}}"
+datadir=/var/log/milter
+python="python2.4"
+exec >>${datadir}/${appname}.log 2>&1
+if test -s ${datadir}/${script}.py; then
+  cd ${datadir} # use version in log dir if it exists for debugging
+else
+  cd /usr/lib/pymilter
+fi
+
+cd /var/log/milter
+exec >>${appname}.log 2>&1
+${python} ${appname}.py &
+echo $! >/var/run/milter/${appname}.pid

test.py

@@ -2,6 +2,7 @@ import unittest
 import testbms
 import testmime
 import testsample
+import testutils
 import os
 
 def suite(): 
@@ -9,6 +10,7 @@ def suite():
   s.addTest(testbms.suite())
   s.addTest(testmime.suite())
   s.addTest(testsample.suite())
+  s.addTest(testutils.suite())
   return s
 
 if __name__ == '__main__':

testbms.py

@@ -277,6 +277,25 @@ class BMSMilterTestCase(unittest.TestCase):
     fp = milter._body
     open("test/test1.tstout","w").write(fp.getvalue())
 
+  def testFindsrs(self):
+    if not bms.srs:
+      import SRS
+      bms.srs = SRS.new(secret='test')
+    sender = bms.srs.forward('foo@bar.com','mail.example.com')
+    sndr = bms.findsrs(StringIO.StringIO(
+"""Received: from [1.16.33.86] (helo=mail.example.com)
+	by bastion4.mail.zen.co.uk with smtp (Exim 4.50) id 1H3IBC-00013b-O9
+	for foo@bar.com; Sat, 06 Jan 2007 20:30:17 +0000
+X-Mailer: "PyMilter-0.8.5"
+	<%s> foo
+MIME-Version: 1.0
+Content-Type: text/plain
+To: foo@bar.com
+From: postmaster@mail.example.com
+""" % sender
+    ))
+    self.assertEqual(sndr,'foo@bar.com')
+
 #  def testReject(self):
 #    "Test content based spam rejection."
 #    milter = TestMilter()

testutils.py

@@ -0,0 +1,48 @@
+import unittest
+import doctest
+import os
+import Milter.utils
+from Milter.cache import AddrCache
+from Milter.dynip import is_dynip
+
+class AddrCacheTestCase(unittest.TestCase):
+
+  def setUp(self):
+    self.fname = 'test.dat'
+
+  def tearDown(self):
+    os.remove(self.fname)
+
+  def testAdd(self):
+    cache = AddrCache(fname=self.fname)
+    cache['foo@bar.com'] = None
+    cache.addperm('baz@bar.com')
+    cache['temp@bar.com'] = 'testing'
+    self.failUnless(cache.has_key('foo@bar.com'))
+    self.failUnless(not cache.has_key('hello@bar.com'))
+    self.failUnless('baz@bar.com' in cache)
+    self.assertEquals(cache['temp@bar.com'],'testing')
+    s = open(self.fname).readlines()
+    self.failUnless(len(s) == 2)
+    self.failUnless(s[0].startswith('foo@bar.com '))
+    self.assertEquals(s[1].strip(),'baz@bar.com')
+    # check that new result overrides old
+    cache['temp@bar.com'] = None
+    self.failUnless(not cache['temp@bar.com'])
+
+  def testDomain(self):
+    fp = open(self.fname,'w')
+    print >>fp,'spammer.com'
+    fp.close()
+    cache = AddrCache(fname=self.fname)
+    cache.load(self.fname,30)
+    self.failUnless('spammer.com' in cache)
+
+def suite(): 
+  s = unittest.makeSuite(AddrCacheTestCase,'test')
+  s.addTest(doctest.DocTestSuite(Milter.utils))
+  s.addTest(doctest.DocTestSuite(Milter.dynip))
+  return s
+
+if __name__ == '__main__':
+  unittest.TextTestRunner().run(suite())