This commit was manufactured by cvs2svn to create tag 'pymilter-0_8_9'.

Sprout from master 2007-09-25 17:07:32 UTC Stuart Gathman <stuart@gathman.org> 'Test on Centos5'
Cherrypick from bmsi 2005-05-31 18:23:49 UTC Stuart Gathman <stuart@gathman.org> 'Development changes since 0.7.2':
    rejects.py
    rhsbl.m4
    sample.py
    test/amazon
    test/big5
    test/bounce
    test/bounce1
    test/bound
    test/honey
    test/missingboundary
    test/samp1
    test/spam44
    test/spam7
    test/spam8
    test/test1
    test/test8
    test/virus1
    test/virus13
    test/virus2
    test/virus3
    test/virus4
    test/virus5
    test/virus6
    test/virus7
    testsample.py

HOWTO

@@ -35,13 +35,13 @@ wish to install pydspam.
 For basic pymilter you'll need:
 
 python-2.4
-milter-0.8.10
+milter-0.8.7
 sendmail-8.13.x	(with milter support enabled)
 
 and for SPF you'll need:
 
-pydns-2.3.3-2.4
-pyspf-2.0.5-1.py24
+pydns-2.3.0-2.4
+pyspf-2.0.3-2.py24
 
 and for SRS you'll need:
 
@@ -65,7 +65,7 @@ Start milter and pysrs with "service milter start", "service pysrs start".
 Tail /var/log/milter/milter.log while SMTP clients connect to your
 sendmail instance.  This should show you what the milter is doing.
 
-By default, milter-0.8.10 rejects on SPF fail.
+By default, milter-0.8.7 rejects on SPF fail.
 
 	Step four.  Tweaking the basic config.
 

Milter/__init__.py

@@ -184,10 +184,8 @@ def runmilter(name,socketname,timeout = 0):
     print "Removing %s" % fname
     try:
       os.unlink(fname)
-    except os.error, x:
-      import errno
-      if x.errno != errno.ENOENT:
-        raise milter.error(x)
+    except:
+      pass
 
   # The default flags set include everything
   # milter.set_flags(milter.ADDHDRS)

Milter/cache.py

@@ -10,11 +10,6 @@
 # CBV results.
 #
 # $Log$
-# Revision 1.8  2007/09/03 16:18:45  customdesigned
-# Delete unparseable timestamps when loading address cache.  These have
-# arisen because of failure to parse MAIL FROM properly.   Will have to
-# tighten up MAIL FROM parsing to match RFC.
-#
 # Revision 1.7  2007/01/25 22:47:26  customdesigned
 # Persist blacklisting from delayed DSNs.
 #
@@ -94,10 +89,8 @@ class AddrCache(object):
     except IOError:
       lock.unlock()
 
-  def has_precise_key(self,sender):
-    """True if precise sender is cached and has not expired.  Don't
-    try looking up wildcard entries.
-    """
+  def has_key(self,sender):
+    "True if sender is cached and has not expired."
     try:
       lsender = sender and sender.lower()
       ts,res = self.cache[lsender]
@@ -105,17 +98,16 @@ class AddrCache(object):
       if not ts or ts > too_old:
         return True
       del self.cache[lsender]
-    except KeyError: pass
-    return False
-
-  def has_key(self,sender):
-    "True if sender is cached and has not expired."
-    if self.has_precise_key(sender):
-      return True
-    try:
-      user,host = sender.split('@',1)
-      return self.has_precise_key(host)
-    except: pass
+      try:
+	user,host = sender.split('@',1)
+	return self.has_key(host)
+      except ValueError:
+        pass
+    except KeyError:
+      try:
+	user,host = sender.split('@',1)
+	return self.has_key(host)
+      except: pass
     return False
 
   __contains__ = has_key

Milter/utils.py

@@ -4,8 +4,6 @@ import socket
 import email.Errors
 from fnmatch import fnmatchcase
 from email.Header import decode_header
-#import email.Utils
-import rfc822
 
 ip4re = re.compile(r'^[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*$')
 
@@ -42,44 +40,6 @@ def iniplist(ipaddr,iplist):
       return True
   return False
 
-def parseaddr(t):
-  """Split email into Fullname and address.
-
-  >>> parseaddr('user@example.com')
-  ('', 'user@example.com')
-  >>> parseaddr('"Full Name" <foo@example.com>')
-  ('Full Name', 'foo@example.com')
-  >>> parseaddr('spam@spammer.com <foo@example.com>')
-  ('spam@spammer.com', 'foo@example.com')
-  >>> parseaddr('God@heaven <@hop1.org,@hop2.net:jeff@spec.org>')
-  ('God@heaven', 'jeff@spec.org')
-  >>> parseaddr('Real Name ((comment)) <addr...@example.com>')
-  ('Real Name', 'addr...@example.com')
-  >>> parseaddr('a(WRONG)@b')
-  ('WRONG', 'a@b')
-  """
-  #return email.Utils.parseaddr(t)
-  res = rfc822.parseaddr(t)
-  # dirty fix for some broken cases
-  if not res[0]:
-    pos = t.find('<')
-    if pos > 0 and t[-1] == '>':
-      addrspec = t[pos+1:-1]
-      pos1 = addrspec.rfind(':')
-      if pos1 > 0:
-        addrspec = addrspec[pos1+1:]
-      return rfc822.parseaddr('"%s" <%s>' % (t[:pos].strip(),addrspec))
-  if not res[1]:
-    pos = t.find('<')
-    if pos > 0 and t[-1] == '>':
-      addrspec = t[pos+1:-1]
-      pos1 = addrspec.rfind(':')
-      if pos1 > 0:
-        addrspec = addrspec[pos1+1:]
-      return rfc822.parseaddr('%s<%s>' % (t[:pos].strip(),addrspec))
-  return res
-    
-
 def parse_addr(t):
   """Split email into user,domain.
 

NEWS

@@ -1,6 +1,4 @@
-See pymilter.spec for recent history.
-
-Here is a history of older changes to Python milter.
+Here is a history of user visible changes to Python milter.
 0.8.8	move AddrCache, parse_addr, iniplist, parse_header to Milter package
 	fix plock for missing source and can't change owner/group
 	add sample spfmilter.py milter

TODO

@@ -1,10 +1,6 @@
-Check ESMTP NOTIFY before sending real DSNs.  Just use CBV if DSNs are
-not wanted.
-
-Support CBV to local domains and cache results so that invalid users
-can be rejected without maintaining valid user lists.
-
-Now that we blacklist IPs for too many bad rcpts, delay SPF until RCPT TO.
+Convert DSN to REJECT unless sender gets SPF pass or best guess pass.  Make
+configurable by SPF result with NOTSPAM policy (reject or deliver without DSN).
+Maybe policy should be NODSN - still verify sender with CBV.
 
 When content filtering is not installed, reject BLACKLISTed MFROM
 immediately.  There is no use waiting until EOM.
@@ -16,8 +12,7 @@ MTA.  The mail is flagged external, so we don't list example.com in
 internal_domains (or we would get "spam from self").  But, if we try to do a
 CBV, we get "fraudulent MX", because the MX is ourself!  So we need to 
 avoid doing CBV on such domains.  Currently, we try to make sure the SPF
-policies don't do CBV.  The real solution is for users to use SMTP AUTH,
-but some of them are stubborn.
+policies don't do CBV.
 
 We now don't check internal domains for incoming mail if there is an
 SPF record.
@@ -82,6 +77,10 @@ Whitelisted senders from trusted relay get PROBATION.  Need to extracted
 SPF result from headers - and in the case of mail internal to relay
 (e.g. bmsi.com), supply 'pass' result.  
 
+For selected domains, check rcpts via CBV before accepting mail.  Cache
+results.  This will kick out dictonary attacks against a mail domain
+behind a gateway sooner.
+
 Add auto-blacklisted senders to blacklist.log with timestamp.
 Add emails blacklisted via CBV so that they are remembered across milter
 restarts.
@@ -98,6 +97,8 @@ e.g. verizon.net).
 Allow verified hostnames for trusted_relay.  E.g. HELO name that
 passes SPF.
 
+Table of sendmail macros for documentation.
+
 When do we get two hello calls?  STARTTLS is one reason.
 
 Option: accept mail from auto-whitelisted senders even with spf-fail,
@@ -179,18 +180,6 @@ Need a test module to feed sample messages to a milter though a live
 sendmail and SMTP.  The mockup currently used is probably not very accurate,
 and doesn't test the threading code.
 
-DONE Table of sendmail macros for documentation.  In API docs on milter.org.
-
-DONE For selected domains, check rcpts via CBV before accepting mail.  Cache
-results.  This will kick out dictonary attacks against a mail domain
-behind a gateway sooner.
-
-DONE Convert DSN to REJECT unless sender gets SPF pass or best guess pass.  Make
-configurable by SPF result with NOTSPAM policy (reject or deliver without DSN).
-Maybe policy should be NODSN - still verify sender with CBV.
-
-DONE Add parseaddr test case for 'foo@bar.com <baz@barf.biz>'
-
 DONE Require signed MFROM for all incoming bounces when signing all outgoing
 mail - except from trusted relays.
 

bms.py

@@ -1,49 +1,6 @@
 #!/usr/bin/env python
 # A simple milter that has grown quite a bit.
 # $Log$
-# Revision 1.126  2008/08/18 17:47:57  customdesigned
-# Log rcpt for SRS rejections.
-#
-# Revision 1.125  2008/08/06 00:52:38  customdesigned
-# CBV policy sends no DSN.  DSN policy sends DSN.
-#
-# Revision 1.124  2008/08/05 18:04:06  customdesigned
-# Send quarantine DSN to SPF PASS only.
-#
-# Revision 1.123  2008/07/29 21:59:29  customdesigned
-# Parse ESMTP params
-#
-# Revision 1.122  2008/05/08 21:35:56  customdesigned
-# Allow explicitly whitelisted email from banned_users.
-#
-# Revision 1.121  2008/04/10 14:59:35  customdesigned
-# Configure gossip TTL.
-#
-# Revision 1.120  2008/04/02 18:59:14  customdesigned
-# Release 0.8.10
-#
-# Revision 1.119  2008/04/01 00:13:10  customdesigned
-# Do not CBV whitelisted addresses.  We already know they are good.
-#
-# Revision 1.118  2008/01/09 20:15:49  customdesigned
-# Handle unquoted fullname when parsing email.
-#
-# Revision 1.117  2007/11/29 14:35:17  customdesigned
-# Packaging tweaks.
-#
-# Revision 1.116  2007/11/01 20:09:14  customdesigned
-# Support temperror policy in access.
-#
-# Revision 1.115  2007/10/10 18:23:54  customdesigned
-# Send quarantine DSN to SPF pass (official or guessed) only.
-# Reject blacklisted email too big for dspam.
-#
-# Revision 1.114  2007/10/10 18:07:50  customdesigned
-# Check porn keywords in From header field.
-#
-# Revision 1.113  2007/09/25 16:37:26  customdesigned
-# Tested on RH7
-#
 # Revision 1.112  2007/09/13 14:51:03  customdesigned
 # Report domain on reputation reject.
 #
@@ -201,19 +158,17 @@ import gc
 import anydbm
 import Milter.dsn as dsn
 from Milter.dynip import is_dynip as dynip
-from Milter.utils import \
-        iniplist,parse_addr,parse_header,ip4re,addr2bin,parseaddr
+from Milter.utils import iniplist,parse_addr,parse_header,ip4re,addr2bin
 from Milter.config import MilterConfigParser
 
 from fnmatch import fnmatchcase
-from email.Utils import getaddresses
+from email.Utils import getaddresses,parseaddr
 
 # Import gossip if available
 try:
   import gossip
   import gossip.client
   import gossip.server
-  gossip_node = None
 except: gossip = None
 
 # Import pysrs if available
@@ -295,7 +250,6 @@ hello_blacklist = ()
 smart_alias = {}
 dspam_dict = None
 dspam_users = {}
-dspam_train = {}
 dspam_userdir = None
 dspam_exempt = {}
 dspam_whitelist = {}
@@ -427,7 +381,6 @@ def read_config(list):
   dspam_users = cp.getaddrdict('dspam','dspam_users')
   dspam_userdir = cp.getdefault('dspam','dspam_userdir')
   dspam_screener = cp.getlist('dspam','dspam_screener')
-  dspam_train = set(cp.getlist('dspam','dspam_train'))
   dspam_reject = cp.getlist('dspam','dspam_reject')
   dspam_internal = cp.getboolean('dspam','dspam_internal')
   if cp.has_option('dspam','dspam_sizelimit'):
@@ -475,7 +428,7 @@ def read_config(list):
     banned_users = cp.getlist('srs','banned_users')
 
   if gossip:
-    global gossip_node, gossip_ttl
+    global gossip_node
     if cp.has_option('gossip','server'):
       server = cp.get('gossip','server')
       host,port = gossip.splitaddr(server)
@@ -485,10 +438,6 @@ def read_config(list):
       for p in cp.getlist('gossip','peers'):
         host,port = gossip.splitaddr(p)
         gossip_node.peers.append(gossip.server.Peer(host,port))
-    if cp.has_option('gossip','ttl'):
-      gossip_ttl = cp.getint('gossip','ttl')
-    else:
-      gossip_ttl = 1
 
 def findsrs(fp):
   lastln = None
@@ -512,12 +461,6 @@ def findsrs(fp):
         lastln = ln
         break
 
-def param2dict(str):
-  pairs = [x.split('=',1) for x in str]
-  for e in pairs:
-    if len(e) < 2: e.append(None)
-  return dict([(k.upper(),v) for k,v in pairs])
-
 class SPFPolicy(object):
   "Get SPF policy by result from sendmail style access file."
   def __init__(self,sender):
@@ -590,12 +533,6 @@ class SPFPolicy(object):
       policy = 'REJECT'
     return policy
 
-  def getTempErrorPolicy(self):
-    policy = self.getPolicy('spf-temperror:')
-    if not policy:
-      policy = 'REJECT'
-    return policy
-
   def getPassPolicy(self):
     policy = self.getPolicy('spf-pass:')
     if not policy:
@@ -605,7 +542,7 @@ class SPFPolicy(object):
 from Milter.cache import AddrCache
 
 cbv_cache = AddrCache(renew=7)
-auto_whitelist = AddrCache(renew=60)
+auto_whitelist = AddrCache(renew=30)
 blacklist = AddrCache(renew=30)
 
 class bmsMilter(Milter.Milter):
@@ -743,9 +680,6 @@ class bmsMilter(Milter.Milter):
   # of each message.
   def envfrom(self,f,*str):
     self.log("mail from",f,str)
-    #param = param2dict(str)
-    #self.envid = param.get('ENVID',None)
-    #self.mail_param = param
     self.fp = StringIO.StringIO()
     self.tempname = None
     self.mailfrom = f
@@ -788,9 +722,9 @@ class bmsMilter(Milter.Milter):
 
     self.user = self.getsymval('{auth_authen}')
     if self.user:
-      # Very simple SMTP AUTH policy by default:
+      # Very simple SMTP AUTH policy by defaul:
       #   any successful authentication is considered INTERNAL
-      # Detailed authorization policy is configured in the access file below.
+      # FIXME: configure allowed MAIL FROM by user
       self.internal_connection = True
       self.log(
         "SMTP AUTH:",self.user, self.getsymval('{auth_type}'),
@@ -876,8 +810,6 @@ class bmsMilter(Milter.Milter):
       else:
         self.dspam = False
         self.log("PROBATION",self.canon_from)
-      if res not in ('permerror','softfail'):
-        self.cbv_needed = None
     elif cbv_cache.has_key(self.canon_from) and cbv_cache[self.canon_from] \
         or domain in blacklist:
       if not self.internal_connection:
@@ -897,8 +829,7 @@ class bmsMilter(Milter.Milter):
     else:
       global gossip
       if gossip and domain and rc == Milter.CONTINUE \
-          and not (self.internal_connection or self.trusted_relay) \
-          and gossip_node:
+          and not (self.internal_connection or self.trusted_relay):
         if self.spf and self.spf.result == 'pass':
           qual = 'SPF'
         elif res == 'pass':
@@ -953,22 +884,13 @@ class bmsMilter(Milter.Milter):
       res,code,txt = q.check()
     q.result = res
     if res in ('unknown','permerror') and q.perm_error and q.perm_error.ext:
-      self.cbv_needed = (q,'permerror') # report SPF syntax error to sender
+      self.cbv_needed = (q,res) # report SPF syntax error to sender
       res,code,txt = q.perm_error.ext   # extended (lax processing) result
       txt = 'EXT: ' + txt
     p = SPFPolicy(q.s)
     # FIXME: try:finally to close policy db, or reuse with lock
-    if res in ('error','temperror'):
-      if self.need_cbv(p.getTempErrorPolicy(),q,'temperror'):
-        self.log('TEMPFAIL: SPF %s %i %s' % (res,code,txt))
-        self.setreply(str(code),'4.3.0',txt,
-          'We cannot accept your email until the DNS server for %s' % q.o,
-          'is operational for TXT record queries.'
-        )
-        return Milter.TEMPFAIL
-      res,code,txt = 'none',250,'EXT: ignoring DNS error'
     hres = None
-    if res != 'pass':
+    if res not in ('pass','error','temperror'):
       if self.mailfrom != '<>':
         # check hello name via spf unless spf pass
         h = spf.query(self.connectip,'',self.hello_name,receiver=receiver)
@@ -1013,9 +935,11 @@ class bmsMilter(Milter.Milter):
                 and hres != 'pass':
         # this bad boy has no credentials whatsoever
         policy = p.getNonePolicy()
-        if policy in ('CBV','DNS'):
-          self.offenses = 3    # ban ip if any bad recipient
-        if self.need_cbv(policy,q,'strike3'):
+        if policy == 'CBV':
+          if self.mailfrom != '<>':
+            self.cbv_needed = (q,ores)  # accept, but inform sender via DSN
+	  self.offenses = 3    # ban ip if any bad recipient
+        elif policy != 'OK':
           self.log('REJECT: no PTR, HELO or SPF')
           self.setreply('550','5.7.1',
     "You must have a valid HELO or publish SPF: http://www.openspf.org ",
@@ -1024,9 +948,13 @@ class bmsMilter(Milter.Milter):
     "that contain your IP don't count), an invalid or dynamic HELO, ",
     "and no SPF record."
           )
-          return self.offense() # ban ip if too many bad MFROMs
+          return Milter.REJECT
     if res in ('deny', 'fail'):
-      if self.need_cbv(p.getFailPolicy(),q,'fail'):
+      policy = p.getFailPolicy()
+      if policy == 'CBV':
+        if self.mailfrom != '<>':
+          self.cbv_needed = (q,res)
+      elif policy != 'OK':
         self.log('REJECT: SPF %s %i %s' % (res,code,txt))
         self.setreply(str(code),'5.7.1',txt)
         # A proper SPF fail error message would read:
@@ -1034,7 +962,11 @@ class bmsMilter(Milter.Milter):
         # "forged.org" in the sender address.  Contact <postmaster@forged.org>.
         return Milter.REJECT
     if res == 'softfail':
-      if self.need_cbv(p.getSoftfailPolicy(),q,'softfail'):
+      policy = p.getSoftfailPolicy()
+      if policy == 'CBV':
+        if self.mailfrom != '<>':
+          self.cbv_needed = (q,res)
+      elif policy != 'OK':
         self.log('REJECT: SPF %s %i %s' % (res,code,txt))
         self.setreply('550','5.7.1',
           'SPF softfail: If you get this Delivery Status Notice, your email',
@@ -1045,7 +977,12 @@ class bmsMilter(Milter.Milter):
         )
         return Milter.REJECT
     if res == 'neutral':
-      if self.need_cbv(p.getNeutralPolicy(),q,'neutral'):
+      policy = p.getNeutralPolicy()
+      if policy == 'CBV':
+        if self.mailfrom != '<>':
+          self.cbv_needed = (q,res)
+          # FIXME: this makes Received-SPF show wrong result
+      elif policy != 'OK':
         self.log('REJECT: SPF neutral for',q.s)
         self.setreply('550','5.7.1',
           'mail from %s must pass SPF: http://openspf.org/why.html' % q.o,
@@ -1057,7 +994,11 @@ class bmsMilter(Milter.Milter):
         )
         return Milter.REJECT
     if res in ('unknown','permerror'):
-      if self.need_cbv(p.getPermErrorPolicy(),q,'permerror'):
+      policy = p.getPermErrorPolicy()
+      if policy == 'CBV':
+        if self.mailfrom != '<>':
+          self.cbv_needed = (q,res)
+      elif policy != 'OK':
         self.log('REJECT: SPF %s %i %s' % (res,code,txt))
         # latest SPF draft recommends 5.5.2 instead of 5.7.1
         self.setreply(str(code),'5.5.2',txt,
@@ -1065,6 +1006,10 @@ class bmsMilter(Milter.Milter):
           'We cannot accept mail from %s until this is corrected.' % q.o
         )
         return Milter.REJECT
+    if res in ('error','temperror'):
+      self.log('TEMPFAIL: SPF %s %i %s' % (res,code,txt))
+      self.setreply(str(code),'4.3.0',txt)
+      return Milter.TEMPFAIL
     kv = {}
     if hres and q.h != q.o:
       kv['helo_spf'] = hres
@@ -1080,15 +1025,6 @@ class bmsMilter(Milter.Milter):
   # track header mods separately from body mods - so use only
   # in emergencies.
   def envrcpt(self,to,*str):
-    try:
-      param = param2dict(str)
-      self.notify = param.get('NOTIFY','FAILURE,DELAY').upper().split(',')
-      if 'NEVER' in self.notify: self.notify = ()
-      #self.rcpt_param = param
-    except:
-      self.log("REJECT: invalid PARAM:",to,str)
-      self.setreply('550','5.7.1','Invalid SRS signature')
-      return Milter.REJECT
     # mail to MAILER-DAEMON is generally spam that bounced
     if to.startswith('<MAILER-DAEMON@'):
       self.log('REJECT: RCPT TO:',to,str)
@@ -1127,13 +1063,10 @@ class bmsMilter(Milter.Milter):
                 self.setreply('550','5.7.1','Invalid SES signature')
                 return Milter.REJECT
               # reject for certain recipients are delayed until after DATA
-	      if auto_whitelist.has_precise_key(self.canon_from):
-		self.log("WHITELIST: DSN from",self.canon_from)
-	      else:
-                if srs_reject_spoofed \
-                  and user.lower() not in ('postmaster','abuse'):
-                  return self.forged_bounce(to)
-                self.data_allowed = not srs_reject_spoofed
+              if srs_reject_spoofed \
+                  and not user.lower() in ('postmaster','abuse'):
+                return self.forged_bounce()
+              self.data_allowed = not srs_reject_spoofed
 
         if not self.internal_connection and domain in private_relay:
           self.log('REJECT: RELAY:',to)
@@ -1156,7 +1089,7 @@ class bmsMilter(Milter.Milter):
           self.log('REJECT: RCPT TO:',to,str)
           if gossip and self.umis:
             gossip_node.feedback(self.umis,1)
-            self.umis = None
+	    self.umis = None
           return self.offense()
         # FIXME: should dspam_exempt be case insensitive?
         if user in block_forward.get(domain,()):
@@ -1246,14 +1179,7 @@ class bmsMilter(Milter.Milter):
           # original sender (encoded in Message-ID) is blacklisted
 
     elif lname == 'from':
-      fname,email = parseaddr(val)
-      # check for porn keywords
-      lval = fname.lower().strip()
-      for w in porn_words:
-        if lval.find(w) >= 0:
-          self.log('REJECT: %s: %s' % (name,val))
-          self.setreply('550','5.7.1','Watch your language')
-          return Milter.REJECT
+      name,email = parseaddr(val)
       if email.lower().startswith('postmaster@'):
         # Yes, if From header comes last, this might not help much.
         # But this is a heuristic - if MTAs would send proper DSNs in
@@ -1274,9 +1200,9 @@ class bmsMilter(Milter.Milter):
         return Milter.REJECT
     return Milter.CONTINUE
 
-  def forged_bounce(self,rcpt='-'):
+  def forged_bounce(self):
     if self.mailfrom != '<>':
-      self.log("REJECT: bogus DSN",rcpt)
+      self.log("REJECT: bogus DSN")
       self.setreply('550','5.7.1',
         "I do not accept normal mail from %s." % self.mailfrom.split('@')[0],
         "All such mail has turned out to be Delivery Status Notifications",
@@ -1284,7 +1210,7 @@ class bmsMilter(Milter.Milter):
         "you need to.  Use another MAIL FROM if you need to send me mail."
       )
     else:
-      self.log('REJECT: bounce with no SRS encoding',rcpt)
+      self.log('REJECT: bounce with no SRS encoding')
       self.setreply('550','5.7.1',
         "I did not send you that message. Please consider implementing SPF",
         "(http://openspf.org) to avoid bouncing mail to spoofed senders.",
@@ -1494,12 +1420,6 @@ class bmsMilter(Milter.Milter):
             elif not self.internal_connection or dspam_internal:
               if len(txt) > dspam_sizelimit:
                 self.log("Large message:",len(txt))
-                if self.blacklist:
-                  self.log('REJECT: BLACKLISTED')
-                  self.setreply('550','5.7.1',
-                        '%s has been blacklisted.'%self.canon_from)
-                  self.fp = None
-                  return Milter.REJECT
                 return False
               if user == 'honeypot' and Dspam.VERSION >= '1.1.9':
                 keep = False    # keep honeypot mail
@@ -1511,12 +1431,9 @@ class bmsMilter(Milter.Milter):
                     return False
                   if self.spf and self.mailfrom != '<>':
                     # check that sender accepts quarantine DSN
-                    if self.spf.result == 'pass':
-                      msg = mime.message_from_file(StringIO.StringIO(txt))
-                      rc = self.send_dsn(self.spf,msg,'quarantine')
-                      del msg
-                    else:
-                      rc = self.send_dsn(self.spf)
+                    msg = mime.message_from_file(StringIO.StringIO(txt))
+                    rc = self.send_dsn(self.spf,msg,'quarantine')
+                    del msg
                     if rc != Milter.CONTINUE:
                       return rc 
                   ds.check_spam(user,txt,self.recipients,quarantine=True,
@@ -1534,13 +1451,8 @@ class bmsMilter(Milter.Milter):
               elif self.blacklist:
                 txt = ds.check_spam(user,txt,self.recipients,
                         force_result=dspam.DSR_ISSPAM)
-              elif user in dspam_train:
-                txt = ds.check_spam(user,txt,self.recipients)
               else:
-                txt = ds.check_spam(user,txt,self.recipients,classify=True)
-                if txt:
-                  self.add_header("X-DSpam-Score",'%f' % ds.probability)
-                  return False
+                txt = ds.check_spam(user,txt,self.recipients)
               if not txt:
                 # DISCARD if quarrantined for any recipient.  It
                 # will be resent to all recipients if they submit
@@ -1571,7 +1483,7 @@ class bmsMilter(Milter.Milter):
           ds.check_spam(screener,txt,self.recipients,
                   force_result=dspam.DSR_ISINNOCENT)
           return False
-        if self.reject_spam and self.spf.result != 'pass':
+        if self.reject_spam:
           self.log("DSPAM:",screener,
                 'REJECT: X-DSpam-Score: %f' % ds.probability)
           self.setreply('550','5.7.1','Your Message looks spammy')
@@ -1581,18 +1493,12 @@ class bmsMilter(Milter.Milter):
         if self.spf and self.mailfrom != '<>':
           # check that sender accepts quarantine DSN
           self.fp.seek(0)
-	  if self.spf.result == 'pass' or self.cbv_needed:
-	    msg = mime.message_from_file(self.fp)
-	    if self.spf.result == 'pass':
-	      rc = self.send_dsn(self.spf,msg,'quarantine')
-	    else:
-	      rc = self.do_needed_cbv(msg)
-	    del msg
-	  else:
-	    rc = self.send_dsn(self.spf)
+          msg = mime.message_from_file(self.fp)
+          rc = self.send_dsn(self.spf,msg,'quarantine')
           if rc != Milter.CONTINUE:
             self.fp = None
             return rc
+          del msg
         if not ds.check_spam(screener,txt,self.recipients,classify=True):
           self.fp = None
           return Milter.DISCARD
@@ -1635,22 +1541,6 @@ class bmsMilter(Milter.Milter):
         quarantine=False)
     self.log("TRAINSPAM:",screener,'X-Dspam-Score: %f' % ds.probability)
 
-  def do_needed_cbv(self,msg):
-    q,template_name = self.cbv_needed
-    rc = self.send_dsn(q,msg,template_name)
-    self.cbv_needed = None
-    return rc
-
-  def need_cbv(self,policy,q,tname):
-    if policy == 'CBV':
-      if self.mailfrom != '<>' and not self.cbv_needed:
-        self.cbv_needed = (q,None)
-    elif policy == 'DSN':
-      if self.mailfrom != '<>' and not self.cbv_needed:
-        self.cbv_needed = (q,tname)
-    elif policy != 'OK': return True
-    return False
-
   def eom(self):
     if not self.fp:
       return Milter.ACCEPT      # no message collected - so no eom processing
@@ -1771,12 +1661,22 @@ class bmsMilter(Milter.Milter):
       except Milter.error:
         self.addheader(name,val)        # older sendmail can't insheader
 
-    # Do not send CBV to internal domains (since we'll just get
-    # the "Fraudulent MX" error).  Whitelisted senders clearly do not
-    # need CBV.  However, whitelisted domains might (to discover 
-    # bogus localparts).  Need a way to tell the difference.
+    # do not send CBV to internal domains (since we'll just get
+    # the "Fraudulent MX" error).
     if self.cbv_needed and not self.internal_domain:
-      rc = self.do_needed_cbv(msg)
+      q,res = self.cbv_needed
+      if res == 'softfail':
+        template_name = 'softfail'
+      elif res in ('fail','deny'):
+        template_name = 'fail'
+      elif res in ('unknown','permerror'):
+        template_name = 'permerror'
+      elif res == 'neutral':
+        template_name = 'neutral'
+      else:
+        template_name = 'strike3'
+      rc = self.send_dsn(q,msg,template_name)
+      self.cbv_needed = None
       if rc == Milter.REJECT:
         # Do not feedback here, because feedback should only occur
         # for messages that have gone to DATA.  Reputation lets us
@@ -1844,24 +1744,22 @@ class bmsMilter(Milter.Milter):
       out.close()
     return Milter.TEMPFAIL
 
-  def send_dsn(self,q,msg=None,template_name=None):
+  def send_dsn(self,q,msg,template_name):
     sender = q.s
     cached = cbv_cache.has_key(sender)
     if cached:
       self.log('CBV:',sender,'(cached)')
       res = cbv_cache[sender]
     else:
-      m = None
-      if template_name:
-        fname = template_name+'.txt'
-        try:
-          template = file(template_name+'.txt').read()
-          m = dsn.create_msg(q,self.recipients,msg,template)
-          self.log('CBV:',sender,'Using:',fname)
-        except IOError: pass
-      if not m:
+      fname = template_name+'.txt'
+      try:
+        template = file(template_name+'.txt').read()
+        self.log('CBV:',sender,'Using:',fname)
+      except IOError:
+        template = None
         self.log('CBV:',sender,'PLAIN')
-      else:
+      m = dsn.create_msg(q,self.recipients,msg,template)
+      if m:
         if srs:
           # Add SRS coded sender to various headers.  When (incorrectly)
           # replying to our DSN, any of these which are preserved
@@ -1885,10 +1783,7 @@ class bmsMilter(Milter.Milter):
         return Milter.TEMPFAIL
       cbv_cache[sender] = res
       self.log('REJECT:',desc)
-      try:
-        self.setreply('550','5.7.1',*desc.splitlines())
-      except TypeError:
-        self.setreply('550','5.7.1',"Callback failure")
+      self.setreply('550','5.7.1',*desc.splitlines())
       return Milter.REJECT
     cbv_cache[sender] = res
     return Milter.CONTINUE

doc/changes.ht

@@ -2,47 +2,6 @@ Title: Recent Changes
 
 <h2> Recent Changes </h2>
 
-<h3> 0.8.10 </h3>
-
-SRS rejections now log the recipient.  
-I have finally implemented plain CBV (no DSN).  The CBV policy
-will do a plain CBV from now on, and the DSN policy is required
-if you want to send a DSN.
-I started checking the MAIL FROM fullname (human readable part
-of an email) for porn keywords.  There is now a banned IP database.
-IPs are banned for too many bad MAIL FROMs or RCPT TOs, and remain banned
-for 7 days.
-
-<h3> 0.8.9 </h3>
-
-I use the <code>%ifarch</code> hack to build milter and milter-spf
-packages as noarch, while pymilter is built as native.
-
-I removed the spf dependency from dsn.py, so pymilter can be used without
-installing pyspf, and added a Milter.dns module to let python milters do
-general DNS lookups without loading pyspf.
-
-<h3> 0.8.8 </h3>
-
-Programs do not belong in the /var/log directory.  I moved the
-milter apps to /usr/lib/pymilter.  Since having the programs and
-data in the same directory is convenient for debugging, it will
-still use an executable present in the datadir.
-
-Several general utility classes and functions are now in the Milter package
-for possible use by other python milters.  In addition to the trivial example
-milter, a simple SPF only milter is included as a realistic example.
-
-The spec file now build 3 RPMs:
-
-<ul>
-<li> pymilter is the milter module and Milter package for use by all python
-     milters.
-<li> milter is the all-singing, all-dancing python milter application, with
-     supporting <code>/etc/init.d</code>, logrotate and other scripts.
-<li> milter-spf is the simple SPF only milter application.
-</ul>
-
 <h3> 0.8.7 </h3>
 
 The spf module has been moved to the 

doc/credits.ht

@@ -5,7 +5,7 @@ Title: Credits
 <a href="mailto:Jim Niemira <urmane@urmane.org>">Jim Niemira</a>
 wrote the original C module and some quick
 and dirty python to use it.  
-<a href="http://gathman.org/vitae">Stuart D. Gathman</a>
+<a href="mailto:Stuart Gathman <stuart@bmsi.com>">Stuart D. Gathman</a>
 took that kludge and added threading and context objects to it, wrote a proper
 OO wrapper (Milter.py) that handles attachments, did lots of testing, packaged
 it with distutils, and generally transformed it from a quick hack to a

doc/policy.ht

@@ -4,7 +4,8 @@ Title: Python Milter Mail Policy
 
 These are the policies implemented by the <code>bms.py</code> milter
 application.  The milter and Milter modules do not implement any policies
-by themselves.  
+by themselves.  Eventually, I'll get the bms.py milter moved to its
+own package.
 
 <h3> Classify connection </h3>
 
@@ -76,119 +77,161 @@ altered accordingly.
 
 <h2> SPF check </h2>
 
-The MAIL FROM, connect IP, and HELO name are checked against
-any SPF records published via DNS for the alleged sender (MAIL FROM)
-to determine the official SPF policy result.
-The offical SPF result is then logged in the Received-SPF header field,
-but certain results are subjected to further processing to create
-an effective result for policy purposes.
+Finally, the MAIL FROM, connect IP, and HELO name are checked against
+any SPF records published via DNS for the alleged sender (MAIL FROM).
+If there is no SPF record, we check for a local substitute under the
+domain defined in the <code>[spf]delegate</code> configuration.
+Further checks depend on the result.
 
-If the official result is 'none', we try to turn it into an effective result of
-'pass' or 'fail'.  First, we check for a local substitute SPF record
-under the domain defined in the <code>[spf]delegate</code> configuration.  
-It is often useful to add local SPF records for correspondents that are
-too clueless to add their own.  If there is no local substitute, we use a "best
-guess" SPF record of "v=spf1 a/24 mx/24 ptr" for MAIL FROM or "v=spf1 a/24
-mx/24" for HELO.  In addition, a HELO that is a subdomain of MAIL FROM and
-resolves to the connect IP results in an effective result of 'pass'.
-
-If there is no local SPF record, and the effective result is still not
-'pass', we check for either a valid HELO name or a valid PTR record for
-the connect IP.  A valid HELO or PTR cannot look like a dynamic name
-as determined by the heuristic in <code>Milter.dynip</code>.
-
-If HELO has an SPF record, and the result is anything but pass, we reject
-the connection:
+<table border=1>
+<tr><th>NONE</th><td>
+If there is no SPF record (official or delegated), then we 
+initiate a "three strikes and your out" regime, which looks for
+<b>some</b> form of validated identification.
+<ol>
+<li>We try a "best guess" SPF record of "v=spf1 a/24 mx/24 ptr".  If this
+    passes, good.  
+<li> We try to validate the HELO name. First check for an SPF record.
+   Otherwise, check whether the connect IP matches any A record for
+   the HELO name, or any A record for any MX name for the HELO name,
+   or is at least in the same /24 subnet as any of the above.
+   (In other words, a HELO SPF "best guess" of "v=spf1 a/24 mx/24".)
+   If so, good.  We consider the HELO validated.  If the HELO SPF
+   check fails, we reject the email.
+</ol>
 <pre>
 2005Jul30 19:45:16 [93991] connect from [221.200.41.54] at ('221.200.41.54', 3581) EXTERNAL DYN
 2005Jul30 19:45:18 [93991] hello from adelphia.net
 2005Jul30 19:45:19 [93991] mail from <wendy.stubbsua@link-it.com> ()
 2005Jul30 19:45:19 [93991] REJECT: hello SPF: fail 550 access denied
 </pre>
-Note that HELO does not have any forwarding issues like MAIL FROM, and so
-any result other than 'pass' or 'none' should be treated like 'fail'.
-
-Only if nothing about the SMTP envelope can be validated does the effective
-result remain 'none.  I call this the "3 strikes" rule.
-
-If the official result is 'permerror' (a syntax error in the sender's
-policy), we use the 'lax' option in pyspf to try various heuristics to guess 
-what they really meant.  For instance, the invalid mechanism "ip:1.2.3.4" is
-treated as "ip4:1.2.3.4".  The result of lax processing is then used
-as the effective result for policy purposes.
-
-With an effective SPF result in hand, we consult the sendmail access
-database to find our receiver policy for the sender.  
-
-<table border=1>
-<tr><th>REJECT</th><td>
-Reject the sender with a 550 5.7.1 SMTP code.  The SMTP rejection
-includes a detailed description of the problem.
+<ol>
+<li> If there is a validated PTR name, and it doesn't look
+   like a dynamic name, good.  We consider the connection validated.
+</ol>
+If any of the above can be validated, we continue on.
+If none of the above can be validated, and the <code>[SPF]reject_noptr</code>
+option is true, we reject the message immediately with the explanation
+that we need some form of valid identification before we accept an email.
+If <code>[SPF]reject_noptr</code> is false, we flag the message as
+needing Call Back Validation.  
+The Call Back Valildation sends a DSN to the purported sender informing
+them of the lack of identification.  If the message is legitimate, the
+sender needs to know that their email setup is broken and should be corrected.
+If the message is forged, the sender is informed of the forgery, 
+and their need to publish an SPF record or at least use a valid HELO name.
+If the purported sender does not accept the DSN,
+then the message is rejected.  The CBV status is cached to avoid
+annoying the purported sender with too many DSNs.  Currently, the DSN
+is repeated to the same sender once per month.
+<p>
+In this example, although 3com.com has no SPF record, we assume that 
+any legitimate mail from them will at least have a valid HELO or PTR.
+<pre>
+2005Jul30 23:52:03 [96777] connect from [222.252.233.200] at ('222.252.233.200', 29934) EXTERNAL DYN
+2005Jul30 23:52:03 [96777] hello from 3mail.3com.com
+2005Jul30 23:52:04 [96777] mail from <etec_nic_family@3mail.3com.com> ()
+2005Jul30 23:52:04 [96777] REJECT: no PTR, HELO or SPF
+</pre>
 </td></tr>
-<tr><th>CBV</th><td>
-Do a Call Back Validation by connecting to an MX of the sender
-and checking that using the sender as the RCPT TO is not rejected.
-We quit the CBV connection before actualling sending a message.
-If the CBV is rejected, our SMTP connection is rejected with the
-same error code and message.  CBV results are cached.
+
+<tr><th>PASS</th><td>
+A pass result normally lets the email continue on, but the domain is 
+tracked for reputation (and may be blocked), and may skip content scanning if
+it matches a whitelist.
+<pre>
+2005Jul24 17:44:26 [2104] mail from <gnucash-devel-bounces@gnucash.org> ('SIZE=4410',)
+2005Jul24 17:44:26 [2104] Received-SPF: pass (mail.bmsi.com: domain of gnucash.org
+	designates 204.107.200.65 as permitted sender)
+	client-ip=204.107.200.65; envelope-from=gnucash-devel-bounces@gnucash.org; helo=cvs.gnucash.org;
+</pre>
 </td></tr>
-<tr><th>DSN</th><td>
-Do a Call Back Validation by connecting to an MX of the sender
-and checking that using the sender as the RCPT TO is not rejected.
-Unlike a CBV, we continue on to data and send a detailed message
-explaining the problem.  This can be useful for reporting PermError
-or SoftFail to the sender.  Keep in mind that for any result other
-than 'pass', the sender could be forged, and your DSN could annoy the
-wrong person.  However, a SoftFail result is requesting such feedback
-for debugging and a PermError result needs to be fixed by the sender ASAP
-whether forged or not.  DSN results are cached so that senders are
-annoyed only weekly.
+
+<tr><th>NEUTRAL</th><td>
+A neutral result normally lets the email continue on, but the domain is not
+tracked for reputation or matched against any whitelists. 
+Highly forged domains listed in <code>[SPF]reject_neutral</code> are
+rejected.
+<pre>
+2005Jul24 17:41:37 [2070] connect from cp500627-a.dbsch1.nb.home.nl at ('84.27.225.3', 3465) EXTERNAL
+2005Jul24 17:41:37 [2070] hello from cp500627-a.dbsch1.nb.home.nl
+2005Jul24 17:41:38 [2070] mail from <nwarjejkw@yahoo.com> ()
+2005Jul24 17:41:38 [2070] REJECT: SPF neutral for nwarjejkw@yahoo.com
+</pre>
 </td></tr>
-<tr><th>OK</th><td>
-Accept the sender.  The message may still be rejected via reputation
-or content filtering.
+
+<tr><th>SOFTFAIL</th><td>
+A softfail result normally lets the email continue on, but the domain is not
+tracked for reputation or matched against any whitelists.  Furthermore,
+the message is flagged as needing Call Back Validation,
+and the highly forged domains listed in <code>[SPF]reject_neutral</code> are
+rejected as well.  
+<p>
+At present, we also require a valid HELO or PTR to avoid rejecting 
+a softfail.  But this should probably change to only require a
+successful CBV.
+<p>
+The Call Back Valildation sends a DSN to the purported sender informing
+them of the softfail.  If the message is legitimate, the sender needs
+to know about the softfail so that their email setup can be corrected.
+If the message is forged, the sender is informed of the forgery, confirming
+that SPF is protecting their reputation and encouraging a rapid transition
+to a strict policy.  If the purported sender does not accept the DSN,
+then the message is rejected.  The CBV status is cached to avoid
+annoying the purported sender with too many DSNs.  Currently, the DSN
+is repeated to the same sender once per month.
+<pre>
+2005Jul24 15:41:33 [801] mail from <Aitp@horafeliz.com> ()
+2005Jul24 15:41:33 [801] Received-SPF: softfail (mail.bmsi.com: transitioning domain of horafeliz.com
+	does not designate 221.184.83.185 as permitted sender)
+	client-ip=221.184.83.185; envelope-from=Aitp@horafeliz.com;
+	helo=p8185-ipad30funabasi.chiba.ocn.ne.jp;
+2005Jul24 15:41:33 [801] rcpt to <david@example.com> ()
+2005Jul24 15:41:35 [801] Subject: Microsoft, Adobe, Macromedia, Corel software. Up to 80% discount.
+2005Jul24 15:41:35 [801] X-Mailer: Microsoft Outlook, Build 10.0.2605
+2005Jul24 15:41:35 [801] CBV: Aitp@horafeliz.com
+2005Jul24 15:41:38 [801] REJECT: CBV: 550 <Aitp@horafeliz.com>: User unknown
+</pre>
 </td></tr>
+
+<tr><th>FAIL</th><td>
+The message is rejected with a reference the SPF why page.
+<pre>
+2005Jul30 19:53:27 [94070] connect from [212.70.52.16] at ('212.70.52.16', 3192) EXTERNAL DYN
+2005Jul30 19:53:27 [94070] hello from winzip.com
+2005Jul30 19:53:27 [94070] mail from <dan@winzip.com> ()
+2005Jul30 19:53:27 [94070] REJECT: SPF fail 550 SPF fail:
+	see http://openspf.com/why.html?sender=dan@winzip.com&ip=212.70.52.16
+</pre>
+</td></tr>
+
+<tr><th>PERMERROR</th><td>
+Permanent errors were called "unknown", and are still show that way
+in the log.  The message is rejected.  Previously, we enabled "lax" parsing
+of the SPF record, but rejecting is better because it informs the
+sender about their problem.  The next milter version will 
+look for a local substitute SPF record (as for a missing SPF record)
+before rejecting.  This will inform the sender of their problem, but
+also let the receiver install a temporary workaround.
+<pre>
+2005Jul24 18:05:37 [2312] mail from <b-mihdbcgaacaa-becibijh-000-@msg.euxiphipops.com> () 
+2005Jul24 18:05:37 [2312] REJECT: SPF unknown 550 SPF Permanent Error:
+	include mechanism missing domain: include
+</pre>
+The SPF record for msg.euxiphipops.com looked like this at the time of the
+above error:
+<pre>
+msg.euxiphipops.com	TXT "v=spf1 mx ptr a include"
+</pre>
+</td></tr>
+
+<tr><th>TEMPERROR</th><td>
+Temporary errors result in a 451 "Try again later" response.  The sender
+should retry the message at a later time.
+<pre>
+2005Jul24 07:33:13 [29846] mail from <quickenloans@rate.quicken.com> ('SIZE=73775', 'BODY=8BITMIME')
+2005Jul24 07:33:43 [29846] TEMPFAIL: SPF error 450 SPF Temporary Error: DNS Timeout
+</pre>
+</td></tr>
+
 </table>
-
-<h3> SPF policy syntax </h3>
-
-First, the full sender is checked:
-<pre>
-SPF-Fail:abeb@adelphia.net     DSN
-</pre>
-This says to accept mail from that adelphia.net user despite the
-SPF fail, but only after annoying them with a DSN about their ISP's broken
-policy. 
-
-If there is no match on the full sender, the domain is checked:
-<pre>
-SPF-Neutral:aol.com     REJECT
-</pre>
-This says to reject mail from AOL with an SPF result of neutral.
-This means AOL users can't use their AOL address with another mail service
-to send us mail.  This is good because the other mail service is 
-likely a badly configured greeting card site or a virus.
-
-Finally, a default policy for the result is checked.  While there are program
-defaults, you should have defaults in the access database for SPF results:
-<pre>
-SPF-Neutral:            CBV
-SPF-Softfail:           DSN
-SPF-PermError:          DSN
-SPF-TempError:          REJECT
-SPF-None:               REJECT
-SPF-Fail:               REJECT
-SPF-Pass:               OK
-</pre>
-
-<h2> Reputation </h2>
-
-If the sender has not been rejected by this point, and if a GOSSiP server is
-configured, we consult GOSSiP for the reputation score of the sender and
-SPF result.  The score is a number from -100 to 100 with a confidence
-percentage from 0 to 100.  A really bad reputation (less than -50 with
-confidence greater than 3) is rejected.  Note that the reputation is tracked
-independently for each SPF result and sender combination.  So aol.com:neutral
-might have a really bad reputation, while aol.com:pass would be ok.
-Furthermore, when a sender finally publishes an SPF policy and starts
-getting SPF pass, their reputation is effectively reset.

milter.cfg

@@ -1,6 +1,4 @@
 [milter]
-# the directory with log and data files
-datadir = /var/log/milter
 # the socket used to communicate with sendmail.  Must match sendmail.cf
 socket=/var/run/milter/pythonsock
 # where to save original copies of defanged and failed messages
@@ -27,10 +25,7 @@ internal_connect = 192.168.0.0/16,127.*
 ;trusted_relay = 1.2.3.4, 66.12.34.56
 
 # Relaying to these domains is allowed from internal connections only.
-# You might want to restrict aol.com, for instance, so that stupid
-# users don't forward their spam to aol for filtering and get your MTA 
-# blacklisted by aol.
-;private_relay = aol.com, yahoo.com
+;private_relay = mycorp.com
 
 # Reject external senders with hello names no legit external sender would use.
 # SPF will do this also, but listing your own domain and mailserver here
@@ -62,7 +57,7 @@ porn_words = penis, breast, pussy, horse cock, porn, xenical, diet pill, d1ck,
 	p-e-n-i-s, hydrocodone, vicodin, xanax, vicod1n, x@nax, diazepam,
 	v1@gra, xan@x, cialis, ci@lis, frëe, xãnax, valíum, vãlium, via-gra,
 	x@n3x, vicod3n, penís, c0d1n, phentermine, en1arge, dip1oma, v1codin,
-	valium, rolex, sexual, fuck, adv1t, vgaira, medz, acai berry
+	valium, rolex, sexual, fuck, adv1t
 # reject mail with these case sensitive strings in the subject
 spam_words = $$$, !!!, XXX, FREE, HGH
 # attachments with these extensions will be replaced with a warning
@@ -191,11 +186,6 @@ blind = 1
 
 # Map email addresses and aliases to dspam users
 ;dspam_users=david,goliath,spam,falsepositive
-# List dspam users which train on all delivered messages, as opposed to
-# "train on error" which trains only when a spam or falsepositive is reported.
-# Training mode will build the dictionary faster, but requires close attention
-# so as not to miss any spam or false positives.
-;dspam_train=goliath
 ;david=david@foocorp.com,david.yelnetz@foocorp.com,david@bar.foocorp.com
 ;goliath=giant@foocorp.com,goliath.philistine@foocorp.com
 # address to forward spam to.  milter will process these and not deliver
@@ -221,8 +211,6 @@ blind = 1
 # Use a dedicated GOSSiP server.  If not specified, a local database
 # will be used.
 ;server=host:11900
-# To include peers of a peer in reputation, set ttl=2
-;ttl=1
 # If a local database is used, also consult these GOSSiP servers about 
 # domains.  Peer reputation is also tracked as to how often they
 # agree with us, and weighted accordingly.

milter.rc

@@ -46,7 +46,7 @@ start() {
 stop() {
 	# Stop daemons.
 	echo -n "Shutting down $prog: "
-	killproc -d 9 milter
+	killproc milter
 	RETVAL=$?
 	echo
 	[ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/milter

pymilter.spec

@@ -1,10 +1,5 @@
-# This spec file contains 2 noarch packages in addition to the pymilter
-# module.  To compile all three on 32-bit Intel, use:
-# rpmbuild -ba --target=i386,noarch pymilter.spec
-
-%define __python python2.4
-%define version 0.8.10
-%define release 2%{?dist}.py24
+%define version 0.8.9
+%define release 1
 # what version of RH are we building for?
 %define redhat7 0
 
@@ -22,12 +17,22 @@
 %define sysvinit milter.rc
 %endif
 # RH9, other systems (single ps line per process)
+%ifos Linux	# whether to use system default python?
+%define python python
+#define python python2.4
+%else
+%define python python
+%endif
 %ifos aix4.1
 %define libdir /var/log/milter
 %else
 %define libdir /usr/lib/pymilter
 %endif
 
+# This spec file contains 2 noarch packages in addition to the pymilter
+# module.  To compile all three, use:
+# rpmbuild -ba --target=i386,noarch pymilter.spec
+
 %ifarch noarch
 Name: milter
 Group: Applications/System
@@ -43,7 +48,7 @@ Prefix: %{_prefix}
 Vendor: Stuart D. Gathman <stuart@bmsi.com>
 Packager: Stuart D. Gathman <stuart@bmsi.com>
 Url: http://www.bmsi.com/python/milter.html
-Requires: %{__python} >= 2.4, pyspf >= 2.0.4, pymilter
+Requires: %{python} >= 2.4, pyspf >= 2.0.4, pymilter
 %ifos Linux
 Requires: chkconfig
 %endif
@@ -85,7 +90,7 @@ cat >$RPM_BUILD_ROOT/etc/logrotate.d/milter <<'EOF'
   compress
 }
 /var/log/milter/banned_ips {
-  rotate 7
+  rotate 3
   daily
   copytruncate
 }
@@ -107,14 +112,23 @@ find /var/log/milter/save -mtime +7 | xargs $R rm
 EOF
 chmod a+x $RPM_BUILD_ROOT/etc/cron.daily/milter
 
-%ifnos aix4.1
+%ifos aix4.1
+cat >$RPM_BUILD_ROOT%{libdir}/start.sh <<'EOF'
+#!/bin/sh
+cd /var/log/milter
+# uncomment to enable sgmlop if installed
+#export PYTHONPATH=/usr/local/lib/python2.1/site-packages
+exec /usr/local/bin/python bms.py >>milter.log 2>&1
+EOF
+%else # not aix4.1
+cp start.sh $RPM_BUILD_ROOT%{libdir}
 mkdir -p $RPM_BUILD_ROOT/etc/rc.d/init.d
 cp %{sysvinit} $RPM_BUILD_ROOT/etc/rc.d/init.d/milter
 cp spfmilter.rc $RPM_BUILD_ROOT/etc/rc.d/init.d/spfmilter
 ed $RPM_BUILD_ROOT/etc/rc.d/init.d/milter <<'EOF'
 /^python=/
 c
-python="%{__python}"
+python="%{python}"
 .
 w
 q
@@ -122,13 +136,23 @@ EOF
 ed $RPM_BUILD_ROOT/etc/rc.d/init.d/spfmilter <<'EOF'
 /^python=/
 c
-python="%{__python}"
+python="%{python}"
+.
+w
+q
+EOF
+ed $RPM_BUILD_ROOT%{libdir}/start.sh <<'EOF'
+/^python=/
+c
+python="%{python}"
 .
 w
 q
 EOF
 %endif	# aix4.1
+chmod a+x $RPM_BUILD_ROOT%{libdir}/start.sh
 
+mkdir -p $RPM_BUILD_ROOT/var/run/milter
 mkdir -p $RPM_BUILD_ROOT/usr/share/sendmail-cf/hack
 cp -p rhsbl.m4 $RPM_BUILD_ROOT/usr/share/sendmail-cf/hack
 
@@ -161,8 +185,10 @@ fi
 
 %files 
 %defattr(-,root,root)
+%config %{libdir}/start.sh
 /etc/logrotate.d/milter
 /etc/cron.daily/milter
+%{libdir}/bms.py?
 %ifos aix4.1
 %defattr(-,smmsp,mail)
 %else
@@ -178,7 +204,6 @@ fi
 %config(noreplace) /var/log/milter/neutral.txt
 %config(noreplace) /var/log/milter/quarantine.txt
 %config(noreplace) /var/log/milter/permerror.txt
-%config(noreplace) /var/log/milter/temperror.txt
 %config(noreplace) /etc/mail/pymilter.cfg
 /usr/share/sendmail-cf/hack/rhsbl.m4
 
@@ -205,8 +230,8 @@ Prefix: %{_prefix}
 Vendor: Stuart D. Gathman <stuart@bmsi.com>
 Packager: Stuart D. Gathman <stuart@bmsi.com>
 Url: http://www.bmsi.com/python/milter.html
-Requires: %{__python} >= 2.4, sendmail >= 8.13
-BuildRequires: %{__python}-devel >= 2.4, sendmail-devel >= 8.13
+Requires: %{python} >= 2.4, sendmail >= 8.13
+BuildRequires: %{python}-devel >= 2.4, sendmail-devel >= 8.13
 
 %description
 This is a python extension module to enable python scripts to
@@ -224,44 +249,18 @@ DSNs, and doing CBV.
 %else # Redhat builds debug packages after 7.3
   LDFLAGS="-g"
 %endif
-env CFLAGS="$RPM_OPT_FLAGS" LDFLAGS="$LDFLAGS" %{__python} setup.py build
+env CFLAGS="$RPM_OPT_FLAGS" LDFLAGS="$LDFLAGS" %{python} setup.py build
 
 %install
 rm -rf $RPM_BUILD_ROOT
-%{__python} setup.py install --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES
-mkdir -p $RPM_BUILD_ROOT/var/run/milter
-mkdir -p $RPM_BUILD_ROOT%{libdir}
-%ifos aix4.1
-cat >$RPM_BUILD_ROOT%{libdir}/start.sh <<'EOF'
-#!/bin/sh
-cd /var/log/milter
-# uncomment to enable sgmlop if installed
-#export PYTHONPATH=/usr/local/lib/python2.1/site-packages
-exec /usr/local/bin/python bms.py >>milter.log 2>&1
-EOF
-%else # not aix4.1
-cp start.sh $RPM_BUILD_ROOT%{libdir}
-ed $RPM_BUILD_ROOT%{libdir}/start.sh <<'EOF'
-/^python=/
-c
-python="%{__python}"
-.
-w
-q
-EOF
-%endif
-chmod a+x $RPM_BUILD_ROOT%{libdir}/start.sh
+%{python} setup.py install --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES
 %if !%{redhat7}
-#grep '.pyc$' INSTALLED_FILES | sed -e 's/c$/o/' >>INSTALLED_FILES
+grep '.pyc$' INSTALLED_FILES | sed -e 's/c$/o/' >>INSTALLED_FILES
 %endif
 
-# start.sh is used by spfmilter and milter, and could be used by
-# other milters running on redhat
 %files -f INSTALLED_FILES
 %defattr(-,root,root)
 %doc README HOWTO ChangeLog NEWS TODO CREDITS sample.py milter-template.py
-%config %{libdir}/start.sh
-%dir %attr(0755,mail,mail) /var/run/milter
 
 %endif # noarch
 
@@ -269,18 +268,6 @@ chmod a+x $RPM_BUILD_ROOT%{libdir}/start.sh
 rm -rf $RPM_BUILD_ROOT
 
 %changelog
-* Mon Aug 25 2008 Stuart Gathman <stuart@bmsi.com> 0.8.10-2
-- /var/run/milter directory must be owned by mail
-* Mon Aug 25 2008 Stuart Gathman <stuart@bmsi.com> 0.8.10-1
-- log rcpt for SRS rejections
-- improved parsing into email and fullname (still 2 self test failures)
-- implement no-DSN CBV, reduce full DSNs
-- check for porn words in MAIL FROM fullname
-- ban IP for too many bad MAIL FROMs or RCPT TOs
-- temperror policy in access
-- no CBV for whitelisted MAIL FROM except permerror, softfail
-- Allow explicitly whitelisted email from banned_users.
-- configure gossip TTL
 * Mon Sep 24 2007 Stuart Gathman <stuart@bmsi.com> 0.8.9-1
 - Use %ifarch hack to build milter and milter-spf packages as noarch
 - Remove spf dependency from dsn.py, add dns.py
@@ -299,7 +286,6 @@ rm -rf $RPM_BUILD_ROOT
 - SPF moved to pyspf RPM
 - wiretap archive option
 - Do plain CBV if missing template
-- SMTP AUTH policy in access
 * Tue May 23 2006 Stuart Gathman <stuart@bmsi.com> 0.8.6-2
 - Support CBV timeout
 - Support fail template, headers in templates

quarantine.txt

@@ -22,6 +22,19 @@ their quarantined mail and may notice your message.  If your message is
 important, please contact them via other means.  You may also try sending
 them a simple plain text message.
 
+If you never sent the above message, then your domain, %(sender_domain)s,
+was forged - i.e. used without your knowlege or authorization by
+someone attempting to steal your mail identity.  This is a very
+serious problem, and you need to provide authentication for your
+SMTP (email) servers to prevent criminals from forging your
+domain.  The simplest step is usually to publish an SPF record
+with your Sender Policy.  
+
+For more information, see: http://www.openspf.org
+
+Your mail admin needs to publish a strict SPF record so that I can reject
+those forgeries instead of bugging you with them.
+
 If you need further assistance, please do not hesitate to contact me.
 
 Kind regards,

setup.py

@@ -16,7 +16,7 @@ if sys.version < '2.2.3':
   DistributionMetadata.download_url = None
 
 # NOTE: importing Milter to obtain version fails when milter.so not built
-setup(name = "pymilter", version = '0.8.10',
+setup(name = "pymilter", version = '0.8.9',
 	description="Python interface to sendmail milter API",
 	long_description="""\
 This is a python extension module to enable python scripts to

temperror.txt

@@ -1,33 +0,0 @@
-To: %(sender)s
-From: postmaster@%(receiver)s
-Subject: Critical DNS configuration error
-Auto-Submitted: auto-generated (configuration error)
-
-This is an automatically generated Delivery Status Notification.
-
-THIS IS A WARNING MESSAGE ONLY.
-
-YOU DO *NOT* NEED TO RESEND YOUR MESSAGE.
-
-Delivery to the following recipients has been delayed.
-
-	%(rcpt)s
-
-Subject: %(subject)s 
-Received-SPF: %(spf_result)s
-
-Your DNS server is not responding to TXT queries.  In other words,
-it is BROKEN.  You need to get somebody to fix it ASAP.  We
-are attempting to do TXT queries to see if you have an SPF record.
-
-See http://openspf.org
-
-We are sending you this message to alert you to the fact that
-you have problems with your DNS.
-
-If you need further assistance, please do not hesitate to
-contact me again.
-
-Kind regards,
-
-postmaster@%(receiver)s

testbms.py

@@ -44,7 +44,7 @@ class TestMilter(bms.bmsMilter):
       self._msg[field] = value
     self.headerschanged = True
 
-  def addheader(self,field,value,idx=-1):
+  def addheader(self,field,value):
     if not self._body:
       raise IOError,"addheader not called from eom()"
     self.log('addheader: %s=%s' % (field,value))