This commit was manufactured by cvs2svn to create tag 'pymilter-0_8_11'.

Sprout from master 2008-10-11 15:58:00 UTC Stuart Gathman <stuart@gathman.org> 'Release 0.8.11'
Cherrypick from bmsi 2005-05-31 18:23:49 UTC Stuart Gathman <stuart@gathman.org> 'Development changes since 0.7.2':
    rejects.py
    rhsbl.m4
    sample.py
    test/amazon
    test/big5
    test/bounce
    test/bounce1
    test/bound
    test/honey
    test/missingboundary
    test/samp1
    test/spam44
    test/spam7
    test/spam8
    test/test1
    test/test8
    test/virus1
    test/virus13
    test/virus2
    test/virus3
    test/virus4
    test/virus5
    test/virus6
    test/virus7
    testsample.py

MANIFEST.in

@@ -16,6 +16,7 @@ include bms.py
 include spf.py
 include cid2spf.py
 include spfquery.py
+include ban2zone.py
 include test.py
 include sample.py
 include milter-template.py

Milter/greylist.py

@@ -0,0 +1,74 @@
+import time
+import shelve
+import thread
+import logging
+import urllib
+
+log = logging.getLogger('milter.greylist')
+
+def quoteAddress(s):
+  '''Quote an address so that it's safe to store in the file-system.
+  Address can either be a domain name, or local part.
+  Returns the quoted address.'''
+
+  s = urllib.quote(s, '@_-+~!.%')
+  if s.startswith('.'): s = '%2e' + s[1:]
+  return s
+
+class Record(object):
+  __slots__ = ( 'firstseen', 'lastseen', 'umis', 'cnt' )
+
+  def __init__(self):
+    now = time.time()
+    self.firstseen = now
+    self.lastseen = now
+    self.cnt = 0
+    self.umis = None
+
+class Greylist(object):
+
+  def __init__(self,dbname,grey_time=10,grey_expire=4,grey_retain=36):
+    self.ignoreLastByte = False
+    self.greylist_time = grey_time * 60         # minutes
+    self.greylist_expire = grey_expire * 3600   # hours
+    self.greylist_retain = grey_retain * 24 * 3600   # days
+    self.dbp = shelve.open(dbname,'c',protocol=2)
+    self.lock = thread.allocate_lock()
+  
+  def check(self,ip,sender,recipient):
+    "Return number of allowed messages for greylist triple."
+    sender = quoteAddress(sender)
+    recipient = quoteAddress(recipient)
+    key = ip + ':' + sender + ':' + recipient
+    self.lock.acquire()
+    try:
+      dbp = self.dbp
+      try:
+        r = dbp[key]
+        now = time.time()
+        if now > r.lastseen + self.greylist_retain:
+          # expired
+          log.debug('Expired greylist: %s',key)
+          r = Record()
+        elif now < r.firstseen + self.greylist_time:
+          # still greylisted
+          log.debug('Early greylist: %s',key)
+          #r = Record()
+          r.lastseen = now
+        elif r.cnt or now < r.firstseen + self.greylist_expire:
+          # in greylist window or active
+          r.lastseen = now
+          r.cnt += 1
+          log.debug('Active greylist(%d): %s',r.cnt,key)
+        else:
+          # passed greylist window
+          log.debug('Late greylist: %s',key)
+          r = Record()
+        dbp[key] = r
+      except:
+        r = Record()
+        dbp[key] = r
+      dbp.sync()
+    finally:
+      self.lock.release()
+    return r.cnt

TODO

@@ -1,3 +1,14 @@
+The recent feature to let a REJECT policy for SPF None be overridden
+by whitelisting is working for CSI and CMS.  However, there could be
+a sender that we want to REJECT even when whitelisted - because they
+normally get a guessed PASS.  Need another policy name - or else just
+add them to local SPF so they won't ever get 'None'.
+
+When policy is OK, do not use cbv_cache for blacklist.
+
+Add postmaster option or general rcpt list to dsn.  Can send dsn to
+user and postmaster on the same connection.
+
 Check ESMTP NOTIFY before sending real DSNs.  Just use CBV if DSNs are
 not wanted.
 

ban2zone.py

@@ -0,0 +1,15 @@
+#!/usr/bin/python2.4
+
+import socket
+import sys
+from glob import glob
+
+banned_ips = set(socket.inet_aton(ip) 
+    for fn in sys.argv[1:]
+    for ip in open(fn))
+banned_ips = list(banned_ips)
+banned_ips.sort()
+for ip in banned_ips:
+  a = socket.inet_ntoa(ip).split('.')
+  a.reverse()
+  print "%s\tIN A 127.0.0.2"%('.'.join(a))

bms.py

@@ -1,6 +1,28 @@
 #!/usr/bin/env python
 # A simple milter that has grown quite a bit.
 # $Log$
+# Revision 1.133  2008/10/09 18:44:54  customdesigned
+# Skip greylisting for good reputation.
+#
+# Revision 1.132  2008/10/09 00:55:13  customdesigned
+# Don't reset greylist timer on early retries.
+#
+# Revision 1.131  2008/10/08 04:57:28  customdesigned
+# Greylisting
+#
+# Revision 1.130  2008/10/02 03:19:00  customdesigned
+# Delay strike3 REJECT and don't reject if whitelisted.
+# Recognize vacation messages as autoreplies.
+#
+# Revision 1.129  2008/09/09 23:24:56  customdesigned
+# Never ban a trusted relay.
+#
+# Revision 1.128  2008/09/09 23:08:16  customdesigned
+# Wasn't reading banned_ips
+#
+# Revision 1.127  2008/08/25 18:32:22  customdesigned
+# Handle missing gossip_node so self tests pass.
+#
 # Revision 1.126  2008/08/18 17:47:57  customdesigned
 # Log rcpt for SRS rejections.
 #
@@ -204,6 +226,7 @@ from Milter.dynip import is_dynip as dynip
 from Milter.utils import \
         iniplist,parse_addr,parse_header,ip4re,addr2bin,parseaddr
 from Milter.config import MilterConfigParser
+from Milter.greylist import Greylist
 
 from fnmatch import fnmatchcase
 from email.Utils import getaddresses
@@ -317,6 +340,7 @@ supply_sender = False
 access_file = None
 timeout = 600
 banned_ips = set()
+greylist = None
 
 logging.basicConfig(
         stream=sys.stdout,
@@ -490,6 +514,21 @@ def read_config(list):
     else:
       gossip_ttl = 1
 
+  # greylist section
+  if cp.has_option('greylist','dbfile'):
+    global greylist
+    grey_db = cp.getdefault('greylist','dbfile')
+    grey_days = 36
+    if cp.has_option('greylist','retain'):
+      grey_days = cp.getint('greylist','retain')
+    grey_expire = 4
+    if cp.has_option('greylist','expire'):
+      grey_expire = cp.getint('greylist','expire')
+    grey_time = 10
+    if cp.has_option('greylist','time'):
+      grey_time = cp.getint('greylist','expire')
+    greylist = Greylist(grey_db,grey_time,grey_expire,grey_days)
+
 def findsrs(fp):
   lastln = None
   for ln in fp:
@@ -729,12 +768,13 @@ class bmsMilter(Milter.Milter):
 
   def offense(self,inc=1):
     self.offenses += inc
-    if self.offenses > 3:
+    if self.offenses > 3 and not self.trusted_relay:
       try:
         ip = addr2bin(self.connectip)
         if ip not in banned_ips:
           banned_ips.add(ip)
           print >>open('banned_ips','a'),self.connectip
+          self.log("BANNED IP:",self.connectip)
       except: pass
     return Milter.REJECT
 
@@ -756,6 +796,7 @@ class bmsMilter(Milter.Milter):
     self.dspam = True
     self.whitelist = False
     self.blacklist = False
+    self.greylist = False
     self.reject_spam = True
     self.data_allowed = True
     self.delayed_failure = None
@@ -857,12 +898,14 @@ class bmsMilter(Milter.Milter):
       return Milter.REJECT
     self.umis = None
     self.spf = None
+    self.policy = None
     if not (self.internal_connection or self.trusted_relay)     \
         and self.connectip and spf:
       rc = self.check_spf()
       if rc != Milter.CONTINUE:
         if rc != Milter.TEMPFAIL: self.offense()
         return rc
+      self.greylist = True
     else:
       rc = Milter.CONTINUE
     # FIXME: parse Received-SPF from trusted_relay for SPF result
@@ -870,6 +913,7 @@ class bmsMilter(Milter.Milter):
     hres = self.spf and self.spf_helo
     # Check whitelist and blacklist
     if auto_whitelist.has_key(self.canon_from):
+      self.greylist = False
       if res == 'pass' or self.trusted_relay:
         self.whitelist = True
         self.log("WHITELIST",self.canon_from)
@@ -880,8 +924,9 @@ class bmsMilter(Milter.Milter):
         self.cbv_needed = None
     elif cbv_cache.has_key(self.canon_from) and cbv_cache[self.canon_from] \
         or domain in blacklist:
+      # FIXME: don't use cbv_cache for blacklist if policy is 'OK'
       if not self.internal_connection:
-        self.offense()
+        self.offense(inc=2)
         if not dspam_userdir:
           if domain in blacklist:
             self.log('REJECT: BLACKLIST',self.canon_from)
@@ -892,9 +937,21 @@ class bmsMilter(Milter.Milter):
             self.log('REJECT:',desc)
             self.setreply('550','5.7.1',*desc.splitlines())
           return Milter.REJECT
+        self.greylist = False   # don't delay - use spam for training
         self.blacklist = True
         self.log("BLACKLIST",self.canon_from)
     else:
+      # REJECT delayed until after checking whitelist
+      if self.policy == 'REJECT':
+          self.log('REJECT: no PTR, HELO or SPF')
+          self.setreply('550','5.7.1',
+    "You must have a valid HELO or publish SPF: http://www.openspf.org ",
+    "Contact your mail administrator IMMEDIATELY!  Your mail server is ",
+    "severely misconfigured.  It has no PTR record (dynamic PTR records ",
+    "that contain your IP don't count), an invalid or dynamic HELO, ",
+    "and no SPF record."
+          )
+          return self.offense() # ban ip if too many bad MFROMs
       global gossip
       if gossip and domain and rc == Milter.CONTINUE \
           and not (self.internal_connection or self.trusted_relay) \
@@ -930,6 +987,8 @@ class bmsMilter(Milter.Milter):
               self.setreply('550','5.7.1',
                 'Your domain has been sending nothing but spam')
               return Milter.REJECT
+            if self.reputation > 40 and self.confidence > 1:
+              self.greylist = False
         except:
           gossip = None
           raise
@@ -1012,19 +1071,11 @@ class bmsMilter(Milter.Milter):
       if self.missing_ptr and ores == 'none' and res != 'pass' \
                 and hres != 'pass':
         # this bad boy has no credentials whatsoever
+        res = 'none'
         policy = p.getNonePolicy()
-        if policy in ('CBV','DNS'):
+        if policy in ('CBV','DSN'):
           self.offenses = 3    # ban ip if any bad recipient
-        if self.need_cbv(policy,q,'strike3'):
-          self.log('REJECT: no PTR, HELO or SPF')
-          self.setreply('550','5.7.1',
-    "You must have a valid HELO or publish SPF: http://www.openspf.org ",
-    "Contact your mail administrator IMMEDIATELY!  Your mail server is ",
-    "severely misconfigured.  It has no PTR record (dynamic PTR records ",
-    "that contain your IP don't count), an invalid or dynamic HELO, ",
-    "and no SPF record."
-          )
-          return self.offense() # ban ip if too many bad MFROMs
+        self.need_cbv(policy,q,'strike3')
     if res in ('deny', 'fail'):
       if self.need_cbv(p.getFailPolicy(),q,'fail'):
         self.log('REJECT: SPF %s %i %s' % (res,code,txt))
@@ -1115,6 +1166,7 @@ class bmsMilter(Milter.Milter):
               self.log("srs rcpt:",newaddr)
             self.dspam = False    # verified as reply to mail we sent
             self.blacklist = False
+            self.greylist = False
             self.delayed_failure = False
           except:
             if not (self.internal_connection or self.trusted_relay):
@@ -1183,8 +1235,18 @@ class bmsMilter(Milter.Milter):
     except:
       self.log("rcpt to",to,str)
       raise
-    self.log("rcpt to",to,str)
+    if self.greylist and greylist and self.canon_from:
+      # no policy for trusted or internal
+      rc = greylist.check(self.connectip,self.canon_from,canon_to)
+      if rc == 0:
+        self.log("GREYLIST:",self.connectip,self.canon_from,canon_to)
+        self.setreply('451','4.7.1',
+          'Greylisted: http://projects.puremagic.com/greylisting/',
+          'Please retry in %.1f minutes'%(greylist.greylist_time/60.0))
+        return Milter.TEMPFAIL
+      self.log("GREYLISTED: %d"%rc)
       
+    self.log("rcpt to",to,str)
     self.smart_alias(to)
     # get recipient after virtusertable aliasing
     #rcpt = self.getsymval("{rcpt_addr}")
@@ -1305,9 +1367,10 @@ class bmsMilter(Milter.Milter):
         if gossip and self.umis:
           gossip_node.feedback(self.umis,1)
         return rc
-    elif self.whitelist_sender and lname == 'subject':
+    elif self.whitelist_sender:
       # check for AutoReplys
-        if reautoreply.match(val):
+      if (lname == 'subject' and reautoreply.match(val)) \
+	or (lname == 'user-agent' and val.lower().startswith('vacation')):
           self.whitelist_sender = False
           self.log('AUTOREPLY: not whitelisted')
 
@@ -1642,6 +1705,7 @@ class bmsMilter(Milter.Milter):
     return rc
 
   def need_cbv(self,policy,q,tname):
+    self.policy = policy
     if policy == 'CBV':
       if self.mailfrom != '<>' and not self.cbv_needed:
         self.cbv_needed = (q,None)
@@ -1860,7 +1924,7 @@ class bmsMilter(Milter.Milter):
           self.log('CBV:',sender,'Using:',fname)
         except IOError: pass
       if not m:
-        self.log('CBV:',sender,'PLAIN')
+        self.log('CBV:',sender,'PLAIN (%s)'%q.result)
       else:
         if srs:
           # Add SRS coded sender to various headers.  When (incorrectly)
@@ -1915,6 +1979,7 @@ def main():
       return
   try:
     from glob import glob
+    global banned_ips
     banned_ips = set(addr2bin(ip) 
         for fn in glob('banned_ips*')
         for ip in open(fn))

doc/links.h

@@ -11,7 +11,7 @@
 <li><a href="credits.html">CREDITS</a>
 <li><a href="http://sourceforge.net"><img src="http://sflogo.sourceforge.net/sflogo.php?group_id=139894&amp;type=1" width="88" height="31" border="0" alt="SourceForge.net Logo" /></a>
 <h3>Links</h3>
-<li><a href="http://www.milter.org/milter_api/api.html">C&nbsp;API</a>
+<li><a href="https://www.milter.org/developers/api/index">C&nbsp;API</a>
 <li><a href="http://www.milter.org/">Milter.Org</a>
 <li><a href="http://www.python.org/">Python.Org</a>
 <li><a href="http://www.sendmail.org/">Sendmail.Org</a>

doc/milter.ht

@@ -13,14 +13,19 @@ ALT="Viewable With Any Browser" BORDER="0"></A>
 </map>
 </P>
 
-<img src="Maxwells.gif" alt="Maxwell's Daemon: pymilter mascot" align=left>
+<table rules="none">
+<tr><td>
+<img src="Maxwells.gif" alt="Maxwell's Daemon: pymilter mascot" align="top">
+Mascot by <a href="http://alphard.ethz.ch/hafner/lebl.htm">Christian Hafner</a>
+</td>
+<td>
 <h1 align=center>Sendmail Milters in Python</h1>
 <h4 align=center>by <a href="mailto:%75%72%6D%61%6E%65%40%6E%65%75%72%61l%61%63%63%65%73%73%2E%63%6F%6D">Jim Niemira</a>
   and <a href="mailto:%73%74%75%61%72%74%40%62%6D%73%69%2E%63%6F%6D">
   Stuart D. Gathman</a><br>
 This web page is written by Stuart D. Gathman<br>and<br>sponsored by
 <a href="http://www.bmsi.com">Business Management Systems, Inc.</a> <br>
-Last updated Mar 30, 2007</h4>
+Last updated Aug 26, 2008</h4>
 
 See the <a href="faq.html">FAQ</a> | <a href="http://sourceforge.net/project/showfiles.php?group_id=139894">Download now</a> |
 <a href="http://bmsi.com/mailman/listinfo/pymilter">Subscribe to mailing list</a> |
@@ -31,7 +36,7 @@ See the <a href="faq.html">FAQ</a> | <a href="http://sourceforge.net/project/sho
 <a href="//www.python.org">
 <img src="python55.gif" align=left alt="A Python"></a>
 <a href="//www.sendmail.org/">Sendmail</a> introduced a
-<a href="http://www.milter.org/milter_api/api.html"> new API</a> beginning with version 8.10 -
+<a href="https://www.milter.org/developers/api/index"> new API</a> beginning with version 8.10 -
 libmilter.  The milter module for <a href="//www.python.org">Python</a>
 provides a python interface to libmilter that exploits all its features.
 <p>
@@ -39,7 +44,11 @@ Sendmail 8.12 officially releases libmilter.
 Version 8.12 seems to be more robust, and includes new privilege
 separation features to enhance security.  Even better, sendmail 8.13
 supports socket maps, which makes <a href="pysrs.html">pysrs</a> much more
-efficient and secure.  I recommend upgrading.
+efficient and secure.   Sendmail 8.14 finally supports modifying 
+MAIL FROM via the milter API.  Unfortunately, I haven't gotten around
+to supporting that yet in python milter.
+</td></tr>
+</table>
 
 <h3><a name=overview>Overview</a></h3>
 
@@ -48,7 +57,7 @@ href="#milter">milters</a>, and the beginnings of a general purpose mail
 filtering system written in Python.
 <p>
 At the lowest level, the 'milter' module provides a thin wrapper around the
-<a href="http://www.milter.org/milter_api/api.html">
+<a href="https://www.milter.org/developers/api/index">
 sendmail libmilter API</a>.  This API lets you register callbacks for 
 a number of events in the
 <a href="http://www.cs.concordia.ca/~group/fig/public/email/relay/milter+ruleset-checks.html">process of sendmail receiving a message via SMTP</a>.  
@@ -121,7 +130,7 @@ copy mail.
 <li> For more ideas, check the <a href="//www.milter.org">Milter Web Page</a>.
 </menu>
 
-<a href="http://www.milter.org/milter_api/api.html">
+<a href="https://www.milter.org/developers/api/index">
 Documentation</a> for the C API is provided with sendmail.  Miltermodule
 provides a thin python wrapper for the C API.  Milter.py provides a simple
 OO wrapper on top of that.
@@ -202,60 +211,8 @@ href="http://www.duh.org/cvsweb.cgi/~checkout~/pmilter/doc/milter-protocol.txt?r
 <h3> Confirmed Installations </h3>
 
 Please <a href="mailto:%73%74%75%61%72%74%40%62%6D%73%69%2E%63%6F%6D">email</a>
-me if you successfully install milter on a system not mentioned below.
-<p>
-<table>
-<tr>
-<th>Operating System</th> <th>Compiler</th> <th>Python</th> <th>Sendmail</th>
-<th>milter</th>
-<tr>
-<td>Mandrake 8.0</td><td>gcc-3.0.1</td><td>2.1.1</td><td>8.12.0</td>
-<td>0.3.3</td><tr>
-<td>Mandrake 8.0</td><td>gcc-2.96</td><td>2.0</td><td>8.11.2</td>
-<td>0.3.6</td><tr>
-<td>RedHat 6.2</td><td>egcs-1.1.2</td><td>2.2.2</td><td>8.11.6</td>
-<td>0.5.4</td><tr>
-<td>RedHat 7.1</td><td>gcc-2.96</td><td>?</td><td>8.12.1</td>
-<td>0.3.5</td><tr>
-<td>RedHat 7.3</td><td>gcc-2.96</td><td>2.2.2</td><td>8.11.6</td>
-<td>0.5.5</td><tr>
-<td>RedHat 7.3</td><td>gcc-2.96</td><td>2.3.3</td><td>8.13.1</td>
-<td>0.7.2</td><tr>
-<td>RedHat 7.3</td><td>gcc-2.96</td><td>2.4.1</td><td>8.13.5</td>
-<td>0.8.4</td><tr>
-<td>RedHat 8.0</td><td>gcc-3.2</td><td>2.2.1</td><td>8.12.6</td>
-<td>0.5.2</td><tr>
-<td>RedHat 9.0</td><td>gcc-3.2.2</td><td>2.4.1</td><td>8.13.1</td>
-<td>0.8.2</td><tr>
-<td>RedHat EL3</td><td>gcc-3.2.3</td><td>2.4.1</td><td>8.13.5</td>
-<td>0.8.4</td><tr>
-<td>Debian Linux</td><td>gcc-2.95.2</td><td>2.1.1</td><td>8.12.0</td>
-<td>0.3.7</td><tr>
-<td>Debian Linux</td><td>gcc-3.2.2</td><td>2.2.2</td><td>8.12.7</td>
-<td>0.5.4</td><tr>
-<td>AIX-4.1.5</td><td>gcc-2.95.2</td><td>2.1.1</td><td>8.11.5</td>
-<td>0.3.3</td><tr>
-<td>AIX-4.1.5</td><td>gcc-2.95.2</td><td>2.1.1</td><td>8.12.1</td>
-<td>0.3.4</td><tr>
-<td>AIX-4.1.5</td><td>gcc-2.95.2</td><td>2.1.3</td><td>8.12.3</td>
-<td>0.4.2</td><tr>
-<td>AIX-4.1.5</td><td>gcc-2.95.2</td><td>2.4.1</td><td>8.13.1</td>
-<td>0.8.4</td><tr>
-<td>Slackware 7.1</td><td>?</td><td>?</td><td>8.12.1</td>
-<td>0.3.8</td><tr>
-<td>Slackware 9.0</td><td>gcc-3.2.2</td><td>2.2.3</td><td>8.12.9</td>
-<td>0.5.4</td><tr>
-<td>OpenBSD</td><td>?</td><td>2.3.3?</td><td>8.13.1?</td>
-<td>0.7.2</td><tr>
-<td>SuSE 7.3</td><td>gcc-2.95.3</td><td>2.1.1</td><td>8.12.2</td>
-<td>0.3.9</td><tr>
-<td>FreeBSD</td><td>gcc-2.95.3</td><td>2.2.1</td><td>8.12.3</td>
-<td>0.4.0</td><tr>
-<td>FreeBSD</td><td>gcc-2.95.3</td><td>2.2.2</td><td>?</td>
-<td>0.5.5</td><tr>
-<td>FreeBSD 4.4</td><td>gcc-2.95.3</td><td>?</td><td>8.12.10</td>
-<td>0.6.6</td>
-</table>
+me if you do <i>not</i> successfully install milter.  The confirmed
+installations are too numerous to list at this point.
 
 <h2> Enough Already! </h2>
 

doc/policy.ht

@@ -82,7 +82,7 @@ to determine the official SPF policy result.
 The offical SPF result is then logged in the Received-SPF header field,
 but certain results are subjected to further processing to create
 an effective result for policy purposes.
-
+<p>
 If the official result is 'none', we try to turn it into an effective result of
 'pass' or 'fail'.  First, we check for a local substitute SPF record
 under the domain defined in the <code>[spf]delegate</code> configuration.  
@@ -91,12 +91,12 @@ too clueless to add their own.  If there is no local substitute, we use a "best
 guess" SPF record of "v=spf1 a/24 mx/24 ptr" for MAIL FROM or "v=spf1 a/24
 mx/24" for HELO.  In addition, a HELO that is a subdomain of MAIL FROM and
 resolves to the connect IP results in an effective result of 'pass'.
-
+<p>
 If there is no local SPF record, and the effective result is still not
 'pass', we check for either a valid HELO name or a valid PTR record for
 the connect IP.  A valid HELO or PTR cannot look like a dynamic name
 as determined by the heuristic in <code>Milter.dynip</code>.
-
+<p>
 If HELO has an SPF record, and the result is anything but pass, we reject
 the connection:
 <pre>
@@ -107,16 +107,16 @@ the connection:
 </pre>
 Note that HELO does not have any forwarding issues like MAIL FROM, and so
 any result other than 'pass' or 'none' should be treated like 'fail'.
-
+<p>
 Only if nothing about the SMTP envelope can be validated does the effective
 result remain 'none.  I call this the "3 strikes" rule.
-
+<p>
 If the official result is 'permerror' (a syntax error in the sender's
 policy), we use the 'lax' option in pyspf to try various heuristics to guess 
 what they really meant.  For instance, the invalid mechanism "ip:1.2.3.4" is
 treated as "ip4:1.2.3.4".  The result of lax processing is then used
 as the effective result for policy purposes.
-
+<p>
 With an effective SPF result in hand, we consult the sendmail access
 database to find our receiver policy for the sender.  
 
@@ -159,7 +159,7 @@ SPF-Fail:abeb@adelphia.net     DSN
 This says to accept mail from that adelphia.net user despite the
 SPF fail, but only after annoying them with a DSN about their ISP's broken
 policy. 
-
+<p>
 If there is no match on the full sender, the domain is checked:
 <pre>
 SPF-Neutral:aol.com     REJECT
@@ -168,7 +168,7 @@ This says to reject mail from AOL with an SPF result of neutral.
 This means AOL users can't use their AOL address with another mail service
 to send us mail.  This is good because the other mail service is 
 likely a badly configured greeting card site or a virus.
-
+<p>
 Finally, a default policy for the result is checked.  While there are program
 defaults, you should have defaults in the access database for SPF results:
 <pre>
@@ -192,3 +192,58 @@ independently for each SPF result and sender combination.  So aol.com:neutral
 might have a really bad reputation, while aol.com:pass would be ok.
 Furthermore, when a sender finally publishes an SPF policy and starts
 getting SPF pass, their reputation is effectively reset.
+
+<h2> Whitelists and Blacklists </h2>
+
+The administrator can whitelist or blacklist senders and sending domains by
+appending them to <code>${datadir}/auto_whitelist.log</code> or
+<code>${datadir}/blacklist.log</code> respectively.  In addition,
+recipients of internal senders (except for automatic replies like vacation
+messages and return receipts) are automatically whitelisted for 60 days, and
+senders that fail CBV or DSN checks are automatically blacklisted for 30 days.
+Whitelisted and blacklisted senders are used to automatically train the
+bayesian content filter before being delivered or rejected, respectively.
+<p>
+Real Soon Now users will be able to maintain their own whitelist and
+blacklist that applies only when they are the recipient.
+
+<h2> Content Filter </h2>
+
+Most messages have been rejected or delivered by now, but spammers
+are always finding new places to send their junk from.  For instance,
+we get around 10000 emails a day, of which around 500 are first time
+spam senders.  A bayesian filter is trained by the whitelists and
+blacklists, and scores the message.  What is likely spam is either
+rejected or quarantined.  If the sender is an effective SPF pass,
+then they get a DSN notifying them that their message has been
+quarantined.  (A DSN failure gets the sender auto blacklisted.)
+Else, if the reject_spam option is set, the message is rejected.  
+Otherwise, a CBV is done (failure gets the sender auto blacklisted)
+and the message is silently quarantined.
+<p>
+Normally, you don't want email messages to silently disappear into
+a black hole, so you should set the reject_spam option.  However,
+if you don't want your correspondent's email to get rejected, you can
+check your quarantine frequently instead.
+
+<h3> Honeypot </h3>
+
+You can also blacklist recipients by listing them as aliases of the
+'honeypot' dspam user.  These are collectively called
+the honeypot.  Any email to these recipients is used to train the
+spam filter as spam and chalk up a reputation demerit for the sender, then
+discarded.  It might be a good idea to blacklist the sender if it has SPF pass
+as well, but I'm afraid of accidents.
+
+<h3> Reputation </h3>
+
+Reputation is tracked by sending domain and effective SPF result.
+The GOSSiP server tracks the spam/ham status of the last 1024 messages
+for each domain:result combination.  When the server is queried during
+the SMTP envelope phase (MAIL FROM), it also queries any configured
+peers, and the scores are combined.  Domains with a history of spam for
+a given SPF result are rejected at MAIL FROM.  The GOSSiP system has
+a command line utility to reset (delete) a reputation for cases where a
+sender that was infected with malware is repaired.  In addition,
+the confidence score of a reputation decays with time, so a bad sender
+will eventually be able to try again without manual intervention.

fail.txt

@@ -0,0 +1,35 @@
+To: %(sender)s
+From: postmaster@%(receiver)s
+Subject: SPF fail (EMAIL FORGERY)
+Auto-Submitted: auto-generated (configuration error)
+
+This is an automatically generated Delivery Status Notification.
+
+*** WARNING! YOU ARE SENDING FROM AN UNAUTHORIZED LOCATION ***
+
+The email administrator for '%(sender_domain)' (YOUR administrator)
+has FORBIDDEN you to send email from this location.  IMMEDIATELY contact your
+email administrator and follow his instructions to properly send mail.
+
+THIS IS A WARNING MESSAGE ONLY.
+
+YOU DO *NOT* NEED TO RESEND YOUR MESSAGE.
+
+Delivery to the following recipients has been delayed.
+
+       %(rcpt)s
+
+Subject: %(subject)s
+Received-SPF: %(spf_result)s
+
+Your sender policy indicated that the above email was forged.
+Because we believe your policy is in error, we have accepted the
+email anyway.  Please ask your email administrator to review
+your SPF policy.  You may also have neglected to follow your
+postmaster's instructions for configuring outgoing email.
+
+If you need further assistance, please do not hesitate to contact me.
+
+Kind regards,
+Stuart D Gathman
+postmaster@%(receiver)s

milter.cfg

@@ -227,3 +227,9 @@ blind = 1
 # domains.  Peer reputation is also tracked as to how often they
 # agree with us, and weighted accordingly.
 ;peers=host1:port,host2
+
+[greylist]
+dbfile=greylist.db
+grey_time=5     # mins (Google retries in 5 mins)
+grey_expire=6   # hours (some legit sites don't retry for 6 hours)
+grey_retain=36  # days (keep "first monday" type mailings on file)

pymilter.spec

@@ -3,8 +3,8 @@
 # rpmbuild -ba --target=i386,noarch pymilter.spec
 
 %define __python python2.4
-%define version 0.8.10
-%define release 2%{?dist}.py24
+%define version 0.8.11
+%define release 1%{?dist}.py24
 # what version of RH are we building for?
 %define redhat7 0
 
@@ -25,7 +25,7 @@
 %ifos aix4.1
 %define libdir /var/log/milter
 %else
-%define libdir /usr/lib/pymilter
+%define libdir %{_libdir}/pymilter
 %endif
 
 %ifarch noarch
@@ -39,9 +39,7 @@ Source: pymilter-%{version}.tar.gz
 License: GPL
 Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-buildroot
-Prefix: %{_prefix}
 Vendor: Stuart D. Gathman <stuart@bmsi.com>
-Packager: Stuart D. Gathman <stuart@bmsi.com>
 Url: http://www.bmsi.com/python/milter.html
 Requires: %{__python} >= 2.4, pyspf >= 2.0.4, pymilter
 %ifos Linux
@@ -49,21 +47,23 @@ Requires: chkconfig
 %endif
 
 %description -n milter
-A complex but effective spam filtering, SPF checking, and reputation tracking
-mail application.  It uses pydspam if installed for bayesian filtering.
+A complex but effective spam filtering, SPF checking, greylisting,
+and reputation tracking mail application.  It uses pydspam if installed for
+bayesian filtering.
 
 %package spf
 Group: Applications/System
 Summary:  BMS spam and reputation milter
 Requires: pyspf >= 2.0.4, pymilter
-Obsoletes: pymilter-spf
+Obsoletes: pymilter-spf < 0.8.10
 
 %description spf
 A simple mail filter to add Received-SPF headers and reject forged mail.
-Rejection policy is configured via sendmail access file.
+Rejection policy is configured via sendmail access file and can be
+tailored by domain.
 
 %prep
-%setup -n pymilter-%{version}
+%setup -q -n pymilter-%{version}
 #patch -p0 -b .bms
 
 %install
@@ -73,7 +73,7 @@ mkdir -p $RPM_BUILD_ROOT/etc/mail
 mkdir $RPM_BUILD_ROOT/var/log/milter/save
 mkdir -p $RPM_BUILD_ROOT%{libdir}
 cp *.txt $RPM_BUILD_ROOT/var/log/milter
-cp bms.py spfmilter.py $RPM_BUILD_ROOT%{libdir}
+cp -p bms.py spfmilter.py ban2zone.py $RPM_BUILD_ROOT%{libdir}
 cp milter.cfg $RPM_BUILD_ROOT/etc/mail/pymilter.cfg
 cp spfmilter.cfg $RPM_BUILD_ROOT/etc/mail
 
@@ -172,6 +172,7 @@ fi
 %dir /var/log/milter
 %dir /var/log/milter/save
 %config %{libdir}/bms.py
+%config %{libdir}/ban2zone.py
 %config(noreplace) /var/log/milter/strike3.txt
 %config(noreplace) /var/log/milter/softfail.txt
 %config(noreplace) /var/log/milter/fail.txt
@@ -201,9 +202,7 @@ Source: %{name}-%{version}.tar.gz
 License: GPL
 Group: Development/Libraries
 BuildRoot: %{_tmppath}/%{name}-buildroot
-Prefix: %{_prefix}
 Vendor: Stuart D. Gathman <stuart@bmsi.com>
-Packager: Stuart D. Gathman <stuart@bmsi.com>
 Url: http://www.bmsi.com/python/milter.html
 Requires: %{__python} >= 2.4, sendmail >= 8.13
 BuildRequires: %{__python}-devel >= 2.4, sendmail-devel >= 8.13
@@ -215,7 +214,7 @@ modules provide for navigating and modifying MIME parts, sending
 DSNs, and doing CBV.
 
 %prep
-%setup
+%setup -q
 #patch -p0 -b .bms
 
 %build
@@ -235,8 +234,6 @@ mkdir -p $RPM_BUILD_ROOT%{libdir}
 cat >$RPM_BUILD_ROOT%{libdir}/start.sh <<'EOF'
 #!/bin/sh
 cd /var/log/milter
-# uncomment to enable sgmlop if installed
-#export PYTHONPATH=/usr/local/lib/python2.1/site-packages
 exec /usr/local/bin/python bms.py >>milter.log 2>&1
 EOF
 %else # not aix4.1
@@ -282,7 +279,7 @@ rm -rf $RPM_BUILD_ROOT
 - Allow explicitly whitelisted email from banned_users.
 - configure gossip TTL
 * Mon Sep 24 2007 Stuart Gathman <stuart@bmsi.com> 0.8.9-1
-- Use %ifarch hack to build milter and milter-spf packages as noarch
+- Use ifarch hack to build milter and milter-spf packages as noarch
 - Remove spf dependency from dsn.py, add dns.py
 * Fri Jan 05 2007 Stuart Gathman <stuart@bmsi.com> 0.8.8-1
 - move AddrCache, parse_addr, iniplist to Milter package

setup.py

@@ -16,7 +16,7 @@ if sys.version < '2.2.3':
   DistributionMetadata.download_url = None
 
 # NOTE: importing Milter to obtain version fails when milter.so not built
-setup(name = "pymilter", version = '0.8.10',
+setup(name = "pymilter", version = '0.8.11',
 	description="Python interface to sendmail milter API",
 	long_description="""\
 This is a python extension module to enable python scripts to