This commit was manufactured by cvs2svn to create tag 'milter-0_8_2'.

Sprout from bmsi 2005-05-31 18:23:49 UTC Stuart Gathman <stuart@gathman.org> 'Development changes since 0.7.2' Cherrypick from master 2005-07-20 14:56:38 UTC Stuart Gathman <stuart@gathman.org> 'Handle corrupt ZIP attachments': COPYING CREDITS MANIFEST.in Milter/__init__.py Milter/dsn.py Milter/dynip.py NEWS TODO bms.py faq.html milter.cfg milter.html milter.spec miltermodule.c mime.py setup.cfg setup.py softfail.txt spf.py spfquery.py strike3.txt test/zip1 test/zip2 test/zip3 test/ziploop testmime.py
2005-07-20 14:56:39 +00:00
16 changed files with 727 additions and 303 deletions
@@ -9,6 +9,9 @@ Other contributors:

 Terence Way
  for providing a Python port of SPF
+Scott Kitterman
+  for doing lots of testing and debugging of SPF against draft standard,
+  and for putting up a web page that validates SPF records using spf.py
 Alexander Kourakos
  for plugging several memory leaks
 George Graf at Vienna University of Economics and Business Administration
@@ -22,6 +25,9 @@ John Draper
  then pointing out that it would be easier to just write the MTA in Python.
 Eric S. Johansson
  for helpful design discussions while working on camram
+Alex Savguira
+  for finding bugs with international headers and
+  suggesting the scan_zip option.
 Business Management Systems - http://www.bmsi.com
  for hosting the website, and providing paying clients who need milter service
  so I can work on it as part of my day job.
@@ -11,6 +11,7 @@ include testdspam.py
 include rejects.py
 include bms.py
 include spf.py
+include cid2spf.py
 include spfquery.py
 include test.py
 include sample.py
@@ -1,8 +1,17 @@
 Here is a history of user visible changes to Python milter.

+0.8.2	Strict processing limits per SPF RFC
+	Fixed several parsing bugs under RFC 
+	Support official IANA SPF record (type99)
+	Honeypot support (requires pydspam-1.1.9)
+	Extended SPF processing results beyond strict RFC limits
+	Support original SES for bounce protection (requires pysrs-0.30.10)
+	Callback exception processing option in milter module
+	Handle corrupt ZIP attachments
 0.8.1	Fix zip in zip loop in mime.py
 	Fix HeaderParseError in bms.py header callback
 	Check internal_domains for outgoing mail
+	Fix inconsistent results from send_dsn
 0.8.0	Move Milter module to subpackage.
 	DSN support for Three strikes rule and SPF SOFTFAIL
 	Move /*mime*/ and dynip to Milter subpackage
@@ -1,6 +1,30 @@
 #!/usr/bin/env python
 # A simple milter that has grown quite a bit.
 # $Log$
+# Revision 1.18  2005/07/17 01:25:44  customdesigned
+# Log as well as use extended result for best guess.
+#
+# Revision 1.17  2005/07/15 20:25:36  customdesigned
+# Use extended results processing for best_guess.
+#
+# Revision 1.16  2005/07/14 03:23:33  customdesigned
+# Make SES package optional.  Initial honeypot support.
+#
+# Revision 1.15  2005/07/06 04:05:40  customdesigned
+# Initial SES integration.
+#
+# Revision 1.14  2005/07/02 23:27:31  customdesigned
+# Don't match hostnames for internal connects.
+#
+# Revision 1.13  2005/07/01 16:30:24  customdesigned
+# Always log trusted Received and Received-SPF headers.
+#
+# Revision 1.12  2005/06/20 22:35:35  customdesigned
+# Setreply for rejectvirus.
+#
+# Revision 1.11  2005/06/17 02:07:20  customdesigned
+# Release 0.8.1
+#
 # Revision 1.10  2005/06/16 18:35:51  customdesigned
 # Ignore HeaderParseError decoding header
 #
@@ -233,6 +257,9 @@ try:
  import SRS
  srsre = re.compile(r'^SRS[01][+-=]',re.IGNORECASE)
 except: SRS = None
+try:
+  import SES
+except: SES = None

 # Import spf if available
 try: import spf
@@ -277,8 +304,9 @@ dspam_internal = True	# True if internal mail should be dspammed
 dspam_reject = ()
 dspam_sizelimit = 180000
 srs = None
+ses = None
 srs_reject_spoofed = False
-srs_fwdomain = None
+srs_domain = None
 spf_reject_neutral = ()
 spf_accept_softfail = ()
 spf_best_guess = False
@@ -446,7 +474,7 @@ def read_config(list):
  if srs_config: cp.read([srs_config])
  srs_secret = cp.getdefault('srs','secret')
  if SRS and srs_secret:
-    global srs,srs_reject_spoofed,srs_fwdomain
+    global ses,srs,srs_reject_spoofed,srs_domain
    database = cp.getdefault('srs','database')
    srs_reject_spoofed = cp.getboolean('srs','reject_spoofed')
    maxage = cp.getint('srs','maxage')
@@ -459,7 +487,13 @@ def read_config(list):
    else:
      srs = SRS.Guarded.Guarded(secret=srs_secret,
        maxage=maxage,hashlength=hashlength,separator=separator)
-    srs_fwdomain = cp.getdefault('srs','fwdomain')
+    if SES:
+      ses = SES.new(secret=srs_secret,expiration=maxage)
+      srs_domain = cp.getlist('srs','ses')
+    else:
+      srs_domain = []
+    srs_domain.append(cp.getdefault('srs','fwdomain'))
+    #print srs_domain

 def parse_addr(t):
  if t.startswith('<') and t.endswith('>'): t = t[1:-1]
@@ -485,7 +519,7 @@ def parse_header(val):
      except UnicodeError: continue
  except UnicodeDecodeError: pass
  except LookupError: pass
-  except email.errors.HeaderParseError: pass
+  except email.Errors.HeaderParseError: pass
  return val

 class bmsMilter(Milter.Milter):
@@ -543,10 +577,6 @@ class bmsMilter(Milter.Milter):
    else: ipaddr = ''
    self.connectip = ipaddr
    self.missing_ptr = dynip(hostname,self.connectip)
-    for pat in internal_connect:
-      if fnmatchcase(hostname,pat):
-	self.internal_connection = True
-	break
    if self.internal_connection:
      connecttype = 'INTERNAL'
    else:
@@ -636,7 +666,7 @@ class bmsMilter(Milter.Milter):
 	else:
 	  self.log("REJECT: zombie PC at ",self.connectip," sending MAIL FROM ",
 	  	self.canon_from)
-	  self.setreply('550','5.7.1','Get rid of your virus!',
+	  self.setreply('550','5.7.1',
 	  'Your PC is using an unauthorized MAIL FROM.',
 	  'It is either badly misconfigured or controlled by organized crime.'
 	  )
@@ -666,7 +696,8 @@ class bmsMilter(Milter.Milter):
    if len(t) == 2: t[1] = t[1].lower()
    receiver = self.receiver
    q = spf.query(self.connectip,'@'.join(t),self.hello_name,receiver=receiver)
-    q.set_default_explanation('SPF fail: see http://spf.pobox.com/why.html')
+    q.set_default_explanation(
+      'SPF fail: see http://spf.pobox.com/why.html?sender=%s&ip=%s' % (q.s,q.i))
    res,code,txt = q.check()
    if res in ('none', 'softfail'):
      if self.mailfrom != '<>':
@@ -690,6 +721,7 @@ class bmsMilter(Milter.Milter):
 	#self.log('SPF: no record published, guessing')
 	q.set_default_explanation(
 		'SPF guess: see http://spf.pobox.com/why.html')
+	q.strict = False
 	# best_guess should not result in fail
 	if self.missing_ptr:
 	  # ignore dynamic PTR for best guess
@@ -697,6 +729,9 @@ class bmsMilter(Milter.Milter):
 	else:
 	  res,code,txt = q.best_guess()
 	receiver += ': guessing'
+        if q.perm_error:
+          res,code,txt = q.perm_error.ext	# extended result
+	  txt = 'EXT: ' + txt
      if self.missing_ptr and res in ('neutral', 'none') and hres != 'pass':
 	if spf_reject_noptr:
 	  self.log('REJECT: no PTR, HELO or SPF')
@@ -743,12 +778,12 @@ class bmsMilter(Milter.Milter):
 	'servers for %s should accomplish this.' % q.o
      )
      return Milter.REJECT
+    if res == 'unknown':
+      self.log('REJECT: SPF %s %i %s' % (res,code,txt))
+      # latest SPF draft recommends 5.5.2 instead of 5.7.1
+      self.setreply(str(code),'5.5.2',txt)
+      return Milter.REJECT
    if res == 'error':
-      if code >= 500:
-        self.log('REJECT: SPF %s %i %s' % (res,code,txt))
-	# latest SPF draft recommends 5.5.2 instead of 5.7.1
-	self.setreply(str(code),'5.5.2',txt)
-	return Milter.REJECT
      self.log('TEMPFAIL: SPF %s %i %s' % (res,code,txt))
      self.setreply(str(code),'4.3.0',txt)
      return Milter.TEMPFAIL
@@ -771,18 +806,29 @@ class bmsMilter(Milter.Milter):
      	or self.canon_from.startswith('mailer-daemon@'):
        if self.recipients and not multiple_bounce_recipients:
 	  self.data_allowed = False
-        if srs and domain == srs_fwdomain:
+        if srs and domain in srs_domain:
 	  oldaddr = '@'.join(parse_addr(to))
 	  try:
-	    newaddr = srs.reverse(oldaddr)
-	    # Currently, a sendmail map reverses SRS.  We just log it here.
-	    self.log("srs rcpt:",newaddr)
+	    if ses:
+	      newaddr = ses.verify(oldaddr)
+	    else:
+	      newaddr = oldaddr,
+            if len(newaddr) > 1:
+	      self.log("ses rcpt:",newaddr[0])
+	    else:
+	      newaddr = srs.reverse(oldaddr)
+	      # Currently, a sendmail map reverses SRS.  We just log it here.
+	      self.log("srs rcpt:",newaddr)
 	  except:
            if not (self.internal_connection or self.trusted_relay):
 	      if srsre.match(oldaddr):
 		self.log("REJECT: srs spoofed:",oldaddr)
 		self.setreply('550','5.7.1','Invalid SRS signature')
 		return Milter.REJECT
+	      if oldaddr.startswith('SES='):
+		self.log("REJECT: ses spoofed:",oldaddr)
+		self.setreply('550','5.7.1','Invalid SES signature')
+		return Milter.REJECT
 	      self.data_allowed = not srs_reject_spoofed
      # non DSN mail to SRS address will bounce due to invalid local part
      self.recipients.append('@'.join(t))
@@ -858,12 +904,6 @@ class bmsMilter(Milter.Milter):
 	or mailer.find('optin') >= 0:
        self.log('REJECT: %s: %s' % (name,val))
        return Milter.REJECT
-    elif self.trust_received and lname == 'received':
-      self.trust_received = False
-      self.log('%s: %s' % (name,val.splitlines()[0]))
-    elif self.trust_spf and lname == 'received-spf':
-      self.trust_spf = False
-      self.log('%s: %s' % (name,val.splitlines()[0]))
    return Milter.CONTINUE

  def forged_bounce(self):
@@ -898,6 +938,12 @@ class bmsMilter(Milter.Milter):
    # log selected headers
    if log_headers or lname in ('subject','x-mailer'):
      self.log('%s: %s' % (name,val))
+    elif self.trust_received and lname == 'received':
+      self.trust_received = False
+      self.log('%s: %s' % (name,val.splitlines()[0]))
+    elif self.trust_spf and lname == 'received-spf':
+      self.trust_spf = False
+      self.log('%s: %s' % (name,val.splitlines()[0]))
    if self.fp:
      try:
        val = val.encode('us-ascii')
@@ -1028,6 +1074,11 @@ class bmsMilter(Milter.Milter):
 	      if len(txt) > dspam_sizelimit:
 		self.log("Large message:",len(txt))
 		return False
+	      if user == 'honeypot' and Dspam.VERSION >= '1.1.9':
+	        ds.check_spam(user,txt,force_result=dspam.DSR_ISSPAM)
+		self.log("HONEYPOT:",rcpt)
+		self.fp = None
+		return False
 	      txt = ds.check_spam(user,txt,self.recipients)
 	      if not txt:
 	        # DISCARD if quarrantined for any recipient.  It
@@ -1181,6 +1232,8 @@ class bmsMilter(Milter.Milter):
    if defanged:
      if self.rejectvirus and not self.hidepath:
 	self.log("REJECT virus from",self.mailfrom)
+	self.setreply('550','5.7.1','Attachment type not allowed.',
+		'You attempted to send an attachment with a banned extension.')
 	self.tempname = None
 	return Milter.REJECT
      self.log("Temp file:",self.tempname)
@@ -141,6 +141,36 @@ is a milter declaration for sendmail.cf with all timeouts specified:
 <pre>
 Xpythonfilter, S=local:/var/log/milter/pythonsock, F=T, T=C:5m;S:20s;R:60s;E:5m
 </pre>
+<li> Q. There is a Python traceback in the log file!  What happened to
+	my email?
+<p>  A. When the milter fails with an untrapped exception, a TEMPFAIL
+result (451) is returned to the sender.  The sender will then retry every
+hour or so for several days.  Hopefully, someone will notice the 
+traceback, and workaround or fix the problem.
+
+<li> Q. I read some notes such as "Check valid domains allowed by internal
+	senders to detect PCs infected with spam trojans." but could not
+	understand the idea. Could you clarify the content ?
+
+<p>  A. The <code>internal_domains</code> configuration specifies which
+MAIL FROM domains are used by internal connections.  If an internal
+PC tries to use some other domain, it is assumed to be a "Zombie".
+<p>
+Here is a sample log line:
+<pre>
+2005Jun22 12:01:04 [12430] REJECT: zombie PC at  192.168.100.171  sending MAIL FROM  debby@fedex.com
+</pre>
+No, fedex.com does not use pymilter, and there is no one named debby at my
+client.  But the idiot using the PC at 192.168.100.171 has downloaded and
+installed some stupid weatherbar/hotbar/aquariumscreensaver that is actually a
+spam bot.
+<p>
+The <code>internal_domains</code> option is simplistic, it assumes all
+valid senders of the domains are internal.  SPF provides a much more general
+check of IP and MAIL FROM for external email.  Pymilter should soon
+have a local policy feature for more general checking of internal mail.
+
+<h3> Using SPF </h3>

 <a name="spf">
 <li> Q. So how do I use the SPF support?  The sample.py milter doesn't seem
@@ -50,7 +50,7 @@ porn_words = penis, breast, pussy, horse cock, porn, xenical, diet pill, d1ck,
 	p-e-n-i-s, hydrocodone, vicodin, xanax, vicod1n, x@nax, diazepam,
 	v1@gra, xan@x, cialis, ci@lis, frëe, xãnax, valíum, vãlium, via-gra,
 	x@n3x, vicod3n, penís, c0d1n, phentermine, en1arge, dip1oma, v1codin,
-	valium, rolex, sexual, fuck
+	valium, rolex, sexual, fuck, adv1t
 # reject mail with these case sensitive strings in the subject
 spam_words = $$$, !!!, XXX, FREE, HGH
 # attachments with these extensions will be replaced with a warning
@@ -152,6 +152,8 @@ blind = 1
 ;spam=spam@foocorp.com
 # address to forward false positives to.  milter will process and not deliver
 ;falsepositive=ham@foocorp.com
+# account which receives only spam: all received messages are marked as spam.
+;honeypot=spam-me@example.com
 # the dspam_screener is a list of dspam users who screen mail for all
 # recipients who are not dspam_users.  Spam goes to the screeners quarantine,
 # and the original recipients are saved so that false positives can be properly
@@ -124,9 +124,9 @@ recommend ignoring it and continuing to implement and improve SPF until a
 working and unencumbered proposal for RFC2822 headers surfaces.

 <p>
-<a href="http://spf.pobox.com">
+<a href="http://openspf.com">
 <img src="SPF.gif" align=left alt="SPF logo"></a>
-Release 0.6.6 adds support for <a href="http://spf.pobox.com/">SPF</a>,
+Release 0.6.6 adds support for <a href="http://openspf.com/">SPF</a>,
 a protocol to prevent forging of the envelope from address.  
 SPF support requires <a href="http://pydns.sourceforge.net/">pydns</a>.
 The included spf.py module is an updated version of the original 1.6
@@ -222,7 +222,7 @@ methods that
 do nothing, and also provides wrappers for the libmilter methods to mutate
 the message.
 <p>
-The 'spf' module provides an implementation of <a href="http://spf.pobox.com">
+The 'spf' module provides an implementation of <a href="http://openspf.com">
 SPF</a> useful for detecting email forgery.
 <p>
 The 'mime' module provides a wrapper for the Python email package that
@@ -1,6 +1,6 @@
 %define name milter
-%define version 0.8.1
-%define release 1.RH7
+%define version 0.8.2
+%define release 2.RH7
 # what version of RH are we building for?
 %define redhat9 0
 %define redhat7 1
@@ -166,10 +166,20 @@ rm -rf $RPM_BUILD_ROOT
 /usr/share/sendmail-cf/hack/rhsbl.m4

 %changelog
+* Fri Jul 15 2005 Stuart Gathman <stuart@bmsi.com> 0.8.2-1
+- Strict processing limits per SPF RFC
+- Fixed several parsing bugs under RFC 
+- Support official IANA SPF record (type99)
+- Honeypot support (requires pydspam-1.1.9)
+- Extended SPF processing results beyond strict RFC limits
+- Support original SES for local bounce protection (requires pysrs-0.30.10)
+- Callback exception processing option in milter module
+- Handle corrupt ZIP attachments
 * Thu Jun 16 2005 Stuart Gathman <stuart@bmsi.com> 0.8.1-1
 - Fix zip in zip loop in mime.py
 - Fix HeaderParseError in bms.py header callback
 - Check internal_domains for outgoing mail
+- Fix inconsistent results from send_dsn
 * Mon Jun 06 2005 Stuart Gathman <stuart@bmsi.com> 0.8.0-3
 - properly log pydspam exceptions
 * Sat Jun 04 2005 Stuart Gathman <stuart@bmsi.com> 0.8.0-2
@@ -34,6 +34,18 @@ $ python setup.py help
     libraries=["milter","smutil","resolv"]

 * $Log$
+ * Revision 1.5  2005/06/24 04:20:07  customdesigned
+ * Report context allocation error.
+ *
+ * Revision 1.4  2005/06/24 04:12:43  customdesigned
+ * Remove unused name argument to generic wrappers.
+ *
+ * Revision 1.3  2005/06/24 03:57:35  customdesigned
+ * Handle close called before connect.
+ *
+ * Revision 1.2  2005/06/02 04:18:55  customdesigned
+ * Update copyright notices after reading article on /.
+ *
 * Revision 1.1.1.2  2005/05/31 18:09:06  customdesigned
 * Release 0.7.1
 *
@@ -194,7 +206,7 @@ $ python setup.py help


 /* Yes, these are static.  If you need multiple different callbacks, */
-/* it's cleaner to use multiple filters. */
+/* it's cleaner to use multiple filters, or convert to OO method calls. */
 static PyObject *connect_callback = NULL;
 static PyObject *helo_callback    = NULL;
 static PyObject *envfrom_callback = NULL;
@@ -239,8 +251,11 @@ _get_context(SMFICTX *ctx) {
    PyEval_AcquireThread(t);	/* lock interp */
    self = PyObject_New(milter_ContextObject,&milter_ContextType);
    if (!self) {
-      /* Can't pass on exception since we are called from libmilter */
-      PyErr_Clear();
+      /* Report and clear exception since we are called from libmilter */
+      if (PyErr_Occurred()) {
+	PyErr_Print();
+	PyErr_Clear();
+      }
      PyThreadState_Clear(t);
      PyEval_ReleaseThread(t);
      PyThreadState_Delete(t);
@@ -331,7 +346,8 @@ CHGHDRS - filter may change/delete headers";

 static PyObject *
 milter_set_flags(PyObject *self, PyObject *args) {
-  if (!PyArg_ParseTuple(args, "i", &description.xxfi_flags)) return NULL;
+  if (!PyArg_ParseTuple(args, "i:set_flags", &description.xxfi_flags))
+    return NULL;
  Py_INCREF(Py_None);
  return Py_None;
 }
@@ -497,6 +513,28 @@ milter_set_close_callback(PyObject *self, PyObject *args) {
  return generic_set_callback(args, "O:set_close_callback", &close_callback);
 }

+static int exception_policy = SMFIS_TEMPFAIL;
+
+static char milter_set_exception_policy__doc__[] =
+"set_exception_policy(i) -> None\n\
+Sets the policy for untrapped Python exceptions during a callback.\n\
+Must be one of TEMPFAIL,REJECT,CONTINUE";
+
+static PyObject *
+milter_set_exception_policy(PyObject *self, PyObject *args) {
+  int i;
+  if (!PyArg_ParseTuple(args, "i:set_exception_policy", &i))
+    return NULL;
+  switch (i) {
+  case SMFIS_REJECT: case SMFIS_TEMPFAIL: case SMFIS_CONTINUE:
+    exception_policy = i;
+    Py_INCREF(Py_None);
+    return Py_None;
+  }
+  PyErr_SetString(MilterError,"invalid exception policy");
+  return NULL;
+}
+
 /** Report and clear any python exception before returning to libmilter. 
  The interpreter is locked when we are called, and we unlock it.  */
 static int _report_exception(milter_ContextObject *self) {
@@ -504,8 +542,15 @@ static int _report_exception(milter_ContextObject *self) {
    PyErr_Print();
    PyErr_Clear();	/* must clear since not returning to python */
    PyEval_ReleaseThread(self->t);
-    smfi_setreply(self->ctx, "451", "4.3.0", "Filter failure");
-    return SMFIS_TEMPFAIL;
+    switch (exception_policy) {
+      case SMFIS_REJECT:
+	smfi_setreply(self->ctx, "554", "5.3.0", "Filter failure");
+	return SMFIS_REJECT;
+      case SMFIS_TEMPFAIL:
+	smfi_setreply(self->ctx, "451", "4.3.0", "Filter failure");
+	return SMFIS_TEMPFAIL;
+    }
+    return SMFIS_CONTINUE;
  }
  PyEval_ReleaseThread(self->t);
  return SMFIS_CONTINUE;
@@ -616,7 +661,7 @@ milter_wrap_helo(SMFICTX *ctx, char *helohost) {
 }

 static int
-generic_env_wrapper(SMFICTX *ctx, PyObject*cb, char **argv, const char *name) {
+generic_env_wrapper(SMFICTX *ctx, PyObject*cb, char **argv) {
   PyObject *arglist;
   milter_ContextObject *self;
   int count = 0;
@@ -653,12 +698,12 @@ generic_env_wrapper(SMFICTX *ctx, PyObject*cb, char **argv, const char *name) {

 static int
 milter_wrap_envfrom(SMFICTX *ctx, char **argv) {
-  return generic_env_wrapper(ctx,envfrom_callback,argv,"milter_wrap_envfrom");
+  return generic_env_wrapper(ctx,envfrom_callback,argv);
 }

 static int
 milter_wrap_envrcpt(SMFICTX *ctx, char **argv) {
-  return generic_env_wrapper(ctx,envrcpt_callback,argv,"milter_wrap_envrcpt");
+  return generic_env_wrapper(ctx,envrcpt_callback,argv);
 }    
  
 static int
@@ -674,7 +719,7 @@ milter_wrap_header(SMFICTX *ctx, char *headerf, char *headerv) {
 }

 static int
-generic_noarg_wrapper(SMFICTX *ctx,PyObject *cb,const char *name) {
+generic_noarg_wrapper(SMFICTX *ctx,PyObject *cb) {
   PyObject *arglist;
   milter_ContextObject *c;
   if (cb == NULL) return SMFIS_CONTINUE;
@@ -686,7 +731,7 @@ generic_noarg_wrapper(SMFICTX *ctx,PyObject *cb,const char *name) {

 static int
 milter_wrap_eoh(SMFICTX *ctx) {
-  return generic_noarg_wrapper(ctx,eoh_callback,"milter_wrap_eoh");
+  return generic_noarg_wrapper(ctx,eoh_callback);
 }   

 static int
@@ -704,18 +749,31 @@ milter_wrap_body(SMFICTX *ctx, u_char *bodyp, size_t bodylen) {

 static int
 milter_wrap_eom(SMFICTX *ctx) {
-  return generic_noarg_wrapper(ctx,eom_callback,"milter_wrap_eom");
+  return generic_noarg_wrapper(ctx,eom_callback);
 }

 static int
 milter_wrap_abort(SMFICTX *ctx) {
  /* libmilter still calls close after abort */
-  return generic_noarg_wrapper(ctx,abort_callback,"milter_wrap_abort");
+  return generic_noarg_wrapper(ctx,abort_callback);
 }

 static int
 milter_wrap_close(SMFICTX *ctx) {
-  int r = generic_noarg_wrapper(ctx,close_callback,"milter_wrap_close");
+  /* xxfi_close can be called out of order - even before connect.  
+   * There may not yet be a private context pointer.  To avoid
+   * creating a ThreadContext and allocating a milter context only
+   * to destroy them, and to avoid invoking the python close_callback when
+   * connect has never been called, we don't use generic_noarg_wrapper here. */
+  PyObject *cb = close_callback;
+  milter_ContextObject *self = smfi_getpriv(ctx);
+  int r = SMFIS_CONTINUE;
+  if (self != NULL && cb != NULL && self->ctx == ctx) {
+    PyObject *arglist;
+    PyEval_AcquireThread(self->t);
+    arglist = Py_BuildValue("(O)", self);
+    r = _generic_wrapper(self, cb, arglist);
+  }
  /* FIXME: It is inefficient to have released the interp lock only to
     acquire it again in _clear_context.  We can tell _generic_return and
     friends not to release the lock by, for instance, setting self->t to NULL.
@@ -1155,6 +1213,8 @@ static PyMethodDef milter_methods[] = {
   { "set_eom_callback",     milter_set_eom_callback,     METH_VARARGS, milter_set_eom_callback__doc__},
   { "set_abort_callback",   milter_set_abort_callback,   METH_VARARGS, milter_set_abort_callback__doc__},
   { "set_close_callback",   milter_set_close_callback,   METH_VARARGS, milter_set_close_callback__doc__},
+   { "set_exception_policy",   milter_set_exception_policy,METH_VARARGS, milter_set_exception_policy__doc__},
+   { "register",             milter_register,             METH_VARARGS, milter_register__doc__},
   { "register",             milter_register,             METH_VARARGS, milter_register__doc__},
   { "main",                 milter_main,                 METH_VARARGS, milter_main__doc__},
   { "setdbg",               milter_setdbg,               METH_VARARGS, milter_setdbg__doc__},
@@ -1,4 +1,7 @@
 # $Log$
+# Revision 1.4  2005/06/17 01:49:39  customdesigned
+# Handle zip within zip.
+#
 # Revision 1.3  2005/06/02 15:00:17  customdesigned
 # Configure banned extensions.  Scan zipfile option with test case.
 #
@@ -193,7 +196,8 @@ class MimeMessage(Message):
      for key,name in tuple(names):	# copy by converting to tuple
 	if name and name.lower().endswith('.zip'):
 	  txt = self.get_payload(decode=True)
-	  names += zipnames(txt)
+	  if txt.strip():
+	    names += zipnames(txt)
    return names

  def ismodified(self):
@@ -304,19 +308,25 @@ See your administrator.

 def check_name(msg,savname=None,ckname=check_ext,scan_zip=False):
  "Replace attachment with a warning if its name is suspicious."
-  for key,name in msg.getnames(scan_zip):
-    badname = ckname(name)
-    if badname:
-      hostname = socket.gethostname()
-      if key == 'zipname':
-        badname = msg.get_filename()
-      msg.set_payload(virus_msg % (badname,hostname,savname))
-      del msg["content-type"]
-      del msg["content-disposition"]
-      del msg["content-transfer-encoding"]
-      name = "WARNING.TXT"
-      msg["Content-Type"] = "text/plain; name="+name
-      break
+  try:
+    for key,name in msg.getnames(scan_zip):
+      badname = ckname(name)
+      if badname:
+        if key == 'zipname':
+          badname = msg.get_filename()
+	break
+    else:
+      return Milter.CONTINUE
+  except zipfile.BadZipfile:
+    # a ZIP that is not a zip is very suspicious
+    badname = msg.get_filename()
+  hostname = socket.gethostname()
+  msg.set_payload(virus_msg % (badname,hostname,savname))
+  del msg["content-type"]
+  del msg["content-disposition"]
+  del msg["content-transfer-encoding"]
+  name = "WARNING.TXT"
+  msg["Content-Type"] = "text/plain; name="+name
  return Milter.CONTINUE

 import email.Iterators
@@ -12,7 +12,7 @@ if sys.version < '2.2.3':
  DistributionMetadata.classifiers = None
  DistributionMetadata.download_url = None

-setup(name = "milter", version = "0.8.1",
+setup(name = "milter", version = "0.8.2",
 	description="Python interface to sendmail milter API",
 	long_description="""\
 This is a python extension module to enable python scripts to
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-"""SPF (Sender-Permitted From) implementation.
+"""SPF (Sender Policy Framework) implementation.

 Copyright (c) 2003, Terence Way
 Portions Copyright (c) 2004,2005 Stuart Gathman <stuart@bmsi.com>
@@ -19,10 +19,11 @@ AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
 SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.

 For more information about SPF, a tool against email forgery, see
-	http://spf.pobox.com
+	http://spf.pobox.com/

 For news, bugfixes, etc. visit the home page for this implementation at
 	http://www.wayforward.net/spf/
+	http://sourceforge.net/projects/pymilter/
 """

 # Changes:
@@ -46,6 +47,108 @@ For news, bugfixes, etc. visit the home page for this implementation at
 # Terrence is not responding to email.
 #
 # $Log$
+# Revision 1.26  2005/07/20 03:12:40  customdesigned
+# When not in strict mode, don't give PermErr for bad mechanism until
+# encountered during evaluation.
+#
+# Revision 1.25  2005/07/19 23:24:42  customdesigned
+# Validate all mechanisms before evaluating.
+#
+# Revision 1.24  2005/07/19 18:11:52  kitterma
+# Fix to change that compares type TXT and type SPF records.  Bug in the change
+# prevented records from being returned if it was published as TXT, but not SPF.
+#
+# Revision 1.23  2005/07/19 15:22:50  customdesigned
+# MX and PTR limits are MUST NOT check limits, and do not result in PermErr.
+# Also, check belongs in mx and ptr specific methods, not in dns() method.
+#
+# Revision 1.22  2005/07/19 05:02:29  customdesigned
+# FQDN test was broken.  Added test case.  Move FQDN test to after
+# macro expansion.
+#
+# Revision 1.21  2005/07/18 20:46:27  kitterma
+# Fixed reference problem in 1.20
+#
+# Revision 1.20  2005/07/18 20:21:47  kitterma
+# Change to dns_spf to go ahead and check for a type 99 (SPF) record even if a
+# TXT record is found and make sure if type SPF is present that they are
+# identical when using strict processing.
+#
+# Revision 1.19  2005/07/18 19:36:00  kitterma
+# Change to require at least one dot in a domain name.  Added PermError
+# description to indicate FQDN should be used.  This is a common error.
+#
+# Revision 1.18  2005/07/18 17:13:37  kitterma
+# Change macro processing to raise PermError on an unknown macro.
+# schlitt-spf-classic-02 para 8.1.  Change exp modifier processing to ignore
+# exp strings with syntax errors.  schlitt-spf-classic-02 para 6.2.
+#
+# Revision 1.17  2005/07/18 14:35:34  customdesigned
+# Remove debugging printf
+#
+# Revision 1.16  2005/07/18 14:34:14  customdesigned
+# Forgot to remove debugging print
+#
+# Revision 1.15  2005/07/15 21:17:36  customdesigned
+# Recursion limit raises AssertionError in strict mode, PermError otherwise.
+#
+# Revision 1.14  2005/07/15 20:34:11  customdesigned
+# Check whether DNS package already supports SPF before patching
+#
+# Revision 1.13  2005/07/15 20:01:22  customdesigned
+# Allow extended results for MX limit
+#
+# Revision 1.12  2005/07/15 19:12:09  customdesigned
+# Official IANA SPF record (type 99) support.
+#
+# Revision 1.11  2005/07/15 18:03:02  customdesigned
+# Fix unknown Received-SPF header broken by result changes
+#
+# Revision 1.10  2005/07/15 16:17:05  customdesigned
+# Start type99 support.
+# Make Scott's "/" support in parse_mechanism more elegant as requested.
+# Add test case for "/" support.
+#
+# Revision 1.9  2005/07/15 03:33:14  kitterma
+# Fix for bug 1238403 - Crash if non-CIDR / present.  Also added
+# validation check for valid IPv4 CIDR range.
+#
+# Revision 1.8  2005/07/14 04:18:01  customdesigned
+# Bring explanations and Received-SPF header into line with
+# the unknown=PermErr and error=TempErr convention.
+# Hope my case-sensitive mech fix doesn't clash with Scotts.
+#
+# Revision 1.7  2005/07/12 21:43:56  kitterma
+# Added processing to clarify some cases of unknown
+# qualifier errors (to distinguish between unknown qualifier and
+# unknown mechanism).
+# Also cleaned up comments from previous updates.
+#
+# Revision 1.6  2005/06/29 14:46:26  customdesigned
+# Distinguish trivial recursion from missing arg for diagnostic purposes.
+#
+# Revision 1.5  2005/06/28 17:48:56  customdesigned
+# Support extended processing results when a PermError should strictly occur.
+#
+# Revision 1.4  2005/06/22 15:54:54  customdesigned
+# Correct spelling.
+#
+# Revision 1.3  2005/06/22 00:08:24  kitterma
+# Changes from draft-mengwong overall DNS lookup and recursion
+# depth limits to draft-schlitt-spf-classic-02 DNS lookup, MX lookup, and
+# PTR lookup limits.  Recursion code is still present and functioning, but
+# it should be impossible to trip it.
+#
+# Revision 1.2  2005/06/21 16:46:09  kitterma
+# Updated definition of SPF, added reference to the sourceforge project site,
+# and deleted obsolete Microsoft Caller ID for Email XML translation routine.
+#
+# Revision 1.1.1.1  2005/06/20 19:57:32  customdesigned
+# Move Python SPF to its own module.
+#
+# Revision 1.5  2005/06/14 20:31:26  customdesigned
+# fix pychecker nits
+#
 # Revision 1.4  2005/06/02 04:18:55  customdesigned
 # Update copyright notices after reading article on /.
 #
@@ -144,135 +247,11 @@ import struct  # for pack() and unpack()
 import time    # for time()

 import DNS	# http://pydns.sourceforge.net
-import xml.sax
-
-# -------------------------------------------------------------------------
-# Convert a MS Caller-ID entry (XML) to a SPF entry
-#
-# (c) 2004 by Ernesto Baschny
-# (c) 2004 Python version by Stuart Gathman
-#
-# Date: 2004-02-25
-#
-# A complete reverse translation (SPF -> CID) might be impossible, since
-# there are no ways to handle:
-# - PTR and EXISTS mechanism 
-# - MX mechanism with an different domain as argument
-# - macros
-# 
-# References:
-# http://www.microsoft.com/mscorp/twc/privacy/spam_callerid.mspx
-# http://spf.pobox.com/
-#
-# Known bugs:
-# - Currently it won't handle the exclusions provided in the A and R
-#   tags (prefix '!'). They will show up "as-is" in the SPF record
-# - I really haven't read the MS-CID specs in-depth, so there are probably
-#   other bugs too :)
-#
-# Ernesto Baschny <ernst@baschny.de>
-#
-
-class CIDParser(xml.sax.ContentHandler):
-  "Convert a MS Caller-ID entry (XML) to a SPF entry."
-
-  def __init__(self,q=None):
-    self.spf = []
-    self.action = '-all'
-    self.has_servers = None
-    self.spf_entry = None
-    if q:
-      self.spf_query = q
-    else:
-      self.spf_query = query(i='127.0.0.1', s='localhost', h='unknown')
-
-  def startElement(self,tag,attr):
-      if tag == 'm':
-	if self.has_servers != None and not self.has_servers:
-	  raise ValueError(
-    "Declared <noMailServers\> and later <m>, this CID entry is not valid."
-	  )
-	self.has_servers = True
-      elif tag == 'noMailServers':
-	if self.has_servers:
-	  raise ValueError(
-    "Declared <m> and later <noMailServers\>, this CID entry is not valid."
-	  )
-	self.has_servers = False
-      elif tag == 'ep':
-	if attr.has_key('testing') and attr.getValue('testing') == 'true':
-	  # A CID with 'testing' found:
-	  # From the MS-specs:
-	  #  "Documents in which such attribute is present with a true
-	  #  value SHOULD be entirely ignored (one should act as if the
-	  #  document were absent)"
-	  # From the SPF-specs:
-	  #  "Neutral (?): The SPF client MUST proceed as if a domain did
-	  #  not publish SPF data."
-	  # So we set SPF action to "neutral":
-	  self.action = '?all'
-      elif tag == 'mx':
-	  # The empty MX-tag, same as SPF's MX-mechanism
-	  self.spf.append('mx')
-      self.tag = tag
-
-  def characters(self,text):
-	tag = self.tag
-	# Remove starting and trailing spaces from text:
-	text = text.strip()
-
-	if tag == 'a' or tag == 'r':
-	    # The A and R tags from MS-CID are both handled by the 
-	    # ipv4/6-mechanisms from SPF:
-	    if text.find(':') < 0:
-	      mechanism = 'ip4'
-	    else:
-	      mechanism = 'ip6'
-	    self.spf.append(mechanism + ':' + text)
-	elif tag == 'indirect':
-	    # MS-CID's indirect is "sort of" the include from SPF:
-	    # Not really true, because the <indirect> tag from MS-CID also 
-	    # provides a fallback in case the included domain doesn't provide
-	    # _ep-records: The inbound MX-servers of the included domains
-	    # are added to the list of allowed outgoing mailservers for the
-	    # domain that declared the _ep-record with the <indirect> tag.
-	    # In SPF you would use the 'mx:domain' to handle this, but this
-	    # wouldn't depend on referred domain having or not SPF-records.
-	    cid_xml = self.cid_txt(text)
-	    if cid_xml:
-	      p = CIDParser()
-	      xml.sax.parseString(cid_xml,p)
-	      if p.has_servers != False:
-		self.spf += p.spf
-	    else:
-	      self.spf.append('mx:' + text)
-
-  def cid_txt(self,domain):
-    q = self.spf_query
-    domain='_ep.' + domain
-    a = q.dns_txt(domain)
-    if not a: return None
-    if a[0].lower().startswith('<ep ') and a[-1].lower().endswith('</ep>'):
-      return ''.join(a)
-    return None
-
-  def endElement(self,tag):
-      if tag == 'ep':
-	# This is the end... assemble what we've got
-	spf_entry = ['v=spf1']
-	if self.has_servers != False:
-	  spf_entry += self.spf
-	spf_entry.append(self.action)
-	self.spf_entry = ' '.join(spf_entry)
-
-  def spf_txt(self,cid_xml):
-    if not cid_xml.startswith('<'):
-      cid_xml = self.cid_txt(cid_xml)
-      if not cid_xml: return None
-    # Parse the beast. Any XML-problem will be reported by xlm.sax
-    self.spf_entry = None
-    xml.sax.parseString(cid_xml,self)
-    return self.spf_entry
+if not hasattr(DNS.Type,'SPF'):
+  # patch in type99 support
+  DNS.Type.SPF = 99
+  DNS.Type.typemap[99] = 'SPF'
+  DNS.Lib.RRunpacker.getSPFdata = DNS.Lib.RRunpacker.getTXTdata

 # 32-bit IPv4 address mask
 MASK = 0xFFFFFFFFL
@@ -286,6 +265,8 @@ RE_CHAR = re.compile(r'%(%|_|-|(\{[a-zA-Z][0-9]*r?[^\}]*\}))')
 # Regular expression to break up a macro expansion
 RE_ARGS = re.compile(r'([0-9]*)(r?)([^0-9a-zA-Z]*)')

+RE_CIDR = re.compile(r'/([1-9]|1[0-9]*|2[0-9]*|3[0-2]*)$')
+
 # Local parts and senders have their delimiters replaced with '.' during
 # macro expansion
 #
@@ -293,11 +274,12 @@ JOINERS = {'l': '.', 's': '.'}

 RESULTS = {'+': 'pass', '-': 'fail', '?': 'neutral', '~': 'softfail',
           'pass': 'pass', 'fail': 'fail', 'unknown': 'unknown',
-	   'neutral': 'neutral', 'softfail': 'softfail',
+	   'error': 'error', 'neutral': 'neutral', 'softfail': 'softfail',
 	   'none': 'none', 'deny': 'fail' }

 EXPLANATIONS = {'pass': 'sender SPF verified', 'fail': 'access denied',
-                'unknown': 'SPF unknown',
+                'unknown': 'permanent error in processing',
+                'error': 'temporary error in processing',
 		'softfail': 'domain in transition',
 		'neutral': 'access neither permitted nor denied',
 		'none': ''
@@ -316,22 +298,27 @@ except NameError:
 	def bool(x): return not not x
 # ...pre 2.2.1

-# standard default SPF record
+# standard default SPF record for best_guess
 DEFAULT_SPF = 'v=spf1 a/24 mx/24 ptr'

 # maximum DNS lookups allowed
-MAX_LOOKUP = 100
+MAX_LOOKUP = 10 #draft-schlitt-spf-classic-02 Para 10.1
+MAX_MX = 10 #draft-schlitt-spf-classic-02 Para 10.1
+MAX_PTR = 10 #draft-schlitt-spf-classic-02 Para 10.1
 MAX_RECURSION = 20
+ALL_MECHANISMS = ('a', 'mx', 'ptr', 'exists', 'include', 'ip4', 'ip6', 'all')
+COMMON_MISTAKES = { 'prt': 'ptr', 'ip': 'ip4', 'ipv4': 'ip4', 'ipv6': 'ip6' }

 class TempError(Exception):
 	"Temporary SPF error"

 class PermError(Exception):
 	"Permanent SPF error"
-	def __init__(self,msg,mech=None):
+	def __init__(self,msg,mech=None,ext=None):
 	  Exception.__init__(self,msg,mech)
 	  self.msg = msg
 	  self.mech = mech
+	  self.ext = ext
 	def __str__(self):
 	  if self.mech:
 	    return '%s: %s'%(self.msg,self.mech)
@@ -372,7 +359,7 @@ class query(object):

 	Also keeps cache: DNS cache.
 	"""
-	def __init__(self, i, s, h,local=None,receiver=None):
+	def __init__(self, i, s, h,local=None,receiver=None,strict=True):
 		self.i, self.s, self.h = i, s, h
 		if not s and h:
 		  self.s = 'postmaster@' + h
@@ -387,6 +374,8 @@ class query(object):
 		self.exps = dict(EXPLANATIONS)
 		self.local = local	# local policy
    		self.lookups = 0
+		# strict can be False, True, or 2 for harsh
+		self.strict = strict

 	def set_default_explanation(self,exp):
 		exps = self.exps
@@ -408,10 +397,44 @@ class query(object):

 	def check(self, spf=None):
 		"""
-	Returns (result, mta-status-code, explanation) where
-	result in ['fail', 'softfail', 'neutral' 'unknown', 'pass', 'error']
+	Returns (result, mta-status-code, explanation) where result
+	in ['fail', 'softfail', 'neutral' 'unknown', 'pass', 'error', 'none']
+
+	Examples:
+	>>> q = query(s='strong-bad@email.example.com',
+	...           h='mx.example.org', i='192.0.2.3')
+	>>> q.check(spf='v=spf1 ?all')
+	('neutral', 250, 'access neither permitted nor denied')
+
+	>>> q.check(spf='v=spf1 ip4:192.0.0.0/8 ?all moo')
+	('unknown', 550, 'SPF Permanent Error: Unknown mechanism found: moo')
+
+	>>> q.check(spf='v=spf1 ip4:192.0.0.0/8 ~all')
+	('pass', 250, 'sender SPF verified')
+
+	>>> q.strict = False
+	>>> q.check(spf='v=spf1 ip4:192.0.0.0/8 -all moo')
+	('pass', 250, 'sender SPF verified')
+
+	>>> q.check(spf='v=spf1 ip4:192.1.0.0/16 moo -all')
+	('unknown', 550, 'SPF Permanent Error: Unknown mechanism found: moo')
+
+	>>> q.check(spf='v=spf1 ip4:192.1.0.0/16 ~all')
+	('softfail', 250, 'domain in transition')
+
+	>>> q.check(spf='v=spf1 -ip4:192.1.0.0/6 ~all')
+	('fail', 550, 'access denied')
+
+	# Assumes DNS available
+	>>> q.check()
+	('none', 250, '')
 		"""
 		self.mech = []		# unknown mechanisms
+		# If not strict, certain PermErrors (mispelled
+		# mechanisms, strict processing limits exceeded)
+		# will continue processing.  However, the exception
+		# that strict processing would raise is saved here
+		self.perm_error = None
 		if self.i.startswith('127.'):
 			return ('pass', 250, 'local connections always pass')

@@ -421,31 +444,106 @@ class query(object):
 			    spf = self.dns_spf(self.d)
 			if self.local and spf:
 			    spf += ' ' + self.local
-			return self.check1(spf, self.d, 0)
+			rc = self.check1(spf, self.d, 0)
+			if self.perm_error:
+			  # extended processing succeeded, but strict failed
+			  self.perm_error.ext = rc
+			  raise self.perm_error
+			return rc
 		except DNS.DNSError,x:
 			return ('error', 450, 'SPF DNS Error: ' + str(x))
 		except TempError,x:
 			return ('error', 450, 'SPF Temporary Error: ' + str(x))
 		except PermError,x:
 		    self.prob = x.msg
-		    self.mech.append(x.mech)
+		    if x.mech:
+		      self.mech.append(x.mech)
 		    # Pre-Lentczner draft treats this as an unknown result
 		    # and equivalent to no SPF record.
-		    # return ('unknown', 550, 'SPF Permanent Error: ' + str(x))
-		    return ('error', 550, 'SPF Permanent Error: ' + str(x))
+		    return ('unknown', 550, 'SPF Permanent Error: ' + str(x))

 	def check1(self, spf, domain, recursion):
 		# spf rfc: 3.7 Processing Limits
 		#
 		if recursion > MAX_RECURSION:
-			self.prob =  'Too many levels of recursion'
-			return ('unknown', 250, 'SPF recursion limit exceeded')
+			# This should never happen in strict mode
+			# because of the other limits we check,
+			# so if it does, there is something wrong with
+			# our code.  It is not a PermError because there is not
+			# necessarily anything wrong with the SPF record.
+			if self.strict:
+			  raise AssertionError('Too many levels of recursion')
+			# As an extended result, however, it should be
+			# a PermError.
+			raise PermError('Too many levels of recursion')
 		try:
 			tmp, self.d = self.d, domain
 			return self.check0(spf,recursion)
 		finally:
 			self.d = tmp

+	def validate_mechanism(self,mech):
+		"""Parse and validate a mechanism.
+	Returns mech,m,arg,cidrlength,result
+
+	Examples:
+	>>> q = query(s='strong-bad@email.example.com',
+	...           h='mx.example.org', i='192.0.2.3')
+	>>> q.validate_mechanism('A')
+	('A', 'a', 'email.example.com', 32, 'pass')
+
+	>>> q.validate_mechanism('?mx:%{d}/27')
+	('?mx:%{d}/27', 'mx', 'email.example.com', 27, 'neutral')
+
+	>>> q.validate_mechanism('-mx::%%%_/.Clara.de/27')
+	('-mx::%%%_/.Clara.de/27', 'mx', ':% /.Clara.de', 27, 'fail')
+
+	>>> q.validate_mechanism('~exists:%{i}.%{s1}.100/86400.rate.%{d}')
+	('~exists:%{i}.%{s1}.100/86400.rate.%{d}', 'exists', '192.0.2.3.com.100/86400.rate.email.example.com', 32, 'softfail')
+		"""
+		# a mechanism
+		m, arg, cidrlength = parse_mechanism(mech, self.d)
+		# map '?' '+' or '-' to 'unknown' 'pass' or 'fail'
+		if m:
+		  result = RESULTS.get(m[0])
+		  if result:
+			# eat '?' '+' or '-'
+			m = m[1:]
+		  else:
+			# default pass
+			result = 'pass'
+		if m in COMMON_MISTAKES:
+		  try:
+		    raise PermError('Unknown mechanism found',mech)
+		  except PermError, x:
+		    if self.strict: raise
+		    m = COMMON_MISTAKES[m]
+		    if not self.perm_error:
+		      self.perm_error = x
+		  
+		if m in ('a', 'mx', 'ptr', 'exists', 'include'):
+		  arg = self.expand(arg)
+		  if not (0 < arg.find('.') < len(arg) - 1):
+		    raise PermError('Invalid domain found (use FQDN)',
+			  arg)
+		  if m == 'include':
+		    if arg == self.d:
+		      if mech != 'include':
+			raise PermError('include has trivial recursion',mech)
+		      raise PermError('include mechanism missing domain',mech)
+		  return mech,m,arg,cidrlength,result
+		if m in ALL_MECHANISMS:
+		  return mech,m,arg,cidrlength,result
+		try:
+		  if m[1:] in ALL_MECHANISMS:
+		    raise PermError(
+		      'Unknown qualifier, IETF draft para 4.6.1, found in',
+		      mech)
+		  raise PermError('Unknown mechanism found',mech)
+		except PermError, x:
+		  if self.strict: raise
+		  return mech,m,arg,cidrlength,x
+
 	def check0(self, spf,recursion):
 		"""Test this query information against SPF text.

@@ -468,95 +566,83 @@ class query(object):
 		# overridden with 'default=' modifier
 		#
 		default = 'neutral'
+		mechs = []

 		# Look for modifiers
 		#
-		for m in spf:
-			m = RE_MODIFIER.split(m)[1:]
-			if len(m) != 2: continue
-
-			if m[0] == 'exp':
-				exps['fail'] = exps['unknown'] = \
-					self.get_explanation(m[1])
-			elif m[0] == 'redirect':
-				redirect = self.expand(m[1])
-			elif m[0] == 'default':
-				# default=- is the same as default=fail
-				default = RESULTS.get(m[1], default)
-
-			# spf rfc: 3.6 Unrecognized Mechanisms and Modifiers
-
-		# Look for mechanisms
-		#
 		for mech in spf:
-		    if RE_MODIFIER.match(mech): continue
-		    m, arg, cidrlength = parse_mechanism(mech, self.d)
+		    m = RE_MODIFIER.split(mech)[1:]
+		    if len(m) != 2:
+		      mechs.append(self.validate_mechanism(mech))
+		      continue

-		    # map '?' '+' or '-' to 'unknown' 'pass' or 'fail'
-		    if m:
-		      result = RESULTS.get(m[0])
-		      if result:
-			    # eat '?' '+' or '-'
-			    m = m[1:]
-		      else:
-			    # default pass
-			    result = 'pass'
+		    if m[0] == 'exp':
+                        try:
+                            self.set_default_explanation(self.get_explanation(m[1]))
+                        except PermError:
+                            pass
+		    elif m[0] == 'redirect':
+		        self.check_lookups()
+			redirect = self.expand(m[1])
+		    elif m[0] == 'default':
+			# default=- is the same as default=fail
+			default = RESULTS.get(m[1], default)

-		    if m in ['a', 'mx', 'ptr', 'prt', 'exists', 'include']:
-			    arg = self.expand(arg)
+		    # spf rfc: 3.6 Unrecognized Mechanisms and Modifiers
+
+		# Evaluate mechanisms
+		#
+		for mech,m,arg,cidrlength,result in mechs:

 		    if m == 'include':
-		      if arg != self.d:
-			res,code,txt = self.check1(self.dns_spf(arg),
-					  arg, recursion + 1)
-			if res == 'pass':
-			  break
-			if res == 'none':
-			  raise PermError(
-			    'No valid SPF record for included domain: %s'%arg,
-			    mech)
-			continue
-		      else:
-			raise PermError('include mechanism missing domain',mech)
+		      self.check_lookups()
+		      res,code,txt = self.check1(self.dns_spf(arg),
+					arg, recursion + 1)
+		      if res == 'pass':
+			break
+		      if res == 'none':
+			raise PermError(
+			  'No valid SPF record for included domain: %s'%arg,
+			  mech)
+		      continue
 		    elif m == 'all':
 			    break

 		    elif m == 'exists':
-			    if len(self.dns_a(arg)) > 0:
-				    break
+		        self.check_lookups()
+			if len(self.dns_a(arg)) > 0:
+				break

 		    elif m == 'a':
-			    if cidrmatch(self.i, self.dns_a(arg),
-					 cidrlength):
-				    break
+		        self.check_lookups()
+			if cidrmatch(self.i, self.dns_a(arg), cidrlength):
+			      break

 		    elif m == 'mx':
-			    if cidrmatch(self.i, self.dns_mx(arg),
-					 cidrlength):
-				    break
+		        self.check_lookups()
+			if cidrmatch(self.i, self.dns_mx(arg), cidrlength):
+			      break

-		    elif m in ('ip4', 'ipv4', 'ip') and arg != self.d:
+		    elif m == 'ip4' and arg != self.d:
 			try:
 			    if cidrmatch(self.i, [arg], cidrlength):
 				break
 			except socket.error:
 			    raise PermError('syntax error',mech)
 			    
-		    elif m in ('ip6', 'ipv6'):
-		    # Until we support IPV6, we should never
-		    # get an IPv6 connection.  So this mech
-		    # will never match.
-			    pass
+		    elif m == 'ip6':
+			# Until we support IPV6, we should never
+			# get an IPv6 connection.  So this mech
+			# will never match.
+			pass

-		    elif m in ('ptr', 'prt'):
-			    if domainmatch(self.validated_ptrs(self.i),
-					   arg):
-				    break
+		    elif m == 'ptr':
+		        self.check_lookups()
+			if domainmatch(self.validated_ptrs(self.i), arg):
+				break

 		    else:
-		      # unknown mechanisms cause immediate unknown
-		      # abort results
-		      raise PermError('Unknown mechanism found',mech)
+		      raise result
 		else:
 		    # no matches
 		    if redirect:
@@ -570,6 +656,17 @@ class query(object):
 		else:
 		    return (result, 250, exps[result])

+	def check_lookups(self):
+	    self.lookups = self.lookups + 1
+	    if self.lookups > MAX_LOOKUP:
+	      try:
+		if self.strict or not self.perm_error:
+		  raise PermError('Too many DNS lookups')
+	      except PermError,x:
+		if self.strict or self.lookups > MAX_LOOKUP*4:
+		  raise x
+		self.perm_error = x
+
 	def get_explanation(self, spec):
 		"""Expand an explanation."""
 		if spec:
@@ -661,8 +758,10 @@ class query(object):
 				letter = macro[2].lower()
 				if letter == 'p':
 					self.getp()
-				expansion = getattr(self, letter, '')
+				expansion = getattr(self, letter, 'Macro Error')
 				if expansion:
+                                        if expansion == 'Macro Error':
+                                            raise PermError('Unknown Macro Encountered') 
 					result += expand_one(expansion,
 						macro[3:-1],
 					        JOINERS.get(letter))
@@ -675,23 +774,27 @@ class query(object):
 		name.  Returns None if not found, or if more than one record
 		is found.
 		"""
+		# for performance, check for most common case of TXT first
 		a = [t for t in self.dns_txt(domain) if t.startswith('v=spf1')]
-		if not a:
-		  if DELEGATE:
+		if len(a) == 1 and self.strict < 2:
+		    return a[0]   			
+		# check official SPF type first when it becomes more popular
+		b = [t for t in self.dns_99(domain) if t.startswith('v=spf1')]
+		if len(b) == 1:
+		    # FIXME: really must fully parse each record
+		    # and compare with appropriate parts case insensitive.
+		    if self.strict >= 2 and len(a) == 1 and a[0] != b[0]:
+		        raise PermError(
+'v=spf1 records of both type TXT and SPF (type 99) present, but not identical')
+		    return b[0]
+		if len(a) == 1:
+		    return a[0]	# return TXT if SPF wasn't found
+		if DELEGATE:	# use local record if neither found
 		    a = [t
 		      for t in self.dns_txt(domain+'._spf.'+DELEGATE)
 			if t.startswith('v=spf1')
 		    ]
-		  if not a:
-		    # No SPF record: convert and return CID if present
-		    p = CIDParser(q=self)
-		    try:
-		      return p.spf_txt(domain)
-		    except xml.sax._exceptions.SAXParseException:
-		      raise PermError("Caller-ID parse error",domain)
-
-		if len(a) == 1:
-			return a[0]
+		    if len(a) == 1: return a[0]
 		return None

 	def dns_txt(self, domainname):
@@ -699,12 +802,23 @@ class query(object):
 		if domainname:
 		  return [''.join(a) for a in self.dns(domainname, 'TXT')]
 		return []
+	def dns_99(self, domainname):
+		"Get a list of type SPF=99 records for a domain name."
+		if domainname:
+		  return [''.join(a) for a in self.dns(domainname, 'SPF')]
+		return []

 	def dns_mx(self, domainname):
 		"""Get a list of IP addresses for all MX exchanges for a
 		domain name.
 		"""
-		return [a for mx in self.dns(domainname, 'MX') \
+# draft-schlitt-spf-classic-02 section 5.4 "mx"
+# To prevent DoS attacks, more than 10 MX names MUST NOT be looked up
+		if self.strict:
+		  max = MAX_MX
+		else:
+		  max = MAX_MX * 4
+		return [a for mx in self.dns(domainname, 'MX')[:max] \
 		          for a in self.dns_a(mx[1])]

 	def dns_a(self, domainname):
@@ -719,7 +833,12 @@ class query(object):
 		"""Figure out the validated PTR domain names for a given IP
 		address.
 		"""
-		return [p for p in self.dns_ptr(i) if i in self.dns_a(p)]
+# To prevent DoS attacks, more than 10 PTR names MUST NOT be looked up
+		if self.strict:
+		  max = MAX_PTR
+		else:
+		  max = MAX_PTR * 4
+		return [p for p in self.dns_ptr(i)[:max] if i in self.dns_a(p)]

 	def dns_ptr(self, i):
 		"""Get a list of domain names for an IP address."""
@@ -739,26 +858,26 @@ class query(object):
 		pre: qtype in ['A', 'AAAA', 'MX', 'PTR', 'TXT', 'SPF']
 		post: isinstance(__return__, types.ListType)
 		"""
-		self.lookups += 1
-		if self.lookups > MAX_LOOKUP:
-			raise PermError('Too many DNS lookups')
 		result = self.cache.get( (name, qtype) )
 		cname = None
 		if not result:
 			req = DNS.DnsRequest(name, qtype=qtype)
 			resp = req.req()
+			#resp.show()
 			for a in resp.answers:
-				# key k: ('wayforward.net', 'A'), value v
-				k, v = (a['name'], a['typename']), a['data']
-				if k == (name, 'CNAME'):
-					cname = v
-				self.cache.setdefault(k, []).append(v)
+			    # key k: ('wayforward.net', 'A'), value v
+			    k, v = (a['name'], a['typename']), a['data']
+			    if k == (name, 'CNAME'):
+				    cname = v
+			    self.cache.setdefault(k, []).append(v)
 			result = self.cache.get( (name, qtype), [])
 		if not result and cname:
 			result = self.dns(cname, qtype)
 		return result

-	def get_header(self,res,receiver):
+	def get_header(self,res,receiver=None):
+	  if not receiver:
+	    receiver = self.r
 	  if res in ('pass','fail','softfail'):
 	    return '%s (%s: %s) client-ip=%s; envelope-from=%s; helo=%s;' % (
 	  	res,receiver,self.get_header_comment(res),self.i,
@@ -789,10 +908,10 @@ class query(object):
 		    	% (self.i,sender)
 		    #"%s does not designate permitted sender hosts" % sender
 		elif res == 'unknown': return \
-		    "error in processing during lookup of domain of %s: %s" \
+		    "permanent error in processing domain of %s: %s" \
 		    	% (sender, self.prob)
 		elif res == 'error': return \
-		    "error in processing during lookup of %s" % sender
+		    "temporary error in processing during lookup of %s" % sender
 		elif res == 'fail': return \
 		    "domain of %s does not designate %s as permitted sender" \
 			% (sender,self.i)
@@ -836,20 +955,32 @@ def parse_mechanism(str, d):
 	>>> parse_mechanism('a/24', 'foo.com')
 	('a', 'foo.com', 24)

-	>>> parse_mechanism('a:bar.com/16', 'foo.com')
-	('a', 'bar.com', 16)
+	>>> parse_mechanism('A:foo:bar.com/16', 'foo.com')
+	('a', 'foo:bar.com', 16)
+
+	>>> parse_mechanism('-exists:%{i}.%{s1}.100/86400.rate.%{d}','foo.com')
+	('-exists', '%{i}.%{s1}.100/86400.rate.%{d}', 32)
+
+	>>> parse_mechanism('mx::%%%_/.Claranet.de/27','foo.com')
+	('mx', ':%%%_/.Claranet.de', 27)
+
+	>>> parse_mechanism('mx:%{d}/27','foo.com')
+	('mx', '%{d}', 27)
+
+	>>> parse_mechanism('iP4:192.0.0.0/8','foo.com')
+	('ip4', '192.0.0.0', 8)
 	"""
-	a = str.split('/')
-	if len(a) == 2:
+	a = RE_CIDR.split(str)
+	if len(a) == 3:
 		a, port = a[0], int(a[1])
 	else:
 		a, port = str, 32

-	b = a.split(':')
+	b = a.split(':',1)
 	if len(b) == 2:
-		return b[0], b[1], port
+		return b[0].lower(), b[1], port
 	else:
-		return a, d, port
+		return a.lower(), d, port

 def reverse_dots(name):
 	"""Reverse dotted IP addresses or domain names.
@@ -1033,7 +1164,9 @@ if __name__ == '__main__':
 			receiver=socket.gethostname())
 	elif len(sys.argv) == 5:
 		i, s, h = sys.argv[2:]
-		q = query(i=i, s=s, h=h, receiver=socket.gethostname())
+		q = query(i=i, s=s, h=h, receiver=socket.gethostname(),
+			strict=False)
 		print q.check(sys.argv[1])
+		if q.perm_error: print q.perm_error.ext
 	else:
 		print USAGE
@@ -23,7 +23,7 @@ SMTP (email) servers to prevent criminals from forging your
 domain.  The simplest step is usually to publish an SPF record
 with your Sender Policy.  

-For more information, see: http://spf.pobox.com
+For more information, see: http://openspf.com

 I hate to annoy you with a DSN (Delivery Status
 Notification) from a possibly forged email, but since you
@@ -0,0 +1,49 @@
+From paulp@go2net.com  Wed Jun  1 22:35:12 2005
+Return-Path: <paulp@go2net.com>
+Received: from mail.bmsi.com (spidey.bmsi.com [192.168.9.81])
+	by bmsred.bmsi.com (8.13.1/8.12.10) with ESMTP id j522ZCQg014058
+	for <stuart@bmsred.bmsi.com>; Wed, 1 Jun 2005 22:35:12 -0400
+Received: from 127.0.0.1 ([220.117.92.241])
+	by mail.bmsi.com (8.13.1/8.13.1) with ESMTP id j522Ynjm028604
+	for stuart@bmsi.com; Wed, 1 Jun 2005 22:34:51 -0400
+Message-Id: <200506020234.j522Ynjm028604@mail.bmsi.com>
+SUBJECT: urgent
+FROM: paulp@go2net.com
+TO: stuart@bmsi.com
+DATE: [[ ¸ñ, 02 6 2005 ¿ÀÀü 11:34:47 ]]
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="--------bound--"
+X-DSpam-Score: 0.081200
+Received-SPF: neutral (mail.bmsi.com: guessing: 220.117.92.241 is neither permitted nor denied by domain of go2net.com)
+Status: RO
+X-Status: 
+X-Keywords: NonJunk         
+
+----------bound--
+Content-Type: text/plain; charset=us-ascii
+Content-Transfer-Encoding: 7bit
+
+Hi 
+
+Sorry, I forgot to send an important
+document to you in that last email. I had an important phone call.
+Please checkout attached doc file when you have a moment.
+
+Best Regards
+
+<!DSPAM:1043AE6B6492860536935410>
+
+
+----------bound--
+Content-Type: application/octet-stream;
+	name="Readme.zip"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment;
+	filename="Readme.zip"
+
+
+----------bound--
+
+
+----------bound----
+
@@ -0,0 +1,51 @@
+From paulp@go2net.com  Wed Jun  1 22:35:12 2005
+Return-Path: <paulp@go2net.com>
+Received: from mail.bmsi.com (spidey.bmsi.com [192.168.9.81])
+	by bmsred.bmsi.com (8.13.1/8.12.10) with ESMTP id j522ZCQg014058
+	for <stuart@bmsred.bmsi.com>; Wed, 1 Jun 2005 22:35:12 -0400
+Received: from 127.0.0.1 ([220.117.92.241])
+	by mail.bmsi.com (8.13.1/8.13.1) with ESMTP id j522Ynjm028604
+	for stuart@bmsi.com; Wed, 1 Jun 2005 22:34:51 -0400
+Message-Id: <200506020234.j522Ynjm028604@mail.bmsi.com>
+SUBJECT: urgent
+FROM: paulp@go2net.com
+TO: stuart@bmsi.com
+DATE: [[ ¸ñ, 02 6 2005 ¿ÀÀü 11:34:47 ]]
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="--------bound--"
+X-DSpam-Score: 0.081200
+Received-SPF: neutral (mail.bmsi.com: guessing: 220.117.92.241 is neither permitted nor denied by domain of go2net.com)
+Status: RO
+X-Status: 
+X-Keywords: NonJunk         
+
+----------bound--
+Content-Type: text/plain; charset=us-ascii
+Content-Transfer-Encoding: 7bit
+
+Hi 
+
+Sorry, I forgot to send an important
+document to you in that last email. I had an important phone call.
+Please checkout attached doc file when you have a moment.
+
+Best Regards
+
+<!DSPAM:1043AE6B6492860536935410>
+
+
+----------bound--
+Content-Type: application/x-msdownload; name="zip.zip"
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="zip.zip"
+
+USsDBAoBAAAAADVVwjLaV2nEGgAAABoAAAAzABUAemlwLmRvYyAgICAgICAgICAgICAgICAg
+ICAgICAgICAgICAgICAgICAgICAgICAuZXhlVVQJAAOmGp9CphqfQlV4BACGA2UAVGhpcyBw
+cm9ncmFtIHdhcyBhIHZpcnVzLgpQSwECFwMKAAAAAAA1VcIy2ldpxBoAAAAaAAAAMwANAAAA
+AAABAAAAtIEAAAAAemlwLmRvYyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg
+ICAgICAuZXhlVVQFAAOmGp9CVXgAAFBLBQYAAAAAAQABAG4AAACAAAAAAAA=
+----------bound--
+
+
+----------bound----
+
@@ -1,4 +1,7 @@
 # $Log$
+# Revision 1.3  2005/06/17 01:49:39  customdesigned
+# Handle zip within zip.
+#
 # Revision 1.2  2005/06/02 15:00:17  customdesigned
 # Configure banned extensions.  Scan zipfile option with test case.
 #
@@ -130,9 +133,16 @@ class MimeTestCase(unittest.TestCase):

  def testZip(self,vname="zip1",fname='zip.zip'):
    self.testDefang(vname,1,'zip.zip')
+    # test scan_zip flag
    msg = mime.message_from_file(open('test/'+vname,"r"))
    mime.defang(msg,scan_zip=False)
    self.failIf(msg.ismodified())
+    # test ignoring empty zip (often found in DSNs)
+    msg = mime.message_from_file(open('test/zip2','r'))
+    mime.defang(msg,scan_zip=True)
+    self.failIf(msg.ismodified())
+    # test corrupt zip (often an EXE named as a ZIP)
+    self.testDefang('zip3',1,'zip.zip')
    # test zip within zip
    self.testDefang('ziploop',1,'stuart@bmsi.com.zip')