Change version to 1.2.3 (these are all bug fixes) and set release date

(LP: #1901445)

CHANGES

@@ -1,7 +1,28 @@
-1.2.2
+1.2.3 2023-02-26
+ - Improve support for non-ASCII email messages.  Anything UTF-8 should work
+   (including correct signing/verification).  For messages that contain header
+   fields with non-ASCII or UTF-8 content, signatures are likely fail
+   verification, but the milter should continue to run.  (Thanks to Casper
+   Bruun for help with this)
+ - Set minimum pymilter and dkimpy versions in setup.py to those that will
+   work reliably with non-ASCII content.
+ - Fixed support for percent in KeyTable - Thanks to Mika Tiainen
+ - Fix formatting for MinimumKeyBits in dkimpy-milter.conf(5)
+   (Closes: #995335)
+ - Reset the i= signature identity in get_identities_sign() (Closes: #981157)
+ - Improve documentation of inter-relationship between Mode, InternalHosts,
+   MacroList, and MacroListVerify options in dkimpy-milter.conf.5 (Closes:
+   #969215)
+ - Fix subdomain signing with top-level organizational domain (LP: #1999434)
+   - Thanks to Matthias Hunstock for the report and the fix
+ - Fix comma separated list processing in dkimpy_milter/config.py
+   (LP: #1901445)
+
+1.2.2 2020-08-09
  - Improve README.md formating for markdown display on pypi
  - Improve documentation in dkimpy-milter.conf (5) and README.md for signing
    for multiple domains (Thanks to Stefano Rivera)
+ - Minimal fix for dnspython 2.0.0 compatibility (still works with 1.16.0)
 
 1.2.1 2020-01-04
  - Fix expand option not to fail if files are missing since socket activation

README.md

@@ -336,3 +336,8 @@ later support Ed25519 signing and verification.  RFC 8301 removed rsa-sha1
 from DKIM.  dkimpy-milter does not sign with rsa-sha1, but still considers
 rsa-sha1 signatures as valid for verification because they are still in
 common use and are not known to be cryptographically broken.
+
+Support for non-ASCII email messages:  Anything UTF-8 should work (including
+correct signing/verification).  For messages that contain header fields with
+non-ASCII or UTF-8 content, signatures are likely fail verification, but the
+milter should continue to run.  RFC 8616 is not supported.

dkimpy_milter/__init__.py

@@ -39,7 +39,7 @@ from dkimpy_milter.util import write_pid
 from dkimpy_milter.util import get_keys
 from dkimpy_milter.util import fold
 
-__version__ = "1.2.0"
+__version__ = "1.2.3"
 FWS = re.compile(r'\r?\n[ \t]+')
 
 
@@ -101,15 +101,28 @@ class dkimMilter(Milter.Base):
         if self.conf.get('Syslog') and self.conf.get('debugLevel') >= 1:
             syslog.syslog("connect from {0} at {1} {2}"
                           .format(hostname, hostaddr, connecttype))
+        if self.conf.get('Syslog') and self.conf.get('debugLevel') >= 3:
+            syslog.syslog("internal_conn: {0}, external_conn: {1}"
+                          .format(self.internal_connection, self.external_connection))
+
         return Milter.CONTINUE
 
     # multiple messages can be received on a single connection
     # envfrom (MAIL FROM in the SMTP protocol) seems to mark the start
     # of each message.
     @Milter.noreply
-    def envfrom(self, f, *str):
+    def envfrom(self, f, *moredata):
+        try:
+            f = str(codecs.encode(f, 'UTF-8', 'replace'), 'UTF-8', 'ignore')
+        except TypeError:
+            f = codecs.encode(f, 'UTF-8', 'replace').decode()
+        try:
+            moredata = str(codecs.encode(str(moredata), 'UTF-8', 'replace'), 'UTF-8', 'ignore')
+        except TypeError:
+            moredata = codecs.encode(str(moredata), 'UTF-8', 'replace').decode()
+
         if self.conf.get('Syslog') and self.conf.get('debugLevel') >= 2:
-            syslog.syslog("mail from: {0} {1}".format(f, str))
+            syslog.syslog("mail from: {0} {1}".format(f, moredata))
         self.fp = io.BytesIO()
         self.mailfrom = f
         t = parse_addr(f)
@@ -141,6 +154,11 @@ class dkimMilter(Milter.Base):
                 self.fdomain = self.author.split('@')[1].lower()
             except IndexError as er:
                 pass # self.author was not a proper email address
+            # This keeps non-ascii characters out of the From domain
+            try:
+                self.fdomain = str(codecs.encode(self.fdomain, 'ascii', 'replace'), 'ascii', 'ignore')
+            except TypeError:
+                self.fdomain = codecs.encode(self.fdomain, 'ascii', 'replace').decode('ascii','ignore')
             if (self.conf.get('Syslog') and
                     self.conf.get('debugLevel') >= 1):
                 syslog.syslog("{0}: {1}".format(name, val))
@@ -148,7 +166,11 @@ class dkimMilter(Milter.Base):
             self.arheaders.append(val)
         if self.fp:
             try:
-                self.fp.write(b"%s: %s\n" % (codecs.encode(name, 'ascii'), codecs.encode(val, 'ascii')))
+                if lname == 'from':
+                    # Non-ascii in email address localpart is legal, so this is a special case
+                    self.fp.write(b"%s: %s\n" % (codecs.encode(name, 'ascii'), codecs.encode(val, 'UTF-8', 'replace')))
+                else:
+                    self.fp.write(b"%s: %s\n" % (codecs.encode(name, 'ascii'), codecs.encode(val, 'ascii')))
             except:
                 # Don't choke on header fields with non-ascii garbage in them.
                 pass
@@ -195,6 +217,8 @@ class dkimMilter(Milter.Base):
             syslog.syslog('self.domain: {0}, self.fdomain: {1}, self.iequals: {2}'.format(self.domain, self.fdomain, self.iequals))
         if ((self.fdomain in self.domain) and not self.conf.get('Mode') == 'v'
                 and not self.external_connection):
+            if (self.conf.get('Syslog') and self.conf.get('debugLevel') >= 3):
+                syslog.syslog("Signing DKIM")
             self.sign_dkim(txt)
         if ((self.has_dkim) and (not self.internal_connection) and
             (self.conf.get('Mode') == 'v' or
@@ -227,7 +251,7 @@ class dkimMilter(Milter.Base):
     def get_identities_sign(self):
         """Determine d= and i= identiies for signature"""
         self.domain = []
-        iequals = None
+        self.iequals = None
         try:
             self.privkeyRSA = self.conf.get('privateRSA')
         except:
@@ -280,6 +304,7 @@ class dkimMilter(Milter.Base):
                         keytabledata = self.conf.get('privateRSATable')[keytablekey]
                         try:
                             self.fdomain = keytabledata[0]
+                            self.domain.append(self.fdomain)
                             self.selectorRSA = keytabledata[1]
                             self.privkeyRSA = keytabledata[2]
                         except:
@@ -290,11 +315,14 @@ class dkimMilter(Milter.Base):
                         keytabledata = self.conf.get('privateEd25519Table')[keytablekey]
                         try:
                             self.fdomain = keytabledata[0]
+                            self.domain.append(self.fdomain)
                             self.selectorEd25519 = keytabledata[1]
                             self.privkeyEd25519 = keytabledata[2]
                         except:
                             if (self.conf.get('Syslog')):
                                     syslog.syslog('Error: Invalid KeyTable data {0}'.format(keytabledata))
+                    if (self.fdomain == '%'):
+                        self.fdomain = self.author.split('@')[1].lower()
                     break
 
     def sign_dkim(self, txt):

dkimpy_milter/config.py

@@ -89,16 +89,23 @@ class HostsDataset(object):
                 self.item = item[1:]
                 self.negative = True
             try:
-                self.item = ipaddress.ip_address(str(self.item, "utf-8"))
+                try:
+                    self.item = ipaddress.ip_address(str(self.item, "utf-8"))
+                except TypeError:
+                    self.item = ipaddress.ip_address(self.item)
                 if isinstance(self.item, ipaddress.IPv4Address):
                     self.isipv4 = True
                 elif isinstance(self.item, ipaddress.IPv6Address):
                     self.isipv6 = True
             except ValueError as e:
                 try:
-                    self.item = ipaddress.ip_network(str
-                                                     (self.item, "utf-8"),
-                                                     strict=False)
+                    try:
+                        self.item = ipaddress.ip_network(str
+                                                         (self.item, "utf-8"),
+                                                         strict=False)
+                    except TypeError:
+                        self.item = ipaddress.ip_network(self.item,
+                                                         strict=False)
                     if isinstance(self.item, ipaddress.IPv4Network):
                         self.isipv4cidr = True
                     elif isinstance(self.item, ipaddress.IPv6Network):
@@ -114,7 +121,10 @@ class HostsDataset(object):
 
     def match(self, connectip):
         '''Check if the connect IP is part of the dataset'''
-        source = ipaddress.ip_address(str(connectip, "utf-8"))
+        try:
+            source = ipaddress.ip_address(str(connectip, "utf-8"))
+        except TypeError:
+            source = ipaddress.ip_address(connectip)
         for item in self.dataset:
             if item.isdomain or item.ishostname:
                 result = self.matchname(source)   # Match host/domains first
@@ -164,13 +174,19 @@ class HostsDataset(object):
             if isinstance(source, ipaddress.IPv4Address):
                 ips = s.dns(name, 'A')
                 for ip in ips:
-                    ip = ipaddress.IPv4Address(str(ip, 'UTF-8'))
+                    try:
+                        ip = ipaddress.IPv4Address(str(ip, 'UTF-8'))
+                    except TypeError:
+                        ip = ipaddress.IPv4Address(ip)
                     if ip == source:
                         results.append(name)
             if isinstance(source, ipaddress.IPv6Address):
                 ips = s.dns(name, 'AAAA')
                 for ip in ips:
-                    ip = ipaddress.IPv6Address(str(ip, 'UTF-8'))
+                    try:
+                        ip = ipaddress.IPv6Address(str(ip, 'UTF-8'))
+                    except TypeError:
+                        ip = ipaddress.IPv6Address(ip)
                 if ip == source:
                     results.append(name)
         return results
@@ -306,9 +322,15 @@ def _dataset_to_list(dataset):
                 return dsd
         # If it's a str and csl, it has one value and we return a list
         if dataset[:4] == 'csl:':
-            return [dataset[4:].strip().strip(',')]
+            datalist = dataset[4:].split(',')
+            for item in datalist:
+                datalist[datalist.index(item)] = item.strip().strip(',')
+            return datalist
         else:
-            return [dataset.strip().strip(',')]
+            datalist = dataset.split(',')
+            for item in datalist:
+                datalist[datalist.index(item)] = item.strip().strip(',')
+            return datalist
         if dataset[-3:] == '.db' or dataset[:3] == 'db:':
             #  This is a Sleepycat (Oracle) DB  dataset, which we dont support
             raise dkim.ParameterError('Unsupported dataset db datase: {0}'
@@ -439,8 +461,14 @@ def _readConfigFile(path, configData=None, configGlobal={}):
     fp.close()
     try:
         configData['AuthservID'] = _make_authserv_id(configData.get('AuthservID', 'HOSTNAME'))
+    except Exception as e:
+        syslog.syslog("Could not make AuthservID: {}".format(e))
+        pass
+
+    try:
         configData['IntHosts'] = HostsDataset(configData['InternalHosts'])
-    except:
+    except Exception as e:
+        syslog.syslog("Could not make HostDataset from InternalHosts: {}".format(e))
         pass
 
     return(configData)

dkimpy_milter/dnsplug.py

@@ -127,7 +127,7 @@ def DNSLookup_dnspython(name,qtype,tcpfallback=True,timeout=5):
       elif qtype == 'PTR':
         retVal.append(((name, qtype), rdata.target.to_text(True)))
       elif qtype == 'TXT' or qtype == 'SPF':
-        retVal.append(((name, qtype), rdata.strings))
+        retVal.append(((name, qtype), list(rdata.strings)))
   except dns.resolver.NoAnswer:
     pass
   except dns.resolver.NXDOMAIN:

man/dkimpy-milter.conf.5

@@ -244,6 +244,10 @@ Naturally, providing a value here overrides the default, so if mail from
 127.0.0.1 should be signed, the list provided here should include that
 address explicitly. [PeerList NOT IMPLEMENTED]
 
+Mail sent via connections from InternalHosts will not have any existing DKIM
+signatures verified.  This is not overridden by MacroList or Mode.  If the
+Mode is 'v', then no actions will be performed.
+
 .TP
 .I KeyFile (string)
 Gives the location of a PEM-formatted private key to be used for RSA signing
@@ -298,6 +302,10 @@ at the time the filter receives a connection from the MTA and its availability
 depends upon the version of milter used to compile the filter and the version
 of the MTA making the connection.
 
+Mail sent via connections where macros that are in MacroList are provided
+will not have any existing DKIM signatures verified.  If the Mode is 'v', then
+no actions will be performed.
+
 .TP
 .I MacroListVerify (dataset)
 Defines a set of MTA-provided
@@ -308,6 +316,10 @@ Entries in this data set follow the same form as those of the
 .I MacroList
 option above.  [this option is not inhereted from OpenDKIM]  
 
+Mail sent via connections where macros that are in MacroListVerify are
+provided will be not DKIM signed.  If the Mode is 's', then no actions will
+be performed.
+
 .TP
 .I Mode (string)
 Selects operating modes.  The string is a concatenation of characters that
@@ -327,7 +339,12 @@ be set:
 (a) Domain, KeyFile, Selector, no KeyTable, no SigningTable;
 (b) KeyTable, SigningTable, no Domain, no KeyFile, no Selector;
 
-TP
+The action to sign or verify is also affected by the InternalHosts, MacroList,
+and MacroListVerify options.  Those options may preclude signing or
+verification in some cases, but will not enable signing or verifying if not
+allowed by Mode.
+
+.TP
 .I MinimumKeyBits (integer)
 Establishes a minimum key size for acceptable RSA signatures.  Signatures with
 smaller key sizes, even if they otherwise pass DKIM validation, will me marked

man/dkimpy-milter.conf.5.in

@@ -244,6 +244,10 @@ Naturally, providing a value here overrides the default, so if mail from
 127.0.0.1 should be signed, the list provided here should include that
 address explicitly. [PeerList NOT IMPLEMENTED]
 
+Mail sent via connections from InternalHosts will not have any existing DKIM
+signatures verified.  This is not overridden by MacroList or Mode.  If the
+Mode is 'v', then no actions will be performed.
+
 .TP
 .I KeyFile (string)
 Gives the location of a PEM-formatted private key to be used for RSA signing
@@ -298,6 +302,10 @@ at the time the filter receives a connection from the MTA and its availability
 depends upon the version of milter used to compile the filter and the version
 of the MTA making the connection.
 
+Mail sent via connections where macros that are in MacroList are provided
+will not have any existing DKIM signatures verified.  If the Mode is 'v', then
+no actions will be performed.
+
 .TP
 .I MacroListVerify (dataset)
 Defines a set of MTA-provided
@@ -308,6 +316,10 @@ Entries in this data set follow the same form as those of the
 .I MacroList
 option above.  [this option is not inhereted from OpenDKIM]  
 
+Mail sent via connections where macros that are in MacroListVerify are
+provided will be not DKIM signed.  If the Mode is 's', then no actions will
+be performed.
+
 .TP
 .I Mode (string)
 Selects operating modes.  The string is a concatenation of characters that
@@ -327,7 +339,12 @@ be set:
 (a) Domain, KeyFile, Selector, no KeyTable, no SigningTable;
 (b) KeyTable, SigningTable, no Domain, no KeyFile, no Selector;
 
-TP
+The action to sign or verify is also affected by the InternalHosts, MacroList,
+and MacroListVerify options.  Those options may preclude signing or
+verification in some cases, but will not enable signing or verifying if not
+allowed by Mode.
+
+.TP
 .I MinimumKeyBits (integer)
 Establishes a minimum key size for acceptable RSA signatures.  Signatures with
 smaller key sizes, even if they otherwise pass DKIM validation, will me marked

setup.py

@@ -83,13 +83,13 @@ class FileMacroExpand(distutils.cmd.Command):
 kw = {}  # Work-around for lack of 'or' requires in setuptools.
 try:
     import dns
-    kw['install_requires'] = ['dkimpy>=1.0', 'pymilter', 'authres>=1.1.0', 'PyNaCl', 'dnspython>=1.16.0']
+    kw['install_requires'] = ['dkimpy>=1.1.0', 'pymilter>=1.0.5', 'authres>=1.1.0', 'PyNaCl', 'dnspython>=1.16.0']
 except ImportError:  # If PyDNS is not installed, prefer dnspython
-    kw['install_requires'] = ['dkimpy>=1.0', 'pymilter', 'authres>=1.1.0', 'PyNaCl', 'Py3DNS']
+    kw['install_requires'] = ['dkimpy>=1.1.0', 'pymilter>=1.0.5', 'authres>=1.1.0', 'PyNaCl', 'Py3DNS']
 
 setup(
     name='dkimpy-milter',
-    version='1.2.1',
+    version='1.2.3',
     author='Scott Kitterman',
     author_email='scott@kitterman.com',
     url='https://launchpad.net/dkimpy-milter',

tests/02_sign_message.miltertest

@@ -26,7 +26,7 @@ function connect_and_send (sockname, headers, body)
    end
 
    -- mt.macro(conn, SMFIC_MAIL, "i", "simple-message")
-   if mt.mailfrom(conn, "<alice@example.net>") ~= nil then
+   if mt.mailfrom(conn, "<alicüe@example.net> (Alicþþÿÿe)") ~= nil then
       error "mt.mailfrom() failed"
    end
    if mt.getreply(conn) ~= SMFIR_CONTINUE then