diff --git a/CHANGES b/CHANGES index 6c4c8ad..c4e2315 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,23 @@ +1.2.3 2023-02-26 + - Improve support for non-ASCII email messages. Anything UTF-8 should work + (including correct signing/verification). For messages that contain header + fields with non-ASCII or UTF-8 content, signatures are likely fail + verification, but the milter should continue to run. (Thanks to Casper + Bruun for help with this) + - Set minimum pymilter and dkimpy versions in setup.py to those that will + work reliably with non-ASCII content. + - Fixed support for percent in KeyTable - Thanks to Mika Tiainen + - Fix formatting for MinimumKeyBits in dkimpy-milter.conf(5) + (Closes: #995335) + - Reset the i= signature identity in get_identities_sign() (Closes: #981157) + - Improve documentation of inter-relationship between Mode, InternalHosts, + MacroList, and MacroListVerify options in dkimpy-milter.conf.5 (Closes: + #969215) + - Fix subdomain signing with top-level organizational domain (LP: #1999434) + - Thanks to Matthias Hunstock for the report and the fix + - Fix comma separated list processing in dkimpy_milter/config.py + (LP: #1901445) + 1.2.2 2020-08-09 - Improve README.md formating for markdown display on pypi - Improve documentation in dkimpy-milter.conf (5) and README.md for signing diff --git a/README.md b/README.md index 49efe6f..8d51b01 100644 --- a/README.md +++ b/README.md @@ -336,3 +336,8 @@ later support Ed25519 signing and verification. RFC 8301 removed rsa-sha1 from DKIM. dkimpy-milter does not sign with rsa-sha1, but still considers rsa-sha1 signatures as valid for verification because they are still in common use and are not known to be cryptographically broken. + +Support for non-ASCII email messages: Anything UTF-8 should work (including +correct signing/verification). For messages that contain header fields with +non-ASCII or UTF-8 content, signatures are likely fail verification, but the +milter should continue to run. RFC 8616 is not supported. diff --git a/dkimpy_milter/__init__.py b/dkimpy_milter/__init__.py index 5118ae8..a6bb26f 100644 --- a/dkimpy_milter/__init__.py +++ b/dkimpy_milter/__init__.py @@ -39,7 +39,7 @@ from dkimpy_milter.util import write_pid from dkimpy_milter.util import get_keys from dkimpy_milter.util import fold -__version__ = "1.2.0" +__version__ = "1.2.3" FWS = re.compile(r'\r?\n[ \t]+') @@ -101,15 +101,28 @@ class dkimMilter(Milter.Base): if self.conf.get('Syslog') and self.conf.get('debugLevel') >= 1: syslog.syslog("connect from {0} at {1} {2}" .format(hostname, hostaddr, connecttype)) + if self.conf.get('Syslog') and self.conf.get('debugLevel') >= 3: + syslog.syslog("internal_conn: {0}, external_conn: {1}" + .format(self.internal_connection, self.external_connection)) + return Milter.CONTINUE # multiple messages can be received on a single connection # envfrom (MAIL FROM in the SMTP protocol) seems to mark the start # of each message. @Milter.noreply - def envfrom(self, f, *str): + def envfrom(self, f, *moredata): + try: + f = str(codecs.encode(f, 'UTF-8', 'replace'), 'UTF-8', 'ignore') + except TypeError: + f = codecs.encode(f, 'UTF-8', 'replace').decode() + try: + moredata = str(codecs.encode(str(moredata), 'UTF-8', 'replace'), 'UTF-8', 'ignore') + except TypeError: + moredata = codecs.encode(str(moredata), 'UTF-8', 'replace').decode() + if self.conf.get('Syslog') and self.conf.get('debugLevel') >= 2: - syslog.syslog("mail from: {0} {1}".format(f, str)) + syslog.syslog("mail from: {0} {1}".format(f, moredata)) self.fp = io.BytesIO() self.mailfrom = f t = parse_addr(f) @@ -141,6 +154,11 @@ class dkimMilter(Milter.Base): self.fdomain = self.author.split('@')[1].lower() except IndexError as er: pass # self.author was not a proper email address + # This keeps non-ascii characters out of the From domain + try: + self.fdomain = str(codecs.encode(self.fdomain, 'ascii', 'replace'), 'ascii', 'ignore') + except TypeError: + self.fdomain = codecs.encode(self.fdomain, 'ascii', 'replace').decode('ascii','ignore') if (self.conf.get('Syslog') and self.conf.get('debugLevel') >= 1): syslog.syslog("{0}: {1}".format(name, val)) @@ -148,7 +166,11 @@ class dkimMilter(Milter.Base): self.arheaders.append(val) if self.fp: try: - self.fp.write(b"%s: %s\n" % (codecs.encode(name, 'ascii'), codecs.encode(val, 'ascii'))) + if lname == 'from': + # Non-ascii in email address localpart is legal, so this is a special case + self.fp.write(b"%s: %s\n" % (codecs.encode(name, 'ascii'), codecs.encode(val, 'UTF-8', 'replace'))) + else: + self.fp.write(b"%s: %s\n" % (codecs.encode(name, 'ascii'), codecs.encode(val, 'ascii'))) except: # Don't choke on header fields with non-ascii garbage in them. pass @@ -195,6 +217,8 @@ class dkimMilter(Milter.Base): syslog.syslog('self.domain: {0}, self.fdomain: {1}, self.iequals: {2}'.format(self.domain, self.fdomain, self.iequals)) if ((self.fdomain in self.domain) and not self.conf.get('Mode') == 'v' and not self.external_connection): + if (self.conf.get('Syslog') and self.conf.get('debugLevel') >= 3): + syslog.syslog("Signing DKIM") self.sign_dkim(txt) if ((self.has_dkim) and (not self.internal_connection) and (self.conf.get('Mode') == 'v' or @@ -227,7 +251,7 @@ class dkimMilter(Milter.Base): def get_identities_sign(self): """Determine d= and i= identiies for signature""" self.domain = [] - iequals = None + self.iequals = None try: self.privkeyRSA = self.conf.get('privateRSA') except: @@ -280,6 +304,7 @@ class dkimMilter(Milter.Base): keytabledata = self.conf.get('privateRSATable')[keytablekey] try: self.fdomain = keytabledata[0] + self.domain.append(self.fdomain) self.selectorRSA = keytabledata[1] self.privkeyRSA = keytabledata[2] except: @@ -290,11 +315,14 @@ class dkimMilter(Milter.Base): keytabledata = self.conf.get('privateEd25519Table')[keytablekey] try: self.fdomain = keytabledata[0] + self.domain.append(self.fdomain) self.selectorEd25519 = keytabledata[1] self.privkeyEd25519 = keytabledata[2] except: if (self.conf.get('Syslog')): syslog.syslog('Error: Invalid KeyTable data {0}'.format(keytabledata)) + if (self.fdomain == '%'): + self.fdomain = self.author.split('@')[1].lower() break def sign_dkim(self, txt): diff --git a/dkimpy_milter/config.py b/dkimpy_milter/config.py index cbbfaea..1cad1cd 100644 --- a/dkimpy_milter/config.py +++ b/dkimpy_milter/config.py @@ -89,16 +89,23 @@ class HostsDataset(object): self.item = item[1:] self.negative = True try: - self.item = ipaddress.ip_address(str(self.item, "utf-8")) + try: + self.item = ipaddress.ip_address(str(self.item, "utf-8")) + except TypeError: + self.item = ipaddress.ip_address(self.item) if isinstance(self.item, ipaddress.IPv4Address): self.isipv4 = True elif isinstance(self.item, ipaddress.IPv6Address): self.isipv6 = True except ValueError as e: try: - self.item = ipaddress.ip_network(str - (self.item, "utf-8"), - strict=False) + try: + self.item = ipaddress.ip_network(str + (self.item, "utf-8"), + strict=False) + except TypeError: + self.item = ipaddress.ip_network(self.item, + strict=False) if isinstance(self.item, ipaddress.IPv4Network): self.isipv4cidr = True elif isinstance(self.item, ipaddress.IPv6Network): @@ -114,7 +121,10 @@ class HostsDataset(object): def match(self, connectip): '''Check if the connect IP is part of the dataset''' - source = ipaddress.ip_address(str(connectip, "utf-8")) + try: + source = ipaddress.ip_address(str(connectip, "utf-8")) + except TypeError: + source = ipaddress.ip_address(connectip) for item in self.dataset: if item.isdomain or item.ishostname: result = self.matchname(source) # Match host/domains first @@ -164,13 +174,19 @@ class HostsDataset(object): if isinstance(source, ipaddress.IPv4Address): ips = s.dns(name, 'A') for ip in ips: - ip = ipaddress.IPv4Address(str(ip, 'UTF-8')) + try: + ip = ipaddress.IPv4Address(str(ip, 'UTF-8')) + except TypeError: + ip = ipaddress.IPv4Address(ip) if ip == source: results.append(name) if isinstance(source, ipaddress.IPv6Address): ips = s.dns(name, 'AAAA') for ip in ips: - ip = ipaddress.IPv6Address(str(ip, 'UTF-8')) + try: + ip = ipaddress.IPv6Address(str(ip, 'UTF-8')) + except TypeError: + ip = ipaddress.IPv6Address(ip) if ip == source: results.append(name) return results @@ -306,9 +322,15 @@ def _dataset_to_list(dataset): return dsd # If it's a str and csl, it has one value and we return a list if dataset[:4] == 'csl:': - return [dataset[4:].strip().strip(',')] + datalist = dataset[4:].split(',') + for item in datalist: + datalist[datalist.index(item)] = item.strip().strip(',') + return datalist else: - return [dataset.strip().strip(',')] + datalist = dataset.split(',') + for item in datalist: + datalist[datalist.index(item)] = item.strip().strip(',') + return datalist if dataset[-3:] == '.db' or dataset[:3] == 'db:': # This is a Sleepycat (Oracle) DB dataset, which we dont support raise dkim.ParameterError('Unsupported dataset db datase: {0}' @@ -439,8 +461,14 @@ def _readConfigFile(path, configData=None, configGlobal={}): fp.close() try: configData['AuthservID'] = _make_authserv_id(configData.get('AuthservID', 'HOSTNAME')) + except Exception as e: + syslog.syslog("Could not make AuthservID: {}".format(e)) + pass + + try: configData['IntHosts'] = HostsDataset(configData['InternalHosts']) - except: + except Exception as e: + syslog.syslog("Could not make HostDataset from InternalHosts: {}".format(e)) pass return(configData) diff --git a/man/dkimpy-milter.conf.5 b/man/dkimpy-milter.conf.5 index a363598..289d09e 100644 --- a/man/dkimpy-milter.conf.5 +++ b/man/dkimpy-milter.conf.5 @@ -244,6 +244,10 @@ Naturally, providing a value here overrides the default, so if mail from 127.0.0.1 should be signed, the list provided here should include that address explicitly. [PeerList NOT IMPLEMENTED] +Mail sent via connections from InternalHosts will not have any existing DKIM +signatures verified. This is not overridden by MacroList or Mode. If the +Mode is 'v', then no actions will be performed. + .TP .I KeyFile (string) Gives the location of a PEM-formatted private key to be used for RSA signing @@ -298,6 +302,10 @@ at the time the filter receives a connection from the MTA and its availability depends upon the version of milter used to compile the filter and the version of the MTA making the connection. +Mail sent via connections where macros that are in MacroList are provided +will not have any existing DKIM signatures verified. If the Mode is 'v', then +no actions will be performed. + .TP .I MacroListVerify (dataset) Defines a set of MTA-provided @@ -308,6 +316,10 @@ Entries in this data set follow the same form as those of the .I MacroList option above. [this option is not inhereted from OpenDKIM] +Mail sent via connections where macros that are in MacroListVerify are +provided will be not DKIM signed. If the Mode is 's', then no actions will +be performed. + .TP .I Mode (string) Selects operating modes. The string is a concatenation of characters that @@ -327,7 +339,12 @@ be set: (a) Domain, KeyFile, Selector, no KeyTable, no SigningTable; (b) KeyTable, SigningTable, no Domain, no KeyFile, no Selector; -TP +The action to sign or verify is also affected by the InternalHosts, MacroList, +and MacroListVerify options. Those options may preclude signing or +verification in some cases, but will not enable signing or verifying if not +allowed by Mode. + +.TP .I MinimumKeyBits (integer) Establishes a minimum key size for acceptable RSA signatures. Signatures with smaller key sizes, even if they otherwise pass DKIM validation, will me marked diff --git a/man/dkimpy-milter.conf.5.in b/man/dkimpy-milter.conf.5.in index 45e3097..48aa210 100644 --- a/man/dkimpy-milter.conf.5.in +++ b/man/dkimpy-milter.conf.5.in @@ -244,6 +244,10 @@ Naturally, providing a value here overrides the default, so if mail from 127.0.0.1 should be signed, the list provided here should include that address explicitly. [PeerList NOT IMPLEMENTED] +Mail sent via connections from InternalHosts will not have any existing DKIM +signatures verified. This is not overridden by MacroList or Mode. If the +Mode is 'v', then no actions will be performed. + .TP .I KeyFile (string) Gives the location of a PEM-formatted private key to be used for RSA signing @@ -298,6 +302,10 @@ at the time the filter receives a connection from the MTA and its availability depends upon the version of milter used to compile the filter and the version of the MTA making the connection. +Mail sent via connections where macros that are in MacroList are provided +will not have any existing DKIM signatures verified. If the Mode is 'v', then +no actions will be performed. + .TP .I MacroListVerify (dataset) Defines a set of MTA-provided @@ -308,6 +316,10 @@ Entries in this data set follow the same form as those of the .I MacroList option above. [this option is not inhereted from OpenDKIM] +Mail sent via connections where macros that are in MacroListVerify are +provided will be not DKIM signed. If the Mode is 's', then no actions will +be performed. + .TP .I Mode (string) Selects operating modes. The string is a concatenation of characters that @@ -327,7 +339,12 @@ be set: (a) Domain, KeyFile, Selector, no KeyTable, no SigningTable; (b) KeyTable, SigningTable, no Domain, no KeyFile, no Selector; -TP +The action to sign or verify is also affected by the InternalHosts, MacroList, +and MacroListVerify options. Those options may preclude signing or +verification in some cases, but will not enable signing or verifying if not +allowed by Mode. + +.TP .I MinimumKeyBits (integer) Establishes a minimum key size for acceptable RSA signatures. Signatures with smaller key sizes, even if they otherwise pass DKIM validation, will me marked diff --git a/setup.py b/setup.py index fcb4e37..e760bec 100644 --- a/setup.py +++ b/setup.py @@ -83,13 +83,13 @@ class FileMacroExpand(distutils.cmd.Command): kw = {} # Work-around for lack of 'or' requires in setuptools. try: import dns - kw['install_requires'] = ['dkimpy>=1.0', 'pymilter', 'authres>=1.1.0', 'PyNaCl', 'dnspython>=1.16.0'] + kw['install_requires'] = ['dkimpy>=1.1.0', 'pymilter>=1.0.5', 'authres>=1.1.0', 'PyNaCl', 'dnspython>=1.16.0'] except ImportError: # If PyDNS is not installed, prefer dnspython - kw['install_requires'] = ['dkimpy>=1.0', 'pymilter', 'authres>=1.1.0', 'PyNaCl', 'Py3DNS'] + kw['install_requires'] = ['dkimpy>=1.1.0', 'pymilter>=1.0.5', 'authres>=1.1.0', 'PyNaCl', 'Py3DNS'] setup( name='dkimpy-milter', - version='1.2.2', + version='1.2.3', author='Scott Kitterman', author_email='scott@kitterman.com', url='https://launchpad.net/dkimpy-milter', diff --git a/tests/02_sign_message.miltertest b/tests/02_sign_message.miltertest index 77907ba..1596240 100644 --- a/tests/02_sign_message.miltertest +++ b/tests/02_sign_message.miltertest @@ -26,7 +26,7 @@ function connect_and_send (sockname, headers, body) end -- mt.macro(conn, SMFIC_MAIL, "i", "simple-message") - if mt.mailfrom(conn, "") ~= nil then + if mt.mailfrom(conn, " (Alicþþÿÿe)") ~= nil then error "mt.mailfrom() failed" end if mt.getreply(conn) ~= SMFIR_CONTINUE then