From 391b5352f396d8fed9b497558a3566c06fc3bdf8 Mon Sep 17 00:00:00 2001 From: Daniel Kahn Gillmor Date: Wed, 20 Feb 2019 16:57:32 -0500 Subject: [PATCH 1/2] Convert mostly to python3 (still need strings/bytes conversions) This covers conversion of the whole project to python3, *except* for the strings/bytes distinction in __init__.py, which i'm leaving for a second commit. The changes in this commit are intended to be relatively uncontroversial, so that the following commit contains the tricky bits. --- dkimpy_milter/__init__.py | 2 +- dkimpy_milter/__main__.py | 2 +- dkimpy_milter/config.py | 20 ++++++++++---------- dkimpy_milter/dnsplug.py | 16 ++++++++-------- setup.py | 8 ++++---- tests/dkimpy-milter | 2 +- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/dkimpy_milter/__init__.py b/dkimpy_milter/__init__.py index 5345fc7..c4c407b 100644 --- a/dkimpy_milter/__init__.py +++ b/dkimpy_milter/__init__.py @@ -1,4 +1,4 @@ -#! /usr/bin/python2 +#! /usr/bin/python3 # Original dkim-milter.py code: # Author: Stuart D. Gathman # Copyright 2007 Business Management Systems, Inc. diff --git a/dkimpy_milter/__main__.py b/dkimpy_milter/__main__.py index 8c5cf9c..f95075f 100644 --- a/dkimpy_milter/__main__.py +++ b/dkimpy_milter/__main__.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2 +#!/usr/bin/python3 from dkimpy_milter import main diff --git a/dkimpy_milter/config.py b/dkimpy_milter/config.py index d562e97..bf6551a 100644 --- a/dkimpy_milter/config.py +++ b/dkimpy_milter/config.py @@ -31,13 +31,13 @@ import stat import dkim import socket import ipaddress -from dnsplug import Session +from .dnsplug import Session # default values defaultConfigData = { 'Syslog': 'yes', 'SyslogFacility': 'mail', - 'UMask': 007, + 'UMask': 0o07, 'Mode': 'sv', 'Socket': 'local:/var/run/dkimpy-milter/dkimpy-milter.sock', 'PidFile': '/var/run/dkimpy-milter/dkimpy-milter.pid', @@ -85,14 +85,14 @@ class HostsDataset(object): self.item = item[1:] self.negative = True try: - self.item = ipaddress.ip_address(unicode(self.item, "utf-8")) + self.item = ipaddress.ip_address(str(self.item, "utf-8")) if isinstance(self.item, ipaddress.IPv4Address): self.isipv4 = True elif isinstance(self.item, ipaddress.IPv6Address): self.isipv6 = True except ValueError as e: try: - self.item = ipaddress.ip_network(unicode + self.item = ipaddress.ip_network(str (self.item, "utf-8"), strict=False) if isinstance(self.item, ipaddress.IPv4Network): @@ -110,7 +110,7 @@ class HostsDataset(object): def match(self, connectip): '''Check if the connect IP is part of the dataset''' - source = ipaddress.ip_address(unicode(connectip, "utf-8")) + source = ipaddress.ip_address(str(connectip, "utf-8")) for item in self.dataset: if item.isdomain or item.ishostname: result = self.matchname(source) # Match host/domains first @@ -160,13 +160,13 @@ class HostsDataset(object): if isinstance(source, ipaddress.IPv4Address): ips = s.dns(name, 'A') for ip in ips: - ip = ipaddress.IPv4Address(unicode(ip, 'UTF-8')) + ip = ipaddress.IPv4Address(str(ip, 'UTF-8')) if ip == source: results.append(name) if isinstance(source, ipaddress.IPv6Address): ips = s.dns(name, 'AAAA') for ip in ips: - ip = ipaddress.IPv6Address(unicode(ip, 'UTF-8')) + ip = ipaddress.IPv6Address(str(ip, 'UTF-8')) if ip == source: results.append(name) return results @@ -225,13 +225,13 @@ def _processConfigFile(filename=None, configdata=None, useSyslog=1, '''Load the specified config file, exit and log errors if it fails, otherwise return a config dictionary.''' - import config + from . import config if configdata is None: configdata = config.defaultConfigData if filename is not None: try: _readConfigFile(filename, configdata) - except Exception, e: + except Exception as e: raise if useSyslog: syslog.syslog(e.args[0]) @@ -342,7 +342,7 @@ def _readConfigFile(path, configData=None, configGlobal={}): # check to see if it's a file try: mode = os.stat(path)[0] - except OSError, e: + except OSError as e: syslog.syslog(syslog.LOG_ERR, 'ERROR stating "%s": %s' % (path, e.strerror)) return(configData) diff --git a/dkimpy_milter/dnsplug.py b/dkimpy_milter/dnsplug.py index aa357ec..33067e9 100644 --- a/dkimpy_milter/dnsplug.py +++ b/dkimpy_milter/dnsplug.py @@ -84,7 +84,7 @@ class Session(object): raise DNSError('Length of CNAME chain exceeds %d' % MAX_CNAME) cnames[name] = cname if cname in cnames: - raise DNSError, 'CNAME loop' + raise DNSError('CNAME loop') result = self.dns(cname, qtype, cnames=cnames) return result @@ -103,16 +103,16 @@ def DNSLookup_pydns(name, qtype, tcpfallback=True, timeout=30): # if resp.header['tc'] == True: if not tcpfallback: - raise DNS.DNSError, 'DNS: Truncated UDP Reply, SPF records should fit in a UDP packet' + raise DNS.DNSError('DNS: Truncated UDP Reply, SPF records should fit in a UDP packet') try: req = DNS.DnsRequest(name, qtype=qtype, protocol='tcp', timeout=timeout) resp = req.req() - except DNS.DNSError, x: - raise DNS.DNSError, 'TCP Fallback error: ' + str(x) + except DNS.DNSError as x: + raise DNS.DNSError('TCP Fallback error: ' + str(x)) return [((a['name'], a['typename']), a['data']) for a in resp.answers] - except IOError, x: - raise DNS.DNSError, 'DNS: ' + str(x) + except IOError as x: + raise DNS.DNSError('DNS: ' + str(x)) def DNSLookup_dnspython(name,qtype,tcpfallback=True,timeout=30): retVal = [] @@ -164,5 +164,5 @@ if __name__ == '__main__': import sys s = Session() for n,t in zip(*[iter(sys.argv[1:])]*2): - print n,t - print s.dns(n,t) + print(n,t) + print(s.dns(n,t)) diff --git a/setup.py b/setup.py index 18c2ec9..c7990d7 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/python3 # dkimpy-milter: A DKIM signing/verification Milter application # Author: Scott Kitterman # Copyright 2018 Scott Kitterman @@ -24,9 +24,9 @@ description = "Domain Keys Identified Mail (DKIM) signing/verifying milter for P kw = {} # Work-around for lack of 'or' requires in setuptools. try: import dns - kw['install_requires'] = ['dkimpy>=0.7', 'pymilter', 'authres>=1.1.0', 'PyNaCl', 'ipaddress', 'dnspython'] + kw['install_requires'] = ['dkimpy>=0.7', 'pymilter', 'authres>=1.1.0', 'PyNaCl', 'dnspython'] except ImportError: # If PyDNS is not installed, prefer dnspython - kw['install_requires'] = ['dkimpy>=0.7', 'pymilter', 'authres>=1.1.0', 'PyNaCl', 'ipaddress', 'PyDNS'] + kw['install_requires'] = ['dkimpy>=0.7', 'pymilter', 'authres>=1.1.0', 'PyNaCl', 'PyDNS'] setup( name='dkimpy-milter', @@ -43,7 +43,7 @@ setup( 'License :: OSI Approved :: GNU General Public License (GPL)', 'Natural Language :: English', 'Operating System :: POSIX', - 'Programming Language :: Python :: 2 :: Only', + 'Programming Language :: Python :: 3 :: Only', 'Topic :: Communications :: Email :: Mail Transport Agents', 'Topic :: Communications :: Email :: Filters', 'Topic :: Security', diff --git a/tests/dkimpy-milter b/tests/dkimpy-milter index 39b64d5..9ee02e9 100755 --- a/tests/dkimpy-milter +++ b/tests/dkimpy-milter @@ -1,2 +1,2 @@ #!/bin/sh -python2 -m dkimpy_milter "$@" +python3 -m dkimpy_milter "$@" From ea09bab1a8fb9eeed3429e9a8411ee42f9c423f3 Mon Sep 17 00:00:00 2001 From: Daniel Kahn Gillmor Date: Wed, 20 Feb 2019 19:01:30 -0500 Subject: [PATCH 2/2] Convert __init__.py to python3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The main work here is about bytes vs. strings. This work was confusing for several reasons: * pymilter thinks that headers are all strings, but body is bytes * dkimpy wants to deal with bytes objects generally (though it accepts a string object as an ed25519 secret key for some reason, despite requiring bytes as an RSA secret key) * authres.AuthenticationResultsHeader object converts easily to a string, but has no direct bytes conversion. meanwhile, it wants its arguments as strings, but will accept them if they are bytes and convert them with something like str(), which leaves weird cruft like "header.a=b'ed25519-sha256'" * dkimpy_milter/utils.py contains fold() which expects bytes * self.fp needs to accumulate the on-the-wire version of the message as a whole (so it needs to be bytes). That means converting the headers. Header names and values are US-ASCII, per ยง2.2 of RFC 5322, so they should be convertible cleanly, but we still have to convert them explicitly so that python knows the right thing to do. At any rate, tests/runtests all passes with these changes, and the output for both Authentication-Results: and DKIM-Signature headers looks the same. --- dkimpy_milter/__init__.py | 61 +++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/dkimpy_milter/__init__.py b/dkimpy_milter/__init__.py index c4c407b..bcdf2a7 100644 --- a/dkimpy_milter/__init__.py +++ b/dkimpy_milter/__init__.py @@ -28,8 +28,9 @@ import dkim import authres import os import tempfile -import StringIO +import io import re +import codecs from Milter.utils import parse_addr, parseaddr import dkimpy_milter.config as config from dkimpy_milter.util import drop_privileges @@ -110,7 +111,7 @@ class dkimMilter(Milter.Base): def envfrom(self, f, *str): if milterconfig.get('Syslog') and milterconfig.get('debugLevel') >= 2: syslog.syslog("mail from: {0} {1}".format(f, str)) - self.fp = StringIO.StringIO() + self.fp = io.BytesIO() self.mailfrom = f t = parse_addr(f) if len(t) == 2: @@ -142,13 +143,13 @@ class dkimMilter(Milter.Base): elif lname == 'authentication-results': self.arheaders.append(val) if self.fp: - self.fp.write("%s: %s\n" % (name, val)) + self.fp.write(b"%s: %s\n" % (codecs.encode(name, 'ascii'), codecs.encode(val, 'ascii'))) return Milter.CONTINUE @Milter.noreply def eoh(self): if self.fp: - self.fp.write("\n") # terminate headers + self.fp.write(b"\n") # terminate headers self.bodysize = 0 return Milter.CONTINUE @@ -195,20 +196,20 @@ class dkimMilter(Milter.Base): h = authres.AuthenticationResultsHeader(authserv_id= self.AuthservID, results=self.arresults) - h = fold(str(h)) + h = fold(codecs.encode(str(h), 'ascii')) if (milterconfig.get('Syslog') and milterconfig.get('debugLevel') >= 2): - syslog.syslog(str(h)) - name, val = str(h).split(': ', 1) + syslog.syslog(codecs.decode(h, 'ascii')) + name, val = codecs.decode(h, 'ascii').split(': ', 1) self.addheader(name, val, 0) return Milter.CONTINUE def sign_dkim(self, txt): - canon = milterconfig.get('Canonicalization') + canon = codecs.encode(milterconfig.get('Canonicalization'), 'ascii') canonicalize = [] - if len(canon.split('/')) == 2: - canonicalize.append(canon.split('/')[0]) - canonicalize.append(canon.split('/')[1]) + if len(canon.split(b'/')) == 2: + canonicalize.append(canon.split(b'/')[0]) + canonicalize.append(canon.split(b'/')[1]) else: canonicalize.append(canon) canonicalize.append(canon) @@ -218,11 +219,12 @@ class dkimMilter(Milter.Base): try: if privateRSA: d = dkim.DKIM(txt) - h = d.sign(milterconfig.get('Selector'), self.fdomain, - privateRSA, canonicalize=(canonicalize[0], - canonicalize[1])) - name, val = h.split(': ', 1) - self.addheader(name, val.strip().replace('\r\n', '\n'), 0) + h = d.sign(codecs.encode(milterconfig.get('Selector'), 'ascii'), codecs.encode(self.fdomain, 'ascii'), + codecs.encode(privateRSA, 'ascii'), + canonicalize=(canonicalize[0], + canonicalize[1])) + name, val = h.split(b': ', 1) + self.addheader(codecs.decode(name, 'ascii'), codecs.decode(val, 'ascii').strip().replace('\r\n', '\n'), 0) if (milterconfig.get('Syslog') and (milterconfig.get('SyslogSuccess') or milterconfig.get('debugLevel') >= 1)): @@ -233,12 +235,12 @@ class dkimMilter(Milter.Base): d.domain.lower())) if privateEd25519: d = dkim.DKIM(txt) - h = d.sign(milterconfig.get('SelectorEd25519'), self.fdomain, + h = d.sign(codecs.encode(milterconfig.get('SelectorEd25519'), 'ascii'), codecs.encode(self.fdomain, 'ascii'), privateEd25519, canonicalize=(canonicalize[0], canonicalize[1]), - signature_algorithm='ed25519-sha256') - name, val = h.split(': ', 1) - self.addheader(name, val.strip().replace('\r\n', '\n'), 0) + signature_algorithm=b'ed25519-sha256') + name, val = h.split(b': ', 1) + self.addheader(codecs.decode(name, 'ascii'), codecs.decode(val, 'ascii').strip().replace('\r\n', '\n'), 0) if (milterconfig.get('Syslog') and (milterconfig.get('SyslogSuccess') or milterconfig.get('debugLevel') >= 1)): @@ -266,20 +268,17 @@ class dkimMilter(Milter.Base): res = d.verify(idx=y, dnsfunc=lambda _x: dnsoverride) else: res = d.verify(idx=y) + algo = codecs.decode(d.signature_fields.get(b'a'), 'ascii') if res: - if d.signature_fields.get(b'a') == 'ed25519-sha256': + if algo == 'ed25519-sha256': self.dkim_comment = ('Good {0} signature' - .format(d.signature_fields - .get(b'a'))) + .format(algo)) else: self.dkim_comment = ('Good {0} bit {1} signature' - .format(d.keysize, - d.signature_fields - .get(b'a'))) + .format(d.keysize, algo)) else: self.dkim_comment = ('Bad {0} bit {1} signature.' - .format(d.keysize, - d.signature_fields.get(b'a'))) + .format(d.keysize, algo)) except dkim.DKIMException as x: self.dkim_comment = str(x) if milterconfig.get('Syslog'): @@ -288,9 +287,9 @@ class dkimMilter(Milter.Base): self.dkim_comment = str(x) if milterconfig.get('Syslog'): syslog.syslog("check_dkim: {0}".format(x)) - self.header_i = d.signature_fields.get(b'i') - self.header_d = d.signature_fields.get(b'd') - self.header_a = d.signature_fields.get(b'a') + self.header_i = codecs.decode(d.signature_fields.get(b'i'), 'ascii') + self.header_d = codecs.decode(d.signature_fields.get(b'd'), 'ascii') + self.header_a = codecs.decode(d.signature_fields.get(b'a'), 'ascii') if res: if (milterconfig.get('Syslog') and (milterconfig.get('SyslogSuccess') or