Stuart Gathman
608 lines
20 KiB
Python
608 lines
20 KiB
Python
# $Log$
|
|
# Revision 1.51 2004/03/25 03:19:10 stuart
|
|
# Correctly defang rfc822 attachments when boundary specified with
|
|
# content-type message/rfc822.
|
|
#
|
|
# Revision 1.50 2003/10/15 22:01:00 stuart
|
|
# Test for and work around email bug with encoded filenames.
|
|
#
|
|
# Revision 1.49 2003/09/04 18:48:13 stuart
|
|
# Support python-2.2.3
|
|
#
|
|
# Revision 1.48 2003/09/02 00:27:27 stuart
|
|
# Should have full milter based dspam support working
|
|
#
|
|
# Revision 1.47 2003/08/26 06:08:18 stuart
|
|
# Use new python boolean since we now require 2.2.2
|
|
#
|
|
# Revision 1.46 2003/08/26 05:01:38 stuart
|
|
# Release 0.6.0
|
|
#
|
|
# Revision 1.45 2003/08/26 04:01:24 stuart
|
|
# Use new email module for parsing mail. Still need mime module to
|
|
# provide various bug fixes to email module, and maintain some compatibility
|
|
# with old milter code.
|
|
#
|
|
|
|
# This module provides a "defang" function to replace naughty attachments
|
|
# with a warning message.
|
|
|
|
# Author: Stuart D. Gathman <stuart@bmsi.com>
|
|
# Copyright 2001 Business Management Systems, Inc.
|
|
# This code is under GPL. See COPYING for details.
|
|
|
|
import StringIO
|
|
import socket
|
|
import Milter
|
|
import email
|
|
import email.Message
|
|
from email.Message import Message
|
|
from email.Generator import Generator
|
|
from email.Utils import quote
|
|
from email import Utils
|
|
|
|
from types import ListType,StringType
|
|
|
|
# Enhance email.Parser
|
|
# - Fix _parsebody to decode message attachments before parsing
|
|
|
|
from email.Parser import Parser
|
|
try: from email.Parser import NLCRE
|
|
except: from email.Parser import nlcre as NLCRE
|
|
|
|
from email import Errors
|
|
|
|
class MimeParser(Parser):
|
|
|
|
# This is a copy of _parsebody from email.Parser, with a fix
|
|
# for message attachments. I couldn't find a smaller way to patch it
|
|
# in a subclass.
|
|
|
|
def _parsebody(self, container, fp, firstbodyline=None):
|
|
# Parse the body, but first split the payload on the content-type
|
|
# boundary if present.
|
|
boundary = container.get_boundary()
|
|
isdigest = (container.get_content_type() == 'multipart/digest')
|
|
# If there's a boundary, split the payload text into its constituent
|
|
# parts and parse each separately. Otherwise, just parse the rest of
|
|
# the body as a single message. Note: any exceptions raised in the
|
|
# recursive parse need to have their line numbers coerced.
|
|
if boundary:
|
|
preamble = epilogue = None
|
|
# Split into subparts. The first boundary we're looking for won't
|
|
# always have a leading newline since we're at the start of the
|
|
# body text, and there's not always a preamble before the first
|
|
# boundary.
|
|
separator = '--' + boundary
|
|
payload = fp.read()
|
|
if firstbodyline is not None:
|
|
payload = firstbodyline + '\n' + payload
|
|
# We use an RE here because boundaries can have trailing
|
|
# whitespace.
|
|
mo = re.search(
|
|
r'(?P<sep>' + re.escape(separator) + r')(?P<ws>[ \t]*)',
|
|
payload)
|
|
if not mo:
|
|
if self._strict:
|
|
raise Errors.BoundaryError(
|
|
"Couldn't find starting boundary: %s" % boundary)
|
|
container.set_payload(payload)
|
|
return
|
|
start = mo.start()
|
|
if start > 0:
|
|
# there's some pre-MIME boundary preamble
|
|
preamble = payload[0:start]
|
|
# Find out what kind of line endings we're using
|
|
start += len(mo.group('sep')) + len(mo.group('ws'))
|
|
mo = NLCRE.search(payload, start)
|
|
if mo:
|
|
start += len(mo.group(0))
|
|
# We create a compiled regexp first because we need to be able to
|
|
# specify the start position, and the module function doesn't
|
|
# support this signature. :(
|
|
cre = re.compile('(?P<sep>\r\n|\r|\n)' +
|
|
re.escape(separator) + '--')
|
|
mo = cre.search(payload, start)
|
|
if mo:
|
|
terminator = mo.start()
|
|
linesep = mo.group('sep')
|
|
if mo.end() < len(payload):
|
|
# There's some post-MIME boundary epilogue
|
|
epilogue = payload[mo.end():]
|
|
elif self._strict:
|
|
raise Errors.BoundaryError(
|
|
"Couldn't find terminating boundary: %s" % boundary)
|
|
else:
|
|
# Handle the case of no trailing boundary. Check that it ends
|
|
# in a blank line. Some cases (spamspamspam) don't even have
|
|
# that!
|
|
mo = re.search('(?P<sep>\r\n|\r|\n){2}$', payload)
|
|
if not mo:
|
|
mo = re.search('(?P<sep>\r\n|\r|\n)$', payload)
|
|
if not mo:
|
|
raise Errors.BoundaryError(
|
|
'No terminating boundary and no trailing empty line')
|
|
linesep = mo.group('sep')
|
|
terminator = len(payload)
|
|
# We split the textual payload on the boundary separator, which
|
|
# includes the trailing newline. If the container is a
|
|
# multipart/digest then the subparts are by default message/rfc822
|
|
# instead of text/plain. In that case, they'll have a optional
|
|
# block of MIME headers, then an empty line followed by the
|
|
# message headers.
|
|
parts = re.split(
|
|
linesep + re.escape(separator) + r'[ \t]*' + linesep,
|
|
payload[start:terminator])
|
|
for part in parts:
|
|
if isdigest:
|
|
if part.startswith(linesep):
|
|
# There's no header block so create an empty message
|
|
# object as the container, and lop off the newline so
|
|
# we can parse the sub-subobject
|
|
msgobj = self._class()
|
|
part = part[len(linesep):]
|
|
else:
|
|
parthdrs, part = part.split(linesep+linesep, 1)
|
|
# msgobj in this case is the "message/rfc822" container
|
|
msgobj = self.parsestr(parthdrs, headersonly=1)
|
|
# while submsgobj is the message itself
|
|
msgobj.set_default_type('message/rfc822')
|
|
maintype = msgobj.get_content_maintype()
|
|
if maintype in ('message', 'multipart'):
|
|
submsgobj = self.parsestr(part)
|
|
msgobj.attach(submsgobj)
|
|
else:
|
|
msgobj.set_payload(part)
|
|
else:
|
|
msgobj = self.parsestr(part)
|
|
container.preamble = preamble
|
|
container.epilogue = epilogue
|
|
container.attach(msgobj)
|
|
elif container.get_main_type() == 'multipart':
|
|
# Very bad. A message is a multipart with no boundary!
|
|
raise Errors.BoundaryError(
|
|
'multipart message with no defined boundary')
|
|
elif container.get_type() == 'message/delivery-status':
|
|
# This special kind of type contains blocks of headers separated
|
|
# by a blank line. We'll represent each header block as a
|
|
# separate Message object
|
|
blocks = []
|
|
while True:
|
|
blockmsg = self._class()
|
|
self._parseheaders(blockmsg, fp)
|
|
if not len(blockmsg):
|
|
# No more header blocks left
|
|
break
|
|
blocks.append(blockmsg)
|
|
container.set_payload(blocks)
|
|
elif container.get_main_type() == 'message':
|
|
# Create a container for the payload, but watch out for there not
|
|
# being any headers left
|
|
container.set_payload(fp.read())
|
|
fp = StringIO.StringIO(container.get_payload(decode=True))
|
|
try:
|
|
msg = self.parse(fp)
|
|
except Errors.HeaderParseError:
|
|
msg = self._class()
|
|
self._parsebody(msg, fp)
|
|
container.set_payload([msg])
|
|
else:
|
|
text = fp.read()
|
|
if firstbodyline is not None:
|
|
text = firstbodyline + '\n' + text
|
|
container.set_payload(text)
|
|
|
|
def unquote(str):
|
|
"""Remove quotes from a string."""
|
|
if len(str) > 1:
|
|
if str.startswith('"'):
|
|
if str.endswith('"'):
|
|
str = str[1:-1]
|
|
else: # remove garbage after trailing quote
|
|
try: str = str[1:str[1:].index('"')+1]
|
|
except: return str
|
|
return str.replace('\\\\', '\\').replace('\\"', '"')
|
|
if str.startswith('<') and str.endswith('>'):
|
|
return str[1:-1]
|
|
return str
|
|
|
|
from types import TupleType
|
|
|
|
def _unquotevalue(value):
|
|
if isinstance(value, TupleType):
|
|
return value[0], value[1], unquote(value[2])
|
|
else:
|
|
return unquote(value)
|
|
|
|
email.Message._unquotevalue = _unquotevalue
|
|
|
|
def _parseparam(str):
|
|
plist = []
|
|
while str[:1] == ';':
|
|
str = str[1:]
|
|
end = str.find(';')
|
|
while end > 0 and (str.count('"',0,end) & 1):
|
|
end = str.find(';',end + 1)
|
|
if end < 0: end = len(str)
|
|
f = str[:end]
|
|
if '=' in f:
|
|
i = f.index('=')
|
|
f = f[:i].strip().lower() + \
|
|
'=' + f[i+1:].strip()
|
|
plist.append(f.strip())
|
|
str = str[end:]
|
|
return plist
|
|
|
|
# Enhance email.Message
|
|
# - Fix getparam to parse attributes IE style
|
|
# - Provide a headerchange event for integration with Milter
|
|
# Headerchange attribute can be assigned a function to be called when
|
|
# changing headers. The signature is:
|
|
# headerchange(msg,name,value) -> None
|
|
# - Track modifications to headers of body or any part independently
|
|
|
|
class MimeMessage(Message):
|
|
"""Version of email.Message.Message compatible with old mime module
|
|
"""
|
|
def __init__(self,fp=None,seekable=1):
|
|
self.headerchange = None
|
|
self.submsg = None
|
|
Message.__init__(self)
|
|
self.fp = fp
|
|
if fp:
|
|
parser = MimeParser(MimeMessage)
|
|
self.startofheaders = fp.tell()
|
|
parser._parseheaders(self,fp)
|
|
self.startofbody = fp.tell()
|
|
parser._parsebody(self,fp)
|
|
for part in self.walk():
|
|
part.modified = False
|
|
|
|
def rewindbody(self):
|
|
return self.fp.seek(self.startofbody)
|
|
|
|
# override param parsing to handle quotes
|
|
def _get_params_preserve(self,failobj=None,header='content-type'):
|
|
"Return all parameter names and values. Use parser that handles quotes."
|
|
missing = []
|
|
value = self.get(header, missing)
|
|
if value is missing:
|
|
return failobj
|
|
params = []
|
|
for p in _parseparam(';' + value):
|
|
try:
|
|
name, val = p.split('=', 1)
|
|
name = name.strip()
|
|
val = val.strip()
|
|
except ValueError:
|
|
# Must have been a bare attribute
|
|
name = p.strip()
|
|
val = ''
|
|
params.append((name, val))
|
|
params = Utils.decode_params(params)
|
|
return params
|
|
|
|
def get_filename(self, failobj=None):
|
|
"""Return the filename associated with the payload if present.
|
|
|
|
The filename is extracted from the Content-Disposition header's
|
|
`filename' parameter, and it is unquoted.
|
|
"""
|
|
missing = []
|
|
filename = self.get_param('filename', missing, 'content-disposition')
|
|
if filename is missing:
|
|
return failobj
|
|
if isinstance(filename, TupleType):
|
|
# It's an RFC 2231 encoded parameter
|
|
newvalue = _unquotevalue(filename)
|
|
if newvalue[0]:
|
|
return unicode(newvalue[2], newvalue[0])
|
|
return unicode(newvalue[2])
|
|
else:
|
|
newvalue = _unquotevalue(filename.strip())
|
|
return newvalue
|
|
|
|
getfilename = get_filename
|
|
ismultipart = Message.is_multipart
|
|
getheaders = Message.get_all
|
|
gettype = Message.get_content_type
|
|
getparam = Message.get_param
|
|
|
|
def getparams(self): return self.get_params([])
|
|
|
|
def getname(self):
|
|
return self.get_param('name')
|
|
|
|
def getnames(self):
|
|
"""Return a list of (attr,name) pairs of attributes that IE might
|
|
interpret as a name - and hence decide to execute this message."""
|
|
names = []
|
|
for attr,val in self.get_params([]):
|
|
if isinstance(val, TupleType):
|
|
# It's an RFC 2231 encoded parameter
|
|
newvalue = _unquotevalue(val)
|
|
if val[0]:
|
|
val = unicode(newvalue[2], newvalue[0])
|
|
else:
|
|
val = unicode(newvalue[2])
|
|
else:
|
|
val = _unquotevalue(val.strip())
|
|
names.append((attr,val))
|
|
return names + [("filename",self.get_filename())]
|
|
|
|
def ismodified(self):
|
|
"True if this message or a subpart has been modified."
|
|
if not self.is_multipart():
|
|
if self.submsg:
|
|
return self.submsg.ismodified()
|
|
return self.modified
|
|
if self.modified: return True
|
|
for i in self.get_payload():
|
|
if i.ismodified(): return True
|
|
return False
|
|
|
|
def dump(self,file,unixfrom=False):
|
|
"Write this message (and all subparts) to a file"
|
|
g = Generator(file)
|
|
g.flatten(self,unixfrom=unixfrom)
|
|
|
|
def getencoding(self):
|
|
return self.get('content-transfer-encoding',None)
|
|
|
|
# Decode body to stream according to transfer encoding, return encoding name
|
|
def decode(self,filter):
|
|
try:
|
|
filter.write(self.get_payload(decode=True))
|
|
except:
|
|
pass
|
|
return self.getencoding()
|
|
|
|
def get_payload_decoded(self):
|
|
return self.get_payload(decode=True)
|
|
|
|
def __setitem__(self, name, value):
|
|
rc = Message.__setitem__(self,name,value)
|
|
self.modified = True
|
|
if self.headerchange: self.headerchange(self,name,value)
|
|
return rc
|
|
|
|
def __delitem__(self, name):
|
|
if self.headerchange: self.headerchange(self,name,None)
|
|
rc = Message.__delitem__(self,name)
|
|
self.modified = True
|
|
return rc
|
|
|
|
def get_payload(self,i=None,decode=False):
|
|
msg = self.submsg
|
|
if msg and msg.ismodified():
|
|
self.set_payload([msg])
|
|
return Message.get_payload(self,i,decode)
|
|
|
|
def set_payload(self, val, charset=None):
|
|
self.modified = True
|
|
try:
|
|
val.seek(0)
|
|
val = val.read()
|
|
except: pass
|
|
Message.set_payload(self,val,charset)
|
|
self.submsg = None
|
|
|
|
def get_submsg(self):
|
|
if self.get_content_type().lower() == 'message/rfc822':
|
|
if not self.submsg:
|
|
txt = self.get_payload()
|
|
if type(txt) == str:
|
|
txt = self.get_payload(decode=True)
|
|
parser = MimeParser(MimeMessage)
|
|
self.submsg = parser.parsestr(txt)
|
|
else:
|
|
self.submsg = txt[0]
|
|
return self.submsg
|
|
return None
|
|
|
|
extlist = ''.join("""
|
|
ade,adp,asd,asx,asp,bas,bat,chm,cmd,com,cpl,crt,dll,exe,hlp,hta,inf,ins,isp,js,
|
|
jse,lnk,mdb,mde,msc,msi,msp,mst,ocx,pcd,pif,reg,scr,sct,shs,url,vb,vbe,vbs,wsc,
|
|
wsf,wsh
|
|
""".split())
|
|
bad_extensions = map(lambda x:'.' + x,extlist.split(','))
|
|
|
|
def check_ext(name):
|
|
"Check a name for dangerous Winblows extensions."
|
|
if not name: return name
|
|
lname = name.lower()
|
|
for ext in bad_extensions:
|
|
if lname.endswith(ext): return name
|
|
return None
|
|
|
|
virus_msg = """This message appeared to contain a virus.
|
|
It was originally named '%s', and has been removed.
|
|
A copy of your original message was saved as '%s:%s'.
|
|
See your administrator.
|
|
"""
|
|
|
|
def check_name(msg,savname=None,ckname=check_ext):
|
|
"Replace attachment with a warning if its name is suspicious."
|
|
for (key,name) in msg.getnames():
|
|
badname = ckname(name)
|
|
if badname:
|
|
hostname = socket.gethostname()
|
|
msg.set_payload(virus_msg % (badname,hostname,savname))
|
|
del msg["content-type"]
|
|
del msg["content-disposition"]
|
|
del msg["content-transfer-encoding"]
|
|
name = "WARNING.TXT"
|
|
msg["Content-Type"] = "text/plain; name="+name
|
|
break
|
|
return Milter.CONTINUE
|
|
|
|
import email.Iterators
|
|
|
|
def check_attachments(msg,check):
|
|
"""Scan attachments.
|
|
msg MimeMessage
|
|
check function(MimeMessage): int
|
|
Return CONTINUE, REJECT, ACCEPT
|
|
"""
|
|
if msg.ismultipart() and not msg.get_content_type() == 'message/rfc822':
|
|
for i in msg.get_payload():
|
|
rc = check_attachments(i,check)
|
|
if rc != Milter.CONTINUE: return rc
|
|
return Milter.CONTINUE
|
|
return check(msg)
|
|
|
|
# save call context for Python without nested_scopes
|
|
class _defang:
|
|
def __init__(self,savname,check):
|
|
self._savname = savname
|
|
self._check = check
|
|
self.scan_rfc822 = True
|
|
self.scan_html = True
|
|
def _chk_name(self,msg):
|
|
rc = check_name(msg,self._savname,self._check)
|
|
if self.scan_html:
|
|
check_html(msg,self._savname) # remove scripts from HTML
|
|
if self.scan_rfc822:
|
|
msg = msg.get_submsg()
|
|
if msg: return check_attachments(msg,self._chk_name)
|
|
return rc
|
|
|
|
# emulate old defang function
|
|
def defang(msg,savname=None,check=check_ext):
|
|
"""Compatible entry point.
|
|
Replace all attachments with dangerous names."""
|
|
check_attachments(msg,_defang(savname,check)._chk_name)
|
|
if msg.ismodified():
|
|
return 1;
|
|
return 0
|
|
|
|
import sgmllib
|
|
|
|
import re
|
|
declname = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*')
|
|
declstringlit = re.compile(r'(\'[^\']*\'|"[^"]*")\s*')
|
|
|
|
class SGMLFilter(sgmllib.SGMLParser):
|
|
"""Parse HTML and pass through all constructs unchanged. It is intended for
|
|
derived classes to implement exceptional processing for selected cases.
|
|
"""
|
|
def __init__(self,out):
|
|
sgmllib.SGMLParser.__init__(self)
|
|
self.out = out
|
|
|
|
def handle_comment(self,comment):
|
|
self.out.write("<!--%s-->" % comment)
|
|
|
|
def unknown_starttag(self,tag,attr):
|
|
if hasattr(self,"get_starttag_text"):
|
|
self.out.write(self.get_starttag_text())
|
|
else:
|
|
self.out.write("<%s" % tag)
|
|
for (key,val) in attr:
|
|
self.out.write(' %s="%s"' % (key,val))
|
|
self.out.write('>')
|
|
|
|
def handle_data(self,data):
|
|
self.out.write(data)
|
|
|
|
def handle_entityref(self,ref):
|
|
self.out.write("&%s;" % ref)
|
|
|
|
def handle_charref(self,ref):
|
|
self.out.write("&#%s;" % ref)
|
|
|
|
def unknown_endtag(self,tag):
|
|
self.out.write("</%s>" % tag)
|
|
|
|
def handle_special(self,data):
|
|
self.out.write("<!%s>" % data)
|
|
|
|
def write(self,buf):
|
|
"Act like a writer. Why doesn't SGMLParser do this by default?"
|
|
self.feed(buf)
|
|
|
|
# Python-2.1 sgmllib rejects illegal declarations. Since various Microsoft
|
|
# products accept and output them, we need to pass them through -
|
|
# at least until we discover that MS will execute them.
|
|
# sgmlop-1.1 will not use this method, but calls handle_special to
|
|
# do what we want.
|
|
def parse_declaration(self, i):
|
|
rawdata = self.rawdata
|
|
n = len(rawdata)
|
|
j = i + 2
|
|
while j < n:
|
|
c = rawdata[j]
|
|
if c == ">":
|
|
# end of declaration syntax
|
|
self.handle_special(rawdata[i+2:j])
|
|
return j + 1
|
|
if c in "\"'":
|
|
m = declstringlit.match(rawdata, j)
|
|
if not m:
|
|
# incomplete or an error?
|
|
return -1
|
|
j = m.end()
|
|
elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
|
|
m = declname.match(rawdata, j)
|
|
if not m:
|
|
# incomplete or an error?
|
|
return -1
|
|
j = m.end()
|
|
else:
|
|
j += 1
|
|
# end of buffer between tokens
|
|
return -1
|
|
|
|
class HTMLScriptFilter(SGMLFilter):
|
|
"Remove scripts from an HTML document."
|
|
def __init__(self,out):
|
|
SGMLFilter.__init__(self,out)
|
|
self.ignoring = 0
|
|
self.modified = False
|
|
self.msg = "<!-- WARNING: embedded script removed -->"
|
|
def start_script(self,unused):
|
|
self.ignoring += 1
|
|
self.modified = True
|
|
self.out.write(self.msg)
|
|
def end_script(self):
|
|
self.ignoring -= 1
|
|
def handle_data(self,data):
|
|
if not self.ignoring: SGMLFilter.handle_data(self,data)
|
|
def handle_comment(self,comment):
|
|
if not self.ignoring: SGMLFilter.handle_comment(self,comment)
|
|
|
|
|
|
def check_html(msg,savname=None):
|
|
"Remove scripts from HTML attachments."
|
|
msgtype = msg.get_content_type().lower()
|
|
# check for more MSIE braindamage
|
|
if msgtype == 'application/octet-stream':
|
|
for (attr,name) in msg.getnames():
|
|
if name and name.lower().endswith(".htm"):
|
|
msgtype = 'text/html'
|
|
if msgtype == 'text/html':
|
|
out = StringIO.StringIO()
|
|
filter = HTMLScriptFilter(out)
|
|
try:
|
|
filter.write(msg.get_payload(decode=True))
|
|
filter.close()
|
|
#except sgmllib.SGMLParseError:
|
|
except:
|
|
#mimetools.copyliteral(msg.get_payload(),open('debug.out','w')
|
|
filter.close()
|
|
hostname = socket.gethostname()
|
|
msg.set_payload(
|
|
"An HTML attachment could not be parsed. The original is saved as '%s:%s'"
|
|
% (hostname,savname))
|
|
del msg["content-type"]
|
|
del msg["content-disposition"]
|
|
del msg["content-transfer-encoding"]
|
|
name = "WARNING.TXT"
|
|
msg["Content-Type"] = "text/plain; name="+name
|
|
return Milter.CONTINUE
|
|
if filter.modified:
|
|
msg.set_payload(out) # remove embedded scripts
|
|
del msg["content-transfer-encoding"]
|
|
email.Encoders.encode_quopri(msg)
|
|
return Milter.CONTINUE
|