From bf4f3ad75f689188819bb2f9b6572db646b677de Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 29 Apr 2014 18:45:37 +0000 Subject: [PATCH] Made several changes to better handle non-ascii UTF-8 in email messages. Used multipart mime when including the original message in an error message to the secretariat. Modified the way IANA review email is parsed. Added non-ascii UTF-8 tests to the IANA review email parser and to the SMTP Exception handling code. Commit ready for merge. - Legacy-Id: 7673 --- ietf/sync/iana.py | 2 +- ietf/sync/tests.py | 9 +++-- ietf/utils/mail.py | 91 +++++++++++++++++++-------------------------- ietf/utils/tests.py | 37 +++++++++++++++--- 4 files changed, 77 insertions(+), 62 deletions(-) diff --git a/ietf/sync/iana.py b/ietf/sync/iana.py index 9552dfc83..c75c4b17f 100644 --- a/ietf/sync/iana.py +++ b/ietf/sync/iana.py @@ -216,7 +216,7 @@ def update_history_with_changes(changes, send_email=True): def parse_review_email(text): - msg = email.message_from_string(text.encode("utf-8")) + msg = email.message_from_string(text) # doc doc_name = "" diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 7fce09716..c005d2117 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import os import json import datetime @@ -130,7 +131,7 @@ class IANASyncTests(TestCase): def test_iana_review_mail(self): draft = make_test_data() - msg = """From: "%(person)s via RT" + msg = u"""From: "%(person)s via RT" Date: Thu, 10 May 2012 12:00:00 +0000 Subject: [IANA #12345] Last Call: <%(draft)s-%(rev)s.txt> (Long text) to Informational RFC @@ -147,7 +148,7 @@ IANA Actions that need completion. Thanks, %(person)s -IANA Fake Test Person +IANA “Fake Test” Person ICANN (END IANA LAST CALL COMMENTS) @@ -156,8 +157,8 @@ ICANN msg = msg % dict(person=Person.objects.get(user__username="iana").name, draft=draft.name, rev=draft.rev) - - doc_name, review_time, by, comment = iana.parse_review_email(msg) + + doc_name, review_time, by, comment = iana.parse_review_email(msg.encode('utf-8')) self.assertEqual(doc_name, draft.name) # self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 5, 0, 0)) diff --git a/ietf/utils/mail.py b/ietf/utils/mail.py index cc65cbd02..62dad0c06 100644 --- a/ietf/utils/mail.py +++ b/ietf/utils/mail.py @@ -304,17 +304,20 @@ def smtp_error_user_warning(thing,request): except smtplib.SMTPException as e: (extype, value, tb) = log_smtp_exception(e) - warning = "An error occured while sending email with\n" - warning += "Subject: %s\n" % e.original_msg.get('Subject','[no subject]') - warning += "To: %s\n" % e.original_msg.get('To','[no to]') - warning += "Cc: %s\n" % e.original_msg.get('Cc','[no cc]') - if isinstance(e,SMTPSomeRefusedRecipients): - warning += e.detailed_refusals() - else: - warning += "SMTP Exception: %s\n"%extype - warning += "Error Message: %s\n\n"%value - warning += "The message was not delivered to anyone." - messages.warning(request,warning,extra_tags='preformatted',fail_silently=True) + if request: + warning = "An error occured while sending email:\n" + if (e.original_msg): + warning += "Subject: %s\n" % e.original_msg.get('Subject','[no subject]') + warning += "To: %s\n" % e.original_msg.get('To','[no to]') + warning += "Cc: %s\n" % e.original_msg.get('Cc','[no cc]') + if isinstance(e,SMTPSomeRefusedRecipients): + warning += e.detailed_refusals() + else: + warning += "SMTP Exception: %s\n"%extype + warning += "Error Message: %s\n\n"%value + warning += "The message was not delivered to anyone." + messages.warning(request,warning,extra_tags='preformatted',fail_silently=True) + raise @contextmanager @@ -324,68 +327,52 @@ def smtp_error_logging(thing): except smtplib.SMTPException as e: (extype, value, tb) = log_smtp_exception(e) - msg = textwrap.dedent("""\ - To: - From: %s - """) % settings.SERVER_EMAIL + msg = MIMEMultipart() + msg['To'] = '' + msg['From'] = settings.SERVER_EMAIL if isinstance(e,SMTPSomeRefusedRecipients): - msg += textwrap.dedent("""\ - Subject: Some recipients were refused while sending mail with Subject: %s + msg['Subject'] = 'Subject: Some recipients were refused while sending mail with Subject: %s' % e.original_msg.get('Subject','[no subject]') + textpart = textwrap.dedent("""\ + This is a message from the datatracker to IETF-Action about an email + delivery failure, when sending email from the datatracker. - This is a message from the datatracker to IETF-Action about an email - delivery failure, when sending email from the datatracker. + %s - %s - - The original message follows: - -------- BEGIN ORIGINAL MESSAGE -------- - %s - --------- END ORIGINAL MESSAGE --------- - """) % (e.original_msg.get('Subject', '[no subject]'),e.detailed_refusals(),e.original_msg.as_string()) + """) % e.detailed_refusals() else: - msg += textwrap.dedent("""\ - Subject: Datatracker error while sending email + msg['Subject'] = 'Datatracker error while sending email' + textpart = textwrap.dedent("""\ + This is a message from the datatracker to IETF-Action about an email + delivery failure, when sending email from the datatracker. - This is a message from the datatracker to IETF-Action about an email - delivery failure, when sending email from the datatracker. + The original message was not delivered to anyone. - The original message was not delivered to anyone. + SMTP Exception: %s - SMTP Exception: %s - - Error Message: %s + Error Message: %s - """) % (extype,value) - if hasattr(e,'original_msg'): - msg += textwrap.dedent("""\ - The original message follows: - -------- BEGIN ORIGINAL MESSAGE -------- - %s - --------- END ORIGINAL MESSAGE --------- - """) % e.original_msg.as_string() + """) % (extype,value) + if hasattr(e,'original_msg'): + textpart += "The original message follows:\n" + msg.attach(MIMEText(textpart,_charset='utf-8')) + if hasattr(e,'original_msg'): + msg.attach(MIMEMessage(e.original_msg)) send_error_to_secretariat(msg) -def send_error_to_secretariat(raw_msg): - - (parsed_msg , headers , bcc) = parse_preformatted(raw_msg) - send_msg = encode_message(parsed_msg.get_payload()) - cc = None - condition_message(parsed_msg['To'], parsed_msg['From'], parsed_msg['Subject'], send_msg, cc, headers) +def send_error_to_secretariat(msg): debugging = getattr(settings, "USING_DEBUG_EMAIL_SERVER", False) and settings.EMAIL_HOST == 'localhost' and settings.EMAIL_PORT == 2025 try: if test_mode or debugging or settings.SERVER_MODE == 'production': - send_smtp(send_msg, bcc) + send_smtp(msg, bcc=None) try: copy_to = settings.EMAIL_COPY_TO except AttributeError: copy_to = "ietf.tracker.archive+%s@gmail.com" % settings.SERVER_MODE if copy_to and not test_mode and not debugging: # if we're running automated tests, this copy is just annoying - if bcc: - send_msg['X-Tracker-Bcc']=bcc - copy_email(send_msg, copy_to,originalBcc=bcc) + copy_email(msg, copy_to,originalBcc=None) except smtplib.SMTPException: log("Exception encountered while sending a ticket to the secretariat") (extype,value) = sys.exc_info()[:2] diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 70553e952..e378120b6 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -1,10 +1,16 @@ +# -*- coding: utf-8 -*- import os.path +from textwrap import dedent +from email.mime.text import MIMEText +from email.mime.image import MIMEImage +from email.mime.multipart import MIMEMultipart + from django.conf import settings from django.test import TestCase from ietf.utils.management.commands import pyflakes -from ietf.utils.mail import send_mail_text, outbox +from ietf.utils.mail import send_mail_text, send_mail_mime, outbox class PyFlakesTestCase(TestCase): @@ -18,16 +24,37 @@ class TestSMTPServer(TestCase): def test_address_rejected(self): - def send_mail(to): - send_mail_text(None, to=to, frm=None, subject="Test for rejection", txt="dummy body") + def send_simple_mail(to): + send_mail_text(None, to=to, frm=None, subject="Test for rejection", txt="dummy body") len_before = len(outbox) - send_mail('good@example.com,poison@example.com') + send_simple_mail('good@example.com,poison@example.com') self.assertEqual(len(outbox),len_before+2) self.assertTrue('Some recipients were refused' in outbox[-1]['Subject']) len_before = len(outbox) - send_mail('poison@example.com') + send_simple_mail('poison@example.com') self.assertEqual(len(outbox),len_before+2) self.assertTrue('error while sending email' in outbox[-1]['Subject']) + def test_rejecting_complex_mail(self): + + def send_complex_mail(to): + msg = MIMEMultipart() + textpart= MIMEText(dedent(u"""\ + Sometimes people send mail with things like “smart quotes” in them. + Sometimes they have attachments with pictures. + """),_charset='utf-8') + msg.attach(textpart) + img = MIMEImage('\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x06\x00\x00\x00\x1f\xf3\xffa\x00\x00\x02\x88IDATx\xda\xa5\x93\xcbO\x13Q\x14\xc6\xbf\xb9\xd32}L\x9fZ\x06\x10Q\x90\x10\x85b\x89\xfe\x01\x06BXK"\xd4hB\xdc\xa0\x06q\xe1c% H1l\xd0\x8dbT6\x1a5\x91\x12#K\x891\xf2\x07\x98\xc8[L\x1ay\xa8@\xdb\xd0\xd2\xe9\x83N;\xbdc\x1f\x11\x03\x04\x17zW\'_\xce\xf9\xdd\xef\x9c\x9c\xc3\xe0?\x1f\xb3S\xf8\xfe\xba\xc2Be\xa9]m\xd6\x9e\xe6x\xde\x9e\xd1\xa4HdF\x0e\xc5G\x89\x8a{X\xec\xfc\x1a\xdc\x1307X\xd4$T\nC\xc6\xfc|\x13\x8d\xa6\x00\xe5O\x16\xd1\xb3\x10}\xbe\x90w\xce\xdbZyeed\x17\xc03(4\x15\x9d(s\x13\xca!\x10\xa6\xb0\x1a\xb6\x9b\x0b\x84\x95\xb4F@\x89\x84\xe5O\x0b\xcdG\xaf\xae\x8dl\x01V\x9f\x1d\xb4q\x16\xde\xa33[\x8d\xe3\x93)\xdc\x7f\x9b\xc4\xf3\x1b\x1c,|\xaex#\n\xb4\x0cH\xb8\xd6\xa8F\xad\x83El# \xc6\x83\xb1\xf2\xa2\x0bK\xfe,`y\xd0\xe6*\x03\xaes\x0c\xea\xaas.\xc6g\x14t\xbcR\xd0P\x03t;\tX\x15\x83\xd5\xf9\xc5\xbe\x926_W6\xe3\xe7ca\xc2Z,82\xb1\xeb\r\x8b\xb1)\x82\xde\xa6\x14\xea\xec4\x0b\xf88K\xd0\xe5fQ_M\xd1s&\x95k\xe9\x87w\xf2\xc0eoM\x16\xe0\x1b*\xdc4XM\x9aL\xfca\x8e\xc5\xbd1\x0e//\xc6`\xd5\xe7Z\x08\xa6[8\xffT\x87\xeb\r\x12\xea\xabr\x80p \x14\xcfo]\xd5f\x01k\x8fl\x9bF3\xaf\xf9=\xb0X\x82\x81.O\xd96\xc4\x9d\x9a\xb8\x11\x89\x17\xb4\xf9s\x80\xe5\x01\xc3\xc4\xfe}FG\\\x064\xaa\xbf/\x0eM3\x92i\x13\xe1\x908Yr3\x9ck\xe1[\xbf\xd6%X\xf4\x9d\xef=z$(\xc1\xa9\xc3Q\xf0\x1c\xddV(\xa7\x18Ly9L\xafq8{\\D0\x14\xbd{\xe4V\xac3\x0bX\xe8\xd7\xdb\xb4,\xf5\x18\xb4j\xe3\xf8\xa2\x1e/\xa6\xac`\x18\x06\x02\x9f\x84\x8a\xa4\x07\x16c\xb1\xbe\xc9\xa2\xf6P\x04-\x8e\x00\x12\xc9\x84(&\xd9\xf2\x8a\x8e\x88\x7fk[\xbet\xe75\x0bzf\x98cI\xd6\xe6\xfc\xba\x06\xd3~\x1d\x12\xe9\x9fK\xcd\x12N\x16\xc4\xa0UQH)\x8a\x95\x08\x9c\xf6^\xc9\xbdk\x95\xe7o\xab\x9c&\xb5\xf2\x84W3\xa6\x9dG\x92\x19_$\xa9\x84\xd6%r\xc9\xde\x97\x1c\xde\xf3\x98\x96\xee\xb0\x16\x99\xd2v\x15\x94\xc6<\xc2Te\xb4\x04Ufe\x85\x8c2\x84<(\xeb\x91\xf7>\xa6\x7fy\xbf\x00\x96T\xff\x11\xf7\xd8R\xb9\x00\x00\x00\x00IEND\xaeB`\x82') + + msg.attach(img) + send_mail_mime(request=None, to=to, frm=settings.DEFAULT_FROM_EMAIL, subject=u'это сложно', msg=msg, cc=None, extra=None) + + len_before = len(outbox) + send_complex_mail('good@example.com') + self.assertEqual(len(outbox),len_before+1) + + len_before = len(outbox) + send_complex_mail('good@example.com,poison@example.com') + self.assertEqual(len(outbox),len_before+2)