From bc05e7a8d0c8dd51ef708b5ed33d512b32d8efd5 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sat, 16 May 2020 20:20:05 +0000 Subject: [PATCH] Refactored many instances of msg.get_payload(decode=True).decode('utf-8') and similar to get_payload_text(msg), which uses msg.get_charset() to get the actual charset to do the decoding, in a consistent manner. Also removed some instances of force_str, but more remain. - Legacy-Id: 17807 --- ietf/doc/mails.py | 8 +++--- ietf/doc/tests_ballot.py | 12 ++++----- ietf/doc/tests_charter.py | 20 +++++++-------- ietf/doc/tests_conflict_review.py | 6 ++--- ietf/doc/tests_draft.py | 4 +-- ietf/doc/tests_irsg_ballot.py | 4 +-- ietf/doc/tests_review.py | 41 +++++++++++++++---------------- ietf/group/tests_info.py | 4 +-- ietf/group/views.py | 4 +-- ietf/ietfauth/tests.py | 12 ++++----- ietf/ipr/tests.py | 11 +++------ ietf/meeting/tests_views.py | 6 ++--- ietf/message/tests.py | 4 +-- ietf/message/utils.py | 6 ++--- ietf/nomcom/tests.py | 12 ++++----- ietf/nomcom/utils.py | 14 ++++------- ietf/review/mailarch.py | 4 ++- ietf/secr/sreq/tests.py | 4 +-- ietf/submit/tests.py | 23 +++++++++-------- ietf/sync/iana.py | 5 ++-- ietf/utils/mail.py | 15 +++++------ ietf/utils/test_utils.py | 4 +-- ietf/utils/tests.py | 6 ++--- 23 files changed, 111 insertions(+), 118 deletions(-) diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py index a6fdfac76..fae909495 100644 --- a/ietf/doc/mails.py +++ b/ietf/doc/mails.py @@ -10,11 +10,11 @@ from django.template.loader import render_to_string from django.utils.html import strip_tags from django.conf import settings from django.urls import reverse as urlreverse -from django.utils.encoding import force_str, force_text +from django.utils.encoding import force_text import debug # pyflakes:ignore -from ietf.utils.mail import send_mail, send_mail_text, get_payload +from ietf.utils.mail import send_mail, send_mail_text from ietf.ipr.utils import iprs_from_docs, related_docs from ietf.doc.models import WriteupDocEvent, LastCallDocEvent, DocAlias, ConsensusDocEvent from ietf.doc.utils import needed_ballot_positions @@ -387,7 +387,7 @@ def generate_issue_ballot_mail(request, doc, ballot): def email_iana(request, doc, to, msg, cc=None): # fix up message and send it with extra info on doc in headers import email - parsed_msg = email.message_from_string(force_str(msg)) + parsed_msg = email.message_from_string(msg) parsed_msg.set_charset('UTF-8') extra = extra_automation_headers(doc) @@ -395,7 +395,7 @@ def email_iana(request, doc, to, msg, cc=None): send_mail_text(request, to, parsed_msg["From"], parsed_msg["Subject"], - get_payload(parsed_msg), + parsed_msg.get_payload(), extra=extra, cc=cc) diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py index f57332f47..76aee087c 100644 --- a/ietf/doc/tests_ballot.py +++ b/ietf/doc/tests_ballot.py @@ -23,7 +23,7 @@ from ietf.iesg.models import TelechatDate from ietf.person.models import Person, PersonalApiKey from ietf.person.factories import PersonFactory from ietf.utils.test_utils import TestCase, login_testing_unauthorized -from ietf.utils.mail import outbox, empty_outbox, get_payload +from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.text import unwrap @@ -174,7 +174,7 @@ class EditPositionTests(TestCase): self.assertEqual(len(outbox), mailbox_before + 1) m = outbox[-1] self.assertIn('COMMENT', m['Subject']) - self.assertIn('New comment', get_payload(m)) + self.assertIn('New comment', get_payload_text(m)) def test_edit_position_as_secretary(self): @@ -460,11 +460,11 @@ class BallotWriteupsTests(TestCase): self.assertTrue('drafts-eval@' in outbox[-1]['To']) self.assertTrue('X-IETF-Draft-string' in outbox[-1]) if case=='none': - self.assertNotIn('call expire', outbox[-1].get_payload(decode=True).decode("utf-8")) + self.assertNotIn('call expire', get_payload_text(outbox[-1])) elif case=='past': - self.assertIn('call expired', outbox[-1].get_payload(decode=True).decode("utf-8")) + self.assertIn('call expired', get_payload_text(outbox[-1])) else: - self.assertIn('call expires', outbox[-1].get_payload(decode=True).decode("utf-8")) + self.assertIn('call expires', get_payload_text(outbox[-1])) self.client.logout() @@ -797,7 +797,7 @@ class MakeLastCallTests(TestCase): self.assertTrue("ietf-announce@" in outbox[-2]['To']) for prefix in ['draft-ietf-mars-test','mars-chairs','aread']: self.assertTrue(prefix+"@" in outbox[-2]['Cc']) - self.assertIn("The following IPR Declarations", get_payload(outbox[-2])) + self.assertIn("The following IPR Declarations", get_payload_text(outbox[-2])) self.assertTrue("Last Call" in outbox[-1]['Subject']) self.assertTrue("drafts-lastcall@icann.org" in outbox[-1]['To']) diff --git a/ietf/doc/tests_charter.py b/ietf/doc/tests_charter.py index 5a8d20e71..d7a25d3c0 100644 --- a/ietf/doc/tests_charter.py +++ b/ietf/doc/tests_charter.py @@ -25,7 +25,7 @@ from ietf.group.models import Group, GroupMilestone from ietf.iesg.models import TelechatDate from ietf.person.models import Person from ietf.utils.test_utils import TestCase -from ietf.utils.mail import outbox, empty_outbox, get_payload +from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import login_testing_unauthorized class ViewCharterTests(TestCase): @@ -187,7 +187,7 @@ class EditCharterTests(TestCase): self.assertIn("Internal WG Review", outbox[-3]['Subject']) self.assertIn("iab@", outbox[-3]['To']) self.assertIn("iesg@", outbox[-3]['To']) - body = get_payload(outbox[-3]) + body = get_payload_text(outbox[-3]) for word in ["A new IETF WG", "Chairs", "Ames Man ", "Secretaries", "Secretary ", "Assigned Area Director", "Areað Irector ", @@ -198,13 +198,13 @@ class EditCharterTests(TestCase): self.assertIn("state changed", outbox[-2]['Subject'].lower()) self.assertIn("iesg-secretary@", outbox[-2]['To']) - body = get_payload(outbox[-2]) + body = get_payload_text(outbox[-2]) for word in ["WG", "Charter", ]: self.assertIn(word, body) self.assertIn("State Update Notice", outbox[-1]['Subject']) self.assertIn("ames-chairs@", outbox[-1]['To']) - body = get_payload(outbox[-1]) + body = get_payload_text(outbox[-1]) for word in ["State changed", "Datatracker URL", ]: self.assertIn(word, body) @@ -224,7 +224,7 @@ class EditCharterTests(TestCase): empty_outbox() r = self.client.post(url, dict(charter_state=str(State.objects.get(used=True,type="charter",slug="intrev").pk), message="test")) self.assertEqual(r.status_code, 302) - self.assertTrue("A new charter" in get_payload(outbox[-3])) + self.assertTrue("A new charter" in get_payload_text(outbox[-3])) def test_change_rg_state(self): @@ -245,7 +245,7 @@ class EditCharterTests(TestCase): self.assertIn("Internal RG Review", outbox[-3]['Subject']) self.assertIn("iab@", outbox[-3]['To']) self.assertIn("irsg@", outbox[-3]['To']) - body = get_payload(outbox[-3]) + body = get_payload_text(outbox[-3]) for word in ["A new IRTF RG", "Mailing list", "somerg@ietf.org", "Charter", "Milestones"]: @@ -254,13 +254,13 @@ class EditCharterTests(TestCase): self.assertIn("state changed", outbox[-2]['Subject'].lower()) self.assertIn("iesg-secretary@", outbox[-2]['To']) - body = get_payload(outbox[-2]) + body = get_payload_text(outbox[-2]) for word in ["RG", "Charter", ]: self.assertIn(word, body) self.assertIn("State Update Notice", outbox[-1]['Subject']) self.assertIn("somerg-chairs@", outbox[-1]['To']) - body = get_payload(outbox[-1]) + body = get_payload_text(outbox[-1]) for word in ["State changed", "Datatracker URL", ]: self.assertIn(word, body) @@ -793,7 +793,7 @@ class EditCharterTests(TestCase): # self.assertTrue("approved" in outbox[0]['Subject'].lower()) self.assertTrue("iesg-secretary" in outbox[0]['To']) - body = get_payload(outbox[0]) + body = get_payload_text(outbox[0]) for word in ["WG", "/wg/ames/about/", "Charter", "/doc/charter-ietf-ames/", ]: self.assertIn(word, body) @@ -801,7 +801,7 @@ class EditCharterTests(TestCase): self.assertTrue("WG Action" in outbox[1]['Subject']) self.assertTrue("ietf-announce" in outbox[1]['To']) self.assertTrue("ames-wg@ietf.org" in outbox[1]['Cc']) - body = get_payload(outbox[1]) + body = get_payload_text(outbox[1]) for word in ["Chairs", "Ames Man ", "Secretaries", "Secretary ", "Assigned Area Director", "Areað Irector ", diff --git a/ietf/doc/tests_conflict_review.py b/ietf/doc/tests_conflict_review.py index f3beccf80..ee4ac4e89 100644 --- a/ietf/doc/tests_conflict_review.py +++ b/ietf/doc/tests_conflict_review.py @@ -22,7 +22,7 @@ from ietf.group.models import Person from ietf.iesg.models import TelechatDate from ietf.name.models import StreamName from ietf.utils.test_utils import TestCase -from ietf.utils.mail import outbox, empty_outbox, get_payload +from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import login_testing_unauthorized @@ -302,9 +302,9 @@ class ConflictReviewTests(TestCase): self.assertIn('iana@', outbox[0]['Cc']) if approve_type == 'appr-noprob': - self.assertIn( 'IESG has no problem', ''.join(wrap(get_payload(outbox[0]), 2**16))) + self.assertIn( 'IESG has no problem', ''.join(wrap(get_payload_text(outbox[0]), 2**16))) else: - self.assertIn( 'NOT be published', ''.join(wrap(get_payload(outbox[0]), 2**16))) + self.assertIn( 'NOT be published', ''.join(wrap(get_payload_text(outbox[0]), 2**16))) def test_approve_reqnopub(self): diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index 5bd705334..3752cfd62 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -30,7 +30,7 @@ from ietf.person.models import Person, Email from ietf.meeting.models import Meeting, MeetingTypeName from ietf.iesg.models import TelechatDate from ietf.utils.test_utils import login_testing_unauthorized -from ietf.utils.mail import outbox, empty_outbox, get_payload +from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import TestCase @@ -387,7 +387,7 @@ class EditInfoTests(TestCase): data["telechat_date"] = next_week.isoformat() r = self.client.post(url,data) self.assertEqual(r.status_code, 302) - self.assertIn("may not leave enough time", get_payload(outbox[-1])) + self.assertIn("may not leave enough time", get_payload_text(outbox[-1])) def test_start_iesg_process_on_draft(self): diff --git a/ietf/doc/tests_irsg_ballot.py b/ietf/doc/tests_irsg_ballot.py index 00a6db1e9..b9059fd75 100644 --- a/ietf/doc/tests_irsg_ballot.py +++ b/ietf/doc/tests_irsg_ballot.py @@ -11,7 +11,7 @@ import datetime from django.urls import reverse as urlreverse -from ietf.utils.mail import outbox, empty_outbox, get_payload +from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import TestCase, unicontent, login_testing_unauthorized from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, RgDraftFactory, RgRfcFactory, BallotDocEventFactory, IRSGBallotDocEventFactory, BallotPositionDocEventFactory from ietf.doc.models import BallotDocEvent, BallotPositionDocEvent @@ -363,7 +363,7 @@ class BaseManipulationTests(): r = self.client.post(url, dict(cc_choices=['doc_authors','doc_group_chairs','doc_group_mail_list'], body="Stuff")) self.assertEqual(r.status_code, 302) self.assertEqual(len(outbox),1) - self.assertNotIn('discuss-criteria', get_payload(outbox[0])) + self.assertNotIn('discuss-criteria', get_payload_text(outbox[0])) def test_close_ballot(self): draft = RgDraftFactory() diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index b558c3f26..1fe1e89c2 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -20,24 +20,23 @@ from pyquery import PyQuery import debug # pyflakes:ignore import ietf.review.mailarch + from ietf.doc.factories import ( NewRevisionDocEventFactory, IndividualDraftFactory, WgDraftFactory, - WgRfcFactory, ReviewFactory, DocumentFactory) -from ietf.doc.models import DocumentAuthor, RelatedDocument, DocEvent, ReviewRequestDocEvent, ReviewAssignmentDocEvent + WgRfcFactory, ReviewFactory, DocumentFactory) +from ietf.doc.models import ( DocumentAuthor, RelatedDocument, DocEvent, ReviewRequestDocEvent, + ReviewAssignmentDocEvent, ) from ietf.group.factories import RoleFactory, ReviewTeamFactory from ietf.group.models import Group from ietf.message.models import Message -from ietf.name.models import ReviewResultName, ReviewRequestStateName, ReviewAssignmentStateName, \ - ReviewTypeName +from ietf.name.models import ReviewResultName, ReviewRequestStateName, ReviewAssignmentStateName, ReviewTypeName +from ietf.person.factories import PersonFactory from ietf.person.models import Email, Person from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory -from ietf.review.models import (ReviewRequest, ReviewerSettings, - ReviewWish, NextReviewerInTeam) +from ietf.review.models import ReviewRequest, ReviewerSettings, ReviewWish, NextReviewerInTeam from ietf.review.policies import get_reviewer_queue_policy - -from ietf.utils.test_utils import TestCase +from ietf.utils.mail import outbox, empty_outbox, parseaddr, on_behalf_of, get_payload_text from ietf.utils.test_utils import login_testing_unauthorized, reload_db_objects -from ietf.utils.mail import outbox, empty_outbox, parseaddr, on_behalf_of -from ietf.person.factories import PersonFactory +from ietf.utils.test_utils import TestCase class ReviewTests(TestCase): def setUp(self): @@ -228,7 +227,7 @@ class ReviewTests(TestCase): self.assertIn('', outbox[0]["To"]) self.assertNotIn("", outbox[0]["To"]) self.assertIn("reviewsecretary2@example.com", outbox[0]["CC"]) - mail_content = outbox[0].get_payload(decode=True).decode("utf-8").lower() + mail_content = get_payload_text(outbox[0]) self.assertIn("closed", mail_content) self.assertIn("review_request_close_comment", mail_content) @@ -330,7 +329,7 @@ class ReviewTests(TestCase): self.assertEqual(len(outbox), 1) self.assertEqual('"Some Reviewer" ', outbox[0]["To"]) - message = outbox[0].get_payload(decode=True).decode("utf-8") + message = get_payload_text(outbox[0]) self.assertIn("Pages: {}".format(doc.pages), message ) self.assertIn("{} has assigned {}".format(secretary.person.ascii, reviewer.person.ascii), message) self.assertIn("This team has completed other reviews", message) @@ -423,7 +422,7 @@ class ReviewTests(TestCase): self.assertEqual(len(outbox), 1) self.assertIn(assignment.reviewer.address, outbox[0]["To"]) self.assertNotIn("", outbox[0]["To"]) - self.assertTrue("Test message" in outbox[0].get_payload(decode=True).decode("utf-8")) + self.assertTrue("Test message" in get_payload_text(outbox[0])) # try again, but now with an expired review request, which should not be allowed (#2277) assignment.state_id = 'assigned' @@ -645,7 +644,7 @@ class ReviewTests(TestCase): self.assertEqual(len(outbox), 1) self.assertIn(assignment.review_request.team.list_email, outbox[0]["To"]) self.assertIn(assignment.reviewer.role_set.filter(group=assignment.review_request.team,name='reviewer').first().person.plain_name(), parseaddr(outbox[0]["From"])[0] ) - self.assertIn("This is a review", outbox[0].get_payload(decode=True).decode("utf-8")) + self.assertIn("This is a review", get_payload_text(outbox[0])) self.assertIn(settings.MAILING_LIST_ARCHIVE_URL, assignment.review.external_url) @@ -658,7 +657,7 @@ class ReviewTests(TestCase): self.assertEqual(parseaddr(outbox[0]["To"]), parseaddr(message.to)) self.assertEqual(parseaddr(outbox[0]["From"]), parseaddr(on_behalf_of(message.frm))) self.assertEqual(parseaddr(outbox[0]["Reply-To"]), parseaddr(message.frm)) - self.assertEqual(outbox[0].get_payload(decode=True).decode(str(outbox[0].get_charset())), message.body) + self.assertEqual(get_payload_text(outbox[0]), message.body) # check the review document page url = urlreverse('ietf.doc.views_doc.document_main', kwargs={ "name": assignment.review.name }) @@ -699,7 +698,7 @@ class ReviewTests(TestCase): self.assertEqual(len(outbox), 1) self.assertIn(assignment.review_request.team.list_email, outbox[0]["To"]) - self.assertIn("This is a review", outbox[0].get_payload(decode=True).decode("utf-8")) + self.assertIn("This is a review", get_payload_text(outbox[0])) self.assertIn(settings.MAILING_LIST_ARCHIVE_URL, assignment.review.external_url) @@ -740,7 +739,7 @@ class ReviewTests(TestCase): self.assertEqual(len(outbox), 1) self.assertIn(assignment.review_request.team.list_email, outbox[0]["To"]) - self.assertIn("This is a review", outbox[0].get_payload(decode=True).decode("utf-8")) + self.assertIn("This is a review", get_payload_text(outbox[0])) self.assertIn(settings.MAILING_LIST_ARCHIVE_URL, assignment.review.external_url) @@ -767,7 +766,7 @@ class ReviewTests(TestCase): self.assertEqual(len(outbox), 2) self.assertIn('Has Issues', outbox[-1]['Subject']) - self.assertIn('settings indicated', outbox[-1].get_payload(decode=True).decode("utf-8")) + self.assertIn('settings indicated', get_payload_text(outbox[-1])) def test_complete_notify_ad_because_checkbox(self): assignment, url = self.setup_complete_review_test() @@ -791,7 +790,7 @@ class ReviewTests(TestCase): self.assertEqual(len(outbox), 2) self.assertIn('Has Issues', outbox[-1]['Subject']) - self.assertIn('reviewer indicated', outbox[-1].get_payload(decode=True).decode("utf-8")) + self.assertIn('reviewer indicated', get_payload_text(outbox[-1])) @patch('requests.get') def test_complete_review_link_to_mailing_list(self, mock): @@ -908,7 +907,7 @@ class ReviewTests(TestCase): self.assertTrue(assignment.review_request.team.list_email in outbox[1]["To"]) self.assertTrue("partial review" in outbox[1]["Subject"].lower()) - body = outbox[1].get_payload(decode=True).decode("utf-8") + body = get_payload_text(outbox[1]) self.assertTrue("This is a review" in body) # This review has a line longer than 80, but less than 100; it should # not be wrapped. @@ -941,7 +940,7 @@ class ReviewTests(TestCase): # This review has a line longer than 100; it should be wrapped to less # than 80. - body = outbox[2].get_payload(decode=True).decode("utf-8") + body = get_payload_text(outbox[2]) self.assertIn('really, really, really', body) self.assertTrue(all( len(line) <= 80 for line in body.splitlines() )) diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 8a748086d..84ca13a1c 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -35,7 +35,7 @@ from ietf.name.models import DocTagName, GroupStateName, GroupTypeName from ietf.person.models import Person, Email from ietf.person.factories import PersonFactory from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent, reload_db_objects def group_urlreverse_list(group, viewname): @@ -629,7 +629,7 @@ class GroupEditTests(TestCase): self.assertTrue('Personnel change' in outbox[0]['Subject']) for prefix in ['ad1','ad2','aread','marschairman','marsdelegate']: self.assertTrue(prefix+'@' in outbox[0]['To']) - self.assertTrue(outbox[0].get_payload(decode=True).decode(str(outbox[0].get_charset())).startswith('Sec Retary')) + self.assertTrue(get_payload_text(outbox[0]).startswith('Sec Retary')) def test_edit_field(self): diff --git a/ietf/group/views.py b/ietf/group/views.py index c5dbff083..7979faa04 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -111,7 +111,7 @@ from ietf.doc.models import LastCallDocEvent from ietf.name.models import ReviewAssignmentStateName -from ietf.utils.mail import send_mail_text, parse_preformatted, get_payload +from ietf.utils.mail import send_mail_text, parse_preformatted from ietf.ietfauth.utils import user_is_person from ietf.dbtemplate.models import DBTemplate @@ -1704,7 +1704,7 @@ def email_open_review_assignments(request, acronym, group_type=None): (msg,_,_) = parse_preformatted(partial_msg) - body = get_payload(msg) + body = msg.get_payload() subject = msg['Subject'] form = EmailOpenAssignmentsForm(initial={ diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py index a78a62854..4a319d46e 100644 --- a/ietf/ietfauth/tests.py +++ b/ietf/ietfauth/tests.py @@ -15,17 +15,17 @@ from django.conf import settings import debug # pyflakes:ignore -from ietf.utils.test_utils import TestCase, login_testing_unauthorized -from ietf.utils.mail import outbox, empty_outbox -from ietf.group.models import Group, Role, RoleName from ietf.group.factories import GroupFactory, RoleFactory +from ietf.group.models import Group, Role, RoleName from ietf.ietfauth.htpasswd import update_htpasswd_file from ietf.mailinglists.models import Subscribed -from ietf.person.models import Person, Email, PersonalApiKey from ietf.person.factories import PersonFactory, EmailFactory +from ietf.person.models import Person, Email, PersonalApiKey from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory from ietf.review.models import ReviewWish, UnavailablePeriod from ietf.utils.decorators import skip_coverage +from ietf.utils.mail import outbox, empty_outbox, get_payload_text +from ietf.utils.test_utils import TestCase, login_testing_unauthorized import ietf.ietfauth.views @@ -99,7 +99,7 @@ class IetfAuthTests(TestCase): def extract_confirm_url(self, confirm_email): # dig out confirm_email link - msg = confirm_email.get_payload(decode=True).decode(confirm_email.get_content_charset()) + msg = get_payload_text(confirm_email) line_start = "http" confirm_url = None for line in msg.split("\n"): @@ -643,7 +643,7 @@ class IetfAuthTests(TestCase): self.assertEqual(len(outbox), len(endpoints)) for mail in outbox: - body = mail.get_payload(decode=True).decode('utf-8') + body = get_payload_text(mail) self.assertIn("API key usage", mail['subject']) self.assertIn(" %s times" % count, body) self.assertIn(date, body) diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py index 6542d4d28..58717b9d7 100644 --- a/ietf/ipr/tests.py +++ b/ietf/ipr/tests.py @@ -23,14 +23,11 @@ from ietf.ipr.models import (IprDisclosureBase,GenericIprDisclosure,HolderIprDis from ietf.ipr.utils import get_genitive, get_ipr_summary from ietf.mailtrigger.utils import gather_address_lists from ietf.message.models import Message -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import TestCase, login_testing_unauthorized from ietf.utils.text import text_to_dict -def extract_message_content(message): - return message.get_payload(decode=True).decode(str(message.get_charset())) - def make_data_from_content(content): q = PyQuery(content) data = dict() @@ -582,10 +579,10 @@ I would like to revoke this declaration. self.assertEqual(r.status_code,302) self.assertEqual(len(outbox),len_before+2) self.assertTrue('george@acme.com' in outbox[len_before]['To']) - self.assertIn('posted on '+datetime.date.today().strftime("%Y-%m-%d"), extract_message_content(outbox[len_before]).replace('\n',' ')) + self.assertIn('posted on '+datetime.date.today().strftime("%Y-%m-%d"), get_payload_text(outbox[len_before]).replace('\n',' ')) self.assertTrue('draft-ietf-mars-test@ietf.org' in outbox[len_before+1]['To']) self.assertTrue('mars-wg@ietf.org' in outbox[len_before+1]['Cc']) - self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), extract_message_content(outbox[len_before+1]).replace('\n',' ')) + self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), get_payload_text(outbox[len_before+1]).replace('\n',' ')) def test_notify_generic(self): RoleFactory(name_id='ad',group__acronym='gen') @@ -601,7 +598,7 @@ I would like to revoke this declaration. r = self.client.post(url, data ) self.assertEqual(r.status_code,302) self.assertEqual(len(outbox),2) - self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), extract_message_content(outbox[1]).replace('\n',' ')) + self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), get_payload_text(outbox[1]).replace('\n',' ')) def test_process_response_email(self): # first send a mail diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 7f90e295f..cadf6f6d5 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -40,7 +40,7 @@ from ietf.meeting.utils import add_event_info_to_session_qs from ietf.meeting.views import session_draft_list from ietf.name.models import SessionStatusName, ImportantDateName, RoleName from ietf.utils.decorators import skip_coverage -from ietf.utils.mail import outbox, empty_outbox, get_payload +from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.utils.text import xslugify @@ -1446,8 +1446,8 @@ class InterimTests(TestCase): self.assertRedirects(r, urlreverse('ietf.meeting.views.interim_announce')) self.assertEqual(len(outbox), len_before + 1) self.assertIn('WG Virtual Meeting', outbox[-1]['Subject']) - self.assertIn('09:00 to 09:20 America/Los_Angeles', get_payload(outbox[-1])) - self.assertIn('(17:00 to 17:20 UTC)', get_payload(outbox[-1])) + self.assertIn('09:00 to 09:20 America/Los_Angeles', get_payload_text(outbox[-1])) + self.assertIn('(17:00 to 17:20 UTC)', get_payload_text(outbox[-1])) def test_interim_approve_by_ad(self): make_meeting_test_data() diff --git a/ietf/message/tests.py b/ietf/message/tests.py index c610cdbb1..4ab790699 100644 --- a/ietf/message/tests.py +++ b/ietf/message/tests.py @@ -12,7 +12,7 @@ from ietf.group.factories import GroupFactory from ietf.message.models import Message, SendQueue from ietf.message.utils import send_scheduled_message_from_send_queue from ietf.person.models import Person -from ietf.utils.mail import outbox, send_mail_text, send_mail_message, get_payload +from ietf.utils.mail import outbox, send_mail_text, send_mail_message, get_payload_text from ietf.utils.test_utils import TestCase class MessageTests(TestCase): @@ -46,7 +46,7 @@ class MessageTests(TestCase): continue self.assertEqual(e1.get_all(k), e2.get_all(k), "Header field: %s" % k) self.longMessage = False - self.assertEqual(get_payload(e1), get_payload(e2)) + self.assertEqual(get_payload_text(e1), get_payload_text(e2)) # self.assertEqual(Message.objects.count(), 0) diff --git a/ietf/message/utils.py b/ietf/message/utils.py index 1acda232a..65c018a39 100644 --- a/ietf/message/utils.py +++ b/ietf/message/utils.py @@ -6,13 +6,13 @@ import re, datetime, email from django.utils.encoding import force_str -from ietf.utils.mail import send_mail_text, send_mail_mime, get_payload +from ietf.utils.mail import send_mail_text, send_mail_mime from ietf.message.models import Message first_dot_on_line_re = re.compile(r'^\.', re.MULTILINE) def infer_message(s): - parsed = email.message_from_string(force_str(s)) + parsed = email.message_from_string(s) m = Message( subject = parsed.get("Subject", ""), @@ -21,7 +21,7 @@ def infer_message(s): cc = parsed.get("Cc", ""), bcc = parsed.get("Bcc", ""), reply_to = parsed.get("Reply-To", ""), - body = get_payload(parsed), + body = parsed.get_payload(), content_type = parsed.get_content_type(), ) diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index eb523bfe1..a8aab5005 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -16,7 +16,7 @@ from django.conf import settings from django.core.files import File from django.contrib.auth.models import User from django.urls import reverse -from django.utils.encoding import force_text, force_str +from django.utils.encoding import force_str import debug # pyflakes:ignore @@ -39,7 +39,7 @@ from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, get_hash from ietf.person.factories import PersonFactory, EmailFactory from ietf.person.models import Email, Person from ietf.stats.models import MeetingRegistration -from ietf.utils.mail import outbox, empty_outbox, get_payload +from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent client_test_cert_files = None @@ -539,7 +539,7 @@ class NomcomViewsTest(TestCase): self.assertEqual('Nomination receipt', outbox[-1]['Subject']) self.assertEqual(self.email_from, outbox[-1]['From']) self.assertIn('plain', outbox[-1]['To']) - self.assertIn('Comments with accents äöå', force_text(outbox[-1].get_payload(decode=True),"utf-8","replace")) + self.assertIn('Comments with accents äöå', get_payload_text(outbox[-1])) # Nominate the same person for the same position again without asking for confirmation @@ -580,7 +580,7 @@ class NomcomViewsTest(TestCase): self.assertEqual('Nomination receipt', outbox[-1]['Subject']) self.assertEqual(self.email_from, outbox[-1]['From']) self.assertIn('plain', outbox[-1]['To']) - self.assertIn('Comments with accents äöå', force_text(outbox[-1].get_payload(decode=True),"utf-8","replace")) + self.assertIn('Comments with accents äöå', get_payload_text(outbox[-1])) # Nominate the same person for the same position again without asking for confirmation @@ -831,12 +831,12 @@ class NomcomViewsTest(TestCase): # We're interested in the confirmation receipt here self.assertEqual(len(outbox),3) self.assertEqual('NomCom comment confirmation', outbox[2]['Subject']) - email_body = get_payload(outbox[2]) + email_body = get_payload_text(outbox[2]) self.assertIn(position, email_body) self.assertNotIn('$', email_body) self.assertEqual(self.email_from, outbox[-2]['From']) self.assertIn('plain', outbox[2]['To']) - self.assertIn('Comments with accents äöå', force_text(outbox[2].get_payload(decode=True),"utf-8","replace")) + self.assertIn('Comments with accents äöå', get_payload_text(outbox[2])) empty_outbox() self.feedback_view(public=True) diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index e92d34d31..32b2ff79e 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -20,13 +20,13 @@ from django.core.exceptions import ObjectDoesNotExist from django.urls import reverse from django.template.loader import render_to_string from django.shortcuts import get_object_or_404 -from django.utils.encoding import force_str, force_text +from django.utils.encoding import force_str from ietf.dbtemplate.models import DBTemplate from ietf.person.models import Email, Person from ietf.mailtrigger.utils import gather_address_lists from ietf.utils.pipe import pipe -from ietf.utils.mail import send_mail_text, send_mail +from ietf.utils.mail import send_mail_text, send_mail, get_payload_text from ietf.utils.log import log from ietf.person.name import unidecode_name @@ -438,18 +438,14 @@ def get_body(message): 'plain')] body = [] for part in text_parts: - charset = get_charset(part, get_charset(message)) - body.append(force_text(part.get_payload(decode=True), - charset, - "replace")) + charset = get_charset(message) + body.append(get_payload_text(part, default_charset=charset)) return "\n".join(body).strip() else: # if it is not multipart, the payload will be a string # representing the message body - body = force_text(message.get_payload(decode=True), - get_charset(message), - "replace") + body = get_payload_text(message) return body.strip() diff --git a/ietf/review/mailarch.py b/ietf/review/mailarch.py index 986b79b36..2ca6c9566 100644 --- a/ietf/review/mailarch.py +++ b/ietf/review/mailarch.py @@ -24,6 +24,8 @@ from pyquery import PyQuery from django.conf import settings from django.utils.encoding import force_bytes, force_str +from ietf.utils.mail import get_payload_text + def list_name_from_email(list_email): if not list_email.endswith("@ietf.org"): return None @@ -81,7 +83,7 @@ def retrieve_messages_from_mbox(mbox_fileobj): for part in msg.walk(): if part.get_content_type() == "text/plain": charset = part.get_content_charset() or "utf-8" - content += part.get_payload(decode=True).decode(charset, "ignore") + content += get_payload_text(part, default_charset=charset) # parse a couple of things for the front end utcdate = None diff --git a/ietf/secr/sreq/tests.py b/ietf/secr/sreq/tests.py index 2c311752a..a7e8d7b8e 100644 --- a/ietf/secr/sreq/tests.py +++ b/ietf/secr/sreq/tests.py @@ -15,7 +15,7 @@ from ietf.meeting.factories import MeetingFactory, SessionFactory from ietf.name.models import TimerangeName from ietf.person.models import Person from ietf.secr.sreq.forms import SessionForm -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.mail import outbox, empty_outbox, get_payload_text from pyquery import PyQuery @@ -320,7 +320,7 @@ class SubmitRequestCase(TestCase): self.assertRedirects(r, reverse('ietf.secr.sreq.views.main')) self.assertEqual(len(outbox),len_before+1) notification = outbox[-1] - notification_payload = notification.get_payload(decode=True).decode(encoding="utf-8", errors="replace") + notification_payload = get_payload_text(notification) sessions = Session.objects.filter(meeting=meeting,group=group) self.assertEqual(len(sessions), 2) session = sessions[0] diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 06808c168..c933aae3e 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -35,7 +35,7 @@ from ietf.person.models import Person from ietf.person.factories import UserFactory, PersonFactory, EmailFactory from ietf.submit.models import Submission, Preapproval from ietf.submit.mail import add_submission_email, process_response_email -from ietf.utils.mail import outbox, empty_outbox, get_payload +from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.models import VersionInfo from ietf.utils.test_utils import login_testing_unauthorized, TestCase from ietf.utils.draft import Draft @@ -194,8 +194,7 @@ class SubmitTests(TestCase): def extract_confirmation_url(self, confirmation_email): # dig out confirmation_email link - charset = confirmation_email.get_content_charset() - msg = confirmation_email.get_payload(decode=True).decode(charset) + msg = get_payload_text(confirmation_email) line_start = "http" confirmation_url = None for line in msg.split("\n"): @@ -298,16 +297,16 @@ class SubmitTests(TestCase): self.assertTrue(draft.relations_that_doc("possibly-replaces").first().target, sug_replaced_alias) self.assertEqual(len(outbox), mailbox_before + 5) self.assertIn(("I-D Action: %s" % name), outbox[-4]["Subject"]) - self.assertIn(author.ascii, get_payload(outbox[-4])) + self.assertIn(author.ascii, get_payload_text(outbox[-4])) self.assertIn(("I-D Action: %s" % name), outbox[-3]["Subject"]) - self.assertIn(author.ascii, get_payload(outbox[-3])) + self.assertIn(author.ascii, get_payload_text(outbox[-3])) self.assertIn("New Version Notification",outbox[-2]["Subject"]) - self.assertIn(name, get_payload(outbox[-2])) - self.assertIn("mars", get_payload(outbox[-2])) + self.assertIn(name, get_payload_text(outbox[-2])) + self.assertIn("mars", get_payload_text(outbox[-2])) # Check "Review of suggested possible replacements for..." mail self.assertIn("review", outbox[-1]["Subject"].lower()) - self.assertIn(name, get_payload(outbox[-1])) - self.assertIn(sug_replaced_alias.name, get_payload(outbox[-1])) + self.assertIn(name, get_payload_text(outbox[-1])) + self.assertIn(sug_replaced_alias.name, get_payload_text(outbox[-1])) self.assertIn("ames-chairs@", outbox[-1]["To"].lower()) self.assertIn("mars-chairs@", outbox[-1]["To"].lower()) @@ -497,17 +496,17 @@ class SubmitTests(TestCase): self.assertEqual(len(outbox), mailbox_before + 3) self.assertTrue(("I-D Action: %s" % name) in outbox[-3]["Subject"]) self.assertTrue(("I-D Action: %s" % name) in draft.message_set.order_by("-time")[0].subject) - self.assertTrue(author.ascii in get_payload(outbox[-3])) + self.assertTrue(author.ascii in get_payload_text(outbox[-3])) self.assertTrue("i-d-announce@" in outbox[-3]['To']) self.assertTrue("New Version Notification" in outbox[-2]["Subject"]) - self.assertTrue(name in get_payload(outbox[-2])) + self.assertTrue(name in get_payload_text(outbox[-2])) interesting_address = {'ietf':'mars', 'irtf':'irtf-chair', 'iab':'iab-chair', 'ise':'rfc-ise'}[draft.stream_id] self.assertTrue(interesting_address in force_text(outbox[-2].as_string())) if draft.stream_id == 'ietf': self.assertTrue(draft.ad.role_email("ad").address in force_text(outbox[-2].as_string())) self.assertTrue(ballot_position.balloter.role_email("ad").address in force_text(outbox[-2].as_string())) self.assertTrue("New Version Notification" in outbox[-1]["Subject"]) - self.assertTrue(name in get_payload(outbox[-1])) + self.assertTrue(name in get_payload_text(outbox[-1])) r = self.client.get(urlreverse('ietf.doc.views_search.recent_drafts')) self.assertEqual(r.status_code, 200) self.assertContains(r, draft.name) diff --git a/ietf/sync/iana.py b/ietf/sync/iana.py index 7e6085e93..beb539306 100644 --- a/ietf/sync/iana.py +++ b/ietf/sync/iana.py @@ -19,7 +19,7 @@ from ietf.doc.mails import email_state_changed from ietf.doc.models import Document, DocEvent, State, StateDocEvent, StateType from ietf.doc.utils import add_state_change_event from ietf.person.models import Person -from ietf.utils.mail import parseaddr +from ietf.utils.mail import parseaddr, get_payload_text from ietf.utils.timezone import local_timezone_to_utc, email_time_to_local_timezone, utc_to_local_timezone @@ -254,8 +254,7 @@ def parse_review_email(text): by = Person.objects.get(name="(System)") # comment - charset = msg.get_content_charset() - body = msg.get_payload(decode=True).decode(charset or 'utf-8').replace("\r", "") + body = get_payload_text(msg).replace("\r", "") begin_search = re.search(r'\(BEGIN\s+IANA\s+(LAST\s+CALL\s+)?COMMENTS?(\s*:\s*[a-zA-Z0-9-\.]*)?\s*\)',body) end_search = re.search(r'\(END\s+IANA\s+(LAST\s+CALL\s+)?COMMENTS?\)',body) diff --git a/ietf/utils/mail.py b/ietf/utils/mail.py index 42149bff5..e7f6c622a 100644 --- a/ietf/utils/mail.py +++ b/ietf/utils/mail.py @@ -18,7 +18,7 @@ from email.mime.text import MIMEText from email.mime.message import MIMEMessage from email.mime.multipart import MIMEMultipart from email.header import Header, decode_header -from email import message_from_string +from email import message_from_bytes, message_from_string from email import charset as Charset from django.conf import settings @@ -404,7 +404,8 @@ def send_mail_mime(request, to, frm, subject, msg, cc=None, extra=None, toUser=F def parse_preformatted(preformatted, extra={}, override={}): """Parse preformatted string containing mail with From:, To:, ...,""" - msg = message_from_string(force_str(preformatted)) + assert isinstance(preformatted, str) + msg = message_from_bytes(preformatted.encode('utf-8')) msg.set_charset('UTF-8') for k, v in override.items(): @@ -460,7 +461,7 @@ def send_mail_preformatted(request, preformatted, extra={}, override={}): extra headers as needed).""" (msg, extra, bcc) = parse_preformatted(preformatted, extra, override) - txt = get_payload(msg) + txt = msg.get_payload() send_mail_text(request, msg['To'], msg["From"], msg["Subject"], txt, extra=extra, bcc=bcc) return msg @@ -617,11 +618,11 @@ def get_email_addresses_from_text(text): -def get_payload(msg, decode=False): - return msg.get_payload(decode=decode) +# def get_payload(msg, decode=False): +# return msg.get_payload(decode=decode) -def get_payload_text(msg, decode=True): - charset = msg.get_charset() +def get_payload_text(msg, decode=True, default_charset="utf-8"): + charset = msg.get_charset() or default_charset payload = msg.get_payload(decode=decode) try: payload = payload.decode(str(charset)) diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index a3d9b1314..b4b28419d 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -51,7 +51,7 @@ from django.utils.text import slugify import debug # pyflakes:ignore -from ietf.utils.mail import get_payload +from ietf.utils.mail import get_payload_text real_database_name = settings.DATABASES["default"]["NAME"] @@ -171,7 +171,7 @@ class TestCase(django.test.TestCase): mlist = [ m for m in mlist if subject in m["Subject"] ] if text: assert isinstance(text, str) - mlist = [ m for m in mlist if text in get_payload(m) ] + mlist = [ m for m in mlist if text in get_payload_text(m) ] if count and len(mlist) != count: sys.stderr.write("Wrong count in assertMailboxContains(). The complete mailbox contains %s emails:\n\n" % len(mailbox)) for m in mailbox: diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index b6556df0b..270e3731a 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -41,7 +41,7 @@ from ietf.submit.tests import submission_file from ietf.utils.bower_storage import BowerStorageFinder from ietf.utils.draft import Draft, getmeta from ietf.utils.log import unreachable, assertion -from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox, get_payload +from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox, get_payload_text from ietf.utils.test_runner import get_template_paths, set_coverage_checking from ietf.utils.test_utils import TestCase @@ -72,7 +72,7 @@ body self.assertSameEmail(recv['Cc'], 'cc1@example.com, cc2@example.com') self.assertSameEmail(recv['Bcc'], None) self.assertEqual(recv['Subject'], 'subject') - self.assertEqual(get_payload(recv), 'body\n') + self.assertEqual(get_payload_text(recv), 'body\n') override = { 'To': 'oto1@example.net, oto2@example.net', @@ -102,7 +102,7 @@ body self.assertSameEmail(recv['Cc'], ', occ2@example.net') self.assertSameEmail(recv['Bcc'], None) self.assertEqual(recv['Subject'], 'osubject') - self.assertEqual(get_payload(recv), 'body\n') + self.assertEqual(get_payload_text(recv), 'body\n') extra = {'Fuzz': [ 'bucket' ]} send_mail_preformatted(request=None, preformatted=msg, extra=extra, override={})