Refactored many instances of msg.get_payload(decode=True).decode('utf-8') and similar to get_payload_text(msg), which uses msg.get_charset() to get the actual charset to do the decoding, in a consistent manner. Also removed some instances of force_str, but more remain.

- Legacy-Id: 17807

ietf/doc/mails.py

@@ -10,11 +10,11 @@ from django.template.loader import render_to_string
 from django.utils.html import strip_tags
 from django.conf import settings
 from django.urls import reverse as urlreverse
-from django.utils.encoding import force_str, force_text
+from django.utils.encoding import force_text
 
 import debug                            # pyflakes:ignore
 
-from ietf.utils.mail import send_mail, send_mail_text, get_payload
+from ietf.utils.mail import send_mail, send_mail_text
 from ietf.ipr.utils import iprs_from_docs, related_docs
 from ietf.doc.models import WriteupDocEvent, LastCallDocEvent, DocAlias, ConsensusDocEvent
 from ietf.doc.utils import needed_ballot_positions
@@ -387,7 +387,7 @@ def generate_issue_ballot_mail(request, doc, ballot):
 def email_iana(request, doc, to, msg, cc=None):
     # fix up message and send it with extra info on doc in headers
     import email
-    parsed_msg = email.message_from_string(force_str(msg))
+    parsed_msg = email.message_from_string(msg)
     parsed_msg.set_charset('UTF-8')
 
     extra = extra_automation_headers(doc)
@@ -395,7 +395,7 @@ def email_iana(request, doc, to, msg, cc=None):
     
     send_mail_text(request, to,
                    parsed_msg["From"], parsed_msg["Subject"],
-                   get_payload(parsed_msg),
+                   parsed_msg.get_payload(),
                    extra=extra,
                    cc=cc)
 

ietf/doc/tests_ballot.py

@@ -23,7 +23,7 @@ from ietf.iesg.models import TelechatDate
 from ietf.person.models import Person, PersonalApiKey
 from ietf.person.factories import PersonFactory
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized
-from ietf.utils.mail import outbox, empty_outbox, get_payload
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.text import unwrap
 
 
@@ -174,7 +174,7 @@ class EditPositionTests(TestCase):
         self.assertEqual(len(outbox), mailbox_before + 1)
         m = outbox[-1]
         self.assertIn('COMMENT', m['Subject'])
-        self.assertIn('New comment', get_payload(m))
+        self.assertIn('New comment', get_payload_text(m))
 
 
     def test_edit_position_as_secretary(self):
@@ -460,11 +460,11 @@ class BallotWriteupsTests(TestCase):
             self.assertTrue('drafts-eval@' in outbox[-1]['To'])
             self.assertTrue('X-IETF-Draft-string' in outbox[-1])
             if case=='none':
-                self.assertNotIn('call expire', outbox[-1].get_payload(decode=True).decode("utf-8"))
+                self.assertNotIn('call expire', get_payload_text(outbox[-1]))
             elif case=='past':
-                self.assertIn('call expired', outbox[-1].get_payload(decode=True).decode("utf-8"))
+                self.assertIn('call expired', get_payload_text(outbox[-1]))
             else:
-                self.assertIn('call expires', outbox[-1].get_payload(decode=True).decode("utf-8"))
+                self.assertIn('call expires', get_payload_text(outbox[-1]))
             self.client.logout()
 
 
@@ -797,7 +797,7 @@ class MakeLastCallTests(TestCase):
         self.assertTrue("ietf-announce@" in outbox[-2]['To'])
         for prefix in ['draft-ietf-mars-test','mars-chairs','aread']:
             self.assertTrue(prefix+"@" in outbox[-2]['Cc'])
-        self.assertIn("The following IPR Declarations", get_payload(outbox[-2]))
+        self.assertIn("The following IPR Declarations", get_payload_text(outbox[-2]))
 
         self.assertTrue("Last Call" in outbox[-1]['Subject'])
         self.assertTrue("drafts-lastcall@icann.org" in outbox[-1]['To'])

ietf/doc/tests_charter.py

@@ -25,7 +25,7 @@ from ietf.group.models import Group, GroupMilestone
 from ietf.iesg.models import TelechatDate
 from ietf.person.models import Person
 from ietf.utils.test_utils import TestCase
-from ietf.utils.mail import outbox, empty_outbox, get_payload
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import login_testing_unauthorized
 
 class ViewCharterTests(TestCase):
@@ -187,7 +187,7 @@ class EditCharterTests(TestCase):
                 self.assertIn("Internal WG Review", outbox[-3]['Subject'])
                 self.assertIn("iab@", outbox[-3]['To'])
                 self.assertIn("iesg@", outbox[-3]['To'])
-                body = get_payload(outbox[-3])
+                body = get_payload_text(outbox[-3])
                 for word in ["A new IETF WG", "Chairs", "Ames Man <ameschairman@example.org>",
                     "Secretaries", "Secretary <amessecretary@example.org>",
                     "Assigned Area Director", "Areað Irector <aread@example.org>",
@@ -198,13 +198,13 @@ class EditCharterTests(TestCase):
 
             self.assertIn("state changed", outbox[-2]['Subject'].lower())
             self.assertIn("iesg-secretary@", outbox[-2]['To'])
-            body = get_payload(outbox[-2])
+            body = get_payload_text(outbox[-2])
             for word in ["WG", "Charter", ]:
                 self.assertIn(word, body)
 
             self.assertIn("State Update Notice", outbox[-1]['Subject'])
             self.assertIn("ames-chairs@", outbox[-1]['To'])
-            body = get_payload(outbox[-1])
+            body = get_payload_text(outbox[-1])
             for word in ["State changed", "Datatracker URL", ]:
                 self.assertIn(word, body)
 
@@ -224,7 +224,7 @@ class EditCharterTests(TestCase):
         empty_outbox()
         r = self.client.post(url, dict(charter_state=str(State.objects.get(used=True,type="charter",slug="intrev").pk), message="test"))
         self.assertEqual(r.status_code, 302)
-        self.assertTrue("A new charter" in get_payload(outbox[-3]))
+        self.assertTrue("A new charter" in get_payload_text(outbox[-3]))
 
     def test_change_rg_state(self):
 
@@ -245,7 +245,7 @@ class EditCharterTests(TestCase):
         self.assertIn("Internal RG Review", outbox[-3]['Subject'])
         self.assertIn("iab@", outbox[-3]['To'])
         self.assertIn("irsg@", outbox[-3]['To'])
-        body = get_payload(outbox[-3])
+        body = get_payload_text(outbox[-3])
         for word in ["A new IRTF RG", 
             "Mailing list", "somerg@ietf.org",
             "Charter", "Milestones"]:
@@ -254,13 +254,13 @@ class EditCharterTests(TestCase):
 
         self.assertIn("state changed", outbox[-2]['Subject'].lower())
         self.assertIn("iesg-secretary@", outbox[-2]['To'])
-        body = get_payload(outbox[-2])
+        body = get_payload_text(outbox[-2])
         for word in ["RG", "Charter", ]:
             self.assertIn(word, body)
 
         self.assertIn("State Update Notice", outbox[-1]['Subject'])
         self.assertIn("somerg-chairs@", outbox[-1]['To'])
-        body = get_payload(outbox[-1])
+        body = get_payload_text(outbox[-1])
         for word in ["State changed", "Datatracker URL", ]:
             self.assertIn(word, body)
 
@@ -793,7 +793,7 @@ class EditCharterTests(TestCase):
         #
         self.assertTrue("approved" in outbox[0]['Subject'].lower())
         self.assertTrue("iesg-secretary" in outbox[0]['To'])
-        body = get_payload(outbox[0])
+        body = get_payload_text(outbox[0])
         for word in ["WG",   "/wg/ames/about/",
             "Charter", "/doc/charter-ietf-ames/", ]:
             self.assertIn(word, body)
@@ -801,7 +801,7 @@ class EditCharterTests(TestCase):
         self.assertTrue("WG Action" in outbox[1]['Subject'])
         self.assertTrue("ietf-announce" in outbox[1]['To'])
         self.assertTrue("ames-wg@ietf.org" in outbox[1]['Cc'])
-        body = get_payload(outbox[1])
+        body = get_payload_text(outbox[1])
         for word in ["Chairs", "Ames Man <ameschairman@example.org>",
             "Secretaries", "Secretary <amessecretary@example.org>",
             "Assigned Area Director", "Areað Irector <aread@example.org>",

ietf/doc/tests_conflict_review.py

@@ -22,7 +22,7 @@ from ietf.group.models import Person
 from ietf.iesg.models import TelechatDate
 from ietf.name.models import StreamName
 from ietf.utils.test_utils import TestCase
-from ietf.utils.mail import outbox, empty_outbox, get_payload
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import login_testing_unauthorized
 
 
@@ -302,9 +302,9 @@ class ConflictReviewTests(TestCase):
         self.assertIn('iana@', outbox[0]['Cc'])
 
         if approve_type == 'appr-noprob':
-            self.assertIn( 'IESG has no problem', ''.join(wrap(get_payload(outbox[0]), 2**16)))
+            self.assertIn( 'IESG has no problem', ''.join(wrap(get_payload_text(outbox[0]), 2**16)))
         else:
-            self.assertIn( 'NOT be published', ''.join(wrap(get_payload(outbox[0]), 2**16)))
+            self.assertIn( 'NOT be published', ''.join(wrap(get_payload_text(outbox[0]), 2**16)))
 
 
     def test_approve_reqnopub(self):

ietf/doc/tests_draft.py

@@ -30,7 +30,7 @@ from ietf.person.models import Person, Email
 from ietf.meeting.models import Meeting, MeetingTypeName
 from ietf.iesg.models import TelechatDate
 from ietf.utils.test_utils import login_testing_unauthorized
-from ietf.utils.mail import outbox, empty_outbox, get_payload
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import TestCase
 
 
@@ -387,7 +387,7 @@ class EditInfoTests(TestCase):
         data["telechat_date"] = next_week.isoformat()
         r = self.client.post(url,data)
         self.assertEqual(r.status_code, 302)
-        self.assertIn("may not leave enough time", get_payload(outbox[-1]))
+        self.assertIn("may not leave enough time", get_payload_text(outbox[-1]))
 
     def test_start_iesg_process_on_draft(self):
 

ietf/doc/tests_irsg_ballot.py

@@ -11,7 +11,7 @@ import datetime
 
 from django.urls import reverse as urlreverse
 
-from ietf.utils.mail import outbox, empty_outbox, get_payload
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import TestCase, unicontent, login_testing_unauthorized
 from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, RgDraftFactory, RgRfcFactory, BallotDocEventFactory, IRSGBallotDocEventFactory, BallotPositionDocEventFactory
 from ietf.doc.models import BallotDocEvent, BallotPositionDocEvent
@@ -363,7 +363,7 @@ class BaseManipulationTests():
         r = self.client.post(url, dict(cc_choices=['doc_authors','doc_group_chairs','doc_group_mail_list'], body="Stuff"))
         self.assertEqual(r.status_code, 302)
         self.assertEqual(len(outbox),1)
-        self.assertNotIn('discuss-criteria', get_payload(outbox[0]))
+        self.assertNotIn('discuss-criteria', get_payload_text(outbox[0]))
 
     def test_close_ballot(self):
         draft = RgDraftFactory()

ietf/doc/tests_review.py

@@ -20,24 +20,23 @@ from pyquery import PyQuery
 import debug                            # pyflakes:ignore
 
 import ietf.review.mailarch
+
 from ietf.doc.factories import ( NewRevisionDocEventFactory, IndividualDraftFactory, WgDraftFactory,
-    WgRfcFactory, ReviewFactory, DocumentFactory)
+                            WgRfcFactory, ReviewFactory, DocumentFactory)
-from ietf.doc.models import DocumentAuthor, RelatedDocument, DocEvent, ReviewRequestDocEvent, ReviewAssignmentDocEvent
+from ietf.doc.models import ( DocumentAuthor, RelatedDocument, DocEvent, ReviewRequestDocEvent,
+                            ReviewAssignmentDocEvent, )
 from ietf.group.factories import RoleFactory, ReviewTeamFactory
 from ietf.group.models import Group
 from ietf.message.models import Message
-from ietf.name.models import ReviewResultName, ReviewRequestStateName, ReviewAssignmentStateName, \
+from ietf.name.models import ReviewResultName, ReviewRequestStateName, ReviewAssignmentStateName, ReviewTypeName
-    ReviewTypeName
+from ietf.person.factories import PersonFactory
 from ietf.person.models import Email, Person
 from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory
-from ietf.review.models import (ReviewRequest, ReviewerSettings,
+from ietf.review.models import ReviewRequest, ReviewerSettings, ReviewWish, NextReviewerInTeam
-                                ReviewWish, NextReviewerInTeam)
 from ietf.review.policies import get_reviewer_queue_policy
-
+from ietf.utils.mail import outbox, empty_outbox, parseaddr, on_behalf_of, get_payload_text
-from ietf.utils.test_utils import TestCase
 from ietf.utils.test_utils import login_testing_unauthorized, reload_db_objects
-from ietf.utils.mail import outbox, empty_outbox, parseaddr, on_behalf_of
+from ietf.utils.test_utils import TestCase
-from ietf.person.factories import PersonFactory
 
 class ReviewTests(TestCase):
     def setUp(self):
@@ -228,7 +227,7 @@ class ReviewTests(TestCase):
         self.assertIn('<reviewer@example.com>', outbox[0]["To"])
         self.assertNotIn("<reviewsecretary@example.com>", outbox[0]["To"])
         self.assertIn("reviewsecretary2@example.com", outbox[0]["CC"])
-        mail_content = outbox[0].get_payload(decode=True).decode("utf-8").lower()
+        mail_content = get_payload_text(outbox[0])
         self.assertIn("closed", mail_content)
         self.assertIn("review_request_close_comment", mail_content)
 
@@ -330,7 +329,7 @@ class ReviewTests(TestCase):
 
         self.assertEqual(len(outbox), 1)
         self.assertEqual('"Some Reviewer" <reviewer@example.com>', outbox[0]["To"])
-        message = outbox[0].get_payload(decode=True).decode("utf-8")
+        message = get_payload_text(outbox[0])
         self.assertIn("Pages: {}".format(doc.pages), message )
         self.assertIn("{} has assigned {}".format(secretary.person.ascii, reviewer.person.ascii), message)
         self.assertIn("This team has completed other reviews", message)
@@ -423,7 +422,7 @@ class ReviewTests(TestCase):
         self.assertEqual(len(outbox), 1)
         self.assertIn(assignment.reviewer.address, outbox[0]["To"])
         self.assertNotIn("<reviewsecretary@example.com>", outbox[0]["To"])
-        self.assertTrue("Test message" in outbox[0].get_payload(decode=True).decode("utf-8"))
+        self.assertTrue("Test message" in get_payload_text(outbox[0]))
 
         # try again, but now with an expired review request, which should not be allowed (#2277)
         assignment.state_id = 'assigned'
@@ -645,7 +644,7 @@ class ReviewTests(TestCase):
         self.assertEqual(len(outbox), 1)
         self.assertIn(assignment.review_request.team.list_email, outbox[0]["To"])
         self.assertIn(assignment.reviewer.role_set.filter(group=assignment.review_request.team,name='reviewer').first().person.plain_name(), parseaddr(outbox[0]["From"])[0] )
-        self.assertIn("This is a review", outbox[0].get_payload(decode=True).decode("utf-8"))
+        self.assertIn("This is a review", get_payload_text(outbox[0]))
 
         self.assertIn(settings.MAILING_LIST_ARCHIVE_URL, assignment.review.external_url)
 
@@ -658,7 +657,7 @@ class ReviewTests(TestCase):
         self.assertEqual(parseaddr(outbox[0]["To"]), parseaddr(message.to))
         self.assertEqual(parseaddr(outbox[0]["From"]), parseaddr(on_behalf_of(message.frm)))
         self.assertEqual(parseaddr(outbox[0]["Reply-To"]), parseaddr(message.frm))
-        self.assertEqual(outbox[0].get_payload(decode=True).decode(str(outbox[0].get_charset())), message.body)
+        self.assertEqual(get_payload_text(outbox[0]), message.body)
 
         # check the review document page
         url = urlreverse('ietf.doc.views_doc.document_main', kwargs={ "name": assignment.review.name })
@@ -699,7 +698,7 @@ class ReviewTests(TestCase):
 
         self.assertEqual(len(outbox), 1)
         self.assertIn(assignment.review_request.team.list_email, outbox[0]["To"])
-        self.assertIn("This is a review", outbox[0].get_payload(decode=True).decode("utf-8"))
+        self.assertIn("This is a review", get_payload_text(outbox[0]))
 
         self.assertIn(settings.MAILING_LIST_ARCHIVE_URL, assignment.review.external_url)
 
@@ -740,7 +739,7 @@ class ReviewTests(TestCase):
 
         self.assertEqual(len(outbox), 1)
         self.assertIn(assignment.review_request.team.list_email, outbox[0]["To"])
-        self.assertIn("This is a review", outbox[0].get_payload(decode=True).decode("utf-8"))
+        self.assertIn("This is a review", get_payload_text(outbox[0]))
 
         self.assertIn(settings.MAILING_LIST_ARCHIVE_URL, assignment.review.external_url)
 
@@ -767,7 +766,7 @@ class ReviewTests(TestCase):
 
         self.assertEqual(len(outbox), 2)
         self.assertIn('Has Issues', outbox[-1]['Subject'])
-        self.assertIn('settings indicated', outbox[-1].get_payload(decode=True).decode("utf-8"))
+        self.assertIn('settings indicated', get_payload_text(outbox[-1]))
 
     def test_complete_notify_ad_because_checkbox(self):
         assignment, url = self.setup_complete_review_test()
@@ -791,7 +790,7 @@ class ReviewTests(TestCase):
 
         self.assertEqual(len(outbox), 2)
         self.assertIn('Has Issues', outbox[-1]['Subject']) 
-        self.assertIn('reviewer indicated', outbox[-1].get_payload(decode=True).decode("utf-8"))
+        self.assertIn('reviewer indicated', get_payload_text(outbox[-1]))
 
     @patch('requests.get')
     def test_complete_review_link_to_mailing_list(self, mock):
@@ -908,7 +907,7 @@ class ReviewTests(TestCase):
 
         self.assertTrue(assignment.review_request.team.list_email in outbox[1]["To"])
         self.assertTrue("partial review" in outbox[1]["Subject"].lower())
-        body = outbox[1].get_payload(decode=True).decode("utf-8")
+        body = get_payload_text(outbox[1])
         self.assertTrue("This is a review" in body)
         # This review has a line longer than 80, but less than 100; it should
         # not be wrapped.
@@ -941,7 +940,7 @@ class ReviewTests(TestCase):
 
         # This review has a line longer than 100; it should be wrapped to less
         # than 80.
-        body = outbox[2].get_payload(decode=True).decode("utf-8")
+        body = get_payload_text(outbox[2])
         self.assertIn('really, really, really', body)
         self.assertTrue(all( len(line) <= 80 for line in body.splitlines() ))
 

ietf/group/tests_info.py

@@ -35,7 +35,7 @@ from ietf.name.models import DocTagName, GroupStateName, GroupTypeName
 from ietf.person.models import Person, Email
 from ietf.person.factories import PersonFactory
 from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory
-from ietf.utils.mail import outbox, empty_outbox
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent, reload_db_objects
 
 def group_urlreverse_list(group, viewname):
@@ -629,7 +629,7 @@ class GroupEditTests(TestCase):
         self.assertTrue('Personnel change' in outbox[0]['Subject'])
         for prefix in ['ad1','ad2','aread','marschairman','marsdelegate']:
             self.assertTrue(prefix+'@' in outbox[0]['To'])
-        self.assertTrue(outbox[0].get_payload(decode=True).decode(str(outbox[0].get_charset())).startswith('Sec Retary'))
+        self.assertTrue(get_payload_text(outbox[0]).startswith('Sec Retary'))
 
 
     def test_edit_field(self):

ietf/group/views.py

@@ -111,7 +111,7 @@ from ietf.doc.models import LastCallDocEvent
 
 
 from ietf.name.models import ReviewAssignmentStateName
-from ietf.utils.mail import send_mail_text, parse_preformatted, get_payload
+from ietf.utils.mail import send_mail_text, parse_preformatted
 
 from ietf.ietfauth.utils import user_is_person
 from ietf.dbtemplate.models import DBTemplate
@@ -1704,7 +1704,7 @@ def email_open_review_assignments(request, acronym, group_type=None):
         
         (msg,_,_) = parse_preformatted(partial_msg)
 
-        body = get_payload(msg)
+        body = msg.get_payload()
         subject = msg['Subject']
 
         form = EmailOpenAssignmentsForm(initial={

ietf/ietfauth/tests.py

@@ -15,17 +15,17 @@ from django.conf import settings
 
 import debug                            # pyflakes:ignore
 
-from ietf.utils.test_utils import TestCase, login_testing_unauthorized
-from ietf.utils.mail import outbox, empty_outbox
-from ietf.group.models import Group, Role, RoleName
 from ietf.group.factories import GroupFactory, RoleFactory
+from ietf.group.models import Group, Role, RoleName
 from ietf.ietfauth.htpasswd import update_htpasswd_file
 from ietf.mailinglists.models import Subscribed
-from ietf.person.models import Person, Email, PersonalApiKey
 from ietf.person.factories import PersonFactory, EmailFactory
+from ietf.person.models import Person, Email, PersonalApiKey
 from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory
 from ietf.review.models import ReviewWish, UnavailablePeriod
 from ietf.utils.decorators import skip_coverage
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
+from ietf.utils.test_utils import TestCase, login_testing_unauthorized
 
 import ietf.ietfauth.views
 
@@ -99,7 +99,7 @@ class IetfAuthTests(TestCase):
 
     def extract_confirm_url(self, confirm_email):
         # dig out confirm_email link
-        msg = confirm_email.get_payload(decode=True).decode(confirm_email.get_content_charset())
+        msg = get_payload_text(confirm_email)
         line_start = "http"
         confirm_url = None
         for line in msg.split("\n"):
@@ -643,7 +643,7 @@ class IetfAuthTests(TestCase):
         
         self.assertEqual(len(outbox), len(endpoints))
         for mail in outbox:
-            body = mail.get_payload(decode=True).decode('utf-8')
+            body = get_payload_text(mail)
             self.assertIn("API key usage", mail['subject'])
             self.assertIn(" %s times" % count, body)
             self.assertIn(date, body)

ietf/ipr/tests.py

@@ -23,14 +23,11 @@ from ietf.ipr.models import (IprDisclosureBase,GenericIprDisclosure,HolderIprDis
 from ietf.ipr.utils import get_genitive, get_ipr_summary
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.message.models import Message
-from ietf.utils.mail import outbox, empty_outbox
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized
 from ietf.utils.text import text_to_dict
 
 
-def extract_message_content(message):
-    return message.get_payload(decode=True).decode(str(message.get_charset()))
-
 def make_data_from_content(content):
     q = PyQuery(content)
     data = dict()
@@ -582,10 +579,10 @@ I would like to revoke this declaration.
         self.assertEqual(r.status_code,302)
         self.assertEqual(len(outbox),len_before+2)
         self.assertTrue('george@acme.com' in outbox[len_before]['To'])
-        self.assertIn('posted on '+datetime.date.today().strftime("%Y-%m-%d"), extract_message_content(outbox[len_before]).replace('\n',' '))
+        self.assertIn('posted on '+datetime.date.today().strftime("%Y-%m-%d"), get_payload_text(outbox[len_before]).replace('\n',' '))
         self.assertTrue('draft-ietf-mars-test@ietf.org' in outbox[len_before+1]['To'])
         self.assertTrue('mars-wg@ietf.org' in outbox[len_before+1]['Cc'])
-        self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), extract_message_content(outbox[len_before+1]).replace('\n',' '))
+        self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), get_payload_text(outbox[len_before+1]).replace('\n',' '))
 
     def test_notify_generic(self):
         RoleFactory(name_id='ad',group__acronym='gen')
@@ -601,7 +598,7 @@ I would like to revoke this declaration.
         r = self.client.post(url, data )
         self.assertEqual(r.status_code,302)
         self.assertEqual(len(outbox),2)
-        self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), extract_message_content(outbox[1]).replace('\n',' '))
+        self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), get_payload_text(outbox[1]).replace('\n',' '))
 
     def test_process_response_email(self):
         # first send a mail

ietf/meeting/tests_views.py

@@ -40,7 +40,7 @@ from ietf.meeting.utils import add_event_info_to_session_qs
 from ietf.meeting.views import session_draft_list
 from ietf.name.models import SessionStatusName, ImportantDateName, RoleName
 from ietf.utils.decorators import skip_coverage
-from ietf.utils.mail import outbox, empty_outbox, get_payload
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent
 from ietf.utils.text import xslugify
 
@@ -1446,8 +1446,8 @@ class InterimTests(TestCase):
         self.assertRedirects(r, urlreverse('ietf.meeting.views.interim_announce'))
         self.assertEqual(len(outbox), len_before + 1)
         self.assertIn('WG Virtual Meeting', outbox[-1]['Subject'])
-        self.assertIn('09:00 to 09:20 America/Los_Angeles', get_payload(outbox[-1]))
+        self.assertIn('09:00 to 09:20 America/Los_Angeles', get_payload_text(outbox[-1]))
-        self.assertIn('(17:00 to 17:20 UTC)', get_payload(outbox[-1]))
+        self.assertIn('(17:00 to 17:20 UTC)', get_payload_text(outbox[-1]))
 
     def test_interim_approve_by_ad(self):
         make_meeting_test_data()

ietf/message/tests.py

@@ -12,7 +12,7 @@ from ietf.group.factories import GroupFactory
 from ietf.message.models import Message, SendQueue
 from ietf.message.utils import send_scheduled_message_from_send_queue
 from ietf.person.models import Person
-from ietf.utils.mail import outbox, send_mail_text, send_mail_message, get_payload
+from ietf.utils.mail import outbox, send_mail_text, send_mail_message, get_payload_text
 from ietf.utils.test_utils import TestCase
 
 class MessageTests(TestCase):
@@ -46,7 +46,7 @@ class MessageTests(TestCase):
                     continue
                 self.assertEqual(e1.get_all(k), e2.get_all(k), "Header field: %s" % k)
             self.longMessage = False
-            self.assertEqual(get_payload(e1), get_payload(e2))
+            self.assertEqual(get_payload_text(e1), get_payload_text(e2))
 
         #
         self.assertEqual(Message.objects.count(), 0)

ietf/message/utils.py

@@ -6,13 +6,13 @@ import re, datetime, email
 
 from django.utils.encoding import force_str
 
-from ietf.utils.mail import send_mail_text, send_mail_mime, get_payload
+from ietf.utils.mail import send_mail_text, send_mail_mime
 from ietf.message.models import Message
 
 first_dot_on_line_re = re.compile(r'^\.', re.MULTILINE)
 
 def infer_message(s):
-    parsed = email.message_from_string(force_str(s))
+    parsed = email.message_from_string(s)
 
     m = Message(
         subject = parsed.get("Subject", ""),
@@ -21,7 +21,7 @@ def infer_message(s):
         cc = parsed.get("Cc", ""),
         bcc = parsed.get("Bcc", ""),
         reply_to = parsed.get("Reply-To", ""),
-        body = get_payload(parsed),
+        body = parsed.get_payload(),
         content_type = parsed.get_content_type(),
     )
 

ietf/nomcom/tests.py

@@ -16,7 +16,7 @@ from django.conf import settings
 from django.core.files import File
 from django.contrib.auth.models import User
 from django.urls import reverse
-from django.utils.encoding import force_text, force_str
+from django.utils.encoding import force_str
 
 import debug                            # pyflakes:ignore
 
@@ -39,7 +39,7 @@ from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, get_hash
 from ietf.person.factories import PersonFactory, EmailFactory
 from ietf.person.models import Email, Person
 from ietf.stats.models import MeetingRegistration
-from ietf.utils.mail import outbox, empty_outbox, get_payload
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent
 
 client_test_cert_files = None
@@ -539,7 +539,7 @@ class NomcomViewsTest(TestCase):
         self.assertEqual('Nomination receipt', outbox[-1]['Subject'])
         self.assertEqual(self.email_from, outbox[-1]['From'])
         self.assertIn('plain', outbox[-1]['To'])
-        self.assertIn('Comments with accents äöå', force_text(outbox[-1].get_payload(decode=True),"utf-8","replace"))
+        self.assertIn('Comments with accents äöå', get_payload_text(outbox[-1]))
 
         # Nominate the same person for the same position again without asking for confirmation 
 
@@ -580,7 +580,7 @@ class NomcomViewsTest(TestCase):
         self.assertEqual('Nomination receipt', outbox[-1]['Subject'])
         self.assertEqual(self.email_from, outbox[-1]['From'])
         self.assertIn('plain', outbox[-1]['To'])
-        self.assertIn('Comments with accents äöå', force_text(outbox[-1].get_payload(decode=True),"utf-8","replace"))
+        self.assertIn('Comments with accents äöå', get_payload_text(outbox[-1]))
 
         # Nominate the same person for the same position again without asking for confirmation 
 
@@ -831,12 +831,12 @@ class NomcomViewsTest(TestCase):
         # We're interested in the confirmation receipt here
         self.assertEqual(len(outbox),3)
         self.assertEqual('NomCom comment confirmation', outbox[2]['Subject'])
-        email_body = get_payload(outbox[2])
+        email_body = get_payload_text(outbox[2])
         self.assertIn(position, email_body)
         self.assertNotIn('$', email_body)
         self.assertEqual(self.email_from, outbox[-2]['From'])
         self.assertIn('plain', outbox[2]['To'])
-        self.assertIn('Comments with accents äöå', force_text(outbox[2].get_payload(decode=True),"utf-8","replace"))
+        self.assertIn('Comments with accents äöå', get_payload_text(outbox[2]))
 
         empty_outbox()
         self.feedback_view(public=True)

ietf/nomcom/utils.py

@@ -20,13 +20,13 @@ from django.core.exceptions import ObjectDoesNotExist
 from django.urls import reverse
 from django.template.loader import render_to_string
 from django.shortcuts import get_object_or_404
-from django.utils.encoding import force_str, force_text
+from django.utils.encoding import force_str
 
 from ietf.dbtemplate.models import DBTemplate
 from ietf.person.models import Email, Person
 from ietf.mailtrigger.utils import gather_address_lists
 from ietf.utils.pipe import pipe
-from ietf.utils.mail import send_mail_text, send_mail
+from ietf.utils.mail import send_mail_text, send_mail, get_payload_text
 from ietf.utils.log import log
 from ietf.person.name import unidecode_name
 
@@ -438,18 +438,14 @@ def get_body(message):
                                                              'plain')]
         body = []
         for part in text_parts:
-            charset = get_charset(part, get_charset(message))
+            charset = get_charset(message)
-            body.append(force_text(part.get_payload(decode=True),
+            body.append(get_payload_text(part, default_charset=charset))
-                                charset,
-                                "replace"))
 
         return "\n".join(body).strip()
 
     else:  # if it is not multipart, the payload will be a string
            # representing the message body
-        body = force_text(message.get_payload(decode=True),
+        body = get_payload_text(message)
-                       get_charset(message),
-                       "replace")
         return body.strip()
 
 

ietf/review/mailarch.py

@@ -24,6 +24,8 @@ from pyquery import PyQuery
 from django.conf import settings
 from django.utils.encoding import force_bytes, force_str
 
+from ietf.utils.mail import get_payload_text
+
 def list_name_from_email(list_email):
     if not list_email.endswith("@ietf.org"):
         return None
@@ -81,7 +83,7 @@ def retrieve_messages_from_mbox(mbox_fileobj):
             for part in msg.walk():
                 if part.get_content_type() == "text/plain":
                     charset = part.get_content_charset() or "utf-8"
-                    content += part.get_payload(decode=True).decode(charset, "ignore")
+                    content += get_payload_text(part, default_charset=charset)
 
             # parse a couple of things for the front end
             utcdate = None

ietf/secr/sreq/tests.py

@@ -15,7 +15,7 @@ from ietf.meeting.factories import MeetingFactory, SessionFactory
 from ietf.name.models import TimerangeName
 from ietf.person.models import Person
 from ietf.secr.sreq.forms import SessionForm
-from ietf.utils.mail import outbox, empty_outbox
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 
 from pyquery import PyQuery
 
@@ -320,7 +320,7 @@ class SubmitRequestCase(TestCase):
         self.assertRedirects(r, reverse('ietf.secr.sreq.views.main'))
         self.assertEqual(len(outbox),len_before+1)
         notification = outbox[-1]
-        notification_payload = notification.get_payload(decode=True).decode(encoding="utf-8", errors="replace")
+        notification_payload = get_payload_text(notification)
         sessions = Session.objects.filter(meeting=meeting,group=group)
         self.assertEqual(len(sessions), 2)
         session = sessions[0]

ietf/submit/tests.py

@@ -35,7 +35,7 @@ from ietf.person.models import Person
 from ietf.person.factories import UserFactory, PersonFactory, EmailFactory
 from ietf.submit.models import Submission, Preapproval
 from ietf.submit.mail import add_submission_email, process_response_email
-from ietf.utils.mail import outbox, empty_outbox, get_payload
+from ietf.utils.mail import outbox, empty_outbox, get_payload_text
 from ietf.utils.models import VersionInfo
 from ietf.utils.test_utils import login_testing_unauthorized, TestCase
 from ietf.utils.draft import Draft
@@ -194,8 +194,7 @@ class SubmitTests(TestCase):
 
     def extract_confirmation_url(self, confirmation_email):
         # dig out confirmation_email link
-        charset = confirmation_email.get_content_charset()
+        msg = get_payload_text(confirmation_email)
-        msg = confirmation_email.get_payload(decode=True).decode(charset)
         line_start = "http"
         confirmation_url = None
         for line in msg.split("\n"):
@@ -298,16 +297,16 @@ class SubmitTests(TestCase):
         self.assertTrue(draft.relations_that_doc("possibly-replaces").first().target, sug_replaced_alias)
         self.assertEqual(len(outbox), mailbox_before + 5)
         self.assertIn(("I-D Action: %s" % name), outbox[-4]["Subject"])
-        self.assertIn(author.ascii, get_payload(outbox[-4]))
+        self.assertIn(author.ascii, get_payload_text(outbox[-4]))
         self.assertIn(("I-D Action: %s" % name), outbox[-3]["Subject"])
-        self.assertIn(author.ascii, get_payload(outbox[-3]))
+        self.assertIn(author.ascii, get_payload_text(outbox[-3]))
         self.assertIn("New Version Notification",outbox[-2]["Subject"])
-        self.assertIn(name, get_payload(outbox[-2]))
+        self.assertIn(name, get_payload_text(outbox[-2]))
-        self.assertIn("mars", get_payload(outbox[-2]))
+        self.assertIn("mars", get_payload_text(outbox[-2]))
         # Check "Review of suggested possible replacements for..." mail
         self.assertIn("review", outbox[-1]["Subject"].lower())
-        self.assertIn(name, get_payload(outbox[-1]))
+        self.assertIn(name, get_payload_text(outbox[-1]))
-        self.assertIn(sug_replaced_alias.name, get_payload(outbox[-1]))
+        self.assertIn(sug_replaced_alias.name, get_payload_text(outbox[-1]))
         self.assertIn("ames-chairs@", outbox[-1]["To"].lower())
         self.assertIn("mars-chairs@", outbox[-1]["To"].lower())
 
@@ -497,17 +496,17 @@ class SubmitTests(TestCase):
         self.assertEqual(len(outbox), mailbox_before + 3)
         self.assertTrue(("I-D Action: %s" % name) in outbox[-3]["Subject"])
         self.assertTrue(("I-D Action: %s" % name) in draft.message_set.order_by("-time")[0].subject)
-        self.assertTrue(author.ascii in get_payload(outbox[-3]))
+        self.assertTrue(author.ascii in get_payload_text(outbox[-3]))
         self.assertTrue("i-d-announce@" in outbox[-3]['To'])
         self.assertTrue("New Version Notification" in outbox[-2]["Subject"])
-        self.assertTrue(name in get_payload(outbox[-2]))
+        self.assertTrue(name in get_payload_text(outbox[-2]))
         interesting_address = {'ietf':'mars', 'irtf':'irtf-chair', 'iab':'iab-chair', 'ise':'rfc-ise'}[draft.stream_id]
         self.assertTrue(interesting_address in force_text(outbox[-2].as_string()))
         if draft.stream_id == 'ietf':
             self.assertTrue(draft.ad.role_email("ad").address in force_text(outbox[-2].as_string()))
             self.assertTrue(ballot_position.balloter.role_email("ad").address in force_text(outbox[-2].as_string()))
         self.assertTrue("New Version Notification" in outbox[-1]["Subject"])
-        self.assertTrue(name in get_payload(outbox[-1]))
+        self.assertTrue(name in get_payload_text(outbox[-1]))
         r = self.client.get(urlreverse('ietf.doc.views_search.recent_drafts'))
         self.assertEqual(r.status_code, 200)
         self.assertContains(r, draft.name)

ietf/sync/iana.py

@@ -19,7 +19,7 @@ from ietf.doc.mails import email_state_changed
 from ietf.doc.models import Document, DocEvent, State, StateDocEvent, StateType
 from ietf.doc.utils import add_state_change_event
 from ietf.person.models import Person
-from ietf.utils.mail import parseaddr
+from ietf.utils.mail import parseaddr, get_payload_text
 from ietf.utils.timezone import local_timezone_to_utc, email_time_to_local_timezone, utc_to_local_timezone
 
 
@@ -254,8 +254,7 @@ def parse_review_email(text):
         by = Person.objects.get(name="(System)")
 
     # comment
-    charset = msg.get_content_charset()
+    body = get_payload_text(msg).replace("\r", "")
-    body = msg.get_payload(decode=True).decode(charset or 'utf-8').replace("\r", "")
 
     begin_search = re.search(r'\(BEGIN\s+IANA\s+(LAST\s+CALL\s+)?COMMENTS?(\s*:\s*[a-zA-Z0-9-\.]*)?\s*\)',body)
     end_search = re.search(r'\(END\s+IANA\s+(LAST\s+CALL\s+)?COMMENTS?\)',body)

ietf/utils/mail.py

@@ -18,7 +18,7 @@ from email.mime.text import MIMEText
 from email.mime.message import MIMEMessage
 from email.mime.multipart import MIMEMultipart
 from email.header import Header, decode_header
-from email import message_from_string
+from email import message_from_bytes, message_from_string
 from email import charset as Charset
 
 from django.conf import settings
@@ -404,7 +404,8 @@ def send_mail_mime(request, to, frm, subject, msg, cc=None, extra=None, toUser=F
 
 def parse_preformatted(preformatted, extra={}, override={}):
     """Parse preformatted string containing mail with From:, To:, ...,"""
-    msg = message_from_string(force_str(preformatted))
+    assert isinstance(preformatted, str)
+    msg = message_from_bytes(preformatted.encode('utf-8'))
     msg.set_charset('UTF-8')
 
     for k, v in override.items():
@@ -460,7 +461,7 @@ def send_mail_preformatted(request, preformatted, extra={}, override={}):
     extra headers as needed)."""
 
     (msg, extra, bcc) = parse_preformatted(preformatted, extra, override)
-    txt = get_payload(msg)
+    txt = msg.get_payload()
     send_mail_text(request, msg['To'], msg["From"], msg["Subject"], txt, extra=extra, bcc=bcc)
     return msg
 
@@ -617,11 +618,11 @@ def get_email_addresses_from_text(text):
     
 
 
-def get_payload(msg, decode=False):
+# def get_payload(msg, decode=False):
-    return msg.get_payload(decode=decode)
+#     return msg.get_payload(decode=decode)
 
-def get_payload_text(msg, decode=True):
+def get_payload_text(msg, decode=True, default_charset="utf-8"):
-    charset = msg.get_charset()
+    charset = msg.get_charset() or default_charset
     payload = msg.get_payload(decode=decode)
     try:
         payload = payload.decode(str(charset))

ietf/utils/test_utils.py

@@ -51,7 +51,7 @@ from django.utils.text import slugify
 
 import debug                            # pyflakes:ignore
 
-from ietf.utils.mail import get_payload
+from ietf.utils.mail import get_payload_text
 
 real_database_name = settings.DATABASES["default"]["NAME"]
 
@@ -171,7 +171,7 @@ class TestCase(django.test.TestCase):
             mlist = [ m for m in mlist if subject in m["Subject"] ]
         if text:
             assert isinstance(text, str)
-            mlist = [ m for m in mlist if text in get_payload(m) ]
+            mlist = [ m for m in mlist if text in get_payload_text(m) ]
         if count and len(mlist) != count:
             sys.stderr.write("Wrong count in assertMailboxContains().  The complete mailbox contains %s emails:\n\n" % len(mailbox))
             for m in mailbox:

ietf/utils/tests.py

@@ -41,7 +41,7 @@ from ietf.submit.tests import submission_file
 from ietf.utils.bower_storage import BowerStorageFinder
 from ietf.utils.draft import Draft, getmeta
 from ietf.utils.log import unreachable, assertion
-from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox, get_payload
+from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox, get_payload_text
 from ietf.utils.test_runner import get_template_paths, set_coverage_checking
 from ietf.utils.test_utils import TestCase
 
@@ -72,7 +72,7 @@ body
         self.assertSameEmail(recv['Cc'], 'cc1@example.com, cc2@example.com')
         self.assertSameEmail(recv['Bcc'], None)
         self.assertEqual(recv['Subject'], 'subject')
-        self.assertEqual(get_payload(recv), 'body\n')
+        self.assertEqual(get_payload_text(recv), 'body\n')
 
         override = {
             'To': 'oto1@example.net, oto2@example.net',
@@ -102,7 +102,7 @@ body
         self.assertSameEmail(recv['Cc'], '<occ1@example.net>, occ2@example.net')
         self.assertSameEmail(recv['Bcc'], None)
         self.assertEqual(recv['Subject'], 'osubject')
-        self.assertEqual(get_payload(recv), 'body\n')
+        self.assertEqual(get_payload_text(recv), 'body\n')
 
         extra = {'Fuzz': [ 'bucket' ]}
         send_mail_preformatted(request=None, preformatted=msg, extra=extra, override={})