altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Refactored many instances of msg.get_payload(decode=True).decode('utf-8') and similar to get_payload_text(msg), which uses msg.get_charset() to get the actual charset to do the decoding, in a consistent manner. Also removed some instances of force_str, but more remain.

Browse source 
- Legacy-Id: 17807
This commit is contained in:
Henrik Levkowetz 2020-05-16 20:20:05 +00:00
parent 0bc19f0c7c
commit bc05e7a8d0
23 changed files with 111 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
ietf/doc/mails.py
View file

@ -10,11 +10,11 @@ from django.template.loader import render_to_string
from django.utils.html import strip_tags
from django.conf import settings
from django.urls import reverse as urlreverse
from django.utils.encoding import force_str, force_text
from django.utils.encoding import force_text
import debug # pyflakes:ignore
from ietf.utils.mail import send_mail, send_mail_text, get_payload
from ietf.utils.mail import send_mail, send_mail_text
from ietf.ipr.utils import iprs_from_docs, related_docs
from ietf.doc.models import WriteupDocEvent, LastCallDocEvent, DocAlias, ConsensusDocEvent
from ietf.doc.utils import needed_ballot_positions
@ -387,7 +387,7 @@ def generate_issue_ballot_mail(request, doc, ballot):
def email_iana(request, doc, to, msg, cc=None):
# fix up message and send it with extra info on doc in headers
import email
parsed_msg = email.message_from_string(force_str(msg))
parsed_msg = email.message_from_string(msg)
parsed_msg.set_charset('UTF-8')
extra = extra_automation_headers(doc)
@ -395,7 +395,7 @@ def email_iana(request, doc, to, msg, cc=None):
send_mail_text(request, to,
parsed_msg["From"], parsed_msg["Subject"],
get_payload(parsed_msg),
parsed_msg.get_payload(),
extra=extra,
cc=cc)

12
ietf/doc/tests_ballot.py
View file

@ -23,7 +23,7 @@ from ietf.iesg.models import TelechatDate
from ietf.person.models import Person, PersonalApiKey
from ietf.person.factories import PersonFactory
from ietf.utils.test_utils import TestCase, login_testing_unauthorized
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.text import unwrap
@ -174,7 +174,7 @@ class EditPositionTests(TestCase):
self.assertEqual(len(outbox), mailbox_before + 1)
m = outbox[-1]
self.assertIn('COMMENT', m['Subject'])
self.assertIn('New comment', get_payload(m))
self.assertIn('New comment', get_payload_text(m))
def test_edit_position_as_secretary(self):
@ -460,11 +460,11 @@ class BallotWriteupsTests(TestCase):
self.assertTrue('drafts-eval@' in outbox[-1]['To'])
self.assertTrue('X-IETF-Draft-string' in outbox[-1])
if case=='none':
self.assertNotIn('call expire', outbox[-1].get_payload(decode=True).decode("utf-8"))
self.assertNotIn('call expire', get_payload_text(outbox[-1]))
elif case=='past':
self.assertIn('call expired', outbox[-1].get_payload(decode=True).decode("utf-8"))
self.assertIn('call expired', get_payload_text(outbox[-1]))
else:
self.assertIn('call expires', outbox[-1].get_payload(decode=True).decode("utf-8"))
self.assertIn('call expires', get_payload_text(outbox[-1]))
self.client.logout()
@ -797,7 +797,7 @@ class MakeLastCallTests(TestCase):
self.assertTrue("ietf-announce@" in outbox[-2]['To'])
for prefix in ['draft-ietf-mars-test','mars-chairs','aread']:
self.assertTrue(prefix+"@" in outbox[-2]['Cc'])
self.assertIn("The following IPR Declarations", get_payload(outbox[-2]))
self.assertIn("The following IPR Declarations", get_payload_text(outbox[-2]))
self.assertTrue("Last Call" in outbox[-1]['Subject'])
self.assertTrue("drafts-lastcall@icann.org" in outbox[-1]['To'])

20
ietf/doc/tests_charter.py
View file

@ -25,7 +25,7 @@ from ietf.group.models import Group, GroupMilestone
from ietf.iesg.models import TelechatDate
from ietf.person.models import Person
from ietf.utils.test_utils import TestCase
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.test_utils import login_testing_unauthorized
class ViewCharterTests(TestCase):
@ -187,7 +187,7 @@ class EditCharterTests(TestCase):
self.assertIn("Internal WG Review", outbox[-3]['Subject'])
self.assertIn("iab@", outbox[-3]['To'])
self.assertIn("iesg@", outbox[-3]['To'])
body = get_payload(outbox[-3])
body = get_payload_text(outbox[-3])
for word in ["A new IETF WG", "Chairs", "Ames Man <ameschairman@example.org>",
"Secretaries", "Secretary <amessecretary@example.org>",
"Assigned Area Director", "Areað Irector <aread@example.org>",
@ -198,13 +198,13 @@ class EditCharterTests(TestCase):
self.assertIn("state changed", outbox[-2]['Subject'].lower())
self.assertIn("iesg-secretary@", outbox[-2]['To'])
body = get_payload(outbox[-2])
body = get_payload_text(outbox[-2])
for word in ["WG", "Charter", ]:
self.assertIn(word, body)
self.assertIn("State Update Notice", outbox[-1]['Subject'])
self.assertIn("ames-chairs@", outbox[-1]['To'])
body = get_payload(outbox[-1])
body = get_payload_text(outbox[-1])
for word in ["State changed", "Datatracker URL", ]:
self.assertIn(word, body)
@ -224,7 +224,7 @@ class EditCharterTests(TestCase):
empty_outbox()
r = self.client.post(url, dict(charter_state=str(State.objects.get(used=True,type="charter",slug="intrev").pk), message="test"))
self.assertEqual(r.status_code, 302)
self.assertTrue("A new charter" in get_payload(outbox[-3]))
self.assertTrue("A new charter" in get_payload_text(outbox[-3]))
def test_change_rg_state(self):
@ -245,7 +245,7 @@ class EditCharterTests(TestCase):
self.assertIn("Internal RG Review", outbox[-3]['Subject'])
self.assertIn("iab@", outbox[-3]['To'])
self.assertIn("irsg@", outbox[-3]['To'])
body = get_payload(outbox[-3])
body = get_payload_text(outbox[-3])
for word in ["A new IRTF RG",
"Mailing list", "somerg@ietf.org",
"Charter", "Milestones"]:
@ -254,13 +254,13 @@ class EditCharterTests(TestCase):
self.assertIn("state changed", outbox[-2]['Subject'].lower())
self.assertIn("iesg-secretary@", outbox[-2]['To'])
body = get_payload(outbox[-2])
body = get_payload_text(outbox[-2])
for word in ["RG", "Charter", ]:
self.assertIn(word, body)
self.assertIn("State Update Notice", outbox[-1]['Subject'])
self.assertIn("somerg-chairs@", outbox[-1]['To'])
body = get_payload(outbox[-1])
body = get_payload_text(outbox[-1])
for word in ["State changed", "Datatracker URL", ]:
self.assertIn(word, body)
@ -793,7 +793,7 @@ class EditCharterTests(TestCase):
#
self.assertTrue("approved" in outbox[0]['Subject'].lower())
self.assertTrue("iesg-secretary" in outbox[0]['To'])
body = get_payload(outbox[0])
body = get_payload_text(outbox[0])
for word in ["WG", "/wg/ames/about/",
"Charter", "/doc/charter-ietf-ames/", ]:
self.assertIn(word, body)
@ -801,7 +801,7 @@ class EditCharterTests(TestCase):
self.assertTrue("WG Action" in outbox[1]['Subject'])
self.assertTrue("ietf-announce" in outbox[1]['To'])
self.assertTrue("ames-wg@ietf.org" in outbox[1]['Cc'])
body = get_payload(outbox[1])
body = get_payload_text(outbox[1])
for word in ["Chairs", "Ames Man <ameschairman@example.org>",
"Secretaries", "Secretary <amessecretary@example.org>",
"Assigned Area Director", "Areað Irector <aread@example.org>",

6
ietf/doc/tests_conflict_review.py
View file

@ -22,7 +22,7 @@ from ietf.group.models import Person
from ietf.iesg.models import TelechatDate
from ietf.name.models import StreamName
from ietf.utils.test_utils import TestCase
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.test_utils import login_testing_unauthorized
@ -302,9 +302,9 @@ class ConflictReviewTests(TestCase):
self.assertIn('iana@', outbox[0]['Cc'])
if approve_type == 'appr-noprob':
self.assertIn( 'IESG has no problem', ''.join(wrap(get_payload(outbox[0]), 2**16)))
self.assertIn( 'IESG has no problem', ''.join(wrap(get_payload_text(outbox[0]), 2**16)))
else:
self.assertIn( 'NOT be published', ''.join(wrap(get_payload(outbox[0]), 2**16)))
self.assertIn( 'NOT be published', ''.join(wrap(get_payload_text(outbox[0]), 2**16)))
def test_approve_reqnopub(self):

4
ietf/doc/tests_draft.py
View file

@ -30,7 +30,7 @@ from ietf.person.models import Person, Email
from ietf.meeting.models import Meeting, MeetingTypeName
from ietf.iesg.models import TelechatDate
from ietf.utils.test_utils import login_testing_unauthorized
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.test_utils import TestCase
@ -387,7 +387,7 @@ class EditInfoTests(TestCase):
data["telechat_date"] = next_week.isoformat()
r = self.client.post(url,data)
self.assertEqual(r.status_code, 302)
self.assertIn("may not leave enough time", get_payload(outbox[-1]))
self.assertIn("may not leave enough time", get_payload_text(outbox[-1]))
def test_start_iesg_process_on_draft(self):

4
ietf/doc/tests_irsg_ballot.py
View file

@ -11,7 +11,7 @@ import datetime
from django.urls import reverse as urlreverse
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.test_utils import TestCase, unicontent, login_testing_unauthorized
from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, RgDraftFactory, RgRfcFactory, BallotDocEventFactory, IRSGBallotDocEventFactory, BallotPositionDocEventFactory
from ietf.doc.models import BallotDocEvent, BallotPositionDocEvent
@ -363,7 +363,7 @@ class BaseManipulationTests():
r = self.client.post(url, dict(cc_choices=['doc_authors','doc_group_chairs','doc_group_mail_list'], body="Stuff"))
self.assertEqual(r.status_code, 302)
self.assertEqual(len(outbox),1)
self.assertNotIn('discuss-criteria', get_payload(outbox[0]))
self.assertNotIn('discuss-criteria', get_payload_text(outbox[0]))
def test_close_ballot(self):
draft = RgDraftFactory()

39
ietf/doc/tests_review.py
View file

@ -20,24 +20,23 @@ from pyquery import PyQuery
import debug # pyflakes:ignore
import ietf.review.mailarch
from ietf.doc.factories import ( NewRevisionDocEventFactory, IndividualDraftFactory, WgDraftFactory,
WgRfcFactory, ReviewFactory, DocumentFactory)
from ietf.doc.models import DocumentAuthor, RelatedDocument, DocEvent, ReviewRequestDocEvent, ReviewAssignmentDocEvent
from ietf.doc.models import ( DocumentAuthor, RelatedDocument, DocEvent, ReviewRequestDocEvent,
ReviewAssignmentDocEvent, )
from ietf.group.factories import RoleFactory, ReviewTeamFactory
from ietf.group.models import Group
from ietf.message.models import Message
from ietf.name.models import ReviewResultName, ReviewRequestStateName, ReviewAssignmentStateName, \
ReviewTypeName
from ietf.name.models import ReviewResultName, ReviewRequestStateName, ReviewAssignmentStateName, ReviewTypeName
from ietf.person.factories import PersonFactory
from ietf.person.models import Email, Person
from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory
from ietf.review.models import (ReviewRequest, ReviewerSettings,
ReviewWish, NextReviewerInTeam)
from ietf.review.models import ReviewRequest, ReviewerSettings, ReviewWish, NextReviewerInTeam
from ietf.review.policies import get_reviewer_queue_policy
from ietf.utils.test_utils import TestCase
from ietf.utils.mail import outbox, empty_outbox, parseaddr, on_behalf_of, get_payload_text
from ietf.utils.test_utils import login_testing_unauthorized, reload_db_objects
from ietf.utils.mail import outbox, empty_outbox, parseaddr, on_behalf_of
from ietf.person.factories import PersonFactory
from ietf.utils.test_utils import TestCase
class ReviewTests(TestCase):
def setUp(self):
@ -228,7 +227,7 @@ class ReviewTests(TestCase):
self.assertIn('<reviewer@example.com>', outbox[0]["To"])
self.assertNotIn("<reviewsecretary@example.com>", outbox[0]["To"])
self.assertIn("reviewsecretary2@example.com", outbox[0]["CC"])
mail_content = outbox[0].get_payload(decode=True).decode("utf-8").lower()
mail_content = get_payload_text(outbox[0])
self.assertIn("closed", mail_content)
self.assertIn("review_request_close_comment", mail_content)
@ -330,7 +329,7 @@ class ReviewTests(TestCase):
self.assertEqual(len(outbox), 1)
self.assertEqual('"Some Reviewer" <reviewer@example.com>', outbox[0]["To"])
message = outbox[0].get_payload(decode=True).decode("utf-8")
message = get_payload_text(outbox[0])
self.assertIn("Pages: {}".format(doc.pages), message )
self.assertIn("{} has assigned {}".format(secretary.person.ascii, reviewer.person.ascii), message)
self.assertIn("This team has completed other reviews", message)
@ -423,7 +422,7 @@ class ReviewTests(TestCase):
self.assertEqual(len(outbox), 1)
self.assertIn(assignment.reviewer.address, outbox[0]["To"])
self.assertNotIn("<reviewsecretary@example.com>", outbox[0]["To"])
self.assertTrue("Test message" in outbox[0].get_payload(decode=True).decode("utf-8"))
self.assertTrue("Test message" in get_payload_text(outbox[0]))
# try again, but now with an expired review request, which should not be allowed (#2277)
assignment.state_id = 'assigned'
@ -645,7 +644,7 @@ class ReviewTests(TestCase):
self.assertEqual(len(outbox), 1)
self.assertIn(assignment.review_request.team.list_email, outbox[0]["To"])
self.assertIn(assignment.reviewer.role_set.filter(group=assignment.review_request.team,name='reviewer').first().person.plain_name(), parseaddr(outbox[0]["From"])[0] )
self.assertIn("This is a review", outbox[0].get_payload(decode=True).decode("utf-8"))
self.assertIn("This is a review", get_payload_text(outbox[0]))
self.assertIn(settings.MAILING_LIST_ARCHIVE_URL, assignment.review.external_url)
@ -658,7 +657,7 @@ class ReviewTests(TestCase):
self.assertEqual(parseaddr(outbox[0]["To"]), parseaddr(message.to))
self.assertEqual(parseaddr(outbox[0]["From"]), parseaddr(on_behalf_of(message.frm)))
self.assertEqual(parseaddr(outbox[0]["Reply-To"]), parseaddr(message.frm))
self.assertEqual(outbox[0].get_payload(decode=True).decode(str(outbox[0].get_charset())), message.body)
self.assertEqual(get_payload_text(outbox[0]), message.body)
# check the review document page
url = urlreverse('ietf.doc.views_doc.document_main', kwargs={ "name": assignment.review.name })
@ -699,7 +698,7 @@ class ReviewTests(TestCase):
self.assertEqual(len(outbox), 1)
self.assertIn(assignment.review_request.team.list_email, outbox[0]["To"])
self.assertIn("This is a review", outbox[0].get_payload(decode=True).decode("utf-8"))
self.assertIn("This is a review", get_payload_text(outbox[0]))
self.assertIn(settings.MAILING_LIST_ARCHIVE_URL, assignment.review.external_url)
@ -740,7 +739,7 @@ class ReviewTests(TestCase):
self.assertEqual(len(outbox), 1)
self.assertIn(assignment.review_request.team.list_email, outbox[0]["To"])
self.assertIn("This is a review", outbox[0].get_payload(decode=True).decode("utf-8"))
self.assertIn("This is a review", get_payload_text(outbox[0]))
self.assertIn(settings.MAILING_LIST_ARCHIVE_URL, assignment.review.external_url)
@ -767,7 +766,7 @@ class ReviewTests(TestCase):
self.assertEqual(len(outbox), 2)
self.assertIn('Has Issues', outbox[-1]['Subject'])
self.assertIn('settings indicated', outbox[-1].get_payload(decode=True).decode("utf-8"))
self.assertIn('settings indicated', get_payload_text(outbox[-1]))
def test_complete_notify_ad_because_checkbox(self):
assignment, url = self.setup_complete_review_test()
@ -791,7 +790,7 @@ class ReviewTests(TestCase):
self.assertEqual(len(outbox), 2)
self.assertIn('Has Issues', outbox[-1]['Subject'])
self.assertIn('reviewer indicated', outbox[-1].get_payload(decode=True).decode("utf-8"))
self.assertIn('reviewer indicated', get_payload_text(outbox[-1]))
@patch('requests.get')
def test_complete_review_link_to_mailing_list(self, mock):
@ -908,7 +907,7 @@ class ReviewTests(TestCase):
self.assertTrue(assignment.review_request.team.list_email in outbox[1]["To"])
self.assertTrue("partial review" in outbox[1]["Subject"].lower())
body = outbox[1].get_payload(decode=True).decode("utf-8")
body = get_payload_text(outbox[1])
self.assertTrue("This is a review" in body)
# This review has a line longer than 80, but less than 100; it should
# not be wrapped.
@ -941,7 +940,7 @@ class ReviewTests(TestCase):
# This review has a line longer than 100; it should be wrapped to less
# than 80.
body = outbox[2].get_payload(decode=True).decode("utf-8")
body = get_payload_text(outbox[2])
self.assertIn('really, really, really', body)
self.assertTrue(all( len(line) <= 80 for line in body.splitlines() ))

4
ietf/group/tests_info.py
View file

@ -35,7 +35,7 @@ from ietf.name.models import DocTagName, GroupStateName, GroupTypeName
from ietf.person.models import Person, Email
from ietf.person.factories import PersonFactory
from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory
from ietf.utils.mail import outbox, empty_outbox
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent, reload_db_objects
def group_urlreverse_list(group, viewname):
@ -629,7 +629,7 @@ class GroupEditTests(TestCase):
self.assertTrue('Personnel change' in outbox[0]['Subject'])
for prefix in ['ad1','ad2','aread','marschairman','marsdelegate']:
self.assertTrue(prefix+'@' in outbox[0]['To'])
self.assertTrue(outbox[0].get_payload(decode=True).decode(str(outbox[0].get_charset())).startswith('Sec Retary'))
self.assertTrue(get_payload_text(outbox[0]).startswith('Sec Retary'))
def test_edit_field(self):

4
ietf/group/views.py
View file

@ -111,7 +111,7 @@ from ietf.doc.models import LastCallDocEvent
from ietf.name.models import ReviewAssignmentStateName
from ietf.utils.mail import send_mail_text, parse_preformatted, get_payload
from ietf.utils.mail import send_mail_text, parse_preformatted
from ietf.ietfauth.utils import user_is_person
from ietf.dbtemplate.models import DBTemplate
@ -1704,7 +1704,7 @@ def email_open_review_assignments(request, acronym, group_type=None):
(msg,_,_) = parse_preformatted(partial_msg)
body = get_payload(msg)
body = msg.get_payload()
subject = msg['Subject']
form = EmailOpenAssignmentsForm(initial={

12
ietf/ietfauth/tests.py
View file

@ -15,17 +15,17 @@ from django.conf import settings
import debug # pyflakes:ignore
from ietf.utils.test_utils import TestCase, login_testing_unauthorized
from ietf.utils.mail import outbox, empty_outbox
from ietf.group.models import Group, Role, RoleName
from ietf.group.factories import GroupFactory, RoleFactory
from ietf.group.models import Group, Role, RoleName
from ietf.ietfauth.htpasswd import update_htpasswd_file
from ietf.mailinglists.models import Subscribed
from ietf.person.models import Person, Email, PersonalApiKey
from ietf.person.factories import PersonFactory, EmailFactory
from ietf.person.models import Person, Email, PersonalApiKey
from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory
from ietf.review.models import ReviewWish, UnavailablePeriod
from ietf.utils.decorators import skip_coverage
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.test_utils import TestCase, login_testing_unauthorized
import ietf.ietfauth.views
@ -99,7 +99,7 @@ class IetfAuthTests(TestCase):
def extract_confirm_url(self, confirm_email):
# dig out confirm_email link
msg = confirm_email.get_payload(decode=True).decode(confirm_email.get_content_charset())
msg = get_payload_text(confirm_email)
line_start = "http"
confirm_url = None
for line in msg.split("\n"):
@ -643,7 +643,7 @@ class IetfAuthTests(TestCase):
self.assertEqual(len(outbox), len(endpoints))
for mail in outbox:
body = mail.get_payload(decode=True).decode('utf-8')
body = get_payload_text(mail)
self.assertIn("API key usage", mail['subject'])
self.assertIn(" %s times" % count, body)
self.assertIn(date, body)

11
ietf/ipr/tests.py
View file

@ -23,14 +23,11 @@ from ietf.ipr.models import (IprDisclosureBase,GenericIprDisclosure,HolderIprDis
from ietf.ipr.utils import get_genitive, get_ipr_summary
from ietf.mailtrigger.utils import gather_address_lists
from ietf.message.models import Message
from ietf.utils.mail import outbox, empty_outbox
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.test_utils import TestCase, login_testing_unauthorized
from ietf.utils.text import text_to_dict
def extract_message_content(message):
return message.get_payload(decode=True).decode(str(message.get_charset()))
def make_data_from_content(content):
q = PyQuery(content)
data = dict()
@ -582,10 +579,10 @@ I would like to revoke this declaration.
self.assertEqual(r.status_code,302)
self.assertEqual(len(outbox),len_before+2)
self.assertTrue('george@acme.com' in outbox[len_before]['To'])
self.assertIn('posted on '+datetime.date.today().strftime("%Y-%m-%d"), extract_message_content(outbox[len_before]).replace('\n',' '))
self.assertIn('posted on '+datetime.date.today().strftime("%Y-%m-%d"), get_payload_text(outbox[len_before]).replace('\n',' '))
self.assertTrue('draft-ietf-mars-test@ietf.org' in outbox[len_before+1]['To'])
self.assertTrue('mars-wg@ietf.org' in outbox[len_before+1]['Cc'])
self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), extract_message_content(outbox[len_before+1]).replace('\n',' '))
self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), get_payload_text(outbox[len_before+1]).replace('\n',' '))
def test_notify_generic(self):
RoleFactory(name_id='ad',group__acronym='gen')
@ -601,7 +598,7 @@ I would like to revoke this declaration.
r = self.client.post(url, data )
self.assertEqual(r.status_code,302)
self.assertEqual(len(outbox),2)
self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), extract_message_content(outbox[1]).replace('\n',' '))
self.assertIn('Secretariat on '+ipr.get_latest_event_submitted().time.strftime("%Y-%m-%d"), get_payload_text(outbox[1]).replace('\n',' '))
def test_process_response_email(self):
# first send a mail

6
ietf/meeting/tests_views.py
View file

@ -40,7 +40,7 @@ from ietf.meeting.utils import add_event_info_to_session_qs
from ietf.meeting.views import session_draft_list
from ietf.name.models import SessionStatusName, ImportantDateName, RoleName
from ietf.utils.decorators import skip_coverage
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent
from ietf.utils.text import xslugify
@ -1446,8 +1446,8 @@ class InterimTests(TestCase):
self.assertRedirects(r, urlreverse('ietf.meeting.views.interim_announce'))
self.assertEqual(len(outbox), len_before + 1)
self.assertIn('WG Virtual Meeting', outbox[-1]['Subject'])
self.assertIn('09:00 to 09:20 America/Los_Angeles', get_payload(outbox[-1]))
self.assertIn('(17:00 to 17:20 UTC)', get_payload(outbox[-1]))
self.assertIn('09:00 to 09:20 America/Los_Angeles', get_payload_text(outbox[-1]))
self.assertIn('(17:00 to 17:20 UTC)', get_payload_text(outbox[-1]))
def test_interim_approve_by_ad(self):
make_meeting_test_data()

4
ietf/message/tests.py
View file

@ -12,7 +12,7 @@ from ietf.group.factories import GroupFactory
from ietf.message.models import Message, SendQueue
from ietf.message.utils import send_scheduled_message_from_send_queue
from ietf.person.models import Person
from ietf.utils.mail import outbox, send_mail_text, send_mail_message, get_payload
from ietf.utils.mail import outbox, send_mail_text, send_mail_message, get_payload_text
from ietf.utils.test_utils import TestCase
class MessageTests(TestCase):
@ -46,7 +46,7 @@ class MessageTests(TestCase):
continue
self.assertEqual(e1.get_all(k), e2.get_all(k), "Header field: %s" % k)
self.longMessage = False
self.assertEqual(get_payload(e1), get_payload(e2))
self.assertEqual(get_payload_text(e1), get_payload_text(e2))
#
self.assertEqual(Message.objects.count(), 0)

6
ietf/message/utils.py
View file

@ -6,13 +6,13 @@ import re, datetime, email
from django.utils.encoding import force_str
from ietf.utils.mail import send_mail_text, send_mail_mime, get_payload
from ietf.utils.mail import send_mail_text, send_mail_mime
from ietf.message.models import Message
first_dot_on_line_re = re.compile(r'^\.', re.MULTILINE)
def infer_message(s):
parsed = email.message_from_string(force_str(s))
parsed = email.message_from_string(s)
m = Message(
subject = parsed.get("Subject", ""),
@ -21,7 +21,7 @@ def infer_message(s):
cc = parsed.get("Cc", ""),
bcc = parsed.get("Bcc", ""),
reply_to = parsed.get("Reply-To", ""),
body = get_payload(parsed),
body = parsed.get_payload(),
content_type = parsed.get_content_type(),
)

12
ietf/nomcom/tests.py
View file

@ -16,7 +16,7 @@ from django.conf import settings
from django.core.files import File
from django.contrib.auth.models import User
from django.urls import reverse
from django.utils.encoding import force_text, force_str
from django.utils.encoding import force_str
import debug # pyflakes:ignore
@ -39,7 +39,7 @@ from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, get_hash
from ietf.person.factories import PersonFactory, EmailFactory
from ietf.person.models import Email, Person
from ietf.stats.models import MeetingRegistration
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent
client_test_cert_files = None
@ -539,7 +539,7 @@ class NomcomViewsTest(TestCase):
self.assertEqual('Nomination receipt', outbox[-1]['Subject'])
self.assertEqual(self.email_from, outbox[-1]['From'])
self.assertIn('plain', outbox[-1]['To'])
self.assertIn('Comments with accents äöå', force_text(outbox[-1].get_payload(decode=True),"utf-8","replace"))
self.assertIn('Comments with accents äöå', get_payload_text(outbox[-1]))
# Nominate the same person for the same position again without asking for confirmation
@ -580,7 +580,7 @@ class NomcomViewsTest(TestCase):
self.assertEqual('Nomination receipt', outbox[-1]['Subject'])
self.assertEqual(self.email_from, outbox[-1]['From'])
self.assertIn('plain', outbox[-1]['To'])
self.assertIn('Comments with accents äöå', force_text(outbox[-1].get_payload(decode=True),"utf-8","replace"))
self.assertIn('Comments with accents äöå', get_payload_text(outbox[-1]))
# Nominate the same person for the same position again without asking for confirmation
@ -831,12 +831,12 @@ class NomcomViewsTest(TestCase):
# We're interested in the confirmation receipt here
self.assertEqual(len(outbox),3)
self.assertEqual('NomCom comment confirmation', outbox[2]['Subject'])
email_body = get_payload(outbox[2])
email_body = get_payload_text(outbox[2])
self.assertIn(position, email_body)
self.assertNotIn('$', email_body)
self.assertEqual(self.email_from, outbox[-2]['From'])
self.assertIn('plain', outbox[2]['To'])
self.assertIn('Comments with accents äöå', force_text(outbox[2].get_payload(decode=True),"utf-8","replace"))
self.assertIn('Comments with accents äöå', get_payload_text(outbox[2]))
empty_outbox()
self.feedback_view(public=True)

14
ietf/nomcom/utils.py
View file

@ -20,13 +20,13 @@ from django.core.exceptions import ObjectDoesNotExist
from django.urls import reverse
from django.template.loader import render_to_string
from django.shortcuts import get_object_or_404
from django.utils.encoding import force_str, force_text
from django.utils.encoding import force_str
from ietf.dbtemplate.models import DBTemplate
from ietf.person.models import Email, Person
from ietf.mailtrigger.utils import gather_address_lists
from ietf.utils.pipe import pipe
from ietf.utils.mail import send_mail_text, send_mail
from ietf.utils.mail import send_mail_text, send_mail, get_payload_text
from ietf.utils.log import log
from ietf.person.name import unidecode_name
@ -438,18 +438,14 @@ def get_body(message):
'plain')]
body = []
for part in text_parts:
charset = get_charset(part, get_charset(message))
body.append(force_text(part.get_payload(decode=True),
charset,
"replace"))
charset = get_charset(message)
body.append(get_payload_text(part, default_charset=charset))
return "\n".join(body).strip()
else: # if it is not multipart, the payload will be a string
# representing the message body
body = force_text(message.get_payload(decode=True),
get_charset(message),
"replace")
body = get_payload_text(message)
return body.strip()

4
ietf/review/mailarch.py
View file

@ -24,6 +24,8 @@ from pyquery import PyQuery
from django.conf import settings
from django.utils.encoding import force_bytes, force_str
from ietf.utils.mail import get_payload_text
def list_name_from_email(list_email):
if not list_email.endswith("@ietf.org"):
return None
@ -81,7 +83,7 @@ def retrieve_messages_from_mbox(mbox_fileobj):
for part in msg.walk():
if part.get_content_type() == "text/plain":
charset = part.get_content_charset() or "utf-8"
content += part.get_payload(decode=True).decode(charset, "ignore")
content += get_payload_text(part, default_charset=charset)
# parse a couple of things for the front end
utcdate = None

4
ietf/secr/sreq/tests.py
View file

@ -15,7 +15,7 @@ from ietf.meeting.factories import MeetingFactory, SessionFactory
from ietf.name.models import TimerangeName
from ietf.person.models import Person
from ietf.secr.sreq.forms import SessionForm
from ietf.utils.mail import outbox, empty_outbox
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from pyquery import PyQuery
@ -320,7 +320,7 @@ class SubmitRequestCase(TestCase):
self.assertRedirects(r, reverse('ietf.secr.sreq.views.main'))
self.assertEqual(len(outbox),len_before+1)
notification = outbox[-1]
notification_payload = notification.get_payload(decode=True).decode(encoding="utf-8", errors="replace")
notification_payload = get_payload_text(notification)
sessions = Session.objects.filter(meeting=meeting,group=group)
self.assertEqual(len(sessions), 2)
session = sessions[0]

23
ietf/submit/tests.py
View file

@ -35,7 +35,7 @@ from ietf.person.models import Person
from ietf.person.factories import UserFactory, PersonFactory, EmailFactory
from ietf.submit.models import Submission, Preapproval
from ietf.submit.mail import add_submission_email, process_response_email
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.mail import outbox, empty_outbox, get_payload_text
from ietf.utils.models import VersionInfo
from ietf.utils.test_utils import login_testing_unauthorized, TestCase
from ietf.utils.draft import Draft
@ -194,8 +194,7 @@ class SubmitTests(TestCase):
def extract_confirmation_url(self, confirmation_email):
# dig out confirmation_email link
charset = confirmation_email.get_content_charset()
msg = confirmation_email.get_payload(decode=True).decode(charset)
msg = get_payload_text(confirmation_email)
line_start = "http"
confirmation_url = None
for line in msg.split("\n"):
@ -298,16 +297,16 @@ class SubmitTests(TestCase):
self.assertTrue(draft.relations_that_doc("possibly-replaces").first().target, sug_replaced_alias)
self.assertEqual(len(outbox), mailbox_before + 5)
self.assertIn(("I-D Action: %s" % name), outbox[-4]["Subject"])
self.assertIn(author.ascii, get_payload(outbox[-4]))
self.assertIn(author.ascii, get_payload_text(outbox[-4]))
self.assertIn(("I-D Action: %s" % name), outbox[-3]["Subject"])
self.assertIn(author.ascii, get_payload(outbox[-3]))
self.assertIn(author.ascii, get_payload_text(outbox[-3]))
self.assertIn("New Version Notification",outbox[-2]["Subject"])
self.assertIn(name, get_payload(outbox[-2]))
self.assertIn("mars", get_payload(outbox[-2]))
self.assertIn(name, get_payload_text(outbox[-2]))
self.assertIn("mars", get_payload_text(outbox[-2]))
# Check "Review of suggested possible replacements for..." mail
self.assertIn("review", outbox[-1]["Subject"].lower())
self.assertIn(name, get_payload(outbox[-1]))
self.assertIn(sug_replaced_alias.name, get_payload(outbox[-1]))
self.assertIn(name, get_payload_text(outbox[-1]))
self.assertIn(sug_replaced_alias.name, get_payload_text(outbox[-1]))
self.assertIn("ames-chairs@", outbox[-1]["To"].lower())
self.assertIn("mars-chairs@", outbox[-1]["To"].lower())
@ -497,17 +496,17 @@ class SubmitTests(TestCase):
self.assertEqual(len(outbox), mailbox_before + 3)
self.assertTrue(("I-D Action: %s" % name) in outbox[-3]["Subject"])
self.assertTrue(("I-D Action: %s" % name) in draft.message_set.order_by("-time")[0].subject)
self.assertTrue(author.ascii in get_payload(outbox[-3]))
self.assertTrue(author.ascii in get_payload_text(outbox[-3]))
self.assertTrue("i-d-announce@" in outbox[-3]['To'])
self.assertTrue("New Version Notification" in outbox[-2]["Subject"])
self.assertTrue(name in get_payload(outbox[-2]))
self.assertTrue(name in get_payload_text(outbox[-2]))
interesting_address = {'ietf':'mars', 'irtf':'irtf-chair', 'iab':'iab-chair', 'ise':'rfc-ise'}[draft.stream_id]
self.assertTrue(interesting_address in force_text(outbox[-2].as_string()))
if draft.stream_id == 'ietf':
self.assertTrue(draft.ad.role_email("ad").address in force_text(outbox[-2].as_string()))
self.assertTrue(ballot_position.balloter.role_email("ad").address in force_text(outbox[-2].as_string()))
self.assertTrue("New Version Notification" in outbox[-1]["Subject"])
self.assertTrue(name in get_payload(outbox[-1]))
self.assertTrue(name in get_payload_text(outbox[-1]))
r = self.client.get(urlreverse('ietf.doc.views_search.recent_drafts'))
self.assertEqual(r.status_code, 200)
self.assertContains(r, draft.name)

5
ietf/sync/iana.py
View file

@ -19,7 +19,7 @@ from ietf.doc.mails import email_state_changed
from ietf.doc.models import Document, DocEvent, State, StateDocEvent, StateType
from ietf.doc.utils import add_state_change_event
from ietf.person.models import Person
from ietf.utils.mail import parseaddr
from ietf.utils.mail import parseaddr, get_payload_text
from ietf.utils.timezone import local_timezone_to_utc, email_time_to_local_timezone, utc_to_local_timezone
@ -254,8 +254,7 @@ def parse_review_email(text):
by = Person.objects.get(name="(System)")
# comment
charset = msg.get_content_charset()
body = msg.get_payload(decode=True).decode(charset or 'utf-8').replace("\r", "")
body = get_payload_text(msg).replace("\r", "")
begin_search = re.search(r'\(BEGIN\s+IANA\s+(LAST\s+CALL\s+)?COMMENTS?(\s*:\s*[a-zA-Z0-9-\.]*)?\s*\)',body)
end_search = re.search(r'\(END\s+IANA\s+(LAST\s+CALL\s+)?COMMENTS?\)',body)

15
ietf/utils/mail.py
View file

@ -18,7 +18,7 @@ from email.mime.text import MIMEText
from email.mime.message import MIMEMessage
from email.mime.multipart import MIMEMultipart
from email.header import Header, decode_header
from email import message_from_string
from email import message_from_bytes, message_from_string
from email import charset as Charset
from django.conf import settings
@ -404,7 +404,8 @@ def send_mail_mime(request, to, frm, subject, msg, cc=None, extra=None, toUser=F
def parse_preformatted(preformatted, extra={}, override={}):
"""Parse preformatted string containing mail with From:, To:, ...,"""
msg = message_from_string(force_str(preformatted))
assert isinstance(preformatted, str)
msg = message_from_bytes(preformatted.encode('utf-8'))
msg.set_charset('UTF-8')
for k, v in override.items():
@ -460,7 +461,7 @@ def send_mail_preformatted(request, preformatted, extra={}, override={}):
extra headers as needed)."""
(msg, extra, bcc) = parse_preformatted(preformatted, extra, override)
txt = get_payload(msg)
txt = msg.get_payload()
send_mail_text(request, msg['To'], msg["From"], msg["Subject"], txt, extra=extra, bcc=bcc)
return msg
@ -617,11 +618,11 @@ def get_email_addresses_from_text(text):
def get_payload(msg, decode=False):
return msg.get_payload(decode=decode)
# def get_payload(msg, decode=False):
# return msg.get_payload(decode=decode)
def get_payload_text(msg, decode=True):
charset = msg.get_charset()
def get_payload_text(msg, decode=True, default_charset="utf-8"):
charset = msg.get_charset() or default_charset
payload = msg.get_payload(decode=decode)
try:
payload = payload.decode(str(charset))

4
ietf/utils/test_utils.py
View file

@ -51,7 +51,7 @@ from django.utils.text import slugify
import debug # pyflakes:ignore
from ietf.utils.mail import get_payload
from ietf.utils.mail import get_payload_text
real_database_name = settings.DATABASES["default"]["NAME"]
@ -171,7 +171,7 @@ class TestCase(django.test.TestCase):
mlist = [ m for m in mlist if subject in m["Subject"] ]
if text:
assert isinstance(text, str)
mlist = [ m for m in mlist if text in get_payload(m) ]
mlist = [ m for m in mlist if text in get_payload_text(m) ]
if count and len(mlist) != count:
sys.stderr.write("Wrong count in assertMailboxContains(). The complete mailbox contains %s emails:\n\n" % len(mailbox))
for m in mailbox:

6
ietf/utils/tests.py
View file

@ -41,7 +41,7 @@ from ietf.submit.tests import submission_file
from ietf.utils.bower_storage import BowerStorageFinder
from ietf.utils.draft import Draft, getmeta
from ietf.utils.log import unreachable, assertion
from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox, get_payload
from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox, get_payload_text
from ietf.utils.test_runner import get_template_paths, set_coverage_checking
from ietf.utils.test_utils import TestCase
@ -72,7 +72,7 @@ body
self.assertSameEmail(recv['Cc'], 'cc1@example.com, cc2@example.com')
self.assertSameEmail(recv['Bcc'], None)
self.assertEqual(recv['Subject'], 'subject')
self.assertEqual(get_payload(recv), 'body\n')
self.assertEqual(get_payload_text(recv), 'body\n')
override = {
'To': 'oto1@example.net, oto2@example.net',
@ -102,7 +102,7 @@ body
self.assertSameEmail(recv['Cc'], '<occ1@example.net>, occ2@example.net')
self.assertSameEmail(recv['Bcc'], None)
self.assertEqual(recv['Subject'], 'osubject')
self.assertEqual(get_payload(recv), 'body\n')
self.assertEqual(get_payload_text(recv), 'body\n')
extra = {'Fuzz': [ 'bucket' ]}
send_mail_preformatted(request=None, preformatted=msg, extra=extra, override={})