From 36cac48063b66c7a094f0ed1090b5011c90c1c2d Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Sun, 30 Jun 2019 20:54:09 +0000 Subject: [PATCH] Fixed a bunch of str/bytes issues. - Legacy-Id: 16341 --- ietf/doc/mails.py | 4 ++-- ietf/doc/tests_charter.py | 8 ++++---- ietf/doc/views_review.py | 10 +++++----- ietf/ietfauth/htpasswd.py | 11 +++++++---- ietf/meeting/models.py | 16 ++++++++-------- ietf/message/utils.py | 2 +- ietf/nomcom/fields.py | 3 ++- ietf/nomcom/test_data.py | 5 +++-- ietf/nomcom/utils.py | 9 ++++----- ietf/review/mailarch.py | 7 ++++--- ietf/utils/test_utils.py | 4 ++-- ietf/utils/textupload.py | 13 ++++++------- 12 files changed, 48 insertions(+), 44 deletions(-) diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py index b2de66f83..aca9a0c8d 100644 --- a/ietf/doc/mails.py +++ b/ietf/doc/mails.py @@ -383,7 +383,7 @@ def generate_issue_ballot_mail(request, doc, ballot): def email_iana(request, doc, to, msg, cc=None): # fix up message and send it with extra info on doc in headers import email - parsed_msg = email.message_from_string(msg.encode("utf-8")) + parsed_msg = email.message_from_string(msg) parsed_msg.set_charset('UTF-8') extra = extra_automation_headers(doc) @@ -391,7 +391,7 @@ def email_iana(request, doc, to, msg, cc=None): send_mail_text(request, to, parsed_msg["From"], parsed_msg["Subject"], - parsed_msg.get_payload().decode(str(parsed_msg.get_charset())), + parsed_msg.get_payload(), extra=extra, cc=cc) diff --git a/ietf/doc/tests_charter.py b/ietf/doc/tests_charter.py index e38cf2c33..49d8c5b5e 100644 --- a/ietf/doc/tests_charter.py +++ b/ietf/doc/tests_charter.py @@ -399,9 +399,9 @@ class EditCharterTests(TestCase): # post prev_rev = charter.rev - latin_1_snippet = '\xe5' * 10 - utf_8_snippet = '\xc3\xa5' * 10 - test_file = StringIO("Windows line\r\nMac line\rUnix line\n" + latin_1_snippet) + latin_1_snippet = b'\xe5' * 10 + utf_8_snippet = b'\xc3\xa5' * 10 + test_file = StringIO("Windows line\r\nMac line\rUnix line\n" + latin_1_snippet.decode('latin-1')) test_file.name = "unnamed" r = self.client.post(url, dict(txt=test_file)) @@ -413,7 +413,7 @@ class EditCharterTests(TestCase): with open(os.path.join(self.charter_dir, charter.canonical_name() + "-" + charter.rev + ".txt")) as f: self.assertEqual(f.read(), - "Windows line\nMac line\nUnix line\n" + utf_8_snippet) + "Windows line\nMac line\nUnix line\n" + utf_8_snippet.decode('utf_8')) def test_submit_initial_charter(self): group = GroupFactory(type_id='wg',acronym='mars',list_email='mars-wg@ietf.org') diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index f641ebd28..bd9fb73ac 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -620,13 +620,13 @@ def complete_review(request, name, assignment_id): # save file on disk if review_submission == "upload": - encoded_content = form.cleaned_data['review_file'] + content = form.cleaned_data['review_file'] else: - encoded_content = form.cleaned_data['review_content'].encode("utf-8") + content = form.cleaned_data['review_content'] filename = os.path.join(review.get_file_path(), '{}.txt'.format(review.name, review.rev)) - with open(filename, 'wb') as destination: - destination.write(encoded_content) + with open(filename, 'w', encoding='utf-8') as destination: + destination.write(content) completion_datetime = datetime.datetime.now() if "completion_date" in form.cleaned_data: @@ -699,7 +699,7 @@ def complete_review(request, name, assignment_id): cc=form.cleaned_data["cc"], body = render_to_string("review/completed_review.txt", { "assignment": assignment, - "content": encoded_content.decode("utf-8"), + "content": content, }), ) msg.related_groups.add(*related_groups) diff --git a/ietf/ietfauth/htpasswd.py b/ietf/ietfauth/htpasswd.py index 145af84c7..7474d7592 100644 --- a/ietf/ietfauth/htpasswd.py +++ b/ietf/ietfauth/htpasswd.py @@ -1,4 +1,7 @@ +# Copyright The IETF Trust 2016-2019, All Rights Reserved + import subprocess, hashlib +from django.utils.encoding import force_bytes from django.conf import settings @@ -6,9 +9,9 @@ def update_htpasswd_file(username, password): if getattr(settings, 'USE_PYTHON_HTDIGEST', None): pass_file = settings.HTPASSWD_FILE realm = settings.HTDIGEST_REALM - prefix = '%s:%s:' % (username, realm) - key = hashlib.md5(prefix + password).hexdigest() - f = open(pass_file, 'r+') + prefix = force_bytes('%s:%s:' % (username, realm)) + key = force_bytes(hashlib.md5(prefix + force_bytes(password)).hexdigest()) + f = open(pass_file, 'r+b') pos = f.tell() line = f.readline() while line: @@ -17,7 +20,7 @@ def update_htpasswd_file(username, password): pos=f.tell() line = f.readline() f.seek(pos) - f.write('%s%s\n' % (prefix, key)) + f.write(b'%s%s\n' % (prefix, key)) f.close() else: p = subprocess.Popen([settings.HTPASSWD_COMMAND, "-b", settings.HTPASSWD_FILE, username, password], stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 5bce1032d..a84e51db0 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -255,7 +255,7 @@ class Meeting(models.Model): days.sort() for ymd in time_slices: time_slices[ymd].sort() - slots[ymd].sort(lambda x,y: cmp(x.time, y.time)) + slots[ymd].sort(key=lambda x: x.time) return days,time_slices,slots # this functions makes a list of timeslices and rooms, and @@ -1080,22 +1080,22 @@ class Session(models.Model): sess1['group_href'] = urljoin(host_scheme, self.group.json_url()) if self.group.parent is not None: sess1['area'] = self.group.parent.acronym.upper() - sess1['description'] = self.group.name.encode('utf-8') + sess1['description'] = self.group.name sess1['group_id'] = str(self.group.pk) reslist = [] for r in self.resources.all(): reslist.append(r.json_dict(host_scheme)) sess1['resources'] = reslist sess1['session_id'] = str(self.pk) - sess1['name'] = self.name.encode('utf-8') - sess1['title'] = self.short_name.encode('utf-8') - sess1['short_name'] = self.short_name.encode('utf-8') + sess1['name'] = self.name + sess1['title'] = self.short_name + sess1['short_name'] = self.short_name sess1['bof'] = str(self.group.is_bof()) - sess1['agenda_note'] = self.agenda_note.encode('utf-8') + sess1['agenda_note'] = self.agenda_note sess1['attendees'] = str(self.attendees) - sess1['status'] = self.status.name.encode('utf-8') + sess1['status'] = self.status.name if self.comments is not None: - sess1['comments'] = self.comments.encode('utf-8') + sess1['comments'] = self.comments sess1['requested_time'] = self.requested.strftime("%Y-%m-%d") # the related person object sometimes does not exist in the dataset. try: diff --git a/ietf/message/utils.py b/ietf/message/utils.py index 9ae32f796..8c0c6934a 100644 --- a/ietf/message/utils.py +++ b/ietf/message/utils.py @@ -41,7 +41,7 @@ def send_scheduled_message_from_send_queue(send_queue): # make body a real message so we can parse it body = ("MIME-Version: 1.0\r\nContent-Type: %s\r\n" % message.content_type) + body - msg = email.message_from_string(body.encode("utf-8")) + msg = email.message_from_string(body) send_mail_mime(None, message.to, message.frm, message.subject, msg, cc=message.cc, bcc=message.bcc) diff --git a/ietf/nomcom/fields.py b/ietf/nomcom/fields.py index eab7111da..f418bb382 100644 --- a/ietf/nomcom/fields.py +++ b/ietf/nomcom/fields.py @@ -1,3 +1,4 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved from django.conf import settings from django.db import models from django.utils.encoding import smart_str @@ -19,7 +20,7 @@ class EncryptedTextField(models.TextField): raise ValueError("Trying to read the NomCom public key: " + str(e)) command = "%s smime -encrypt -in /dev/stdin %s" % (settings.OPENSSL_COMMAND, cert_file) - code, out, error = pipe(command, comments) + code, out, error = pipe(command, comments.encode()) if code != 0: log("openssl error: %s:\n Error %s: %s" %(command, code, error)) if not error: diff --git a/ietf/nomcom/test_data.py b/ietf/nomcom/test_data.py index c04621c91..2cb998242 100644 --- a/ietf/nomcom/test_data.py +++ b/ietf/nomcom/test_data.py @@ -1,3 +1,4 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved import tempfile import os @@ -38,7 +39,7 @@ POSITIONS = [ def generate_cert(): """Function to generate cert""" - config = """ + config = b""" [ req ] distinguished_name = req_distinguished_name string_mask = utf8only @@ -89,7 +90,7 @@ def check_comments(encryped, plain, privatekey_file): decrypted_file.close() encrypted_file.close() - decrypted_comments = open(decrypted_file.name, 'r').read().decode('utf8') + decrypted_comments = open(decrypted_file.name, 'r').read() os.unlink(encrypted_file.name) os.unlink(decrypted_file.name) diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index 43be9b204..2146c262a 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -87,7 +87,7 @@ def get_user_email(user): return user._email_cache def get_hash_nominee_position(date, nominee_position_id): - return hashlib.md5('%s%s%s' % (settings.SECRET_KEY, date, nominee_position_id)).hexdigest() + return hashlib.md5(('%s%s%s' % (settings.SECRET_KEY, date, nominee_position_id)).encode()).hexdigest() def initialize_templates_for_group(group): @@ -160,7 +160,7 @@ def retrieve_nomcom_private_key(request, year): command = "%s bf -d -in /dev/stdin -k \"%s\" -a" code, out, error = pipe(command % (settings.OPENSSL_COMMAND, - settings.SECRET_KEY), private_key) + settings.SECRET_KEY), private_key.encode()) if code != 0: log("openssl error: %s:\n Error %s: %s" %(command, code, error)) return out @@ -399,9 +399,8 @@ def make_nomineeposition_for_newperson(nomcom, candidate_name, candidate_email, def getheader(header_text, default="ascii"): """Decode the specified header""" - headers = decode_header(header_text) - header_sections = [str(text, charset or default) - for text, charset in headers] + tuples = decode_header(header_text) + header_sections = [ text.decode(charset or default) if isinstance(text, bytes) else text for text, charset in tuples] return "".join(header_sections) diff --git a/ietf/review/mailarch.py b/ietf/review/mailarch.py index 13582503c..f905a603c 100644 --- a/ietf/review/mailarch.py +++ b/ietf/review/mailarch.py @@ -11,6 +11,7 @@ import debug # pyflakes:ignore from pyquery import PyQuery from django.conf import settings +from django.utils.encoding import force_bytes def list_name_from_email(list_email): if not list_email.endswith("@ietf.org"): @@ -23,9 +24,9 @@ def hash_list_message_id(list_name, msgid): # https://www.mail-archive.com/faq.html#listserver except the list # name (without "@ietf.org") is used instead of the full address, # and rightmost "=" signs are (optionally) stripped - sha = hashlib.sha1(msgid) - sha.update(list_name) - return base64.urlsafe_b64encode(sha.digest()).rstrip("=") + sha = hashlib.sha1(force_bytes(msgid)) + sha.update(force_bytes(list_name)) + return base64.urlsafe_b64encode(sha.digest()).rstrip(b"=") def construct_query_urls(review_req, query=None): list_name = list_name_from_email(review_req.team.list_email) diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index c7134b970..bf1e3ce61 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -73,7 +73,7 @@ def unicontent(r): def textcontent(r): text = BeautifulSoup(r.content, 'lxml').get_text() - text = re.sub('(\n\s+){2,}', '\n\n', text) + text = re.sub(r'(\n\s+){2,}', '\n\n', text) return text def reload_db_objects(*objects): @@ -163,7 +163,7 @@ class TestCase(django.test.TestCase): if subject: mlist = [ m for m in mlist if subject in m["Subject"] ] if text: - mlist = [ m for m in mlist if text in m.get_payload(decode=True) ] + mlist = [ m for m in mlist if text in m.get_payload() ] if count and len(mlist) != count: sys.stderr.write("Wrong count in assertMailboxContains(). The complete mailbox contains %s emails:\n\n" % len(mailbox)) for m in mailbox: diff --git a/ietf/utils/textupload.py b/ietf/utils/textupload.py index 2cf90549c..751a75450 100644 --- a/ietf/utils/textupload.py +++ b/ietf/utils/textupload.py @@ -17,7 +17,7 @@ def get_cleaned_text_file_content(uploaded_file): if uploaded_file.size and uploaded_file.size > 10 * 1000 * 1000: raise ValidationError("Text file too large (size %s)." % uploaded_file.size) - content = "".join(uploaded_file.chunks()) + content = b"".join(uploaded_file.chunks()) # try to fixup encoding import magic @@ -39,13 +39,12 @@ def get_cleaned_text_file_content(uploaded_file): raise ValidationError("File has unknown encoding.") encoding = match.group(1) - if "ascii" not in encoding: - try: - content = content.decode(encoding) - except Exception as e: - raise ValidationError("Error decoding file (%s). Try submitting with UTF-8 encoding or remove non-ASCII characters." % str(e)) + try: + content = content.decode(encoding) + except Exception as e: + raise ValidationError("Error decoding file (%s). Try submitting with UTF-8 encoding or remove non-ASCII characters." % str(e)) # turn line-endings into Unix style content = content.replace("\r\n", "\n").replace("\r", "\n") - return content.encode("utf-8") + return content