From 8c6eb3a30a12e1e1277e917332227650d4701a12 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Mon, 15 Jul 2019 19:14:04 +0000 Subject: [PATCH] Python2/3 compatibility: Changed the use of open() and StringIO to io.open() etc. - Legacy-Id: 16458 --- changelog.py | 6 +- ietf/api/__init__.py | 22 ++- ietf/api/serializer.py | 8 +- ietf/community/views.py | 14 +- ietf/cookies/tests.py | 149 +++++++++--------- ietf/doc/feeds.py | 7 +- ietf/doc/fields.py | 10 +- ietf/doc/mails.py | 25 +-- .../commands/generate_draft_bibxml_files.py | 10 +- .../0016_set_document_docalias_fk.py | 5 +- ietf/doc/models.py | 40 +++-- ietf/doc/resources.py | 2 + ietf/doc/templatetags/ietf_filters.py | 23 +-- ietf/doc/tests.py | 48 +++--- ietf/doc/tests_charter.py | 41 ++--- ietf/doc/tests_conflict_review.py | 25 +-- ietf/doc/tests_draft.py | 23 +-- ietf/doc/tests_material.py | 14 +- ietf/doc/tests_review.py | 19 ++- ietf/doc/tests_status_change.py | 10 +- ietf/doc/utils.py | 26 +-- ietf/doc/utils_charter.py | 16 +- ietf/doc/views_charter.py | 18 ++- ietf/doc/views_conflict_review.py | 11 +- ietf/doc/views_doc.py | 23 ++- ietf/doc/views_material.py | 6 +- ietf/doc/views_review.py | 5 +- ietf/doc/views_status_change.py | 15 +- ietf/group/models.py | 24 ++- ietf/group/tests_info.py | 12 +- ietf/group/tests_review.py | 6 +- ietf/group/utils.py | 6 +- ietf/group/views.py | 31 ++-- ietf/idindex/generate_all_id2_txt.py | 10 +- ietf/idindex/generate_all_id_txt.py | 9 +- ietf/idindex/generate_id_abstracts_txt.py | 9 +- ietf/idindex/generate_id_index_txt.py | 9 +- ietf/idindex/index.py | 16 +- ietf/idindex/tests.py | 17 +- ietf/iesg/agenda.py | 7 +- ietf/iesg/tests.py | 12 +- ietf/iesg/views.py | 22 +-- ietf/ietfauth/htpasswd.py | 7 +- ietf/ietfauth/tests.py | 25 +-- ietf/ipr/feeds.py | 8 +- ietf/ipr/fields.py | 8 +- ietf/ipr/management/commands/process_email.py | 8 +- ietf/ipr/tests.py | 10 +- ietf/ipr/views.py | 8 +- ietf/liaisons/fields.py | 10 +- ietf/liaisons/forms.py | 12 +- ietf/liaisons/tests.py | 21 ++- ietf/meeting/forms.py | 7 +- ietf/meeting/helpers.py | 11 +- ietf/meeting/models.py | 40 +++-- ietf/meeting/tests_api.py | 6 +- ietf/meeting/tests_views.py | 58 +++---- ietf/meeting/utils.py | 12 +- ietf/meeting/views.py | 24 +-- ietf/name/generate_fixtures.py | 4 +- ietf/nomcom/forms.py | 11 +- .../management/commands/feedback_email.py | 8 +- ietf/nomcom/test_data.py | 10 +- ietf/nomcom/tests.py | 36 +++-- ietf/nomcom/utils.py | 26 +-- ietf/person/factories.py | 12 +- ietf/person/fields.py | 12 +- .../commands/deactivate_email_addresses.py | 5 +- ietf/person/models.py | 37 +++-- ietf/person/tests.py | 6 +- ietf/person/utils.py | 13 +- ietf/release/views.py | 5 +- ietf/review/mailarch.py | 26 ++- ietf/review/utils.py | 11 +- ietf/secr/drafts/tests_views.py | 8 +- ietf/secr/drafts/views.py | 6 +- ietf/secr/meetings/blue_sheets.py | 43 +++-- ietf/secr/proceedings/proc_utils.py | 5 +- ietf/secr/proceedings/tests.py | 8 +- ietf/secr/proceedings/utils.py | 4 +- ietf/secr/sreq/tests.py | 13 +- ietf/secr/utils/group.py | 8 +- ietf/stats/backfill_data.py | 23 +-- ietf/submit/checkers.py | 24 ++- ietf/submit/forms.py | 19 ++- .../management/commands/manualpost_email.py | 8 +- ietf/submit/tests.py | 54 ++++--- ietf/submit/utils.py | 13 +- ietf/sync/iana.py | 21 ++- ietf/sync/rfceditor.py | 21 ++- ietf/sync/tests.py | 12 +- ietf/utils/admin.py | 11 +- ietf/utils/draft.py | 21 ++- ietf/utils/fields.py | 10 +- ietf/utils/html.py | 9 +- ietf/utils/log.py | 19 ++- ietf/utils/mail.py | 46 ++++-- .../commands/check_referential_integrity.py | 28 ++-- .../management/commands/coverage_changes.py | 15 +- .../management/commands/create_group_wikis.py | 7 +- ietf/utils/management/commands/dumprelated.py | 8 +- .../management/commands/import_htpasswd.py | 4 +- ietf/utils/management/commands/makefixture.py | 10 +- ietf/utils/management/commands/tests.py | 15 +- ietf/utils/markup_txt.py | 8 +- ietf/utils/pdf.py | 8 +- ietf/utils/test_runner.py | 17 +- ietf/utils/test_smtpserver.py | 31 ++-- ietf/utils/test_utils.py | 23 ++- ietf/utils/tests.py | 104 ++++++------ ietf/utils/tests_restapi.py | 24 ++- ietf/utils/text.py | 10 +- ietf/utils/urls.py | 8 +- ietf/virtualenv-manage.py | 9 +- ietf/wsgi.py | 10 +- 115 files changed, 1361 insertions(+), 692 deletions(-) diff --git a/changelog.py b/changelog.py index f5f867f0e..c7eb53a8e 100644 --- a/changelog.py +++ b/changelog.py @@ -1,5 +1,9 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals import re +import six from tzparse import tzparse from datetime import datetime as Datetime @@ -44,7 +48,7 @@ def parse(logfile): inf_line = r"^ \*\*(.*)\*\* *" entries = [] - if type(logfile) == type(''): + if isinstance(logfile, six.string_types): logfile = open(logfile) entry = None for line in logfile: diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index d2b205e59..ff0b57bd3 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -1,7 +1,14 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved -import re +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime -from urllib.parse import urlencode +import re +import six + +from six.moves.urllib.parse import urlencode from django.conf import settings from django.core.exceptions import ObjectDoesNotExist @@ -75,7 +82,7 @@ class TimedeltaField(ApiField): if value is None: return None - if isinstance(value, str): + if isinstance(value, six.string_types): match = TIMEDELTA_REGEX.search(value) if match: @@ -90,7 +97,7 @@ class TimedeltaField(ApiField): value = super(TimedeltaField, self).hydrate(bundle) if value and not hasattr(value, 'seconds'): - if isinstance(value, str): + if isinstance(value, six.string_types): try: match = TIMEDELTA_REGEX.search(value) @@ -112,14 +119,17 @@ class ToOneField(tastypie.fields.ToOneField): def dehydrate(self, bundle, for_list=True): foreign_obj = None + previous_obj = None + attrib = None if callable(self.attribute): previous_obj = bundle.obj foreign_obj = self.attribute(bundle) - elif isinstance(self.attribute, str): + elif isinstance(self.attribute, six.string_types): foreign_obj = bundle.obj for attr in self._attrs: + attrib = attr previous_obj = foreign_obj try: foreign_obj = getattr(foreign_obj, attr, None) @@ -131,7 +141,7 @@ class ToOneField(tastypie.fields.ToOneField): if callable(self.attribute): raise ApiFieldError("The related resource for resource %s could not be found." % (previous_obj)) else: - raise ApiFieldError("The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attr)) + raise ApiFieldError("The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attrib)) return None fk_resource = self.get_related_resource(foreign_obj) diff --git a/ietf/api/serializer.py b/ietf/api/serializer.py index 3fdfc428c..0dfdb6aed 100644 --- a/ietf/api/serializer.py +++ b/ietf/api/serializer.py @@ -1,6 +1,12 @@ # Copyright The IETF Trust 2018-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import hashlib import json +import six from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist, FieldError @@ -148,7 +154,7 @@ class AdminJsonSerializer(Serializer): if hasattr(field_value, "_meta"): self._current[name] = self.expand_related(field_value, name) else: - self._current[name] = str(field_value) + self._current[name] = six.ensure_text(field_value) except ObjectDoesNotExist: pass except AttributeError: diff --git a/ietf/community/views.py b/ietf/community/views.py index 09ffd361b..375287e29 100644 --- a/ietf/community/views.py +++ b/ietf/community/views.py @@ -1,16 +1,22 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import csv -import uuid import datetime import json +import six +import uuid from django.http import HttpResponse, HttpResponseForbidden, HttpResponseRedirect, Http404 from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required from django.utils.html import strip_tags +import debug # pyflakes:ignore + from ietf.community.models import SearchRule, EmailSubscription from ietf.community.forms import SearchRuleTypeForm, SearchRuleForm, AddDocumentsForm, SubscriptionForm from ietf.community.utils import lookup_community_list, can_manage_community_list @@ -174,7 +180,7 @@ def export_to_csv(request, username=None, acronym=None, group_type=None): response['Content-Disposition'] = 'attachment; filename=%s' % filename - writer = csv.writer(response, dialect=csv.excel, delimiter=',') + writer = csv.writer(response, dialect=csv.excel, delimiter=str(',')) header = [ "Name", @@ -196,7 +202,7 @@ def export_to_csv(request, username=None, acronym=None, group_type=None): row.append(e.time.strftime("%Y-%m-%d") if e else "") row.append(strip_tags(doc.friendly_state())) row.append(doc.group.acronym if doc.group else "") - row.append(str(doc.ad) if doc.ad else "") + row.append(six.ensure_text(doc.ad) if doc.ad else "") e = doc.latest_event() row.append(e.time.strftime("%Y-%m-%d") if e else "") writer.writerow([v.encode("utf-8") for v in row]) @@ -221,7 +227,7 @@ def feed(request, username=None, acronym=None, group_type=None): host = request.get_host() feed_url = 'https://%s%s' % (host, request.get_full_path()) - feed_id = uuid.uuid5(uuid.NAMESPACE_URL, feed_url) + feed_id = uuid.uuid5(uuid.NAMESPACE_URL, str(feed_url)) title = '%s RSS Feed' % clist.long_name() if significant: subtitle = 'Significant document changes' diff --git a/ietf/cookies/tests.py b/ietf/cookies/tests.py index 42ea437b2..302cab100 100644 --- a/ietf/cookies/tests.py +++ b/ietf/cookies/tests.py @@ -1,6 +1,11 @@ # Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from pyquery import PyQuery -from http.cookies import SimpleCookie +from six.moves.http_cookies import SimpleCookie from django.urls import reverse as urlreverse @@ -22,7 +27,7 @@ class CookieTests(TestCase): def test_settings_defaults_from_cookies(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '7', 'expires_soon' : 7, 'left_menu': 'on', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '7', str('expires_soon') : 7, str('left_menu'): 'on', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) self.assertListEqual([], list(r.cookies.keys())) @@ -33,7 +38,7 @@ class CookieTests(TestCase): self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/on"]').contents(), ['On']) def test_settings_values_from_cookies_garbage(self): - self.client.cookies = SimpleCookie({'full_draft': 'foo', 'new_enough' : 'foo', 'expires_soon' : 'foo', 'left_menu': 'foo', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'foo', str('new_enough') : 'foo', str('expires_soon') : 'foo', str('left_menu'): 'foo', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -43,7 +48,7 @@ class CookieTests(TestCase): self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off']) def test_settings_values_from_cookies_random(self): - self.client.cookies = SimpleCookie({'full_draft': 'zappa', 'new_enough' : '365', 'expires_soon' : '5', 'left_menu': 'zappa', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'zappa', str('new_enough') : '365', str('expires_soon') : '5', str('left_menu'): 'zappa', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -58,7 +63,7 @@ class CookieTests(TestCase): # self.assertNotRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon') def test_settings_values_from_cookies_1(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '90', 'expires_soon' : 7, 'left_menu': 'off', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '90', str('expires_soon') : 7, str('left_menu'): 'off', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) self.assertListEqual([], list(r.cookies.keys())) @@ -72,7 +77,7 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*7 days') def test_settings_values_from_cookies_2(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '60', 'expires_soon' : 14, 'left_menu': 'on', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '60', str('expires_soon') : 14, str('left_menu'): 'on', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) self.assertListEqual([], list(r.cookies.keys())) @@ -86,7 +91,7 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days') def test_settings_values_from_cookies_3(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '30', 'expires_soon' : 21, 'left_menu': 'off'}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '30', str('expires_soon') : 21, str('left_menu'): 'off'}) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) self.assertListEqual([], list(r.cookies.keys())) @@ -100,7 +105,7 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*21 days') def test_settings_values_from_cookies_4(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '21', 'expires_soon' : 30, 'left_menu': 'on', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '21', str('expires_soon') : 30, str('left_menu'): 'on', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) self.assertListEqual([], list(r.cookies.keys())) @@ -114,7 +119,7 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*30 days') def test_settings_values_from_cookies_5(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '14', 'expires_soon' : 60, 'left_menu': 'off', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '14', str('expires_soon') : 60, str('left_menu'): 'off', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) self.assertListEqual([], list(r.cookies.keys())) @@ -128,7 +133,7 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*60 days') def test_settings_values_from_cookies_6(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '7', 'expires_soon' : 90, 'left_menu': 'on', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '7', str('expires_soon') : 90, str('left_menu'): 'on', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) self.assertListEqual([], list(r.cookies.keys())) @@ -142,11 +147,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*90 days') def test_full_draft(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.full_draft")) # no value: reset self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['full_draft'].value, '') - self.assertListEqual(['full_draft'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('full_draft')].value, '') + self.assertListEqual([str('full_draft')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -156,21 +161,21 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days') def test_full_draft_on(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.full_draft", kwargs=dict(enabled="on"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['full_draft'].value, 'on') - self.assertListEqual(['full_draft'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('full_draft')].value, 'on') + self.assertListEqual([str('full_draft')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*full_draft.*on') def test_full_draft_off(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.full_draft", kwargs=dict(enabled="off"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['full_draft'].value, 'off') - self.assertListEqual(['full_draft'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('full_draft')].value, 'off') + self.assertListEqual([str('full_draft')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) # self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -178,7 +183,7 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*full_draft.*off') def test_full_draft_foo(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.full_draft", kwargs=dict(enabled="foo"))) self.assertEqual(r.status_code, 200) self.assertListEqual([], list(r.cookies.keys())) @@ -189,11 +194,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*full_draft.*off') def test_left_menu(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'on', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'on', }) r = self.client.get(urlreverse("ietf.cookies.views.left_menu")) # no value: reset self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['left_menu'].value, '') - self.assertListEqual(['left_menu'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('left_menu')].value, '') + self.assertListEqual([str('left_menu')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off']) @@ -201,25 +206,25 @@ class CookieTests(TestCase): self.assertEqual(q('div a.active[href="/accounts/settings/expires_soon/14"]').contents(), ['14 days']) def test_left_menu_on(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'off', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'off', }) r = self.client.get(urlreverse("ietf.cookies.views.left_menu", kwargs=dict(enabled="on"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['left_menu'].value, 'on') - self.assertListEqual(['left_menu'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('left_menu')].value, 'on') + self.assertListEqual([str('left_menu')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/on"]').contents(), ['On']) def test_left_menu_off(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'off', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'off', }) r = self.client.get(urlreverse("ietf.cookies.views.left_menu", kwargs=dict(enabled="off"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['left_menu'].value, 'off') - self.assertListEqual(['left_menu'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('left_menu')].value, 'off') + self.assertListEqual([str('left_menu')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off']) def test_left_menu_foo(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'off', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'off', }) r = self.client.get(urlreverse("ietf.cookies.views.left_menu", kwargs=dict(enabled="foo"))) self.assertEqual(r.status_code, 200) self.assertListEqual([], list(r.cookies.keys())) @@ -227,11 +232,11 @@ class CookieTests(TestCase): self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off']) def test_new_enough(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough")) # no value: reset self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '') - self.assertListEqual(['new_enough'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('new_enough')].value, '') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -241,11 +246,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days') def test_new_enough_7(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '14', 'expires_soon' : 21}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '14', str('expires_soon') : 21}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="7"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '7') - self.assertListEqual(['new_enough'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('new_enough')].value, '7') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/7"]').contents(), ['7 days']) @@ -255,11 +260,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*21 days') def test_new_enough_14(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '7', 'expires_soon' : 99}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '7', str('expires_soon') : 99}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="14"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '14') - self.assertListEqual(['new_enough'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('new_enough')].value, '14') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -269,11 +274,11 @@ class CookieTests(TestCase): # self.assertNotRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon') def test_new_enough_21(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '14', 'expires_soon' : 90}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '14', str('expires_soon') : 90}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="21"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '21') - self.assertListEqual(['new_enough'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('new_enough')].value, '21') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/21"]').contents(), ['21 days']) @@ -283,11 +288,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*90 days') def test_new_enough_30(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 7}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 7}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="30"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '30') - self.assertListEqual(['new_enough'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('new_enough')].value, '30') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/30"]').contents(), ['30 days']) @@ -297,11 +302,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*7 days') def test_new_enough_60(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="60"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '60') - self.assertListEqual(['new_enough'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('new_enough')].value, '60') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/60"]').contents(), ['60 days']) @@ -311,11 +316,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days') def test_new_enough_90(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '22', 'expires_soon' : 60}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '22', str('expires_soon') : 60}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="90"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '90') - self.assertListEqual(['new_enough'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('new_enough')].value, '90') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/90"]').contents(), ['90 days']) @@ -325,11 +330,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*60 days') def test_expires_soon(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '14', 'new_enough' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '14', str('new_enough') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon")) # no value: reset self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '') - self.assertListEqual(['expires_soon'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('expires_soon')].value, '') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -339,11 +344,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*14 days') def test_expires_soon_7(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'expires_soon' : '14', 'new_enough' : 21}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('expires_soon') : '14', str('new_enough') : 21}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="7"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '7') - self.assertListEqual(['expires_soon'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('expires_soon')].value, '7') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/21"]').contents(), ['21 days']) @@ -353,11 +358,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*21 days') def test_expires_soon_14(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'expires_soon' : '7', 'new_enough' : 99}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('expires_soon') : '7', str('new_enough') : 99}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="14"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '14') - self.assertListEqual(['expires_soon'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('expires_soon')].value, '14') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href^="/accounts/settings/new_enough/"]').contents(), []) @@ -367,11 +372,11 @@ class CookieTests(TestCase): # self.assertNotRegexpMatches(r.content, r'ietf-highlight-y.*new_enough') def test_expires_soon_21(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'expires_soon' : '14', 'new_enough' : 90}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('expires_soon') : '14', str('new_enough') : 90}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="21"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '21') - self.assertListEqual(['expires_soon'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('expires_soon')].value, '21') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/90"]').contents(), ['90 days']) @@ -381,11 +386,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*90 days') def test_expires_soon_30(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '14', 'new_enough' : 7}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '14', str('new_enough') : 7}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="30"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '30') - self.assertListEqual(['expires_soon'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('expires_soon')].value, '30') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/7"]').contents(), ['7 days']) @@ -395,11 +400,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*7 days') def test_expires_soon_60(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '14', 'new_enough' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '14', str('new_enough') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="60"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '60') - self.assertListEqual(['expires_soon'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('expires_soon')].value, '60') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -409,11 +414,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*14 days') def test_expires_soon_90(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '22', 'new_enough' : 60}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '22', str('new_enough') : 60}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="90"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '90') - self.assertListEqual(['expires_soon'], list(r.cookies.keys())) + self.assertEqual(r.cookies[str('expires_soon')].value, '90') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/60"]').contents(), ['60 days']) diff --git a/ietf/doc/feeds.py b/ietf/doc/feeds.py index 16e51a5e5..24a79568c 100644 --- a/ietf/doc/feeds.py +++ b/ietf/doc/feeds.py @@ -1,6 +1,11 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import datetime +import six from django.contrib.syndication.views import Feed, FeedDoesNotExist from django.utils.feedgenerator import Atom1Feed, Rss201rev2Feed @@ -44,7 +49,7 @@ class DocumentChangesFeed(Feed): return item.time def item_author_name(self, item): - return str(item.by) + return six.ensure_text(item.by) def item_link(self, item): return urlreverse('ietf.doc.views_doc.document_history', kwargs=dict(name=item.doc.canonical_name())) + "#history-%s" % item.pk diff --git a/ietf/doc/fields.py b/ietf/doc/fields.py index 0cdb1830d..e8f76b222 100644 --- a/ietf/doc/fields.py +++ b/ietf/doc/fields.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import json +import six from django.utils.html import escape from django import forms @@ -53,9 +57,9 @@ class SearchableDocumentsField(forms.CharField): def prepare_value(self, value): if not value: value = "" - if isinstance(value, int): + if isinstance(value, six.integer_types): value = str(value) - if isinstance(value, str): + if isinstance(value, six.string_types): items = self.parse_select2_value(value) # accept both names and pks here names = [ i for i in items if not i.isdigit() ] @@ -79,7 +83,7 @@ class SearchableDocumentsField(forms.CharField): "model_name": self.model.__name__.lower() }) - return ",".join(str(o.pk) for o in value) + return ",".join(six.ensure_text(o.pk) for o in value) def clean(self, value): value = super(SearchableDocumentsField, self).clean(value) diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py index aca9a0c8d..9d9526751 100644 --- a/ietf/doc/mails.py +++ b/ietf/doc/mails.py @@ -1,16 +1,23 @@ # Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- # generation of mails -import textwrap, datetime + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import six +import textwrap from django.template.loader import render_to_string from django.utils.html import strip_tags from django.conf import settings from django.urls import reverse as urlreverse +from django.utils.encoding import force_str import debug # pyflakes:ignore -from ietf.utils.mail import send_mail, send_mail_text +from ietf.utils.mail import send_mail, send_mail_text, get_payload from ietf.ipr.utils import iprs_from_docs, related_docs from ietf.doc.models import WriteupDocEvent, LastCallDocEvent, DocAlias, ConsensusDocEvent from ietf.doc.utils import needed_ballot_positions @@ -121,7 +128,7 @@ def generate_ballot_writeup(request, doc): e.doc = doc e.rev = doc.rev e.desc = "Ballot writeup was generated" - e.text = str(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana})) + e.text = six.ensure_text(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana})) # caller is responsible for saving, if necessary return e @@ -133,7 +140,7 @@ def generate_ballot_rfceditornote(request, doc): e.doc = doc e.rev = doc.rev e.desc = "RFC Editor Note for ballot was generated" - e.text = str(render_to_string("doc/mail/ballot_rfceditornote.txt")) + e.text = six.ensure_text(render_to_string("doc/mail/ballot_rfceditornote.txt")) e.save() return e @@ -178,7 +185,7 @@ def generate_last_call_announcement(request, doc): e.doc = doc e.rev = doc.rev e.desc = "Last call announcement was generated" - e.text = str(mail) + e.text = six.ensure_text(mail) # caller is responsible for saving, if necessary return e @@ -198,7 +205,7 @@ def generate_approval_mail(request, doc): e.doc = doc e.rev = doc.rev e.desc = "Ballot approval text was generated" - e.text = str(mail) + e.text = six.ensure_text(mail) # caller is responsible for saving, if necessary return e @@ -281,7 +288,7 @@ def generate_publication_request(request, doc): approving_body = "IRSG" consensus_body = doc.group.acronym.upper() else: - approving_body = str(doc.stream) + approving_body = six.ensure_text(doc.stream) consensus_body = approving_body e = doc.latest_event(WriteupDocEvent, type="changed_rfc_editor_note_text") @@ -383,7 +390,7 @@ def generate_issue_ballot_mail(request, doc, ballot): def email_iana(request, doc, to, msg, cc=None): # fix up message and send it with extra info on doc in headers import email - parsed_msg = email.message_from_string(msg) + parsed_msg = email.message_from_string(force_str(msg)) parsed_msg.set_charset('UTF-8') extra = extra_automation_headers(doc) @@ -391,7 +398,7 @@ def email_iana(request, doc, to, msg, cc=None): send_mail_text(request, to, parsed_msg["From"], parsed_msg["Subject"], - parsed_msg.get_payload(), + get_payload(parsed_msg), extra=extra, cc=cc) diff --git a/ietf/doc/management/commands/generate_draft_bibxml_files.py b/ietf/doc/management/commands/generate_draft_bibxml_files.py index 171026b67..78a052927 100644 --- a/ietf/doc/management/commands/generate_draft_bibxml_files.py +++ b/ietf/doc/management/commands/generate_draft_bibxml_files.py @@ -1,4 +1,10 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import sys import os @@ -10,14 +16,14 @@ from ietf.doc.models import Document def write(fn, new): try: - f = open(fn) + f = io.open(fn) old = f.read().decode('utf-8') f.close except IOError: old = "" if old.strip() != new.strip(): sys.stdout.write(os.path.basename(fn)+'\n') - f = open(fn, "wb") + f = io.open(fn, "wb") f.write(new.encode('utf-8')) f.close() diff --git a/ietf/doc/migrations/0016_set_document_docalias_fk.py b/ietf/doc/migrations/0016_set_document_docalias_fk.py index 31b244591..e931f1306 100644 --- a/ietf/doc/migrations/0016_set_document_docalias_fk.py +++ b/ietf/doc/migrations/0016_set_document_docalias_fk.py @@ -3,6 +3,9 @@ # Generated by Django 1.11.20 on 2019-05-08 14:04 +from __future__ import absolute_import, print_function, unicode_literals + +import six import sys from tqdm import tqdm @@ -15,7 +18,7 @@ def forward(apps, schema_editor): n = getattr(o, a+'_id') if n: i = nameid[n] - if not isinstance(i, int): + if not isinstance(i, six.integer_types): raise ValueError("Inappropriate value: %s: nameid[%s]: %s" % (o.__class__.__name__, n, i)) if getattr(o, a+'2_id') != i: setattr(o, a+'2_id', i) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 0c79fbf02..65c3e4730 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1,10 +1,13 @@ # Copyright The IETF Trust 2010-2019, All Rights Reserved # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals import datetime import logging +import io import os import rfc2html +import six from django.db import models from django.core import checks @@ -14,6 +17,7 @@ from django.core.validators import URLValidator, RegexValidator from django.urls import reverse as urlreverse from django.contrib.contenttypes.models import ContentType from django.conf import settings +from django.utils.encoding import python_2_unicode_compatible from django.utils.html import mark_safe import debug # pyflakes:ignore @@ -33,6 +37,7 @@ from ietf.utils.models import ForeignKey logger = logging.getLogger('django') +@python_2_unicode_compatible class StateType(models.Model): slug = models.CharField(primary_key=True, max_length=30) # draft, draft-iesg, charter, ... label = models.CharField(max_length=255, help_text="Label that should be used (e.g. in admin) for state drop-down for this type of state") # State, IESG state, WG state, ... @@ -54,6 +59,7 @@ def check_statetype_slugs(app_configs, **kwargs): )) return errors +@python_2_unicode_compatible class State(models.Model): type = ForeignKey(StateType) slug = models.SlugField() @@ -406,7 +412,7 @@ class DocumentInfo(models.Model): def relations_that(self, relationship): """Return the related-document objects that describe a given relationship targeting self.""" - if isinstance(relationship, str): + if isinstance(relationship, six.string_types): relationship = ( relationship, ) if not isinstance(relationship, tuple): raise TypeError("Expected a string or tuple, received %s" % type(relationship)) @@ -429,7 +435,7 @@ class DocumentInfo(models.Model): def relations_that_doc(self, relationship): """Return the related-document objects that describe a given relationship from self to other documents.""" - if isinstance(relationship, str): + if isinstance(relationship, six.string_types): relationship = ( relationship, ) if not isinstance(relationship, tuple): raise TypeError("Expected a string or tuple, received %s" % type(relationship)) @@ -481,7 +487,7 @@ class DocumentInfo(models.Model): if ext != '.txt' and os.path.exists(txtpath): path = txtpath try: - with open(path, 'rb') as file: + with io.open(path, 'rb') as file: raw = file.read() except IOError: return None @@ -523,6 +529,7 @@ class DocumentInfo(models.Model): STATUSCHANGE_RELATIONS = ('tops','tois','tohist','toinf','tobcp','toexp') +@python_2_unicode_compatible class RelatedDocument(models.Model): source = ForeignKey('Document') target = ForeignKey('DocAlias') @@ -530,7 +537,7 @@ class RelatedDocument(models.Model): def action(self): return self.relationship.name def __str__(self): - return "%s %s %s" % (self.source.name, self.relationship.name.lower(), self.target.name) + return u"%s %s %s" % (self.source.name, self.relationship.name.lower(), self.target.name) def is_downref(self): @@ -596,11 +603,12 @@ class DocumentAuthorInfo(models.Model): abstract = True ordering = ["document", "order"] +@python_2_unicode_compatible class DocumentAuthor(DocumentAuthorInfo): document = ForeignKey('Document') def __str__(self): - return "%s %s (%s)" % (self.document.name, self.person, self.order) + return u"%s %s (%s)" % (self.document.name, self.person, self.order) validate_docname = RegexValidator( @@ -609,6 +617,7 @@ validate_docname = RegexValidator( 'invalid' ) +@python_2_unicode_compatible class Document(DocumentInfo): name = models.CharField(max_length=255, validators=[validate_docname,], unique=True) # immutable @@ -844,21 +853,24 @@ class DocumentURL(models.Model): desc = models.CharField(max_length=255, default='', blank=True) url = models.URLField(max_length=2083) # 2083 is the legal max for URLs +@python_2_unicode_compatible class RelatedDocHistory(models.Model): source = ForeignKey('DocHistory') target = ForeignKey('DocAlias', related_name="reversely_related_document_history_set") relationship = ForeignKey(DocRelationshipName) def __str__(self): - return "%s %s %s" % (self.source.doc.name, self.relationship.name.lower(), self.target.name) + return u"%s %s %s" % (self.source.doc.name, self.relationship.name.lower(), self.target.name) +@python_2_unicode_compatible class DocHistoryAuthor(DocumentAuthorInfo): # use same naming convention as non-history version to make it a bit # easier to write generic code document = ForeignKey('DocHistory', related_name="documentauthor_set") def __str__(self): - return "%s %s (%s)" % (self.document.doc.name, self.person, self.order) + return u"%s %s (%s)" % (self.document.doc.name, self.person, self.order) +@python_2_unicode_compatible class DocHistory(DocumentInfo): doc = ForeignKey(Document, related_name="history_set") # the name here is used to capture the canonical name at the time @@ -868,7 +880,7 @@ class DocHistory(DocumentInfo): name = models.CharField(max_length=255) def __str__(self): - return str(self.doc.name) + return six.ensure_text(self.doc.name) def canonical_name(self): if hasattr(self, '_canonical_name'): @@ -903,6 +915,7 @@ class DocHistory(DocumentInfo): verbose_name = "document history" verbose_name_plural = "document histories" +@python_2_unicode_compatible class DocAlias(models.Model): """This is used for documents that may appear under multiple names, and in particular for RFCs, which for continuity still keep the @@ -917,7 +930,7 @@ class DocAlias(models.Model): return self.docs.first() def __str__(self): - return "%s-->%s" % (self.name, ','.join([str(d.name) for d in self.docs.all() if isinstance(d, Document) ])) + return u"%s-->%s" % (self.name, ','.join([six.ensure_text(d.name) for d in self.docs.all() if isinstance(d, Document) ])) document_link = admin_link("document") class Meta: verbose_name = "document alias" @@ -1006,6 +1019,7 @@ EVENT_TYPES = [ ("downref_approved", "Downref approved"), ] +@python_2_unicode_compatible class DocEvent(models.Model): """An occurrence for a document, used for tracking who, when and what.""" time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened", db_index=True) @@ -1023,7 +1037,7 @@ class DocEvent(models.Model): return DocHistory.objects.filter(time__lte=self.time,doc__name=self.doc.name).order_by('-time', '-pk').first() def __str__(self): - return "%s %s by %s at %s" % (self.doc.name, self.get_type_display().lower(), self.by.plain_name(), self.time) + return u"%s %s by %s at %s" % (self.doc.name, self.get_type_display().lower(), self.by.plain_name(), self.time) def save(self, *args, **kwargs): super(DocEvent, self).save(*args, **kwargs) @@ -1046,6 +1060,7 @@ class ConsensusDocEvent(DocEvent): consensus = models.NullBooleanField(default=None) # IESG events +@python_2_unicode_compatible class BallotType(models.Model): doc_type = ForeignKey(DocTypeName, blank=True, null=True) slug = models.SlugField() @@ -1056,7 +1071,7 @@ class BallotType(models.Model): positions = models.ManyToManyField(BallotPositionName, blank=True) def __str__(self): - return "%s: %s" % (self.name, self.doc_type.name) + return u"%s: %s" % (self.name, self.doc_type.name) class Meta: ordering = ['order'] @@ -1176,6 +1191,7 @@ class SubmissionDocEvent(DocEvent): submission = ForeignKey(ietf.submit.models.Submission) # dumping store for removed events +@python_2_unicode_compatible class DeletedEvent(models.Model): content_type = ForeignKey(ContentType) json = models.TextField(help_text="Deleted object in JSON format, with attribute names chosen to be suitable for passing into the relevant create method.") @@ -1183,7 +1199,7 @@ class DeletedEvent(models.Model): time = models.DateTimeField(default=datetime.datetime.now) def __str__(self): - return "%s by %s %s" % (self.content_type, self.by, self.time) + return u"%s by %s %s" % (self.content_type, self.by, self.time) class EditedAuthorsDocEvent(DocEvent): """ Capture the reasoning or authority for changing a document author list. diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index 08f5489fc..b80d2b409 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -1,6 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2015-10-19 12:29 PDT + + from ietf.api import ModelResource from ietf.api import ToOneField from tastypie.fields import ToManyField, CharField diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 90851df25..a8662bbc4 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -1,8 +1,13 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import bleach import datetime import re +import six from email.utils import parseaddr @@ -45,24 +50,24 @@ def parse_email_list(value): Splitting a string of email addresses should return a list: - >>> parse_email_list('joe@example.org, fred@example.com') + >>> six.ensure_str(parse_email_list('joe@example.org, fred@example.com')) 'joe@example.org, fred@example.com' Parsing a non-string should return the input value, rather than fail: - >>> parse_email_list(['joe@example.org', 'fred@example.com']) + >>> [ six.ensure_str(e) for e in parse_email_list(['joe@example.org', 'fred@example.com']) ] ['joe@example.org', 'fred@example.com'] Null input values should pass through silently: - >>> parse_email_list('') + >>> six.ensure_str(parse_email_list('')) '' >>> parse_email_list(None) """ - if value and isinstance(value, (bytes,str)): # testing for 'value' being true isn't necessary; it's a fast-out route + if value and isinstance(value, (six.binary_type, six.text_type)): # testing for 'value' being true isn't necessary; it's a fast-out route addrs = re.split(", ?", value) ret = [] for addr in addrs: @@ -96,7 +101,7 @@ def make_one_per_line(value): """ Turn a comma-separated list into a carriage-return-seperated list. - >>> make_one_per_line("a, b, c") + >>> six.ensure_str(make_one_per_line("a, b, c")) 'a\\nb\\nc' Pass through non-strings: @@ -107,7 +112,7 @@ def make_one_per_line(value): >>> make_one_per_line(None) """ - if value and isinstance(value, (bytes,str)): + if value and isinstance(value, (six.binary_type, six.text_type)): return re.sub(", ?", "\n", value) else: return value @@ -143,7 +148,7 @@ def sanitize(value): @register.filter(name='bracket') def square_brackets(value): """Adds square brackets around text.""" - if isinstance(value, (bytes,str)): + if isinstance(value, (six.binary_type, six.text_type)): if value == "": value = " " return "[ %s ]" % value @@ -193,7 +198,7 @@ def rfcnospace(string): @register.filter def prettystdname(string): from ietf.doc.utils import prettify_std_name - return prettify_std_name(str(string or "")) + return prettify_std_name(six.ensure_text(string or "")) @register.filter(name='rfcurl') def rfclink(string): @@ -336,7 +341,7 @@ def expires_soon(x,request): @register.filter(name='startswith') def startswith(x, y): - return str(x).startswith(y) + return six.ensure_text(x).startswith(y) @register.filter def has_role(user, role_names): diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index b5160bc4e..bfac87497 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -1,19 +1,25 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import os import shutil import datetime +import io import sys -import urllib.parse import bibtexparser + if sys.version_info[0] == 2 and sys.version_info[1] < 7: import unittest2 as unittest else: import unittest + +from six.moves.http_cookies import SimpleCookie from pyquery import PyQuery +from six.moves.urllib.parse import urlparse, parse_qs from tempfile import NamedTemporaryFile -from http.cookies import SimpleCookie from django.urls import reverse as urlreverse from django.conf import settings @@ -141,71 +147,71 @@ class SearchTests(TestCase): # exact match r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) # prefix match r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(draft.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) # non-prefix match r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(draft.name.split("-")[1:])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) # other doctypes than drafts doc = Document.objects.get(name='charter-ietf-mars') r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name='charter-ietf-ma'))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) doc = Document.objects.filter(name__startswith='conflict-review-').first() r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) doc = Document.objects.filter(name__startswith='status-change-').first() r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) doc = Document.objects.filter(name__startswith='agenda-').first() r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) doc = Document.objects.filter(name__startswith='minutes-').first() r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) doc = Document.objects.filter(name__startswith='slides-').first() r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) # match with revision r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name + "-" + prev_rev))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev))) # match with non-existing revision r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name + "-09"))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) # match with revision and extension r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name + "-" + prev_rev + ".txt"))) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev))) # no match r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="draft-ietf-doesnotexist-42"))) self.assertEqual(r.status_code, 302) - parsed = urllib.parse.urlparse(r["Location"]) + parsed = urlparse(r["Location"]) self.assertEqual(parsed.path, urlreverse('ietf.doc.views_search.search')) - self.assertEqual(urllib.parse.parse_qs(parsed.query)["name"][0], "draft-ietf-doesnotexist-42") + self.assertEqual(parse_qs(parsed.query)["name"][0], "draft-ietf-doesnotexist-42") def test_frontpage(self): r = self.client.get("/") @@ -486,7 +492,7 @@ Man Expires September 22, 2015 [Page 3] settings.INTERNET_DRAFT_PATH = self.id_dir self.saved_internet_all_drafts_archive_dir = settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR = self.id_dir - f = open(os.path.join(self.id_dir, 'draft-ietf-mars-test-01.txt'), 'w') + f = io.open(os.path.join(self.id_dir, 'draft-ietf-mars-test-01.txt'), 'w') f.write(self.draft_text) f.close() @@ -528,21 +534,21 @@ Man Expires September 22, 2015 [Page 3] self.assertNotContains(r, "Show full document text") self.assertContains(r, "Deimos street") - self.client.cookies = SimpleCookie({'full_draft': 'on'}) + self.client.cookies = SimpleCookie({str('full_draft'): str('on')}) r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 200) self.assertContains(r, "Active Internet-Draft") self.assertNotContains(r, "Show full document text") self.assertContains(r, "Deimos street") - self.client.cookies = SimpleCookie({'full_draft': 'off'}) + self.client.cookies = SimpleCookie({str('full_draft'): str('off')}) r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 200) self.assertContains(r, "Active Internet-Draft") self.assertContains(r, "Show full document text") self.assertNotContains(r, "Deimos street") - self.client.cookies = SimpleCookie({'full_draft': 'foo'}) + self.client.cookies = SimpleCookie({str('full_draft'): str('foo')}) r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 200) self.assertContains(r, "Active Internet-Draft") @@ -1021,7 +1027,7 @@ class EmailAliasesTests(TestCase): def setUp(self): WgDraftFactory(name='draft-ietf-mars-test',group__acronym='mars') WgDraftFactory(name='draft-ietf-ames-test',group__acronym='ames') - self.doc_alias_file = NamedTemporaryFile(delete=False, encoding='utf-8', mode='w+') + self.doc_alias_file = NamedTemporaryFile(delete=False, mode='w+') self.doc_alias_file.write("""# Generated by hand at 2015-02-12_16:26:45 virtual.ietf.org anything draft-ietf-mars-test@ietf.org xfilter-draft-ietf-mars-test diff --git a/ietf/doc/tests_charter.py b/ietf/doc/tests_charter.py index 0bad514ea..c44cd01eb 100644 --- a/ietf/doc/tests_charter.py +++ b/ietf/doc/tests_charter.py @@ -1,8 +1,14 @@ # -*- coding: utf-8 -*- # Copyright The IETF Trust 2011-2019, All Rights Reserved -import os, shutil, datetime -from io import StringIO + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os +import shutil + from pyquery import PyQuery from django.conf import settings @@ -21,7 +27,7 @@ from ietf.group.models import Group, GroupMilestone from ietf.iesg.models import TelechatDate from ietf.person.models import Person from ietf.utils.test_utils import TestCase -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.mail import outbox, empty_outbox, get_payload from ietf.utils.test_utils import login_testing_unauthorized class EditCharterTests(TestCase): @@ -35,7 +41,7 @@ class EditCharterTests(TestCase): shutil.rmtree(self.charter_dir) def write_charter_file(self, charter): - with open(os.path.join(self.charter_dir, "%s-%s.txt" % (charter.canonical_name(), charter.rev)), "w") as f: + with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (charter.canonical_name(), charter.rev)), "w") as f: f.write("This is a charter.") def test_startstop_process(self): @@ -128,24 +134,24 @@ class EditCharterTests(TestCase): self.assertIn("Internal WG Review", outbox[-3]['Subject']) self.assertIn("iab@", outbox[-3]['To']) self.assertIn("iesg@", outbox[-3]['To']) - self.assertIn("A new IETF WG", outbox[-3].get_payload()) - body = outbox[-3].get_payload() - for word in ["Chairs", "Ames Man ", + body = get_payload(outbox[-3]) + for word in ["A new IETF WG", "Chairs", "Ames Man ", "Secretaries", "Secretary ", "Assigned Area Director", "Areað Irector ", "Mailing list", "ames-wg@ietf.org", "Charter", "Milestones"]: + self.assertIn(word, body) self.assertIn("state changed", outbox[-2]['Subject'].lower()) self.assertIn("iesg-secretary@", outbox[-2]['To']) - body = outbox[-2].get_payload() + body = get_payload(outbox[-2]) for word in ["WG", "Charter", ]: self.assertIn(word, body) self.assertIn("State Update Notice", outbox[-1]['Subject']) self.assertIn("ames-chairs@", outbox[-1]['To']) - body = outbox[-1].get_payload() + body = get_payload(outbox[-1]) for word in ["State changed", "Datatracker URL", ]: self.assertIn(word, body) @@ -165,7 +171,7 @@ class EditCharterTests(TestCase): empty_outbox() r = self.client.post(url, dict(charter_state=str(State.objects.get(used=True,type="charter",slug="intrev").pk), message="test")) self.assertEqual(r.status_code, 302) - self.assertTrue("A new charter" in outbox[-3].get_payload()) + self.assertTrue("A new charter" in get_payload(outbox[-3])) def test_abandon_bof(self): charter = CharterFactory(group__state_id='bof',group__type_id='wg') @@ -389,7 +395,7 @@ class EditCharterTests(TestCase): self.assertEqual(len(q('form input[name=txt]')), 1) # faulty post - test_file = StringIO("\x10\x11\x12") # post binary file + test_file = io.StringIO("\x10\x11\x12") # post binary file test_file.name = "unnamed" r = self.client.post(url, dict(txt=test_file)) @@ -401,7 +407,7 @@ class EditCharterTests(TestCase): latin_1_snippet = b'\xe5' * 10 utf_8_snippet = b'\xc3\xa5' * 10 - test_file = StringIO("Windows line\r\nMac line\rUnix line\n" + latin_1_snippet.decode('latin-1')) + test_file = io.StringIO("Windows line\r\nMac line\rUnix line\n" + latin_1_snippet.decode('latin-1')) test_file.name = "unnamed" r = self.client.post(url, dict(txt=test_file)) @@ -411,9 +417,8 @@ class EditCharterTests(TestCase): self.assertEqual(charter.rev, next_revision(prev_rev)) self.assertTrue("new_revision" in charter.latest_event().type) - with open(os.path.join(self.charter_dir, charter.canonical_name() + "-" + charter.rev + ".txt")) as f: - self.assertEqual(f.read(), - "Windows line\nMac line\nUnix line\n" + utf_8_snippet.decode('utf_8')) + with io.open(os.path.join(self.charter_dir, charter.canonical_name() + "-" + charter.rev + ".txt")) as f: + self.assertEqual(f.read(), "Windows line\nMac line\nUnix line\n" + utf_8_snippet.decode('utf_8')) def test_submit_initial_charter(self): group = GroupFactory(type_id='wg',acronym='mars',list_email='mars-wg@ietf.org') @@ -428,7 +433,7 @@ class EditCharterTests(TestCase): self.assertEqual(len(q('form input[name=txt]')), 1) # create charter - test_file = StringIO("Simple test") + test_file = io.StringIO("Simple test") test_file.name = "unnamed" r = self.client.post(url, dict(txt=test_file)) @@ -658,7 +663,7 @@ class EditCharterTests(TestCase): # self.assertTrue("approved" in outbox[0]['Subject'].lower()) self.assertTrue("iesg-secretary" in outbox[0]['To']) - body = outbox[0].get_payload() + body = get_payload(outbox[0]) for word in ["WG", "/wg/ames/about/", "Charter", "/doc/charter-ietf-ames/", ]: self.assertIn(word, body) @@ -666,7 +671,7 @@ class EditCharterTests(TestCase): self.assertTrue("WG Action" in outbox[1]['Subject']) self.assertTrue("ietf-announce" in outbox[1]['To']) self.assertTrue("ames-wg@ietf.org" in outbox[1]['Cc']) - body = outbox[1].get_payload() + body = get_payload(outbox[1]) for word in ["Chairs", "Ames Man ", "Secretaries", "Secretary ", "Assigned Area Director", "Areað Irector ", diff --git a/ietf/doc/tests_conflict_review.py b/ietf/doc/tests_conflict_review.py index 112623652..eb1c83371 100644 --- a/ietf/doc/tests_conflict_review.py +++ b/ietf/doc/tests_conflict_review.py @@ -1,16 +1,21 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- -import debug # pyflakes:ignore + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import shutil from pyquery import PyQuery -from io import StringIO from textwrap import wrap from django.conf import settings from django.urls import reverse as urlreverse +import debug # pyflakes:ignore + from ietf.doc.factories import IndividualDraftFactory, ConflictReviewFactory from ietf.doc.models import Document, DocEvent, NewRevisionDocEvent, BallotPositionDocEvent, TelechatDocEvent, State from ietf.doc.utils import create_ballot_if_not_open @@ -19,7 +24,7 @@ from ietf.group.models import Person from ietf.iesg.models import TelechatDate from ietf.name.models import StreamName from ietf.utils.test_utils import TestCase -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.mail import outbox, empty_outbox, get_payload from ietf.utils.test_utils import login_testing_unauthorized @@ -299,9 +304,9 @@ class ConflictReviewTests(TestCase): self.assertIn('iana@', outbox[0]['Cc']) if approve_type == 'appr-noprob': - self.assertIn( 'IESG has no problem', ''.join(wrap(outbox[0].get_payload(), 2**16))) + self.assertIn( 'IESG has no problem', ''.join(wrap(get_payload(outbox[0]), 2**16))) else: - self.assertIn( 'NOT be published', ''.join(wrap(outbox[0].get_payload(), 2**16))) + self.assertIn( 'NOT be published', ''.join(wrap(get_payload(outbox[0]), 2**16))) def test_approve_reqnopub(self): @@ -338,7 +343,7 @@ class ConflictReviewSubmitTests(TestCase): self.assertEqual(r.status_code,302) doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission') self.assertEqual(doc.rev,'00') - with open(path) as f: + with io.open(path) as f: self.assertEqual(f.read(),"Some initial review text\n") f.close() self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc) @@ -352,7 +357,7 @@ class ConflictReviewSubmitTests(TestCase): # doc.rev is u'00' per the test setup - double-checking that here - if it fails, the breakage is in setUp self.assertEqual(doc.rev,'00') path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - with open(path,'w') as f: + with io.open(path,'w') as f: f.write('This is the old proposal.') f.close() @@ -365,21 +370,21 @@ class ConflictReviewSubmitTests(TestCase): # faulty posts trying to use file upload # Copied from wgtracker tests - is this really testing the server code, or is it testing # how client.post populates Content-Type? - test_file = StringIO("\x10\x11\x12") # post binary file + test_file = io.StringIO("\x10\x11\x12") # post binary file test_file.name = "unnamed" r = self.client.post(url, dict(txt=test_file,submit_response="1")) self.assertEqual(r.status_code, 200) self.assertContains(r, "does not appear to be a text file") # sane post uploading a file - test_file = StringIO("This is a new proposal.") + test_file = io.StringIO("This is a new proposal.") test_file.name = "unnamed" r = self.client.post(url,dict(txt=test_file,submit_response="1")) self.assertEqual(r.status_code, 302) doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission') self.assertEqual(doc.rev,'01') path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - with open(path) as f: + with io.open(path) as f: self.assertEqual(f.read(),"This is a new proposal.") f.close() self.assertTrue( "submission-01" in doc.latest_event(NewRevisionDocEvent).desc) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index fcc7118cd..a6952ac0d 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import os import shutil import datetime @@ -26,7 +29,7 @@ from ietf.person.models import Person, Email from ietf.meeting.models import Meeting, MeetingTypeName from ietf.iesg.models import TelechatDate from ietf.utils.test_utils import login_testing_unauthorized -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.mail import outbox, empty_outbox, get_payload from ietf.utils.test_utils import TestCase @@ -372,7 +375,7 @@ class EditInfoTests(TestCase): data["telechat_date"] = next_week.isoformat() r = self.client.post(url,data) self.assertEqual(r.status_code, 302) - self.assertTrue("may not leave enough time" in outbox[-1].get_payload()) + self.assertIn("may not leave enough time", get_payload(outbox[-1])) def test_start_iesg_process_on_draft(self): @@ -563,7 +566,7 @@ class ExpireIDsTests(TestCase): settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir def write_draft_file(self, name, size): - f = open(os.path.join(self.id_dir, name), 'w') + f = io.open(os.path.join(self.id_dir, name), 'w') f.write("a" * size) f.close() @@ -1399,9 +1402,9 @@ class ChangeStreamStateTests(TestCase): self.assertEqual(draft.docevent_set.count() - events_before, 2) self.assertEqual(len(outbox), mailbox_before + 1) self.assertTrue("tags changed" in outbox[-1]["Subject"].lower()) - self.assertTrue("mars-chairs@ietf.org" in str(outbox[-1])) - self.assertTrue("marsdelegate@example.org" in str(outbox[-1])) - self.assertTrue("plain@example.com" in str(outbox[-1])) + self.assertTrue("mars-chairs@ietf.org" in outbox[-1].as_string()) + self.assertTrue("marsdelegate@example.org" in outbox[-1].as_string()) + self.assertTrue("plain@example.com" in outbox[-1].as_string()) def test_set_initial_state(self): role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') @@ -1436,8 +1439,8 @@ class ChangeStreamStateTests(TestCase): self.assertTrue(due - datetime.timedelta(days=1) <= reminder[0].due <= due + datetime.timedelta(days=1)) self.assertEqual(len(outbox), 1) self.assertTrue("state changed" in outbox[0]["Subject"].lower()) - self.assertTrue("mars-chairs@ietf.org" in str(outbox[0])) - self.assertTrue("marsdelegate@ietf.org" in str(outbox[0])) + self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string()) + self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string()) def test_set_state(self): role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') @@ -1481,8 +1484,8 @@ class ChangeStreamStateTests(TestCase): self.assertTrue(due - datetime.timedelta(days=1) <= reminder[0].due <= due + datetime.timedelta(days=1)) self.assertEqual(len(outbox), 1) self.assertTrue("state changed" in outbox[0]["Subject"].lower()) - self.assertTrue("mars-chairs@ietf.org" in str(outbox[0])) - self.assertTrue("marsdelegate@ietf.org" in str(outbox[0])) + self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string()) + self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string()) def test_pubreq_validation(self): role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') diff --git a/ietf/doc/tests_material.py b/ietf/doc/tests_material.py index cc414d9d7..47c71f90d 100644 --- a/ietf/doc/tests_material.py +++ b/ietf/doc/tests_material.py @@ -1,10 +1,14 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import os import shutil import datetime -from io import StringIO +import io + from pyquery import PyQuery import debug # pyflakes:ignore @@ -85,7 +89,7 @@ class GroupMaterialTests(TestCase): self.assertEqual(r.status_code, 200) content = "%PDF-1.5\n..." - test_file = StringIO(content) + test_file = io.StringIO(content) test_file.name = "unnamed.pdf" # faulty post @@ -110,7 +114,7 @@ class GroupMaterialTests(TestCase): self.assertEqual(doc.title, "Test File - with fancy title") self.assertEqual(doc.get_state_slug(), "active") - with open(os.path.join(self.materials_dir, "slides", doc.name + "-" + doc.rev + ".pdf")) as f: + with io.open(os.path.join(self.materials_dir, "slides", doc.name + "-" + doc.rev + ".pdf")) as f: self.assertEqual(f.read(), content) # check that posting same name is prevented @@ -165,7 +169,7 @@ class GroupMaterialTests(TestCase): login_testing_unauthorized(self, "secretary", url) content = "some text" - test_file = StringIO(content) + test_file = io.StringIO(content) test_file.name = "unnamed.txt" # post @@ -179,6 +183,6 @@ class GroupMaterialTests(TestCase): self.assertEqual(doc.title, "New title") self.assertEqual(doc.get_state_slug(), "active") - with open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + ".txt")) as f: + with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + ".txt")) as f: self.assertEqual(f.read(), content) diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index de62d9116..1137cf141 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -1,11 +1,14 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime, os, shutil +import io import tarfile, tempfile, mailbox import email.mime.multipart, email.mime.text, email.utils -from io import StringIO from mock import patch from requests import Response @@ -98,8 +101,10 @@ class ReviewTests(TestCase): self.assertEqual(len(outbox),2) self.assertTrue('reviewteam Early' in outbox[0]['Subject']) +# debug.show("outbox[0]['To']") self.assertTrue('reviewsecretary@' in outbox[0]['To']) self.assertTrue('reviewteam3 Early' in outbox[1]['Subject']) +# debug.show("outbox[1]['To']") self.assertTrue('reviewsecretary3@' in outbox[1]['To']) # set the reviewteamsetting for the secretary email alias, then do the post again @@ -557,7 +562,7 @@ class ReviewTests(TestCase): # Test failure to return mailarch results no_result_path = os.path.join(self.review_dir, "mailarch_no_result.html") - with open(no_result_path, "w") as f: + with io.open(no_result_path, "w") as f: f.write('Content-Type: text/html\n\n
No results found
') ietf.review.mailarch.construct_query_urls = lambda review_req, query=None: { "query_data_url": "file://" + os.path.abspath(no_result_path) } @@ -614,7 +619,7 @@ class ReviewTests(TestCase): # complete by uploading file empty_outbox() - test_file = StringIO("This is a review\nwith two lines") + test_file = io.StringIO("This is a review\nwith two lines") test_file.name = "unnamed" r = self.client.post(url, data={ @@ -635,7 +640,7 @@ class ReviewTests(TestCase): self.assertTrue(assignment.review_request.team.acronym.lower() in assignment.review.name) self.assertTrue(assignment.review_request.doc.rev in assignment.review.name) - with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: + with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: self.assertEqual(f.read(), "This is a review\nwith two lines") self.assertEqual(len(outbox), 1) @@ -685,7 +690,7 @@ class ReviewTests(TestCase): self.assertEqual(assignment.state_id, "completed") self.assertNotEqual(assignment.completed_on, None) - with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: + with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: self.assertEqual(f.read(), "This is a review\nwith two lines") self.assertEqual(len(outbox), 1) @@ -772,7 +777,7 @@ class ReviewTests(TestCase): assignment = reload_db_objects(assignment) self.assertEqual(assignment.state_id, "completed") - with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: + with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: self.assertEqual(f.read(), "This is a review\nwith two lines") self.assertEqual(len(outbox), 0) @@ -872,7 +877,7 @@ class ReviewTests(TestCase): event = ReviewAssignmentDocEvent.objects.get(type="closed_review_assignment", review_assignment=assignment) self.assertEqual(event.time, datetime.datetime(2012, 12, 24, 12, 13, 14)) - with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: + with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: self.assertEqual(f.read(), "This is a review\nwith two lines") self.assertEqual(len(outbox), 0) diff --git a/ietf/doc/tests_status_change.py b/ietf/doc/tests_status_change.py index 2665ec702..01dd42457 100644 --- a/ietf/doc/tests_status_change.py +++ b/ietf/doc/tests_status_change.py @@ -1,6 +1,10 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import shutil @@ -421,7 +425,7 @@ class StatusChangeSubmitTests(TestCase): self.assertEqual(r.status_code,302) doc = Document.objects.get(name='status-change-imaginary-mid-review') self.assertEqual(doc.rev,'00') - with open(path) as f: + with io.open(path) as f: self.assertEqual(f.read(),"Some initial review text\n") self.assertTrue( "mid-review-00" in doc.latest_event(NewRevisionDocEvent).desc) @@ -434,7 +438,7 @@ class StatusChangeSubmitTests(TestCase): # doc.rev is u'00' per the test setup - double-checking that here - if it fails, the breakage is in setUp self.assertEqual(doc.rev,'00') path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - with open(path,'w') as f: + with io.open(path,'w') as f: f.write('This is the old proposal.') f.close() # Put the old proposal into IESG review (exercises ballot tab when looking at an older revision below) @@ -466,7 +470,7 @@ class StatusChangeSubmitTests(TestCase): doc = Document.objects.get(name='status-change-imaginary-mid-review') self.assertEqual(doc.rev,'01') path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - with open(path) as f: + with io.open(path) as f: self.assertEqual(f.read(),"This is a new proposal.") f.close() self.assertTrue( "mid-review-01" in doc.latest_event(NewRevisionDocEvent).desc) diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 601efc0d8..91a2e79be 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -1,14 +1,20 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- -import os -import re -import urllib.request, urllib.parse, urllib.error -import math + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import hashlib +import io import json +import math +import os +import re +import six + from collections import defaultdict +from six.moves.urllib.parse import quote from django.conf import settings from django.contrib import messages @@ -313,7 +319,7 @@ def add_links_in_new_revision_events(doc, events, diff_revisions): links += "" if prev != None: - links += ' (diff from previous)' % (settings.RFCDIFF_BASE_URL, urllib.parse.quote(prev, safe="~"), urllib.parse.quote(diff_url, safe="~")) + links += ' (diff from previous)' % (settings.RFCDIFF_BASE_URL, quote(prev, safe="~"), quote(diff_url, safe="~")) # replace the bold filename part e.desc = re.sub(r"(.+-[0-9][0-9].txt)", links, e.desc) @@ -333,7 +339,7 @@ def add_events_message_info(events): def get_unicode_document_content(key, filename, codec='utf-8', errors='ignore'): try: - with open(filename, 'rb') as f: + with io.open(filename, 'rb') as f: raw_content = f.read().decode(codec,errors) except IOError: if settings.DEBUG: @@ -347,7 +353,7 @@ def get_unicode_document_content(key, filename, codec='utf-8', errors='ignore'): def get_document_content(key, filename, split=True, markup=True): log.unreachable("2017-12-05") try: - with open(filename, 'rb') as f: + with io.open(filename, 'rb') as f: raw_content = f.read() except IOError: if settings.DEBUG: @@ -541,7 +547,7 @@ def rebuild_reference_relations(doc,filename=None): filename=os.path.join(settings.INTERNET_DRAFT_PATH,doc.filename_with_rev()) try: - with open(filename, 'rb') as file: + with io.open(filename, 'rb') as file: refs = draft.Draft(file.read().decode('utf8'), filename).get_refs() except IOError as e: return { 'errors': ["%s :%s" % (e.strerror, filename)] } @@ -661,7 +667,7 @@ def get_initial_notify(doc,extra=None): receivers = [] if extra: - if isinstance(extra,str): + if isinstance(extra, six.string_types): extra = extra.split(', ') receivers.extend(extra) @@ -767,7 +773,7 @@ def get_search_cache_key(params): from ietf.doc.views_search import SearchForm fields = set(SearchForm.base_fields) - set(['sort',]) kwargs = dict([ (k,v) for (k,v) in list(params.items()) if k in fields ]) - key = "doc:document:search:" + hashlib.sha512(json.dumps(kwargs, sort_keys=True).encode()).hexdigest() + key = "doc:document:search:" + hashlib.sha512(json.dumps(kwargs, sort_keys=True).encode('utf-8')).hexdigest() return key def label_wrap(label, items, joiner=',', max=50): diff --git a/ietf/doc/utils_charter.py b/ietf/doc/utils_charter.py index beb96864e..3dae431a6 100644 --- a/ietf/doc/utils_charter.py +++ b/ietf/doc/utils_charter.py @@ -1,5 +1,15 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved -import re, datetime, os, shutil +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os +import re +import shutil +import six from django.conf import settings from django.urls import reverse as urlreverse @@ -53,7 +63,7 @@ def next_approved_revision(rev): def read_charter_text(doc): filename = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) try: - with open(filename, 'r') as f: + with io.open(filename, 'r') as f: return f.read() except IOError: return "Error: couldn't read charter text" @@ -142,7 +152,7 @@ def generate_ballot_writeup(request, doc): e.doc = doc e.rev = doc.rev, e.desc = "Ballot writeup was generated" - e.text = str(render_to_string("doc/charter/ballot_writeup.txt")) + e.text = six.ensure_text(render_to_string("doc/charter/ballot_writeup.txt")) # caller is responsible for saving, if necessary return e diff --git a/ietf/doc/views_charter.py b/ietf/doc/views_charter.py index 30e915cf5..2438f4241 100644 --- a/ietf/doc/views_charter.py +++ b/ietf/doc/views_charter.py @@ -1,7 +1,15 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- -import os, datetime, textwrap, json + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import json +import os +import six +import textwrap from django.http import HttpResponseRedirect, HttpResponseNotFound, HttpResponseForbidden, Http404 from django.shortcuts import get_object_or_404, redirect, render @@ -413,7 +421,7 @@ def submit(request, name, option=None): # Save file on disk filename = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (charter.canonical_name(), charter.rev)) - with open(filename, 'w', encoding='utf-8') as destination: + with io.open(filename, 'w', encoding='utf-8') as destination: if form.cleaned_data['txt']: destination.write(form.cleaned_data['txt']) else: @@ -442,7 +450,7 @@ def submit(request, name, option=None): filename = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (charter_canonical_name, charter_rev)) try: - with open(filename, 'r') as f: + with io.open(filename, 'r') as f: init["content"] = f.read() except IOError: pass @@ -807,8 +815,8 @@ def charter_with_milestones_txt(request, name, rev): charter_text = "" try: - with open(os.path.join(settings.CHARTER_PATH, filename), 'r') as f: - charter_text = str(f.read(), errors='ignore') + with io.open(os.path.join(settings.CHARTER_PATH, filename), 'r') as f: + charter_text = six.ensure_text(f.read(), errors='ignore') except IOError: charter_text = "Error reading charter text %s" % filename diff --git a/ietf/doc/views_conflict_review.py b/ietf/doc/views_conflict_review.py index 0b741e6c3..04ed5675c 100644 --- a/ietf/doc/views_conflict_review.py +++ b/ietf/doc/views_conflict_review.py @@ -1,7 +1,12 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- -import datetime, os + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os from django import forms from django.shortcuts import render, get_object_or_404, redirect @@ -159,7 +164,7 @@ class UploadForm(forms.Form): def save(self, review): filename = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (review.canonical_name(), review.rev)) - with open(filename, 'w', encoding='utf-8') as destination: + with io.open(filename, 'w', encoding='utf-8') as destination: if self.cleaned_data['txt']: destination.write(self.cleaned_data['txt']) else: @@ -223,7 +228,7 @@ def submit(request, name): else: filename = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (review.canonical_name(), review.rev)) try: - with open(filename, 'r') as f: + with io.open(filename, 'r') as f: init["content"] = f.read() except IOError: pass diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index f4b4cff20..cbdcfefe7 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1,6 +1,6 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- - +# # Parts Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -33,7 +33,18 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os, datetime, urllib.request, urllib.parse, urllib.error, json, glob, re + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import glob +import io +import json +import os +import re +import six + +from six.moves.urllib.parse import quote from django.http import HttpResponse, Http404 , HttpResponseForbidden from django.shortcuts import render, get_object_or_404, redirect @@ -312,7 +323,7 @@ def document_main(request, name, rev=None): if doc.stream_id == "ietf" and group.type_id == "wg" and group.list_archive: search_archive = group.list_archive - search_archive = urllib.parse.quote(search_archive, safe="~") + search_archive = quote(search_archive, safe="~") # conflict reviews conflict_reviews = [d.document.name for d in doc.related_that("conflrev")] @@ -661,7 +672,7 @@ def check_doc_email_aliases(): pattern = re.compile(r'^expand-(.*?)(\..*?)?@.*? +(.*)$') good_count = 0 tot_count = 0 - with open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file: + with io.open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file: for line in virtual_file.readlines(): m = pattern.match(line) tot_count += 1 @@ -677,7 +688,7 @@ def get_doc_email_aliases(name): else: pattern = re.compile(r'^expand-(.*?)(\..*?)?@.*? +(.*)$') aliases = [] - with open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file: + with io.open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file: for line in virtual_file.readlines(): m = pattern.match(line) if m: @@ -1263,7 +1274,7 @@ def add_sessionpresentation(request,name): if doc.group: sessions = sorted(sessions,key=lambda x:0 if x.group==doc.group else 1) - session_choices = [(s.pk,str(s)) for s in sessions] + session_choices = [(s.pk, six.ensure_text(s)) for s in sessions] if request.method == 'POST': version_form = VersionForm(request.POST,choices=version_choices) diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py index fdd7ab705..29c6174b9 100644 --- a/ietf/doc/views_material.py +++ b/ietf/doc/views_material.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + # views for managing group materials (slides, ...) +import io import os import re @@ -140,7 +144,7 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None): f = form.cleaned_data["material"] file_ext = os.path.splitext(f.name)[1] - with open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + file_ext), 'wb+') as dest: + with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + file_ext), 'wb+') as dest: for chunk in f.chunks(): dest.write(chunk) diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index 1cf5c0510..d4ac64db6 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -2,6 +2,9 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import datetime import requests @@ -625,7 +628,7 @@ def complete_review(request, name, assignment_id): content = form.cleaned_data['review_content'] filename = os.path.join(review.get_file_path(), '{}.txt'.format(review.name, review.rev)) - with open(filename, 'w', encoding='utf-8') as destination: + with io.open(filename, 'w', encoding='utf-8') as destination: destination.write(content) completion_datetime = datetime.datetime.now() diff --git a/ietf/doc/views_status_change.py b/ietf/doc/views_status_change.py index b78aa91e7..14113f5d5 100644 --- a/ietf/doc/views_status_change.py +++ b/ietf/doc/views_status_change.py @@ -1,7 +1,14 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- -import datetime, os, re + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os +import re +import six from django import forms from django.shortcuts import render, get_object_or_404, redirect @@ -124,7 +131,7 @@ class UploadForm(forms.Form): def save(self, doc): filename = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - with open(filename, 'w', encoding='utf-8') as destination: + with io.open(filename, 'w', encoding='utf-8') as destination: if self.cleaned_data['txt']: destination.write(self.cleaned_data['txt']) else: @@ -188,7 +195,7 @@ def submit(request, name): else: filename = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) try: - with open(filename, 'r') as f: + with io.open(filename, 'r') as f: init["content"] = f.read() except IOError: pass @@ -634,7 +641,7 @@ def generate_last_call_text(request, doc): e.doc = doc e.rev = doc.rev e.desc = 'Last call announcement was generated' - e.text = str(new_text) + e.text = six.ensure_text(new_text) e.save() return e diff --git a/ietf/group/models.py b/ietf/group/models.py index ec36e8014..aecbf3287 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -1,19 +1,22 @@ # Copyright The IETF Trust 2010-2019, All Rights Reserved # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals import datetime import email.utils import jsonfield import os import re +import six -from urllib.parse import urljoin +from six.moves.urllib.parse import urljoin from django.conf import settings from django.core.validators import RegexValidator from django.db import models from django.db.models.deletion import CASCADE from django.dispatch import receiver +from django.utils.encoding import python_2_unicode_compatible from simple_history.models import HistoricalRecords @@ -27,6 +30,7 @@ from ietf.utils import log from ietf.utils.models import ForeignKey, OneToOneField +@python_2_unicode_compatible class GroupInfo(models.Model): time = models.DateTimeField(default=datetime.datetime.now) name = models.CharField(max_length=80) @@ -91,7 +95,7 @@ class Group(GroupInfo): return e[0] if e else None def has_role(self, user, role_names): - if isinstance(role_names, str) or isinstance(role_names, str): + if isinstance(role_names, six.string_types): role_names = [role_names] return user.is_authenticated and self.role_set.filter(name__in=role_names, person__user=user).exists() @@ -247,14 +251,16 @@ class GroupHistory(GroupInfo): class Meta: verbose_name_plural="group histories" +@python_2_unicode_compatible class GroupURL(models.Model): group = ForeignKey(Group) name = models.CharField(max_length=255) url = models.URLField() def __str__(self): - return "%s (%s)" % (self.url, self.name) + return u"%s (%s)" % (self.url, self.name) +@python_2_unicode_compatible class GroupMilestoneInfo(models.Model): group = ForeignKey(Group) # a group has two sets of milestones, current milestones @@ -281,6 +287,7 @@ class GroupMilestoneHistory(GroupMilestoneInfo): time = models.DateTimeField() milestone = ForeignKey(GroupMilestone, related_name="history_set") +@python_2_unicode_compatible class GroupStateTransitions(models.Model): """Captures that a group has overriden the default available document state transitions for a certain state.""" @@ -289,7 +296,7 @@ class GroupStateTransitions(models.Model): next_states = models.ManyToManyField('doc.State', related_name='previous_groupstatetransitions_states') def __str__(self): - return '%s "%s" -> %s' % (self.group.acronym, self.state.name, [s.name for s in self.next_states.all()]) + return u'%s "%s" -> %s' % (self.group.acronym, self.state.name, [s.name for s in self.next_states.all()]) GROUP_EVENT_CHOICES = [ ("changed_state", "Changed state"), @@ -301,6 +308,7 @@ GROUP_EVENT_CHOICES = [ ("status_update", "Status update"), ] +@python_2_unicode_compatible class GroupEvent(models.Model): """An occurrence for a group, used for tracking who, when and what.""" group = ForeignKey(Group) @@ -310,7 +318,7 @@ class GroupEvent(models.Model): desc = models.TextField() def __str__(self): - return "%s %s at %s" % (self.by.plain_name(), self.get_type_display().lower(), self.time) + return u"%s %s at %s" % (self.by.plain_name(), self.get_type_display().lower(), self.time) class Meta: ordering = ['-time', 'id'] @@ -321,13 +329,14 @@ class ChangeStateGroupEvent(GroupEvent): class MilestoneGroupEvent(GroupEvent): milestone = ForeignKey(GroupMilestone) +@python_2_unicode_compatible class Role(models.Model): name = ForeignKey(RoleName) group = ForeignKey(Group) person = ForeignKey(Person) email = ForeignKey(Email, help_text="Email address used by person for this role.") def __str__(self): - return "%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym or self.group.name) + return u"%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym or self.group.name) def formatted_ascii_email(self): return email.utils.formataddr((self.person.plain_ascii(), self.email.address)) @@ -338,6 +347,7 @@ class Role(models.Model): class Meta: ordering = ['name_id', ] +@python_2_unicode_compatible class RoleHistory(models.Model): # RoleHistory doesn't have a time field as it's not supposed to be # used on its own - there should always be a GroupHistory @@ -348,7 +358,7 @@ class RoleHistory(models.Model): person = ForeignKey(Person) email = ForeignKey(Email, help_text="Email address used by person for this role.") def __str__(self): - return "%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym) + return u"%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym) class Meta: verbose_name_plural = "role histories" diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 9ee3c0bcb..11c227579 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -1,12 +1,16 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import os import shutil import calendar import datetime import io import bleach +import six from pyquery import PyQuery from tempfile import NamedTemporaryFile @@ -115,7 +119,7 @@ class GroupPagesTests(TestCase): chair = Email.objects.filter(role__group=group, role__name="chair")[0] - with open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: + with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: f.write("This is a charter.") url = urlreverse('ietf.group.views.wg_summary_area', kwargs=dict(group_type="wg")) @@ -229,7 +233,7 @@ class GroupPagesTests(TestCase): group = CharterFactory().group draft = WgDraftFactory(group=group) - with open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: + with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: f.write("This is a charter.") milestone = GroupMilestone.objects.create( @@ -582,7 +586,7 @@ class GroupEditTests(TestCase): self.assertTrue(len(q('form .has-error')) > 0) # edit info - with open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: + with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: f.write("This is a charter.") area = group.parent ad = Person.objects.get(name="Areað Irector") @@ -1264,7 +1268,7 @@ class StatusUpdateTests(TestCase): response = self.client.get(url) self.assertEqual(response.status_code,200) q=PyQuery(response.content) - self.assertTrue(bleach.linkify(escape(event.desc)) in str(q('pre'))) + self.assertTrue(bleach.linkify(escape(event.desc)) in six.ensure_text(q('pre'))) self.assertFalse(q('a#edit_button')) self.client.login(username=chair.person.user.username,password='%s+password'%chair.person.user.username) response = self.client.get(url) diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py index ea8e80e89..4d5f99c2a 100644 --- a/ietf/group/tests_review.py +++ b/ietf/group/tests_review.py @@ -1,8 +1,12 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import debug # pyflakes:ignore +import six from pyquery import PyQuery @@ -260,7 +264,7 @@ class ReviewTests(TestCase): q = PyQuery(r.content) generated_text = q("[name=body]").text() self.assertTrue(review_req1.doc.name in generated_text) - self.assertTrue(str(Person.objects.get(user__username="marschairman")) in generated_text) + self.assertTrue(six.ensure_text(Person.objects.get(user__username="marschairman")) in generated_text) empty_outbox() r = self.client.post(url, { diff --git a/ietf/group/utils.py b/ietf/group/utils.py index 2a6003de3..e6660cccf 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -1,6 +1,10 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os from django.db.models import Q @@ -53,7 +57,7 @@ def get_charter_text(group): filename = os.path.join(c.get_file_path(), "%s-%s.txt" % (c.canonical_name(), c.rev)) try: - with open(filename) as f: + with io.open(filename) as f: return f.read() except IOError: return 'Error Loading Group Charter' diff --git a/ietf/group/views.py b/ietf/group/views.py index 462f6f790..ab15db3df 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Copyright The IETF Trust 2009-2019, All Rights Reserved - - +# # Portion Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -34,12 +33,18 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os -import re + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import itertools +import io import json import math -import itertools -import datetime +import os +import re +import six + from tempfile import mkstemp from collections import OrderedDict, defaultdict from simple_history.utils import update_change_reason @@ -109,7 +114,7 @@ from ietf.doc.models import LastCallDocEvent from ietf.name.models import ReviewAssignmentStateName -from ietf.utils.mail import send_mail_text, parse_preformatted +from ietf.utils.mail import send_mail_text, parse_preformatted, get_payload from ietf.ietfauth.utils import user_is_person from ietf.dbtemplate.models import DBTemplate @@ -204,7 +209,7 @@ def check_group_email_aliases(): pattern = re.compile(r'expand-(.*?)(-\w+)@.*? +(.*)$') tot_count = 0 good_count = 0 - with open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file: + with io.open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file: for line in virtual_file.readlines(): m = pattern.match(line) tot_count += 1 @@ -630,7 +635,7 @@ def get_group_email_aliases(acronym, group_type): pattern = re.compile(r'expand-(.*?)(-\w+)@.*? +(.*)$') aliases = [] - with open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file: + with io.open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file: for line in virtual_file.readlines(): m = pattern.match(line) if m: @@ -691,7 +696,7 @@ def dependencies(request, acronym, group_type=None, output_type="pdf"): dothandle, dotname = mkstemp() os.close(dothandle) - dotfile = open(dotname, "w") + dotfile = io.open(dotname, "w") dotfile.write(make_dot(group)) dotfile.close() @@ -708,7 +713,7 @@ def dependencies(request, acronym, group_type=None, output_type="pdf"): pipe("%s -f -l 10 -o %s %s" % (settings.UNFLATTEN_BINARY, unflatname, dotname)) pipe("%s -T%s -o %s %s" % (settings.DOT_BINARY, output_type, outname, unflatname)) - outhandle = open(outname, "rb") + outhandle = io.open(outname, "rb") out = outhandle.read() outhandle.close() @@ -1443,7 +1448,7 @@ def manage_review_requests(request, acronym, group_type=None, assignment_status= saving = form_action.startswith("save") # check for conflicts - review_requests_dict = { str(r.pk): r for r in review_requests } + review_requests_dict = { six.ensure_text(r.pk): r for r in review_requests } posted_reqs = set(request.POST.getlist("reviewrequest", [])) current_reqs = set(review_requests_dict.keys()) @@ -1599,7 +1604,7 @@ def email_open_review_assignments(request, acronym, group_type=None): (msg,_,_) = parse_preformatted(partial_msg) - body = msg.get_payload() + body = get_payload(msg) subject = msg['Subject'] form = EmailOpenAssignmentsForm(initial={ diff --git a/ietf/idindex/generate_all_id2_txt.py b/ietf/idindex/generate_all_id2_txt.py index edaf12ab4..67fef35d2 100755 --- a/ietf/idindex/generate_all_id2_txt.py +++ b/ietf/idindex/generate_all_id2_txt.py @@ -1,5 +1,7 @@ -# Copyright The IETF Trust 2010-2019, All Rights Reserved #!/usr/bin/env python +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- +# # Portions Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -32,11 +34,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import os +import six os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings") import django django.setup() from ietf.idindex.index import all_id2_txt -print(all_id2_txt().encode('utf-8'), end=' ') +six.print_(all_id2_txt().encode('utf-8'), end=' ') diff --git a/ietf/idindex/generate_all_id_txt.py b/ietf/idindex/generate_all_id_txt.py index e9d5f8cf5..f7d805be9 100755 --- a/ietf/idindex/generate_all_id_txt.py +++ b/ietf/idindex/generate_all_id_txt.py @@ -1,5 +1,6 @@ -# Copyright The IETF Trust 2009-2019, All Rights Reserved #!/usr/bin/env python +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Portions Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -32,11 +33,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import os +import six os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings") import django django.setup() from ietf.idindex.index import all_id_txt -print(all_id_txt().encode("utf-8"), end=' ') +six.print_(all_id_txt().encode("utf-8"), end=' ') diff --git a/ietf/idindex/generate_id_abstracts_txt.py b/ietf/idindex/generate_id_abstracts_txt.py index e21a96303..d93b4b8a7 100755 --- a/ietf/idindex/generate_id_abstracts_txt.py +++ b/ietf/idindex/generate_id_abstracts_txt.py @@ -1,5 +1,6 @@ -# Copyright The IETF Trust 2009-2019, All Rights Reserved #!/usr/bin/env python +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Portions Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -32,11 +33,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import os +import six os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings") import django django.setup() from ietf.idindex.index import id_index_txt -print(id_index_txt(with_abstracts=True).encode('utf-8'), end=' ') +six.print_(id_index_txt(with_abstracts=True).encode('utf-8'), end=' ') diff --git a/ietf/idindex/generate_id_index_txt.py b/ietf/idindex/generate_id_index_txt.py index 7fe969a45..c201a9cb7 100755 --- a/ietf/idindex/generate_id_index_txt.py +++ b/ietf/idindex/generate_id_index_txt.py @@ -1,5 +1,6 @@ -# Copyright The IETF Trust 2009-2019, All Rights Reserved #!/usr/bin/env python +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Portions Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -32,11 +33,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import os +import six os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings") import django django.setup() from ietf.idindex.index import id_index_txt -print(id_index_txt().encode('utf-8'), end=' ') +six.print_(id_index_txt().encode('utf-8'), end=' ') diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py index 0291eda09..b79308952 100644 --- a/ietf/idindex/index.py +++ b/ietf/idindex/index.py @@ -1,22 +1,26 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + # code to generate plain-text index files that are placed on # www.ietf.org in the same directory as the I-Ds -import datetime, os - -import debug # pyflakes:ignore - +import datetime +import os import pytz +import six from django.conf import settings from django.template.loader import render_to_string -from ietf.doc.templatetags.ietf_filters import clean_whitespace +import debug # pyflakes:ignore + from ietf.doc.models import Document, DocEvent, DocumentAuthor, RelatedDocument, DocAlias, State from ietf.doc.models import LastCallDocEvent, NewRevisionDocEvent from ietf.doc.models import IESG_SUBSTATE_TAGS +from ietf.doc.templatetags.ietf_filters import clean_whitespace from ietf.group.models import Group from ietf.person.models import Person, Email from ietf.utils import log @@ -191,7 +195,7 @@ def all_id2_txt(): area = d.group.parent.acronym fields.append(area) # 9 responsible AD name - fields.append(str(d.ad) if d.ad else "") + fields.append(six.ensure_text(d.ad) if d.ad else "") # 10 fields.append(d.intended_std_level.name if d.intended_std_level else "") # 11 diff --git a/ietf/idindex/tests.py b/ietf/idindex/tests.py index 10144d42c..c073fbe0b 100644 --- a/ietf/idindex/tests.py +++ b/ietf/idindex/tests.py @@ -1,14 +1,19 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- -import os -import datetime -import shutil -import debug # pyflakes:ignore +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os +import shutil +import six from django.conf import settings +import debug # pyflakes:ignore + from ietf.doc.factories import WgDraftFactory from ietf.doc.models import Document, DocAlias, RelatedDocument, State, LastCallDocEvent, NewRevisionDocEvent from ietf.group.factories import GroupFactory @@ -28,7 +33,7 @@ class IndexTests(TestCase): shutil.rmtree(self.id_dir) def write_draft_file(self, name, size): - with open(os.path.join(self.id_dir, name), 'w') as f: + with io.open(os.path.join(self.id_dir, name), 'w') as f: f.write("a" * size) def test_all_id_txt(self): @@ -97,7 +102,7 @@ class IndexTests(TestCase): self.assertEqual(t[6], draft.latest_event(type="new_revision").time.strftime("%Y-%m-%d")) self.assertEqual(t[7], draft.group.acronym) self.assertEqual(t[8], draft.group.parent.acronym) - self.assertEqual(t[9], str(draft.ad)) + self.assertEqual(t[9], six.ensure_text(draft.ad)) self.assertEqual(t[10], draft.intended_std_level.name) self.assertEqual(t[11], "") self.assertEqual(t[12], ".pdf,.txt") diff --git a/ietf/iesg/agenda.py b/ietf/iesg/agenda.py index 15f9b0f5f..381d3fa65 100644 --- a/ietf/iesg/agenda.py +++ b/ietf/iesg/agenda.py @@ -1,9 +1,12 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + # utilities for constructing agendas for IESG telechats -import codecs +import io import datetime from collections import OrderedDict @@ -146,7 +149,7 @@ def fill_in_agenda_administrivia(date, sections): for s, key, filename in extra_info_files: try: - with codecs.open(filename, 'r', 'utf-8', 'replace') as f: + with io.open(filename, 'r', encoding='utf-8', errors='replace') as f: t = f.read().strip() except IOError: t = "(Error reading %s)" % filename diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py index c18a35687..3735f3005 100644 --- a/ietf/iesg/tests.py +++ b/ietf/iesg/tests.py @@ -1,9 +1,15 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io import os import shutil -import datetime +import tarfile + from pyquery import PyQuery from django.conf import settings @@ -448,15 +454,13 @@ class IESGAgendaTests(TestCase): d1_filename = "%s-%s.txt" % (d1.name, d1.rev) d2_filename = "%s-%s.txt" % (d2.name, d2.rev) - with open(os.path.join(self.draft_dir, d1_filename), "w") as f: + with io.open(os.path.join(self.draft_dir, d1_filename), "w") as f: f.write("test content") url = urlreverse("ietf.iesg.views.telechat_docs_tarfile", kwargs=dict(date=get_agenda_date().isoformat())) r = self.client.get(url) self.assertEqual(r.status_code, 200) - import tarfile, io - tar = tarfile.open(None, fileobj=io.BytesIO(r.content)) names = tar.getnames() self.assertIn(d1_filename, names) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index ce478b7b1..35b9d0b96 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -1,5 +1,6 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved - +# -*- coding: utf-8 -*- +# # Portion Copyright (C) 2008-2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -32,16 +33,17 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os + +from __future__ import absolute_import, print_function, unicode_literals + import datetime -import tarfile import io -import time import itertools import json - -import debug # pyflakes:ignore - +import os +import six +import tarfile +import time from django import forms from django.conf import settings @@ -53,6 +55,8 @@ from django.utils.encoding import force_bytes #from django.views.decorators.cache import cache_page #from django.views.decorators.vary import vary_on_cookie +import debug # pyflakes:ignore + from ietf.doc.models import Document, State, LastCallDocEvent, ConsensusDocEvent, DocEvent, IESG_BALLOT_ACTIVE_STATES from ietf.doc.utils import update_telechat, augment_events_with_revision from ietf.group.models import GroupMilestone, Role @@ -312,9 +316,9 @@ def agenda_documents_txt(request): row = ( d.computed_telechat_date.isoformat(), d.name, - str(d.intended_std_level), + six.ensure_text(d.intended_std_level), "1" if d.stream_id in ("ise", "irtf") else "0", - str(d.area_acronym()).lower(), + six.ensure_text(d.area_acronym()).lower(), d.ad.plain_name() if d.ad else "None Assigned", d.rev, ) diff --git a/ietf/ietfauth/htpasswd.py b/ietf/ietfauth/htpasswd.py index 7474d7592..cc42a5d8a 100644 --- a/ietf/ietfauth/htpasswd.py +++ b/ietf/ietfauth/htpasswd.py @@ -1,5 +1,10 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import subprocess, hashlib from django.utils.encoding import force_bytes @@ -11,7 +16,7 @@ def update_htpasswd_file(username, password): realm = settings.HTDIGEST_REALM prefix = force_bytes('%s:%s:' % (username, realm)) key = force_bytes(hashlib.md5(prefix + force_bytes(password)).hexdigest()) - f = open(pass_file, 'r+b') + f = io.open(pass_file, 'r+b') pos = f.tell() line = f.readline() while line: diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py index 99ae23c59..ea7eea7e8 100644 --- a/ietf/ietfauth/tests.py +++ b/ietf/ietfauth/tests.py @@ -2,8 +2,11 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import io import os, shutil, time, datetime -from urllib.parse import urlsplit +from six.moves.urllib.parse import urlsplit from pyquery import PyQuery from unittest import skipIf @@ -46,7 +49,7 @@ class IetfAuthTests(TestCase): self.saved_htpasswd_file = settings.HTPASSWD_FILE self.htpasswd_dir = self.tempdir('htpasswd') settings.HTPASSWD_FILE = os.path.join(self.htpasswd_dir, "htpasswd") - open(settings.HTPASSWD_FILE, 'a').close() # create empty file + io.open(settings.HTPASSWD_FILE, 'a').close() # create empty file self.saved_htdigest_realm = getattr(settings, "HTDIGEST_REALM", None) settings.HTDIGEST_REALM = "test-realm" @@ -109,11 +112,11 @@ class IetfAuthTests(TestCase): return confirm_url def username_in_htpasswd_file(self, username): - with open(settings.HTPASSWD_FILE) as f: + with io.open(settings.HTPASSWD_FILE) as f: for l in f: if l.startswith(username + ":"): return True - with open(settings.HTPASSWD_FILE) as f: + with io.open(settings.HTPASSWD_FILE) as f: print(f.read()) return False @@ -581,31 +584,30 @@ class IetfAuthTests(TestCase): self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index')) for key in person.apikeys.all()[:3]: - url = key.endpoint # bad method - r = self.client.put(url, {'apikey':key.hash()}) + r = self.client.put(key.endpoint, {'apikey':key.hash()}) self.assertEqual(r.status_code, 405) # missing apikey - r = self.client.post(url, {'dummy':'dummy',}) + r = self.client.post(key.endpoint, {'dummy':'dummy',}) self.assertContains(r, 'Missing apikey parameter', status_code=400) # invalid apikey - r = self.client.post(url, {'apikey':BAD_KEY, 'dummy':'dummy',}) + r = self.client.post(key.endpoint, {'apikey':BAD_KEY, 'dummy':'dummy',}) self.assertContains(r, 'Invalid apikey', status_code=400) # too long since regular login person.user.last_login = datetime.datetime.now() - datetime.timedelta(days=settings.UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS+1) person.user.save() - r = self.client.post(url, {'apikey':key.hash(), 'dummy':'dummy',}) + r = self.client.post(key.endpoint, {'apikey':key.hash(), 'dummy':'dummy',}) self.assertContains(r, 'Too long since last regular login', status_code=400) person.user.last_login = datetime.datetime.now() person.user.save() # endpoint mismatch key2 = PersonalApiKey.objects.create(person=person, endpoint='/') - r = self.client.post(url, {'apikey':key2.hash(), 'dummy':'dummy',}) + r = self.client.post(key.endpoint, {'apikey':key2.hash(), 'dummy':'dummy',}) self.assertContains(r, 'Apikey endpoint mismatch', status_code=400) key2.delete() @@ -632,8 +634,7 @@ class IetfAuthTests(TestCase): time.sleep(2) for i in range(count): for key in person.apikeys.all(): - url = key.endpoint - self.client.post(url, {'apikey':key.hash(), 'dummy': 'dummy', }) + self.client.post(key.endpoint, {'apikey':key.hash(), 'dummy': 'dummy', }) date = str(datetime.date.today()) empty_outbox() diff --git a/ietf/ipr/feeds.py b/ietf/ipr/feeds.py index 7d1c8ac81..7482abf5e 100644 --- a/ietf/ipr/feeds.py +++ b/ietf/ipr/feeds.py @@ -1,4 +1,10 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import six from django.contrib.syndication.views import Feed from django.utils.feedgenerator import Atom1Feed @@ -22,7 +28,7 @@ class LatestIprDisclosuresFeed(Feed): return mark_safe(item.title) def item_description(self, item): - return str(item.title) + return six.ensure_text(item.title) def item_pubdate(self, item): return item.time diff --git a/ietf/ipr/fields.py b/ietf/ipr/fields.py index 775205772..d2a24268c 100644 --- a/ietf/ipr/fields.py +++ b/ietf/ipr/fields.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import json +import six from django.utils.html import escape from django import forms @@ -48,7 +52,7 @@ class SearchableIprDisclosuresField(forms.CharField): def prepare_value(self, value): if not value: value = "" - if isinstance(value, str): + if isinstance(value, six.string_types): pks = self.parse_select2_value(value) # if the user posted a non integer value we need to remove it for key in pks: @@ -64,7 +68,7 @@ class SearchableIprDisclosuresField(forms.CharField): # patterns may not have been fully constructed there yet self.widget.attrs["data-ajax-url"] = urlreverse('ietf.ipr.views.ajax_search') - return ",".join(str(e.pk) for e in value) + return ",".join(six.ensure_text(e.pk) for e in value) def clean(self, value): value = super(SearchableIprDisclosuresField, self).clean(value) diff --git a/ietf/ipr/management/commands/process_email.py b/ietf/ipr/management/commands/process_email.py index c307ebc8f..0bb200ff6 100644 --- a/ietf/ipr/management/commands/process_email.py +++ b/ietf/ipr/management/commands/process_email.py @@ -1,4 +1,10 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import sys from django.core.management.base import BaseCommand, CommandError @@ -20,7 +26,7 @@ class Command(BaseCommand): if not email: msg = sys.stdin.read() else: - msg = open(email, "r").read() + msg = io.open(email, "r").read() try: process_response_email(msg) diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py index c7683cad7..38186752c 100644 --- a/ietf/ipr/tests.py +++ b/ietf/ipr/tests.py @@ -1,10 +1,14 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime -import urllib.request, urllib.parse, urllib.error + from pyquery import PyQuery +from six.moves.urllib.parse import quote from django.urls import reverse as urlreverse @@ -172,11 +176,11 @@ class IprTests(TestCase): self.assertContains(r, ipr.title) # find by doc title - r = self.client.get(url + "?submit=doctitle&doctitle=%s" % urllib.parse.quote(draft.title)) + r = self.client.get(url + "?submit=doctitle&doctitle=%s" % quote(draft.title)) self.assertContains(r, ipr.title) # find by ipr title - r = self.client.get(url + "?submit=iprtitle&iprtitle=%s" % urllib.parse.quote(ipr.title)) + r = self.client.get(url + "?submit=iprtitle&iprtitle=%s" % quote(ipr.title)) self.assertContains(r, ipr.title) def test_feed(self): diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index 32f9bc4cd..607b609ad 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -1,8 +1,12 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import itertools +import six from django.conf import settings from django.contrib import messages @@ -452,7 +456,7 @@ def by_draft_txt(request): lines = [ "# Machine-readable list of IPR disclosures by draft name" ] for name, iprs in docipr.items(): - lines.append(name + "\t" + "\t".join(str(ipr_id) for ipr_id in sorted(iprs))) + lines.append(name + "\t" + "\t".join(six.ensure_text(ipr_id) for ipr_id in sorted(iprs))) return HttpResponse("\n".join(lines), content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET) @@ -474,7 +478,7 @@ def by_draft_recursive_txt(request): lines = [ "# Machine-readable list of IPR disclosures by draft name" ] for name, iprs in docipr.items(): - lines.append(name + "\t" + "\t".join(str(ipr_id) for ipr_id in sorted(iprs))) + lines.append(name + "\t" + "\t".join(six.ensure_text(ipr_id) for ipr_id in sorted(iprs))) return HttpResponse("\n".join(lines), content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET) diff --git a/ietf/liaisons/fields.py b/ietf/liaisons/fields.py index 94f58bff9..b70572ff8 100644 --- a/ietf/liaisons/fields.py +++ b/ietf/liaisons/fields.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import json +import six from django.utils.html import escape from django import forms @@ -47,9 +51,9 @@ class SearchableLiaisonStatementsField(forms.CharField): def prepare_value(self, value): if not value: value = "" - if isinstance(value, int): + if isinstance(value, six.integer_types): value = str(value) - if isinstance(value, str): + if isinstance(value, six.string_types): pks = self.parse_select2_value(value) value = self.model.objects.filter(pk__in=pks) if isinstance(value, LiaisonStatement): @@ -61,7 +65,7 @@ class SearchableLiaisonStatementsField(forms.CharField): # patterns may not have been fully constructed there yet self.widget.attrs["data-ajax-url"] = urlreverse("ietf.liaisons.views.ajax_select2_search_liaison_statements") - return ",".join(str(o.pk) for o in value) + return ",".join(six.ensure_text(o.pk) for o in value) def clean(self, value): value = super(SearchableLiaisonStatementsField, self).clean(value) diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index f997e5f3c..eeb128a0c 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -1,8 +1,14 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import datetime, os import operator +import six + from email.utils import parseaddr from form_utils.forms import BetterModelForm @@ -198,7 +204,7 @@ class CustomModelMultipleChoiceField(forms.ModelMultipleChoiceField): if isinstance(value, QuerySet): return value if (hasattr(value, '__iter__') and - not isinstance(value, str) and + not isinstance(value, six.text_type) and not hasattr(value, '_meta')): return [super(CustomModelMultipleChoiceField, self).prepare_value(v) for v in value] return super(CustomModelMultipleChoiceField, self).prepare_value(value) @@ -375,7 +381,7 @@ class LiaisonModelForm(BetterModelForm): if created: DocAlias.objects.create(name=attach.name).docs.add(attach) LiaisonStatementAttachment.objects.create(statement=self.instance,document=attach) - attach_file = open(os.path.join(settings.LIAISON_ATTACH_PATH, attach.name + extension), 'wb') + attach_file = io.open(os.path.join(settings.LIAISON_ATTACH_PATH, attach.name + extension), 'wb') attach_file.write(attached_file.read()) attach_file.close() @@ -446,7 +452,7 @@ class IncomingLiaisonForm(LiaisonModelForm): self.fields['from_contact'].initial = self.person.role_set.filter(group=queryset[0]).first().email.address self.fields['from_contact'].widget.attrs['readonly'] = True self.fields['from_groups'].queryset = queryset - self.fields['from_groups'].widget.submitter = str(self.person) + self.fields['from_groups'].widget.submitter = six.ensure_text(self.person) # if there's only one possibility make it the default if len(queryset) == 1: diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index 3d56cccf0..c5cd80b27 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -1,5 +1,14 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved -import datetime, os, shutil +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os +import shutil +import six import debug # pyflakes:ignore @@ -232,7 +241,7 @@ class ManagementCommandTests(TestCase): LiaisonStatementFactory(deadline=datetime.date.today()+datetime.timedelta(days=1)) - out = StringIO() + out = six.StringIO() mailbox_before = len(outbox) call_command('check_liaison_deadlines',stdout=out) self.assertEqual(len(outbox), mailbox_before + 1) @@ -242,7 +251,7 @@ class ManagementCommandTests(TestCase): RoleFactory(name_id='liaiman',group__type_id='sdo') - out = StringIO() + out = six.StringIO() mailbox_before = len(outbox) call_command('remind_update_sdo_list',stdout=out) self.assertTrue(len(outbox) > mailbox_before) @@ -432,7 +441,7 @@ class LiaisonManagementTests(TestCase): self.assertEqual(new_liaison.attachments.count(), attachments_before + 1) attachment = new_liaison.attachments.order_by("-name")[0] self.assertEqual(attachment.title, "attachment") - with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: + with io.open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: written_content = f.read() test_file.seek(0) @@ -736,7 +745,7 @@ class LiaisonManagementTests(TestCase): self.assertEqual(l.attachments.count(), 1) attachment = l.attachments.all()[0] self.assertEqual(attachment.title, "attachment") - with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: + with io.open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: written_content = f.read() test_file.seek(0) @@ -815,7 +824,7 @@ class LiaisonManagementTests(TestCase): self.assertEqual(l.attachments.count(), 1) attachment = l.attachments.all()[0] self.assertEqual(attachment.title, "attachment") - with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: + with io.open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: written_content = f.read() test_file.seek(0) diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index a50965bec..8cea55790 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -1,8 +1,11 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os -import codecs import datetime from django import forms @@ -287,7 +290,7 @@ class InterimSessionModelForm(forms.ModelForm): directory = os.path.dirname(path) if not os.path.exists(directory): os.makedirs(directory) - with codecs.open(path, "w", encoding='utf-8') as file: + with io.open(path, "w", encoding='utf-8') as file: file.write(self.cleaned_data['agenda']) diff --git a/ietf/meeting/helpers.py b/ietf/meeting/helpers.py index fde7887af..50cc4f041 100644 --- a/ietf/meeting/helpers.py +++ b/ietf/meeting/helpers.py @@ -1,6 +1,11 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import datetime +import io import os import re from tempfile import mkstemp @@ -211,7 +216,7 @@ def read_session_file(type, num, doc): # FIXME: uploaded_filename should be replaced with a function call that computes names that are fixed path = os.path.join(settings.AGENDA_PATH, "%s/%s/%s" % (num, type, doc.uploaded_filename)) if os.path.exists(path): - with open(path) as f: + with io.open(path) as f: return f.read(), path else: return None, path @@ -224,13 +229,13 @@ def convert_draft_to_pdf(doc_name): outpath = os.path.join(settings.INTERNET_DRAFT_PDF_PATH, doc_name + ".pdf") try: - infile = open(inpath, "r") + infile = io.open(inpath, "r") except IOError: return t,tempname = mkstemp() os.close(t) - tempfile = open(tempname, "w") + tempfile = io.open(tempname, "w") pageend = 0; newpage = 0; diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index a84e51db0..464b2ae15 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -1,11 +1,13 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals # old meeting models can be found in ../proceedings/models.py import pytz import datetime -from urllib.parse import urljoin +import io +from six.moves.urllib.parse import urljoin import os import re import string @@ -19,6 +21,7 @@ from django.conf import settings # mostly used by json_dict() #from django.template.defaultfilters import slugify, date as date_format, time as time_format from django.template.defaultfilters import date as date_format +from django.utils.encoding import python_2_unicode_compatible from django.utils.text import slugify from ietf.dbtemplate.models import DBTemplate @@ -54,6 +57,7 @@ def fmt_date(o): d = datetime_safe.new_date(o) return d.strftime(DATE_FORMAT) +@python_2_unicode_compatible class Meeting(models.Model): # number is either the number for IETF meetings, or some other # identifier for interim meetings/IESG retreats/liaison summits/... @@ -109,7 +113,7 @@ class Meeting(models.Model): def __str__(self): if self.type_id == "ietf": - return "IETF-%s" % (self.number) + return u"IETF-%s" % (self.number) else: return self.number @@ -276,7 +280,7 @@ class Meeting(models.Model): try: tzfn = os.path.join(settings.TZDATA_ICS_PATH, self.time_zone + ".ics") if os.path.exists(tzfn): - with open(tzfn) as tzf: + with io.open(tzfn) as tzf: icstext = tzf.read() vtimezone = re.search("(?sm)(\nBEGIN:VTIMEZONE.*\nEND:VTIMEZONE\n)", icstext).group(1).strip() if vtimezone: @@ -310,6 +314,7 @@ class Meeting(models.Model): # === Rooms, Resources, Floorplans ============================================= +@python_2_unicode_compatible class ResourceAssociation(models.Model): name = ForeignKey(RoomResourceName) icon = models.CharField(max_length=64) # icon to be found in /static/img @@ -326,6 +331,7 @@ class ResourceAssociation(models.Model): res1['resource_id'] = self.pk return res1 +@python_2_unicode_compatible class Room(models.Model): meeting = ForeignKey(Meeting) modified = models.DateTimeField(auto_now=True) @@ -345,7 +351,7 @@ class Room(models.Model): # end floorplan-related stuff def __str__(self): - return "%s size: %s" % (self.name, self.capacity) + return u"%s size: %s" % (self.name, self.capacity) def delete_timeslots(self): for ts in self.timeslot_set.all(): @@ -415,6 +421,7 @@ def floorplan_path(instance, filename): root, ext = os.path.splitext(filename) return "%s/floorplan-%s-%s%s" % (settings.FLOORPLAN_MEDIA_DIR, instance.meeting.number, xslugify(instance.name), ext) +@python_2_unicode_compatible class FloorPlan(models.Model): name = models.CharField(max_length=255) short = models.CharField(max_length=3, default='') @@ -427,10 +434,11 @@ class FloorPlan(models.Model): ordering = ['-id',] # def __str__(self): - return 'floorplan-%s-%s' % (self.meeting.number, xslugify(self.name)) + return u'floorplan-%s-%s' % (self.meeting.number, xslugify(self.name)) # === Schedules, Sessions, Timeslots and Assignments =========================== +@python_2_unicode_compatible class TimeSlot(models.Model): """ Everything that would appear on the meeting agenda of a meeting is @@ -475,9 +483,9 @@ class TimeSlot(models.Model): def __str__(self): location = self.get_location() if not location: - location = "(no location)" + location = u"(no location)" - return "%s: %s-%s %s, %s" % (self.meeting.number, self.time.strftime("%m-%d %H:%M"), (self.time + self.duration).strftime("%H:%M"), self.name, location) + return u"%s: %s-%s %s, %s" % (self.meeting.number, self.time.strftime("%m-%d %H:%M"), (self.time + self.duration).strftime("%H:%M"), self.name, location) def end_time(self): return self.time + self.duration @@ -598,6 +606,7 @@ class TimeSlot(models.Model): # end of TimeSlot +@python_2_unicode_compatible class Schedule(models.Model): """ Each person may have multiple agendas saved. @@ -617,7 +626,7 @@ class Schedule(models.Model): # considering copiedFrom = ForeignKey('Schedule', blank=True, null=True) def __str__(self): - return "%s:%s(%s)" % (self.meeting, self.name, self.owner) + return u"%s:%s(%s)" % (self.meeting, self.name, self.owner) def base_url(self): return "/meeting/%s/agenda/%s/%s" % (self.meeting.number, self.owner_email(), self.name) @@ -707,6 +716,7 @@ class Schedule(models.Model): self.delete() # to be renamed SchedTimeSessAssignments (stsa) +@python_2_unicode_compatible class SchedTimeSessAssignment(models.Model): """ This model provides an N:M relationship between Session and TimeSlot. @@ -726,7 +736,7 @@ class SchedTimeSessAssignment(models.Model): ordering = ["timeslot__time", "timeslot__type__slug", "session__group__parent__name", "session__group__acronym", "session__name", ] def __str__(self): - return "%s [%s<->%s]" % (self.schedule, self.session, self.timeslot) + return u"%s [%s<->%s]" % (self.schedule, self.session, self.timeslot) @property def room_name(self): @@ -809,6 +819,7 @@ class SchedTimeSessAssignment(models.Model): return "-".join(components).lower() +@python_2_unicode_compatible class Constraint(models.Model): """ Specifies a constraint on the scheduling. @@ -829,7 +840,7 @@ class Constraint(models.Model): active_status = None def __str__(self): - return "%s %s target=%s person=%s" % (self.source, self.name.name.lower(), self.target, self.person) + return u"%s %s target=%s person=%s" % (self.source, self.name.name.lower(), self.target, self.person) def brief_display(self): if self.target and self.person: @@ -857,6 +868,7 @@ class Constraint(models.Model): return ct1 +@python_2_unicode_compatible class SessionPresentation(models.Model): session = ForeignKey('Session') document = ForeignKey(Document) @@ -869,11 +881,12 @@ class SessionPresentation(models.Model): unique_together = (('session', 'document'),) def __str__(self): - return "%s -> %s-%s" % (self.session, self.document.name, self.rev) + return u"%s -> %s-%s" % (self.session, self.document.name, self.rev) constraint_cache_uses = 0 constraint_cache_initials = 0 +@python_2_unicode_compatible class Session(models.Model): """Session records that a group should have a session on the meeting (time and location is stored in a TimeSlot) - if multiple @@ -1113,7 +1126,7 @@ class Session(models.Model): if doc: path = os.path.join(settings.AGENDA_PATH, self.meeting.number, "agenda", doc.uploaded_filename) if os.path.exists(path): - with open(path) as f: + with io.open(path) as f: return f.read() else: return "No agenda file found" @@ -1147,6 +1160,7 @@ class Session(models.Model): else: return self.group.acronym +@python_2_unicode_compatible class ImportantDate(models.Model): meeting = ForeignKey(Meeting) date = models.DateField() @@ -1155,7 +1169,7 @@ class ImportantDate(models.Model): ordering = ["-meeting_id","date", ] def __str__(self): - return '%s : %s : %s' % ( self.meeting, self.name, self.date ) + return u'%s : %s : %s' % ( self.meeting, self.name, self.date ) class SlideSubmission(models.Model): session = ForeignKey(Session) diff --git a/ietf/meeting/tests_api.py b/ietf/meeting/tests_api.py index 51f5c1ae1..dc9b788bf 100644 --- a/ietf/meeting/tests_api.py +++ b/ietf/meeting/tests_api.py @@ -1,8 +1,12 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime -from urllib.parse import urlsplit + +from six.moves.urllib.parse import urlsplit from django.urls import reverse as urlreverse diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index d81a6e062..a9a296923 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -1,17 +1,22 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- -import os -import shutil + +from __future__ import absolute_import, print_function, unicode_literals + import datetime -import urllib.parse +import io +import os import random +import shutil +import six from unittest import skipIf from mock import patch from pyquery import PyQuery from io import StringIO, BytesIO from bs4 import BeautifulSoup +from six.moves.urllib.parse import urlparse from django.urls import reverse as urlreverse from django.conf import settings @@ -68,7 +73,7 @@ class MeetingTests(TestCase): if not os.path.exists(dirname): os.makedirs(dirname) - with open(path, "w") as f: + with io.open(path, "w") as f: f.write(content) def write_materials_files(self, meeting, session): @@ -381,10 +386,9 @@ class MeetingTests(TestCase): response = self.client.get(url) self.assertContains(response, 'test acknowledgements') - - @patch('urllib.request.urlopen') + @patch('six.moves.urllib.request.urlopen') def test_proceedings_attendees(self, mock_urlopen): - mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') + mock_urlopen.return_value = six.BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') make_meeting_test_data() meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96") finalize(meeting) @@ -394,12 +398,12 @@ class MeetingTests(TestCase): q = PyQuery(response.content) self.assertEqual(1,len(q("#id_attendees tbody tr"))) - @patch('urllib.request.urlopen') + @patch('six.moves.urllib.request.urlopen') def test_proceedings_overview(self, mock_urlopen): '''Test proceedings IETF Overview page. Note: old meetings aren't supported so need to add a new meeting then test. ''' - mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') + mock_urlopen.return_value = six.BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') make_meeting_test_data() meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96") finalize(meeting) @@ -476,7 +480,7 @@ class MeetingTests(TestCase): session.sessionpresentation_set.create(document=doc) file,_ = submission_file(name=doc.name,format='txt',templatename='test_submission.txt',group=session.group,rev="00") filename = os.path.join(doc.get_file_path(),file.name) - with open(filename,'w') as draftbits: + with io.open(filename,'w') as draftbits: draftbits.write(file.getvalue()) url = urlreverse('ietf.meeting.views.session_draft_tarfile', kwargs={'num':session.meeting.number,'acronym':session.group.acronym}) @@ -492,7 +496,7 @@ class MeetingTests(TestCase): session.sessionpresentation_set.create(document=doc) file,_ = submission_file(name=doc.name,format='txt',templatename='test_submission.txt',group=session.group,rev="00") filename = os.path.join(doc.get_file_path(),file.name) - with open(filename,'w') as draftbits: + with io.open(filename,'w') as draftbits: draftbits.write(file.getvalue()) url = urlreverse('ietf.meeting.views.session_draft_pdf', kwargs={'num':session.meeting.number,'acronym':session.group.acronym}) @@ -590,7 +594,7 @@ class EditTests(TestCase): }) self.assertEqual(r.status_code, 302) # Verify that we actually got redirected to a new place. - self.assertNotEqual(urllib.parse.urlparse(r.url).path, url) + self.assertNotEqual(urlparse(r.url).path, url) # get schedule = meeting.get_schedule_by_name("foo") @@ -636,7 +640,7 @@ class EditTests(TestCase): 'saveas': "saveas", }) self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r.url).path, url) + self.assertEqual(urlparse(r.url).path, url) # TODO: Verify that an error message was in fact returned. r = self.client.post(url, { @@ -645,7 +649,7 @@ class EditTests(TestCase): }) # TODO: Verify that an error message was in fact returned. self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r.url).path, url) + self.assertEqual(urlparse(r.url).path, url) # Non-ASCII alphanumeric characters r = self.client.post(url, { @@ -654,7 +658,7 @@ class EditTests(TestCase): }) # TODO: Verify that an error message was in fact returned. self.assertEqual(r.status_code, 302) - self.assertEqual(urllib.parse.urlparse(r.url).path, url) + self.assertEqual(urlparse(r.url).path, url) def test_edit_timeslots(self): @@ -1661,7 +1665,7 @@ class IphoneAppJsonTests(TestCase): self.assertEqual(r.status_code,200) class FinalizeProceedingsTests(TestCase): - @patch('urllib.request.urlopen') + @patch('six.moves.urllib.request.urlopen') def test_finalize_proceedings(self, mock_urlopen): mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') make_meeting_test_data() @@ -1711,7 +1715,7 @@ class MaterialsTests(TestCase): soup = BeautifulSoup(page, 'html.parser') for a in soup('a'): href = a.get('href') - path = urllib.parse.urlparse(href).path + path = urlparse(href).path if (path and path not in seen and path.startswith(top)): follow(path) follow(url) @@ -1723,7 +1727,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn('Upload', str(q("title"))) + self.assertIn('Upload', six.ensure_text(q("title"))) self.assertFalse(session.sessionpresentation_set.exists()) test_file = StringIO('%PDF-1.4\n%âãÏÓ\nthis is some text for a test') test_file.name = "not_really.pdf" @@ -1734,7 +1738,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn('Revise', str(q("title"))) + self.assertIn('Revise', six.ensure_text(q("title"))) test_file = StringIO('%PDF-1.4\n%âãÏÓ\nthis is some different text for a test') test_file.name = "also_not_really.pdf" r = self.client.post(url,dict(file=test_file)) @@ -1758,7 +1762,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn('Upload', str(q("title"))) + self.assertIn('Upload', six.ensure_text(q("title"))) self.assertFalse(session.sessionpresentation_set.exists()) test_file = StringIO('%PDF-1.4\n%âãÏÓ\nthis is some text for a test') test_file.name = "not_really.pdf" @@ -1776,7 +1780,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn('Upload', str(q("title"))) + self.assertIn('Upload', six.ensure_text(q("title"))) def test_upload_minutes_agenda(self): @@ -1791,7 +1795,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn('Upload', str(q("Title"))) + self.assertIn('Upload', six.ensure_text(q("Title"))) self.assertFalse(session.sessionpresentation_set.exists()) self.assertFalse(q('form input[type="checkbox"]')) @@ -1845,7 +1849,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn('Revise', str(q("Title"))) + self.assertIn('Revise', six.ensure_text(q("Title"))) test_file = BytesIO(b'this is some different text for a test') test_file.name = "also_not_really.txt" r = self.client.post(url,dict(file=test_file,apply_to_all=True)) @@ -1879,7 +1883,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn('Upload', str(q("Title"))) + self.assertIn('Upload', six.ensure_text(q("Title"))) self.assertFalse(session.sessionpresentation_set.exists()) self.assertFalse(q('form input[type="checkbox"]')) @@ -1900,7 +1904,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn('Upload', str(q("title"))) + self.assertIn('Upload', six.ensure_text(q("title"))) self.assertFalse(session.sessionpresentation_set.filter(document__type_id=doctype)) test_file = BytesIO(b'this is some text for a test') test_file.name = "not_really.txt" @@ -1923,7 +1927,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn('Upload', str(q("title"))) + self.assertIn('Upload', six.ensure_text(q("title"))) self.assertFalse(session1.sessionpresentation_set.filter(document__type_id='slides')) test_file = BytesIO(b'this is not really a slide') test_file.name = 'not_really.txt' @@ -1951,7 +1955,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertTrue(r.status_code, 200) q = PyQuery(r.content) - self.assertIn('Revise', str(q("title"))) + self.assertIn('Revise', six.ensure_text(q("title"))) test_file = BytesIO(b'new content for the second slide deck') test_file.name = 'doesnotmatter.txt' r = self.client.post(url,dict(file=test_file,title='rename the presentation',apply_to_all=False)) diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 296e9ac08..4a845ec33 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -1,8 +1,14 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import json -import urllib.request, urllib.error, urllib.parse +import six.moves.urllib.request +from six.moves.urllib.error import HTTPError from django.conf import settings from django.template.loader import render_to_string @@ -88,8 +94,8 @@ def create_proceedings_templates(meeting): # Get meeting attendees from registration system url = settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number) try: - attendees = json.load(urllib.request.urlopen(url)) - except (ValueError, urllib.error.HTTPError): + attendees = json.load(six.moves.urllib.request.urlopen(url)) + except (ValueError, HTTPError): attendees = [] if attendees: diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 8c80905d0..083c961d0 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -1,18 +1,24 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import csv import datetime import glob +import io import json import os import pytz import re +import six import tarfile -import urllib.request, urllib.parse, urllib.error + from calendar import timegm from collections import OrderedDict, Counter, deque +from six.moves.urllib.parse import unquote from tempfile import mkstemp from wsgiref.handlers import format_date_time @@ -199,7 +205,7 @@ def materials_document(request, document, num=None, ext=None): _, basename = os.path.split(filename) if not os.path.exists(filename): raise Http404("File not found: %s" % filename) - with open(filename, 'rb') as file: + with io.open(filename, 'rb') as file: bytes = file.read() mtype, chset = get_mime_type(bytes) @@ -518,12 +524,12 @@ def agenda(request, num=None, name=None, base=None, ext=None, owner=None, utc="" def agenda_csv(schedule, filtered_assignments): response = HttpResponse(content_type="text/csv; charset=%s"%settings.DEFAULT_CHARSET) - writer = csv.writer(response, delimiter=',', quoting=csv.QUOTE_ALL) + writer = csv.writer(response, delimiter=str(','), quoting=csv.QUOTE_ALL) headings = ["Date", "Start", "End", "Session", "Room", "Area", "Acronym", "Type", "Description", "Session ID", "Agenda", "Slides"] def write_row(row): - encoded_row = [v.encode('utf-8') if isinstance(v, str) else v for v in row] + encoded_row = [v.encode('utf-8') if isinstance(v, six.text_type) else v for v in row] while len(encoded_row) < len(headings): encoded_row.append(None) # produce empty entries at the end as necessary @@ -681,7 +687,7 @@ def session_draft_tarfile(request, num, acronym): tarstream = tarfile.open('','w:gz',response) mfh, mfn = mkstemp() os.close(mfh) - manifest = open(mfn, "w") + manifest = io.open(mfn, "w") for doc_name in drafts: pdf_path = os.path.join(settings.INTERNET_DRAFT_PDF_PATH, doc_name + ".pdf") @@ -709,7 +715,7 @@ def session_draft_pdf(request, num, acronym): curr_page = 1 pmh, pmn = mkstemp() os.close(pmh) - pdfmarks = open(pmn, "w") + pdfmarks = io.open(pmn, "w") pdf_list = "" for draft in drafts: @@ -730,7 +736,7 @@ def session_draft_pdf(request, num, acronym): code, out, err = pipe(gs + " -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -sOutputFile=" + pdfn + " " + pdf_list + " " + pmn) assertion('code == 0') - pdf = open(pdfn,"rb") + pdf = io.open(pdfn,"rb") pdf_contents = pdf.read() pdf.close() @@ -873,7 +879,7 @@ def ical_agenda(request, num=None, name=None, acronym=None, session_id=None): raise Http404 q = request.META.get('QUERY_STRING','') or "" - filter = set(urllib.parse.unquote(q).lower().split(',')) + filter = set(unquote(q).lower().split(',')) include = [ i for i in filter if not (i.startswith('-') or i.startswith('~')) ] include_types = set(["plenary","other"]) exclude = [] @@ -1583,7 +1589,7 @@ def propose_session_slides(request, session_id, num): name = 'slides-%s-%s' % (session.meeting.number, session.docname_token()) name = name + '-' + slugify(title).replace('_', '-')[:128] filename = '%s-00%s'% (name, ext) - destination = open(os.path.join(settings.SLIDE_STAGING_PATH, filename),'wb+') + destination = io.open(os.path.join(settings.SLIDE_STAGING_PATH, filename),'wb+') for chunk in file.chunks(): destination.write(chunk) destination.close() diff --git a/ietf/name/generate_fixtures.py b/ietf/name/generate_fixtures.py index 707c54aff..c453b6c53 100644 --- a/ietf/name/generate_fixtures.py +++ b/ietf/name/generate_fixtures.py @@ -1,8 +1,10 @@ +# Copyright The IETF Trust 2011-2019, All Rights Reserved #!/usr/bin/python # simple script for exporting name related base data for the tests # boiler plate +import io import os, sys import django @@ -17,7 +19,7 @@ from django.core.serializers import serialize def output(name, seq): try: - f = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/%s.json" % name), 'w') + f = io.open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/%s.json" % name), 'w') f.write(serialize("json", seq, indent=1)) f.close() except: diff --git a/ietf/nomcom/forms.py b/ietf/nomcom/forms.py index dd9c18686..2f801a842 100644 --- a/ietf/nomcom/forms.py +++ b/ietf/nomcom/forms.py @@ -1,4 +1,11 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import six + from django.conf import settings from django import forms from django.urls import reverse @@ -33,12 +40,12 @@ class PositionNomineeField(forms.ChoiceField): results = [] for position in positions: accepted_nominees = [np.nominee for np in NomineePosition.objects.filter(position=position,state='accepted').exclude(nominee__duplicated__isnull=False)] - nominees = [('%s_%s' % (position.id, i.id), str(i)) for i in accepted_nominees] + nominees = [('%s_%s' % (position.id, i.id), six.ensure_text(i)) for i in accepted_nominees] if nominees: results.append((position.name+" (Accepted)", nominees)) for position in positions: other_nominees = [np.nominee for np in NomineePosition.objects.filter(position=position).exclude(state='accepted').exclude(nominee__duplicated__isnull=False)] - nominees = [('%s_%s' % (position.id, i.id), str(i)) for i in other_nominees] + nominees = [('%s_%s' % (position.id, i.id), six.ensure_text(i)) for i in other_nominees] if nominees: results.append((position.name+" (Declined or Pending)", nominees)) kwargs['choices'] = results diff --git a/ietf/nomcom/management/commands/feedback_email.py b/ietf/nomcom/management/commands/feedback_email.py index 3c4ae9f96..7774a6c08 100644 --- a/ietf/nomcom/management/commands/feedback_email.py +++ b/ietf/nomcom/management/commands/feedback_email.py @@ -1,4 +1,10 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import sys from django.core.management.base import BaseCommand, CommandError @@ -31,7 +37,7 @@ class Command(BaseCommand): if not email: msg = sys.stdin.read() else: - msg = open(email, "r").read() + msg = io.open(email, "r").read() try: nomcom = NomCom.objects.get(group__acronym__icontains=year, diff --git a/ietf/nomcom/test_data.py b/ietf/nomcom/test_data.py index 2cb998242..072f43e92 100644 --- a/ietf/nomcom/test_data.py +++ b/ietf/nomcom/test_data.py @@ -1,4 +1,10 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import tempfile import os @@ -90,7 +96,7 @@ def check_comments(encryped, plain, privatekey_file): decrypted_file.close() encrypted_file.close() - decrypted_comments = open(decrypted_file.name, 'r').read() + decrypted_comments = io.open(decrypted_file.name, 'rb').read().decode('utf-8') os.unlink(encrypted_file.name) os.unlink(decrypted_file.name) @@ -112,7 +118,7 @@ def nomcom_test_data(): nomcom_test_cert_file, privatekey_file = generate_cert() nomcom.public_key.storage = FileSystemStorage(location=settings.NOMCOM_PUBLIC_KEYS_DIR) - nomcom.public_key.save('cert', File(open(nomcom_test_cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(nomcom_test_cert_file.name, 'r'))) # chair and member create_person(group, "chair", username=CHAIR_USER, email_address='%s%s'%(CHAIR_USER,EMAIL_DOMAIN)) diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index 262ae006d..49ada547b 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1,11 +1,17 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- -#import tempfile + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime +import io import random +import six import shutil -import urllib.parse + from pyquery import PyQuery +from six.moves.urllib.parse import urlparse from django.db import IntegrityError from django.db.models import Max @@ -35,7 +41,7 @@ from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, get_hash from ietf.person.factories import PersonFactory, EmailFactory from ietf.person.models import Email, Person from ietf.stats.models import MeetingRegistration -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.mail import outbox, empty_outbox, get_payload from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent client_test_cert_files = None @@ -334,7 +340,7 @@ class NomcomViewsTest(TestCase): response = self.client.post(self.private_merge_nominee_url, test_data) self.assertEqual(response.status_code, 302) redirect_url = response["Location"] - redirect_path = urllib.parse.urlparse(redirect_url).path + redirect_path = urlparse(redirect_url).path self.assertEqual(redirect_path, reverse('ietf.nomcom.views.private_index', kwargs={"year": NOMCOM_YEAR})) response = self.client.get(redirect_url) @@ -407,7 +413,7 @@ class NomcomViewsTest(TestCase): q = PyQuery(r.content) reminder_date = '%s-09-30' % self.year - f = open(self.cert_file.name) + f = io.open(self.cert_file.name) response = self.client.post(self.edit_nomcom_url, { 'public_key': f, 'reminderdates_set-TOTAL_FORMS': q('input[name="reminderdates_set-TOTAL_FORMS"]').val(), @@ -520,7 +526,7 @@ class NomcomViewsTest(TestCase): self.assertEqual('Nomination receipt', outbox[-1]['Subject']) self.assertEqual(self.email_from, outbox[-1]['From']) self.assertIn('plain', outbox[-1]['To']) - self.assertIn('Comments with accents äöå', str(outbox[-1].get_payload(decode=True),"utf-8","replace")) + self.assertIn('Comments with accents äöå', six.ensure_text(outbox[-1].get_payload(decode=True),"utf-8","replace")) # Nominate the same person for the same position again without asking for confirmation @@ -561,7 +567,7 @@ class NomcomViewsTest(TestCase): self.assertEqual('Nomination receipt', outbox[-1]['Subject']) self.assertEqual(self.email_from, outbox[-1]['From']) self.assertIn('plain', outbox[-1]['To']) - self.assertIn('Comments with accents äöå', str(outbox[-1].get_payload(decode=True),"utf-8","replace")) + self.assertIn('Comments with accents äöå', six.ensure_text(outbox[-1].get_payload(decode=True),"utf-8","replace")) # Nominate the same person for the same position again without asking for confirmation @@ -621,7 +627,7 @@ class NomcomViewsTest(TestCase): # save the cert file in tmp #nomcom.public_key.storage.location = tempfile.gettempdir() - nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) response = self.client.get(nominate_url) self.assertEqual(response.status_code, 200) @@ -687,7 +693,7 @@ class NomcomViewsTest(TestCase): # save the cert file in tmp #nomcom.public_key.storage.location = tempfile.gettempdir() - nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) response = self.client.get(nominate_url) self.assertEqual(response.status_code, 200) @@ -763,7 +769,7 @@ class NomcomViewsTest(TestCase): # save the cert file in tmp #nomcom.public_key.storage.location = tempfile.gettempdir() - nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) response = self.client.get(self.add_questionnaire_url) self.assertEqual(response.status_code, 200) @@ -801,12 +807,12 @@ class NomcomViewsTest(TestCase): # We're interested in the confirmation receipt here self.assertEqual(len(outbox),3) self.assertEqual('NomCom comment confirmation', outbox[2]['Subject']) - email_body = outbox[2].get_payload() + email_body = get_payload(outbox[2]) self.assertIn(position, email_body) self.assertNotIn('$', email_body) self.assertEqual(self.email_from, outbox[-2]['From']) self.assertIn('plain', outbox[2]['To']) - self.assertIn('Comments with accents äöå', str(outbox[2].get_payload(decode=True),"utf-8","replace")) + self.assertIn('Comments with accents äöå', six.ensure_text(outbox[2].get_payload(decode=True),"utf-8","replace")) empty_outbox() self.feedback_view(public=True) @@ -842,7 +848,7 @@ class NomcomViewsTest(TestCase): # save the cert file in tmp #nomcom.public_key.storage.location = tempfile.gettempdir() - nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) response = self.client.get(feedback_url) self.assertEqual(response.status_code, 200) @@ -957,7 +963,7 @@ class FeedbackTest(TestCase): # save the cert file in tmp #nomcom.public_key.storage.location = tempfile.gettempdir() - nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) comment_text = 'Plain text. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' comments = nomcom.encrypt(comment_text) @@ -979,7 +985,7 @@ class ReminderTest(TestCase): self.nomcom = get_nomcom_by_year(NOMCOM_YEAR) self.cert_file, self.privatekey_file = get_cert_files() #self.nomcom.public_key.storage.location = tempfile.gettempdir() - self.nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + self.nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) gen = Position.objects.get(nomcom=self.nomcom,name='GEN') rai = Position.objects.get(nomcom=self.nomcom,name='RAI') diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index 736acb5c5..1d76fa11d 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -1,8 +1,14 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import hashlib import os import re +import six import tempfile from email import message_from_string @@ -16,7 +22,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.urls import reverse from django.template.loader import render_to_string from django.shortcuts import get_object_or_404 -from django.utils.encoding import smart_str +from django.utils.encoding import force_str from ietf.dbtemplate.models import DBTemplate from ietf.person.models import Email, Person @@ -87,7 +93,7 @@ def get_user_email(user): return user._email_cache def get_hash_nominee_position(date, nominee_position_id): - return hashlib.md5(('%s%s%s' % (settings.SECRET_KEY, date, nominee_position_id)).encode()).hexdigest() + return hashlib.md5(('%s%s%s' % (settings.SECRET_KEY, date, nominee_position_id)).encode('utf-8')).hexdigest() def initialize_templates_for_group(group): @@ -160,7 +166,7 @@ def retrieve_nomcom_private_key(request, year): command = "%s bf -d -in /dev/stdin -k \"%s\" -a" code, out, error = pipe(command % (settings.OPENSSL_COMMAND, - settings.SECRET_KEY), private_key.encode()) + settings.SECRET_KEY), private_key.encode('utf-8')) if code != 0: log("openssl error: %s:\n Error %s: %s" %(command, code, error)) return out @@ -172,7 +178,7 @@ def store_nomcom_private_key(request, year, private_key): else: command = "%s bf -e -in /dev/stdin -k \"%s\" -a" code, out, error = pipe(command % (settings.OPENSSL_COMMAND, - settings.SECRET_KEY), private_key.encode()) + settings.SECRET_KEY), private_key.encode('utf-8')) if code != 0: log("openssl error: %s:\n Error %s: %s" %(command, code, error)) if error: @@ -182,7 +188,7 @@ def store_nomcom_private_key(request, year, private_key): def validate_private_key(key): key_file = tempfile.NamedTemporaryFile(delete=False) - key_file.write(key.encode()) + key_file.write(key.encode('utf-8')) key_file.close() command = "%s rsa -in %s -check -noout" @@ -400,7 +406,7 @@ def getheader(header_text, default="ascii"): """Decode the specified header""" tuples = decode_header(header_text) - header_sections = [ text.decode(charset or default) if isinstance(text, bytes) else text for text, charset in tuples] + header_sections = [ text.decode(charset or default) if isinstance(text, six.binary_type) else text for text, charset in tuples] return "".join(header_sections) @@ -427,7 +433,7 @@ def get_body(message): body = [] for part in text_parts: charset = get_charset(part, get_charset(message)) - body.append(str(part.get_payload(decode=True), + body.append(six.ensure_text(part.get_payload(decode=True), charset, "replace")) @@ -435,16 +441,14 @@ def get_body(message): else: # if it is not multipart, the payload will be a string # representing the message body - body = str(message.get_payload(decode=True), + body = six.ensure_text(message.get_payload(decode=True), get_charset(message), "replace") return body.strip() def parse_email(text): - if isinstance(text, str): - text = smart_str(text) - msg = message_from_string(text) + msg = message_from_string(force_str(text)) body = get_body(msg) subject = getheader(msg['Subject']) diff --git a/ietf/person/factories.py b/ietf/person/factories.py index e92b132c6..0dca61037 100644 --- a/ietf/person/factories.py +++ b/ietf/person/factories.py @@ -2,12 +2,16 @@ # -*- coding: utf-8 -*- -import os +from __future__ import absolute_import, print_function, unicode_literals + import factory import faker -import shutil -import random import faker.config +import os +import random +import shutil +import six + from unidecode import unidecode from django.conf import settings @@ -54,7 +58,7 @@ class PersonFactory(factory.DjangoModelFactory): user = factory.SubFactory(UserFactory) name = factory.LazyAttribute(lambda p: normalize_name('%s %s'%(p.user.first_name, p.user.last_name))) - ascii = factory.LazyAttribute(lambda p: str(unidecode_name(p.name))) + ascii = factory.LazyAttribute(lambda p: six.ensure_text(unidecode_name(p.name))) class Params: with_bio = factory.Trait(biography = "\n\n".join(fake.paragraphs())) diff --git a/ietf/person/fields.py b/ietf/person/fields.py index 89c3e8259..f2ab5d756 100644 --- a/ietf/person/fields.py +++ b/ietf/person/fields.py @@ -1,8 +1,14 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import json +import six from collections import Counter -from urllib.parse import urlencode +from six.moves.urllib.parse import urlencode from django.utils.html import escape from django import forms @@ -73,7 +79,7 @@ class SearchablePersonsField(forms.CharField): def prepare_value(self, value): if not value: value = "" - if isinstance(value, str): + if isinstance(value, six.string_types): pks = self.parse_select2_value(value) if self.model == Person: value = self.model.objects.filter(pk__in=pks) @@ -162,7 +168,7 @@ class PersonEmailChoiceField(forms.ModelChoiceField): def label_from_instance(self, email): if self.label_with == "person": - return str(email.person) + return six.ensure_text(email.person) elif self.label_with == "email": return email.address else: diff --git a/ietf/person/management/commands/deactivate_email_addresses.py b/ietf/person/management/commands/deactivate_email_addresses.py index e5ec9e348..04357dc9f 100644 --- a/ietf/person/management/commands/deactivate_email_addresses.py +++ b/ietf/person/management/commands/deactivate_email_addresses.py @@ -2,7 +2,10 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + import flufl.bounce +import io import mailbox import sys @@ -67,7 +70,7 @@ class Command(BaseCommand): self.stderr.write('No person is associated with <%s>\n' % (a, )) else: self.stderr.write('Address not found: <%s>\n' % (a, )) - with open('./failed', 'a') as failed: + with io.open('./failed', 'a') as failed: failed.write(messages[a].as_string(unixfrom=True)) failed.write('\n') diff --git a/ietf/person/models.py b/ietf/person/models.py index 2ecf15398..5025a900b 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -1,22 +1,27 @@ # Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import datetime import email.utils import email.header +import six import uuid from hashids import Hashids -from urllib.parse import urljoin +from six.moves.urllib.parse import urljoin from django.conf import settings - -from django.core.validators import validate_email -from django.core.exceptions import ObjectDoesNotExist -from django.db import models from django.contrib.auth.models import User +from django.core.exceptions import ObjectDoesNotExist +from django.core.validators import validate_email +from django.db import models from django.template.loader import render_to_string -from django.utils.encoding import smart_bytes +from django.utils.encoding import python_2_unicode_compatible, smart_bytes from django.utils.text import slugify + from simple_history.models import HistoricalRecords import debug # pyflakes:ignore @@ -30,6 +35,7 @@ from ietf.utils import log from ietf.utils.models import ForeignKey, OneToOneField +@python_2_unicode_compatible class Person(models.Model): history = HistoricalRecords() user = OneToOneField(User, blank=True, null=True, on_delete=models.SET_NULL) @@ -79,7 +85,11 @@ class Person(models.Model): def plain_ascii(self): if not hasattr(self, '_cached_plain_ascii'): if self.ascii: - ascii = unidecode_name(self.ascii) + if isinstance(self.ascii, six.binary_type): + uname = six.ensure_text(self.ascii) + ascii = unidecode_name(uname) + else: + ascii = unidecode_name(self.ascii) else: ascii = unidecode_name(self.name) prefix, first, middle, last, suffix = name_parts(ascii) @@ -96,7 +106,7 @@ class Person(models.Model): may be an object or the group acronym.""" if group: from ietf.group.models import Group - if isinstance(group, str) or isinstance(group, str): + if isinstance(group, six.string_types): group = Group.objects.get(acronym=group) e = Email.objects.filter(person=self, role__group=group, role__name=role_name) else: @@ -225,6 +235,7 @@ class Person(models.Model): ct1['ascii'] = self.ascii return ct1 +@python_2_unicode_compatible class Alias(models.Model): """This is used for alternative forms of a name. This is the primary lookup point for names, and should always contain the @@ -252,6 +263,7 @@ class Alias(models.Model): class Meta: verbose_name_plural = "Aliases" +@python_2_unicode_compatible class Email(models.Model): history = HistoricalRecords() address = models.CharField(max_length=64, primary_key=True, validators=[validate_email]) @@ -323,6 +335,7 @@ PERSON_API_KEY_ENDPOINTS = [ ("/api/meeting/session/video/url", "/api/meeting/session/video/url"), ] +@python_2_unicode_compatible class PersonalApiKey(models.Model): person = ForeignKey(Person, related_name='apikeys') endpoint = models.CharField(max_length=128, null=False, blank=False, choices=PERSON_API_KEY_ENDPOINTS) @@ -335,11 +348,8 @@ class PersonalApiKey(models.Model): @classmethod def validate_key(cls, s): import struct, hashlib, base64 - try: - key = base64.urlsafe_b64decode(s) - except TypeError: - return None - + assert isinstance(s, six.binary_type) + key = base64.urlsafe_b64decode(s) id, salt, hash = struct.unpack(KEY_STRUCT, key) k = cls.objects.filter(id=id) if not k.exists(): @@ -372,6 +382,7 @@ PERSON_EVENT_CHOICES = [ ("email_address_deactivated", "Email address deactivated"), ] +@python_2_unicode_compatible class PersonEvent(models.Model): person = ForeignKey(Person) time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened") diff --git a/ietf/person/tests.py b/ietf/person/tests.py index 21ec57091..d4a2d9f67 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -2,7 +2,11 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + import datetime +import six + from pyquery import PyQuery from io import StringIO from django.urls import reverse as urlreverse @@ -89,7 +93,7 @@ class PersonTests(TestCase): empty_outbox() p = PersonFactory(name="Föö Bär") PersonFactory(name=p.name) - self.assertTrue("possible duplicate" in str(outbox[0]["Subject"]).lower()) + self.assertTrue("possible duplicate" in six.ensure_text(outbox[0]["Subject"]).lower()) def test_merge(self): url = urlreverse("ietf.person.views.merge") diff --git a/ietf/person/utils.py b/ietf/person/utils.py index 9d250bc17..c45ff423d 100755 --- a/ietf/person/utils.py +++ b/ietf/person/utils.py @@ -1,8 +1,13 @@ # Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import datetime import os import pprint +import six import sys import syslog @@ -50,13 +55,13 @@ def merge_persons(source, target, file=sys.stdout, verbose=False): objs, opts, user, admin_site, using) deletable_objects_summary = deletable_objects[1] if len(deletable_objects_summary) > 1: # should only inlcude one object (Person) - print("Not Deleting Person: {}({})".format(source.ascii,source.pk), file=file) - print("Related objects remain:", file=file) + six.print_("Not Deleting Person: {}({})".format(source.ascii,source.pk), file=file) + six.print_("Related objects remain:", file=file) pprint.pprint(deletable_objects[1], stream=file) success = False else: success = True - print("Deleting Person: {}({})".format(source.ascii,source.pk), file=file) + six.print_("Deleting Person: {}({})".format(source.ascii,source.pk), file=file) source.delete() return success, changes @@ -109,7 +114,7 @@ def move_related_objects(source, target, file, verbose=False): field_name = related_object.field.name queryset = getattr(source, accessor).all() if verbose: - print("Merging {}:{}".format(accessor,queryset.count()),file=file) + six.print_("Merging {}:{}".format(accessor,queryset.count()), file=file) kwargs = { field_name:target } queryset.update(**kwargs) diff --git a/ietf/release/views.py b/ietf/release/views.py index 86b4d04e9..742dcba4f 100644 --- a/ietf/release/views.py +++ b/ietf/release/views.py @@ -2,6 +2,9 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import re import json @@ -43,7 +46,7 @@ def get_coverage_data(): with gzip.open(settings.TEST_COVERAGE_MASTER_FILE, "rb") as file: coverage_data = json.load(file) else: - with open(settings.TEST_COVERAGE_MASTER_FILE) as file: + with io.open(settings.TEST_COVERAGE_MASTER_FILE) as file: coverage_data = json.load(file) cache.set(cache_key, coverage_data, 60*60*24) return coverage_data diff --git a/ietf/review/mailarch.py b/ietf/review/mailarch.py index f905a603c..fc50f1075 100644 --- a/ietf/review/mailarch.py +++ b/ietf/review/mailarch.py @@ -1,11 +1,25 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + + # various utilities for working with the mailarch mail archive at # mailarchive.ietf.org -import datetime, tarfile, mailbox, tempfile, hashlib, base64, email.utils -import urllib.request, urllib.parse, urllib.error -import urllib.request, urllib.error, urllib.parse, contextlib -import debug # pyflakes:ignore +import contextlib +import datetime +import tarfile +import mailbox +import tempfile +import hashlib +import base64 +import email.utils + +from six.moves.urllib.parse import urlencode +from six.moves.urllib.request import urlopen + import debug # pyflakes:ignore from pyquery import PyQuery @@ -36,7 +50,7 @@ def construct_query_urls(review_req, query=None): if not query: query = review_req.doc.name - encoded_query = "?" + urllib.parse.urlencode({ + encoded_query = "?" + urlencode({ "qdr": "c", # custom time frame "start_date": (datetime.date.today() - datetime.timedelta(days=180)).isoformat(), "email_list": list_name, @@ -95,7 +109,7 @@ def retrieve_messages(query_data_url): """Retrieve and return selected content from mailarch.""" res = [] - with contextlib.closing(urllib.request.urlopen(query_data_url, timeout=15)) as fileobj: + with contextlib.closing(urlopen(query_data_url, timeout=15)) as fileobj: content_type = fileobj.info()["Content-type"] if not content_type.startswith("application/x-tar"): if content_type.startswith("text/html"): diff --git a/ietf/review/utils.py b/ietf/review/utils.py index 6f6380c5f..236f85519 100644 --- a/ietf/review/utils.py +++ b/ietf/review/utils.py @@ -1,9 +1,14 @@ -# -*- coding: utf-8 -*- # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import itertools +import re +import six -import datetime, re, itertools from collections import defaultdict, namedtuple from django.db.models import Q, Max, F @@ -959,7 +964,7 @@ def make_assignment_choices(email_queryset, review_req): if stats: explanations.append(", ".join(stats)) - label = str(e.person) + label = six.ensure_text(e.person) if explanations: label = "{}: {}".format(label, "; ".join(explanations)) diff --git a/ietf/secr/drafts/tests_views.py b/ietf/secr/drafts/tests_views.py index 592f16d68..ca0968737 100644 --- a/ietf/secr/drafts/tests_views.py +++ b/ietf/secr/drafts/tests_views.py @@ -1,5 +1,11 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime +import io import os import shutil from collections import OrderedDict @@ -130,7 +136,7 @@ class SecrDraftsTestCase(TestCase): def test_resurrect(self): draft = WgDraftFactory() path = os.path.join(self.repository_dir, draft.filename_with_rev()) - with open(path, 'w') as file: + with io.open(path, 'w') as file: file.write('test') expire_draft(draft) email_url = urlreverse('ietf.secr.drafts.views.email', kwargs={'id':draft.name}) + "?action=resurrect" diff --git a/ietf/secr/drafts/views.py b/ietf/secr/drafts/views.py index 5fbe61c1c..e6a3bfe27 100644 --- a/ietf/secr/drafts/views.py +++ b/ietf/secr/drafts/views.py @@ -1,8 +1,12 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import glob +import io import os import shutil from dateutil.parser import parse @@ -53,7 +57,7 @@ def handle_uploaded_file(f): ''' Save uploaded draft files to temporary directory ''' - destination = open(os.path.join(settings.IDSUBMIT_MANUAL_STAGING_DIR, f.name), 'wb+') + destination = io.open(os.path.join(settings.IDSUBMIT_MANUAL_STAGING_DIR, f.name), 'wb+') for chunk in f.chunks(): destination.write(chunk) destination.close() diff --git a/ietf/secr/meetings/blue_sheets.py b/ietf/secr/meetings/blue_sheets.py index 676542bad..49d8755ca 100644 --- a/ietf/secr/meetings/blue_sheets.py +++ b/ietf/secr/meetings/blue_sheets.py @@ -1,5 +1,13 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io + from django.conf import settings +from django.utils.encoding import force_bytes r''' RTF quick reference (from Word2007RTFSpec9.doc): @@ -17,9 +25,9 @@ RTF quick reference (from Word2007RTFSpec9.doc): ''' def create_blue_sheets(meeting, groups): - file = open(settings.SECR_BLUE_SHEET_PATH, 'w') + file = io.open(settings.SECR_BLUE_SHEET_PATH, 'wb') - header = '''{\\rtf1\\ansi\\ansicpg1252\\uc1 \\deff0\\deflang1033\\deflangfe1033 + header = b'''{\\rtf1\\ansi\\ansicpg1252\\uc1 \\deff0\\deflang1033\\deflangfe1033 {\\fonttbl{\\f0\\froman\\fcharset0\\fprq2{\\*\\panose 02020603050405020304}Times New Roman;}} {\\colortbl;\\red0\\green0\\blue0;\\red0\\green0\\blue255;\\red0\\green255\\blue255;\\red0\\green255\\blue0; \\red255\\green0\\blue255;\\red255\\green0\\blue0;\\red255\\green255\\blue0;\\red255\\green255\\blue255; @@ -32,7 +40,7 @@ def create_blue_sheets(meeting, groups): file.write(header) for group in groups: - group_header = ''' {\\header \\pard\\plain \\s15\\nowidctlpar\\widctlpar\\tqc\\tx4320\\tqr\\tx8640\\adjustright \\fs20\\cgrid + group_header = b''' {\\header \\pard\\plain \\s15\\nowidctlpar\\widctlpar\\tqc\\tx4320\\tqr\\tx8640\\adjustright \\fs20\\cgrid { Mailing List: %s \\tab\\tab Meeting # %s %s (%s) \\par } \\pard \\s15\\nowidctlpar\\widctlpar\\tqc\\tx4320\\tqr\\tx8640\\adjustright {\\b\\fs24 @@ -60,30 +68,31 @@ def create_blue_sheets(meeting, groups): \\par } \\pard \\fi-90\\li90\\nowidctlpar\\widctlpar\\adjustright {\\fs16 -''' % (group.list_email, - meeting.number, - group.acronym, - group.type, - meeting.number, - group.acronym, - group.type, - meeting.number, - group.name, - group.list_email) +''' % (force_bytes(group.list_email), + force_bytes(meeting.number), + force_bytes(group.acronym), + force_bytes(group.type), + force_bytes(meeting.number), + force_bytes(group.acronym), + force_bytes(group.type), + force_bytes(meeting.number), + force_bytes(group.name), + force_bytes(group.list_email), + ) file.write(group_header) for x in range(1,117): - line = '''\\par %s._________________________________________________ \\tab _____________________________________________________ + line = b'''\\par %s._________________________________________________ \\tab _____________________________________________________ \\par - ''' % x + ''' % force_bytes(x) file.write(line) - footer = '''} + footer = b'''} \\pard \\nowidctlpar\\widctlpar\\adjustright {\\fs16 \\sect } \\sectd \\pgnrestart\\linex0\\endnhere\\titlepg\\sectdefaultcl ''' file.write(footer) - file.write('\n}') + file.write(b'\n}') file.close() diff --git a/ietf/secr/proceedings/proc_utils.py b/ietf/secr/proceedings/proc_utils.py index d31c15eba..6c2b9c384 100644 --- a/ietf/secr/proceedings/proc_utils.py +++ b/ietf/secr/proceedings/proc_utils.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + ''' proc_utils.py @@ -10,7 +13,7 @@ import datetime import os import re import subprocess -from urllib.parse import urlencode +from six.moves.urllib.parse import urlencode import debug # pyflakes:ignore diff --git a/ietf/secr/proceedings/tests.py b/ietf/secr/proceedings/tests.py index e6271fcf3..f4d21ee6b 100644 --- a/ietf/secr/proceedings/tests.py +++ b/ietf/secr/proceedings/tests.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import debug # pyflakes:ignore +import io import json import os import shutil @@ -54,7 +58,7 @@ class VideoRecordingTestCase(TestCase): def test_get_urls_from_json(self): path = os.path.join(settings.BASE_DIR, "../test/data/youtube-playlistitems.json") - with open(path) as f: + with io.open(path) as f: doc = json.load(f) urls = _get_urls_from_json(doc) self.assertEqual(len(urls),2) @@ -110,7 +114,7 @@ class RecordingTestCase(TestCase): path = os.path.join(settings.MEETING_RECORDINGS_DIR,'ietf' + timeslot.meeting.number,filename) if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) - with open(path, "w") as f: + with io.open(path, "w") as f: f.write('dummy') def get_filename_for_timeslot(self, timeslot): diff --git a/ietf/secr/proceedings/utils.py b/ietf/secr/proceedings/utils.py index 2ae0ead9f..9b8c7f258 100644 --- a/ietf/secr/proceedings/utils.py +++ b/ietf/secr/proceedings/utils.py @@ -1,5 +1,7 @@ +# Copyright The IETF Trust 2016-2019, All Rights Reserved import glob +import io import os from django.conf import settings @@ -34,7 +36,7 @@ def handle_upload_file(file,filename,meeting,subdir, request=None, encoding=None for f in old_files: os.remove(f) - destination = open(os.path.join(path,filename), 'wb+') + destination = io.open(os.path.join(path,filename), 'wb+') if extension in settings.MEETING_VALID_MIME_TYPE_EXTENSIONS['text/html']: file.open() text = file.read() diff --git a/ietf/secr/sreq/tests.py b/ietf/secr/sreq/tests.py index 4108b48f3..2725feae5 100644 --- a/ietf/secr/sreq/tests.py +++ b/ietf/secr/sreq/tests.py @@ -1,6 +1,13 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved -from django.urls import reverse +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime +import six + +from django.urls import reverse import debug # pyflakes:ignore @@ -177,14 +184,14 @@ class SubmitRequestCase(TestCase): r = self.client.post(url,post_data) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Confirm' in str(q("title"))) + self.assertTrue('Confirm' in six.ensure_text(q("title"))) # confirm post_data['submit'] = 'Submit' r = self.client.post(confirm_url,post_data) self.assertRedirects(r, reverse('ietf.secr.sreq.views.main')) self.assertEqual(len(outbox),len_before+1) notification = outbox[-1] - notification_payload = str(notification.get_payload(decode=True),"utf-8","replace") + notification_payload = six.ensure_text(notification.get_payload(decode=True),"utf-8","replace") session = Session.objects.get(meeting=meeting,group=group) self.assertEqual(session.resources.count(),1) self.assertEqual(session.people_constraints.count(),1) diff --git a/ietf/secr/utils/group.py b/ietf/secr/utils/group.py index b6542b3f4..1a0b3df0e 100644 --- a/ietf/secr/utils/group.py +++ b/ietf/secr/utils/group.py @@ -1,5 +1,11 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + # Python imports +import io import os # Django imports @@ -27,7 +33,7 @@ def get_charter_text(group): ''' charter = group.charter path = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (charter.canonical_name(), charter.rev)) - f = open(path,'r') + f = io.open(path,'r') text = f.read() f.close() diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py index 15b100c02..0d170eeb3 100755 --- a/ietf/stats/backfill_data.py +++ b/ietf/stats/backfill_data.py @@ -1,12 +1,15 @@ #!/usr/bin/env python # Copyright The IETF Trust 2017-2019, All Rights Reserved +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals - +import io import sys import os import os.path import argparse +import six import time basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) @@ -15,7 +18,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py") if os.path.exists(virtualenv_activation): - exec(compile(open(virtualenv_activation, "rb").read(), virtualenv_activation, 'exec'), dict(__file__=virtualenv_activation)) + exec(compile(io.open(virtualenv_activation, "rb").read(), virtualenv_activation, 'exec'), dict(__file__=virtualenv_activation)) import django django.setup() @@ -43,7 +46,7 @@ if args.document: docs_qs = docs_qs.filter(docalias__name=args.document) ts = time.strftime("%Y-%m-%d_%H:%M%z") -logfile = open('backfill-authorstats-%s.log'%ts, 'w') +logfile = io.open('backfill-authorstats-%s.log'%ts, 'w') print("Writing log to %s" % os.path.abspath(logfile.name)) def say(msg): @@ -53,7 +56,7 @@ def say(msg): logfile.write(msg) logfile.write('\n') -def str(text): +def unicode(text): if text is None: return text # order matters here: @@ -83,10 +86,10 @@ for doc in docs_qs.prefetch_related("docalias", "formal_languages", "documentaut say("Skipping %s, no txt file found at %s" % (doc.name, path)) continue - with open(path, 'rb') as f: + with io.open(path, 'rb') as f: say("\nProcessing %s" % doc.name) sys.stdout.flush() - d = Draft(str(f.read()), path) + d = Draft(unicode(f.read()), path) updated = False @@ -128,10 +131,10 @@ for doc in docs_qs.prefetch_related("docalias", "formal_languages", "documentaut # it's an extra author - skip those extra authors seen = set() for full, _, _, _, _, email, country, company in d.get_author_list(): - assert full is None or isinstance(full, str) - assert email is None or isinstance(email, str) - assert country is None or isinstance(country, str) - assert company is None or isinstance(company, str) + assert full is None or isinstance(full, six.text_type) + assert email is None or isinstance(email, six.text_type) + assert country is None or isinstance(country, six.text_type) + assert company is None or isinstance(company, six.text_type) #full, email, country, company = [ unicode(s) for s in [full, email, country, company, ] ] if email in seen: continue diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index 97818dc8f..5036680ae 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -1,14 +1,18 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import re -import sys -from xym import xym import shutil +import six +import sys import tempfile -import io +from xym import xym from django.conf import settings import debug # pyflakes:ignore @@ -144,19 +148,23 @@ class DraftYangChecker(object): # This places the yang models as files in workdir saved_stdout = sys.stdout saved_stderr = sys.stderr - sys.stdout = io.StringIO() - sys.stderr = io.StringIO() + sys.stdout = six.StringIO() + sys.stderr = six.StringIO() extractor.extract_yang_model(file.readlines()) model_list = extractor.get_extracted_models(False, True) out = sys.stdout.getvalue() err = sys.stderr.getvalue() - sys.stdout = saved_stdout - sys.stderr = saved_stderr # signature change in xym: except Exception as exc: + sys.stdout = saved_stdout + sys.stderr = saved_stderr msg = "Exception when running xym on %s: %s" % (name, exc) log(msg) + raise return None, msg, 0, 0, info + finally: + sys.stdout = saved_stdout + sys.stderr = saved_stderr if not model_list: # Found no yang models, don't deliver any YangChecker result return None, "", 0, 0, info @@ -198,7 +206,7 @@ class DraftYangChecker(object): settings.SUBMIT_YANG_IANA_MODEL_DIR, ]) if os.path.exists(path): - with open(path) as file: + with io.open(path) as file: text = file.readlines() # pyang cmd_template = settings.SUBMIT_PYANG_COMMAND diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py index 7ede96b55..d9b6dab2e 100644 --- a/ietf/submit/forms.py +++ b/ietf/submit/forms.py @@ -1,11 +1,19 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import re import datetime import email import pytz -import xml2rfc +import six import tempfile +import xml2rfc + from email.utils import formataddr from unidecode import unidecode @@ -13,6 +21,7 @@ from django import forms from django.conf import settings from django.utils.html import mark_safe from django.urls import reverse as urlreverse +from django.utils.encoding import force_str import debug # pyflakes:ignore @@ -140,7 +149,7 @@ class SubmissionBaseUploadForm(forms.Form): # over to the xml parser. XXX FIXME: investigate updating # xml2rfc to be able to work with file handles to in-memory # files. - with open(tfn, 'wb+') as tf: + with io.open(tfn, 'wb+') as tf: for chunk in xml_file.chunks(): tf.write(chunk) os.environ["XML_LIBRARY"] = settings.XML_LIBRARY @@ -185,10 +194,10 @@ class SubmissionBaseUploadForm(forms.Form): self.revision = None self.filename = draftname self.title = self.xmlroot.findtext('front/title').strip() - if type(self.title) is str: + if type(self.title) is six.text_type: self.title = unidecode(self.title) self.abstract = (self.xmlroot.findtext('front/abstract') or '').strip() - if type(self.abstract) is str: + if type(self.abstract) is six.text_type: self.abstract = unidecode(self.abstract) author_info = self.xmlroot.findall('front/author') for author in author_info: @@ -508,7 +517,7 @@ class SubmissionEmailForm(forms.Form): '''Returns a ietf.message.models.Message object''' self.message_text = self.cleaned_data['message'] try: - message = email.message_from_string(self.message_text) + message = email.message_from_string(force_str(self.message_text)) except Exception as e: self.add_error('message', e) return None diff --git a/ietf/submit/management/commands/manualpost_email.py b/ietf/submit/management/commands/manualpost_email.py index 771da84be..10d3ed3d5 100644 --- a/ietf/submit/management/commands/manualpost_email.py +++ b/ietf/submit/management/commands/manualpost_email.py @@ -1,4 +1,10 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import sys from django.core.management.base import BaseCommand, CommandError @@ -20,7 +26,7 @@ class Command(BaseCommand): if not email: msg = sys.stdin.read() else: - msg = open(email, "r").read() + msg = io.open(email, "r").read() try: process_response_email(msg) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 49470cf9c..4463cb367 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -1,11 +1,16 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import email +import io import os import re import shutil +import six import sys @@ -14,6 +19,7 @@ from pyquery import PyQuery from django.conf import settings from django.urls import reverse as urlreverse +from django.utils.encoding import force_str import debug # pyflakes:ignore @@ -40,7 +46,7 @@ from ietf.utils.draft import Draft def submission_file(name, rev, group, format, templatename, author=None, email=None, title=None, year=None, ascii=True): # construct appropriate text draft - f = open(os.path.join(settings.BASE_DIR, "submit", templatename)) + f = io.open(os.path.join(settings.BASE_DIR, "submit", templatename)) template = f.read() f.close() @@ -290,16 +296,16 @@ class SubmitTests(TestCase): self.assertTrue(draft.relations_that_doc("possibly-replaces").first().target, sug_replaced_alias) self.assertEqual(len(outbox), mailbox_before + 5) self.assertIn(("I-D Action: %s" % name), outbox[-4]["Subject"]) - self.assertIn(author.ascii, str(outbox[-4])) + self.assertIn(author.ascii, six.ensure_text(outbox[-4])) self.assertIn(("I-D Action: %s" % name), outbox[-3]["Subject"]) - self.assertIn(author.ascii, str(outbox[-3])) + self.assertIn(author.ascii, six.ensure_text(outbox[-3])) self.assertIn("New Version Notification",outbox[-2]["Subject"]) - self.assertIn(name, str(outbox[-2])) - self.assertIn("mars", str(outbox[-2])) + self.assertIn(name, six.ensure_text(outbox[-2])) + self.assertIn("mars", six.ensure_text(outbox[-2])) # Check "Review of suggested possible replacements for..." mail self.assertIn("review", outbox[-1]["Subject"].lower()) - self.assertIn(name, str(outbox[-1])) - self.assertIn(sug_replaced_alias.name, str(outbox[-1])) + self.assertIn(name, six.ensure_text(outbox[-1])) + self.assertIn(sug_replaced_alias.name, six.ensure_text(outbox[-1])) self.assertIn("ames-chairs@", outbox[-1]["To"].lower()) self.assertIn("mars-chairs@", outbox[-1]["To"].lower()) @@ -379,7 +385,7 @@ class SubmitTests(TestCase): # write the old draft in a file so we can check it's moved away old_rev = draft.rev - with open(os.path.join(self.repository_dir, "%s-%s.txt" % (name, old_rev)), 'w') as f: + with io.open(os.path.join(self.repository_dir, "%s-%s.txt" % (name, old_rev)), 'w') as f: f.write("a" * 2000) old_docevents = list(draft.docevent_set.all()) @@ -407,7 +413,7 @@ class SubmitTests(TestCase): self.assertTrue("unknown-email-" not in confirm_email["To"]) if change_authors: # Since authors changed, ensure chairs are copied (and that the message says why) - self.assertTrue("chairs have been copied" in str(confirm_email)) + self.assertTrue("chairs have been copied" in six.ensure_text(confirm_email)) if group_type in ['wg','rg','ag']: self.assertTrue("mars-chairs@" in confirm_email["To"].lower()) elif group_type == 'area': @@ -417,7 +423,7 @@ class SubmitTests(TestCase): if stream_type=='ise': self.assertTrue("rfc-ise@" in confirm_email["To"].lower()) else: - self.assertNotIn("chairs have been copied", str(confirm_email)) + self.assertNotIn("chairs have been copied", six.ensure_text(confirm_email)) self.assertNotIn("mars-chairs@", confirm_email["To"].lower()) confirmation_url = self.extract_confirmation_url(confirm_email) @@ -486,17 +492,17 @@ class SubmitTests(TestCase): self.assertEqual(len(outbox), mailbox_before + 3) self.assertTrue(("I-D Action: %s" % name) in outbox[-3]["Subject"]) self.assertTrue(("I-D Action: %s" % name) in draft.message_set.order_by("-time")[0].subject) - self.assertTrue(author.ascii in str(outbox[-3])) + self.assertTrue(author.ascii in six.ensure_text(outbox[-3])) self.assertTrue("i-d-announce@" in outbox[-3]['To']) self.assertTrue("New Version Notification" in outbox[-2]["Subject"]) - self.assertTrue(name in str(outbox[-2])) + self.assertTrue(name in six.ensure_text(outbox[-2])) interesting_address = {'ietf':'mars', 'irtf':'irtf-chair', 'iab':'iab-chair', 'ise':'rfc-ise'}[draft.stream_id] - self.assertTrue(interesting_address in str(outbox[-2])) + self.assertTrue(interesting_address in six.ensure_text(outbox[-2])) if draft.stream_id == 'ietf': - self.assertTrue(draft.ad.role_email("ad").address in str(outbox[-2])) - self.assertTrue(ballot_position.ad.role_email("ad").address in str(outbox[-2])) + self.assertTrue(draft.ad.role_email("ad").address in six.ensure_text(outbox[-2])) + self.assertTrue(ballot_position.ad.role_email("ad").address in six.ensure_text(outbox[-2])) self.assertTrue("New Version Notification" in outbox[-1]["Subject"]) - self.assertTrue(name in str(outbox[-1])) + self.assertTrue(name in six.ensure_text(outbox[-1])) r = self.client.get(urlreverse('ietf.doc.views_search.recent_drafts')) self.assertEqual(r.status_code, 200) self.assertContains(r, draft.name) @@ -556,7 +562,7 @@ class SubmitTests(TestCase): # both submitter and author get email self.assertTrue(author.email().address.lower() in confirm_email["To"]) self.assertTrue("submitter@example.com" in confirm_email["To"]) - self.assertFalse("chairs have been copied" in str(confirm_email)) + self.assertFalse("chairs have been copied" in six.ensure_text(confirm_email)) confirmation_url = self.extract_confirmation_url(outbox[-1]) @@ -885,14 +891,14 @@ class SubmitTests(TestCase): self.assertEqual(Submission.objects.filter(name=name).count(), 1) self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev)))) - self.assertTrue(name in open(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))).read()) + self.assertTrue(name in io.open(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))).read()) self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev)))) - self.assertTrue(name in open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read()) - self.assertTrue('' in open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read()) + self.assertTrue(name in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read()) + self.assertTrue('' in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read()) self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev)))) - self.assertTrue('This is PDF' in open(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev))).read()) + self.assertTrue('This is PDF' in io.open(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev))).read()) self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.ps" % (name, rev)))) - self.assertTrue('This is PostScript' in open(os.path.join(self.staging_dir, "%s-%s.ps" % (name, rev))).read()) + self.assertTrue('This is PostScript' in io.open(os.path.join(self.staging_dir, "%s-%s.ps" % (name, rev))).read()) def test_expire_submissions(self): s = Submission.objects.create(name="draft-ietf-mars-foo", @@ -1174,7 +1180,7 @@ Please submit my draft at http://test.com/mydraft.txt Thank you """.format(datetime.datetime.now().ctime()) - message = email.message_from_string(message_string) + message = email.message_from_string(force_str(message_string)) submission, submission_email_event = ( add_submission_email(request=None, remote_ip ="192.168.0.1", @@ -1257,7 +1263,7 @@ ZSBvZiBsaW5lcyAtIGJ1dCBpdCBjb3VsZCBiZSBhIGRyYWZ0Cg== --------------090908050800030909090207-- """.format(frm, datetime.datetime.now().ctime()) - message = email.message_from_string(message_string) + message = email.message_from_string(force_str(message_string)) submission, submission_email_event = ( add_submission_email(request=None, remote_ip ="192.168.0.1", diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index f1d46e9b0..b95d636b7 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -1,9 +1,14 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime +import io import os import re +import six # pyflakes:ignore import xml2rfc from django.conf import settings @@ -456,7 +461,7 @@ def ensure_person_email_info_exists(name, email, docname): person = Person() person.name = name person.name_from_draft = name - log.assertion('isinstance(person.name, str)') + log.assertion('isinstance(person.name, six.text_type)') person.ascii = unidecode_name(person.name) person.save() else: @@ -605,7 +610,7 @@ def save_files(form): name = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (form.filename, form.revision, ext)) file_name[ext] = name - with open(name, 'wb+') as destination: + with io.open(name, 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) return file_name @@ -641,7 +646,7 @@ def get_draft_meta(form, saved_files): # Some meta-information, such as the page-count, can only # be retrieved from the generated text file. Provide a # parsed draft object to get at that kind of information. - with open(file_name['txt']) as txt_file: + with io.open(file_name['txt']) as txt_file: form.parsed_draft = Draft(txt_file.read(), txt_file.name) else: @@ -667,7 +672,7 @@ def get_draft_meta(form, saved_files): if s is None: return "" - if isinstance(s, str): + if isinstance(s, six.text_type): return s else: try: diff --git a/ietf/sync/iana.py b/ietf/sync/iana.py index cc14c7b60..972f2dda6 100644 --- a/ietf/sync/iana.py +++ b/ietf/sync/iana.py @@ -1,13 +1,20 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import base64 import datetime import email import json import re -import urllib.request, urllib.error, urllib.parse -from django.utils.http import urlquote +from six.moves.urllib.request import Request, urlopen + from django.conf import settings +from django.utils.encoding import force_str +from django.utils.http import urlquote import debug # pyflakes:ignore @@ -23,7 +30,7 @@ from ietf.utils.timezone import local_timezone_to_utc, email_time_to_local_timez #CHANGES_URL = "https://datatracker.dev.icann.org:8080/data-tracker/changes" def fetch_protocol_page(url): - f = urllib.request.urlopen(settings.IANA_SYNC_PROTOCOLS_URL) + f = urlopen(settings.IANA_SYNC_PROTOCOLS_URL) text = f.read() f.close() return text @@ -67,12 +74,12 @@ def update_rfc_log_from_protocol_page(rfc_names, rfc_must_published_later_than): def fetch_changes_json(url, start, end): url += "?start=%s&end=%s" % (urlquote(local_timezone_to_utc(start).strftime("%Y-%m-%d %H:%M:%S")), urlquote(local_timezone_to_utc(end).strftime("%Y-%m-%d %H:%M:%S"))) - request = urllib.request.Request(url) + request = Request(url) # HTTP basic auth username = "ietfsync" password = settings.IANA_SYNC_PASSWORD request.add_header("Authorization", "Basic %s" % base64.encodestring("%s:%s" % (username, password)).replace("\n", "")) - f = urllib.request.urlopen(request) + f = urlopen(request) text = f.read() f.close() return text @@ -234,8 +241,8 @@ def strip_version_extension(text): text = text[:-3] return text -def parse_review_email(bytes): - msg = email.message_from_bytes(bytes) +def parse_review_email(text): + msg = email.message_from_string(force_str(text)) # doc doc_name = find_document_name(msg["Subject"]) or "" doc_name = strip_version_extension(doc_name) diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index c4a60ceac..6caf9ec87 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -1,12 +1,17 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- -import re + +from __future__ import absolute_import, print_function, unicode_literals + import base64 import datetime -import urllib.request, urllib.parse, urllib.error -import urllib.request, urllib.error, urllib.parse +import re import socket +import six + +from six.moves.urllib.request import Request, urlopen +from six.moves.urllib.parse import urlencode from xml.dom import pulldom, Node from django.conf import settings @@ -40,7 +45,7 @@ def get_child_text(parent_node, tag_name): def fetch_queue_xml(url): socket.setdefaulttimeout(30) - return urllib.request.urlopen(url) + return urlopen(url) def parse_queue(response): """Parse RFC Editor queue XML into a bunch of tuples + warnings.""" @@ -228,7 +233,7 @@ def update_drafts_from_queue(drafts): def fetch_index_xml(url): socket.setdefaulttimeout(30) - return urllib.request.urlopen(url) + return urlopen(url) def parse_index(response): """Parse RFC Editor index XML into a bunch of tuples.""" @@ -517,7 +522,7 @@ def post_approved_draft(url, name): the data from the Datatracker and start processing it. Returns response and error (empty string if no error).""" - request = urllib.request.Request(url) + request = Request(url) request.add_header("Content-type", "application/x-www-form-urlencoded") request.add_header("Accept", "text/plain") # HTTP basic auth @@ -531,7 +536,7 @@ def post_approved_draft(url, name): log("Posting RFC-Editor notifcation of approved draft '%s' to '%s'" % (name, url)) text = error = "" try: - f = urllib.request.urlopen(request, data=urllib.parse.urlencode({ 'draft': name }), timeout=20) + f = urlopen(request, data=urlencode({ 'draft': name }), timeout=20) text = f.read() status_code = f.getcode() f.close() @@ -547,6 +552,6 @@ def post_approved_draft(url, name): # catch everything so we don't leak exceptions, convert them # into string instead log("Exception on RFC-Editor notification for draft '%s': '%s'" % (name, e)) - error = str(e) + error = six.ensure_text(e) return text, error diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 8f66503f5..df2d33a53 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -1,16 +1,21 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import os +import io import json import datetime -import io import quopri import shutil from django.conf import settings from django.urls import reverse as urlreverse +import debug # pyflakes:ignore + from ietf.doc.factories import WgDraftFactory from ietf.doc.models import Document, DocAlias, DocEvent, DeletedEvent, DocTagName, RelatedDocument, State, StateDocEvent from ietf.doc.utils import add_state_change_event @@ -177,7 +182,7 @@ ICANN rtime = 7*subjects.index(subject) + 5*tags.index(tag) + embedded_names.index(embedded_name) person=Person.objects.get(user__username="iana") fromaddr = person.email().formatted_email() - msg = msg_template % dict(person=quopri.encodestring(person.name.encode()), + msg = msg_template % dict(person=quopri.encodestring(person.name.encode('utf-8')), fromaddr=fromaddr, draft=draft.name, rev=draft.rev, @@ -185,7 +190,6 @@ ICANN rtime=rtime, subject=subject, embedded_name=embedded_name,) - doc_name, review_time, by, comment = iana.parse_review_email(msg.encode('utf-8')) self.assertEqual(doc_name, draft.name) @@ -232,7 +236,7 @@ class RFCSyncTests(TestCase): settings.INTERNET_DRAFT_ARCHIVE_DIR = self.save_archive_dir def write_draft_file(self, name, size): - with open(os.path.join(self.id_dir, name), 'w') as f: + with io.open(os.path.join(self.id_dir, name), 'w') as f: f.write("a" * size) def test_rfc_index(self): diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index 4aebed393..e01f1388c 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -1,4 +1,11 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import six + from django.contrib import admin from ietf.utils.models import VersionInfo @@ -9,10 +16,10 @@ def name(obj): if callable(obj.name): name = obj.name() else: - name = str(obj.name) + name = six.ensure_text(obj.name) if name: return name - return str(obj) + return six.ensure_text(obj) def admin_link(field, label=None, ordering="", display=name, suffix=""): if not label: diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index ba539d3e1..adf1468ae 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -1,6 +1,11 @@ -# Copyright The IETF Trust 2009-2019, All Rights Reserved #!/usr/bin/python +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- # -*- python -*- + + +from __future__ import absolute_import, print_function, unicode_literals + """ NAME %(program)s - Extract meta-information from an IETF draft. @@ -36,9 +41,11 @@ COPYRIGHT import datetime import getopt +import io import os import os.path import re +import six import stat import sys import time @@ -106,7 +113,7 @@ def _err(string): # ---------------------------------------------------------------------- def _gettext(file): - file = open(file) + file = io.open(file) text = file.read() file.close() @@ -128,7 +135,7 @@ def acronym_match(s, l): class Draft(): def __init__(self, text, source, name_from_source=False): - assert isinstance(text, str) + assert isinstance(text, six.text_type) self.source = source self.rawtext = text self.name_from_source = name_from_source @@ -1203,7 +1210,7 @@ def getmeta(fn): return timestamp = time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime(os.stat(filename)[stat.ST_MTIME])) - with open(filename, 'rb') as file: + with io.open(filename, 'rb') as file: try: draft = Draft(file.read().decode('utf8'), filename) except UnicodeDecodeError: @@ -1311,7 +1318,7 @@ def _main(outfile=sys.stdout): # Option processing # ---------------------------------------------------------------------- options = "" - for line in re.findall(r"\n +(if|elif) +opt in \[(.+)\]:\s+#(.+)\n", open(sys.argv[0]).read()): + for line in re.findall(r"\n +(if|elif) +opt in \[(.+)\]:\s+#(.+)\n", io.open(sys.argv[0]).read()): if not options: options += "OPTIONS\n" options += " %-16s %s\n" % (line[1].replace('"', ''), line[2]) @@ -1357,7 +1364,7 @@ def _main(outfile=sys.stdout): company_domain = {} if opt_getauthors: - gadata = open("/www/tools.ietf.org/tools/getauthors/getauthors.data") + gadata = io.open("/www/tools.ietf.org/tools/getauthors/getauthors.data") for line in gadata: if line.startswith("company:"): try: @@ -1376,7 +1383,7 @@ def _main(outfile=sys.stdout): import gzip file = gzip.open(file) else: - file = open(file) + file = io.open(file) basename = os.path.basename(file.name) if basename.startswith("draft-"): diff --git a/ietf/utils/fields.py b/ietf/utils/fields.py index 89bca6b8c..8663cab58 100644 --- a/ietf/utils/fields.py +++ b/ietf/utils/fields.py @@ -1,7 +1,12 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals -import re import datetime +import six +import re import debug # pyflakes:ignore @@ -18,7 +23,7 @@ class MultiEmailField(forms.Field): if not value: return [] - if isinstance(value, str): + if isinstance(value, six.string_types): values = value.split(',') return [ x.strip() for x in values if x.strip() ] else: @@ -26,7 +31,6 @@ class MultiEmailField(forms.Field): def validate(self, value): "Check if value consists only of valid emails." - # Use the parent's handling of required fields, etc. super(MultiEmailField, self).validate(value) diff --git a/ietf/utils/html.py b/ietf/utils/html.py index b191c1388..3bc223939 100644 --- a/ietf/utils/html.py +++ b/ietf/utils/html.py @@ -1,12 +1,17 @@ # Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Taken from http://code.google.com/p/soclone/source/browse/trunk/soclone/utils/html.py - """Utilities for working with HTML.""" + + +from __future__ import absolute_import, print_function, unicode_literals + import bleach import copy import lxml.etree import lxml.html import lxml.html.clean +import six import debug # pyflakes:ignore @@ -54,7 +59,7 @@ class Cleaner(lxml.html.clean.Cleaner): # Copied from lxml 4.2.0 and modified to insert charset meta: def clean_html(self, html): result_type = type(html) - if isinstance(html, str): + if isinstance(html, six.string_types): doc = lxml.html.fromstring(html) else: doc = copy.deepcopy(html) diff --git a/ietf/utils/log.py b/ietf/utils/log.py index 7d15b9d04..f250aaccb 100644 --- a/ietf/utils/log.py +++ b/ietf/utils/log.py @@ -1,9 +1,14 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import sys import logging import inspect import os.path +import six import traceback try: @@ -28,23 +33,27 @@ def getclass(frame): def getcaller(): parent, pfile, pline, pfunction, lines, index = inspect.stack()[2] - moduleinfo = inspect.getmoduleinfo(pfile) - pmodule = moduleinfo[0] if moduleinfo else None + pmodule = inspect.getmodulename(pfile) pclass = getclass(parent) return (pmodule, pclass, pfunction, pfile, pline) -def log(msg): +def log(msg, e=None): "Uses syslog by preference. Logs the given calling point and message." global logfunc def _flushfunc(): pass _logfunc = logfunc if settings.SERVER_MODE == 'test': - return +## Comment in when debugging for instance test smtp server failures: +# if e: +# _logfunc = debug.say +# _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) +# else: + return elif settings.DEBUG == True: _logfunc = debug.say _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) - if isinstance(msg, str): + if isinstance(msg, six.text_type): msg = msg.encode('unicode_escape') try: mod, cls, func, file, line = getcaller() diff --git a/ietf/utils/mail.py b/ietf/utils/mail.py index 6f0cb060a..4e3961bd5 100644 --- a/ietf/utils/mail.py +++ b/ietf/utils/mail.py @@ -1,9 +1,14 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import copy import datetime import logging import re +import six import smtplib import sys import textwrap @@ -25,6 +30,7 @@ from django.core.exceptions import ImproperlyConfigured, ValidationError from django.core.validators import validate_email from django.template.loader import render_to_string from django.template import Context,RequestContext +from django.utils.encoding import force_text, force_str, force_bytes import debug # pyflakes:ignore @@ -115,12 +121,12 @@ def send_smtp(msg, bcc=None): # advertise the AUTH capability. server.ehlo() server.login(settings.EMAIL_HOST_USER, settings.EMAIL_HOST_PASSWORD) - unhandled = server.sendmail(frm, to, msg.as_bytes()) + unhandled = server.sendmail(frm, to, force_bytes(msg.as_string())) if unhandled != {}: raise SMTPSomeRefusedRecipients(message="%d addresses were refused"%len(unhandled),original_msg=msg,refusals=unhandled) except Exception as e: # need to improve log message - log("Exception while trying to send email from '%s' to %s subject '%s'" % (frm, to, msg.get('Subject', '[no subject]'))) + log("Exception while trying to send email from '%s' to %s subject '%s'" % (frm, to, msg.get('Subject', '[no subject]')), e=e) if isinstance(e, smtplib.SMTPException): e.original_msg=msg raise @@ -131,7 +137,7 @@ def send_smtp(msg, bcc=None): server.quit() except smtplib.SMTPServerDisconnected: pass - subj = msg.get('Subject', '[no subject]') + subj = force_text(msg.get('Subject', '[no subject]')) log("sent email from '%s' to %s id %s subject '%s'" % (frm, to, msg.get('Message-ID', ''), subj)) def copy_email(msg, to, toUser=False, originalBcc=None): @@ -180,7 +186,7 @@ def send_mail(request, to, frm, subject, template, context, *args, **kwargs): return send_mail_text(request, to, frm, subject, txt, *args, **kwargs) def encode_message(txt): - assert isinstance(txt, str) + assert isinstance(txt, six.text_type) return MIMEText(txt.encode('utf-8'), 'plain', 'UTF-8') def send_mail_text(request, to, frm, subject, txt, cc=None, extra=None, toUser=False, bcc=None, copy=True): @@ -219,7 +225,13 @@ def formataddr(addrtuple): address field. Does what's needed, and returns a string value suitable for use in a To: or Cc: email header field. """ - return simple_formataddr(addrtuple, charset='utf-8') + if six.PY2: + name, addr = addrtuple + if name and not isascii(name): + name = str(Header(name, 'utf-8')) + return simple_formataddr((name, addr)) + else: + return simple_formataddr(addrtuple) def parseaddr(addr): """ @@ -230,7 +242,7 @@ def parseaddr(addr): """ - addr = ''.join( [ ( s.decode(m) if m else s.decode()) if isinstance(s, bytes) else s for (s,m) in decode_header(addr) ] ) + addr = ''.join( [ ( s.decode(m) if m else s.decode()) if isinstance(s, six.binary_type) else s for (s,m) in decode_header(addr) ] ) name, addr = simple_parseaddr(addr) return name, addr @@ -269,7 +281,7 @@ def condition_message(to, frm, subject, msg, cc, extra): if name: to_hdr.append('"%s"' % name) to_hdr.append("<%s>," % addr) - to_str = to_hdr.encode() + to_str = to_hdr.encode('utf-8') if to_str and to_str[-1] == ',': to_str=to_str[:-1] # It's important to use this string, and not assign the Header object. @@ -352,7 +364,7 @@ def send_mail_mime(request, to, frm, subject, msg, cc=None, extra=None, toUser=F def parse_preformatted(preformatted, extra={}, override={}): """Parse preformatted string containing mail with From:, To:, ...,""" - msg = message_from_string(preformatted) + msg = message_from_string(force_str(preformatted)) msg.set_charset('UTF-8') for k, v in override.items(): @@ -408,7 +420,7 @@ def send_mail_preformatted(request, preformatted, extra={}, override={}): extra headers as needed).""" (msg, extra, bcc) = parse_preformatted(preformatted, extra, override) - txt = msg.get_payload() + txt = get_payload(msg) send_mail_text(request, msg['To'], msg["From"], msg["Subject"], txt, extra=extra, bcc=bcc) return msg @@ -440,10 +452,10 @@ def exception_components(e): def log_smtp_exception(e): (extype, value, tb) = exception_components(e) - log("SMTP Exception: %s : %s" % (extype,value)) + log("SMTP Exception: %s : %s" % (extype,value), e) if isinstance(e,SMTPSomeRefusedRecipients): - log(" SomeRefused: %s"%(e.summary_refusals())) - log(" Traceback: %s" % tb) + log(" SomeRefused: %s"%(e.summary_refusals()), e) + log(" Traceback: %s" % tb, e) return (extype, value, tb) def build_warning_message(request, e): @@ -540,9 +552,17 @@ def get_email_addresses_from_text(text): validate_email(addr) return True except ValidationError: - logger.error(f'Bad data: get_email_addresses_from_text() got an invalid email address tuple: {email}, in "{text}".') + logger.error('Bad data: get_email_addresses_from_text() got an ' + 'invalid email address tuple: {email}, in "{text}".'.format(email=email, text=text)) return False # whitespace normalization -- getaddresses doesn't do this text = re.sub(r'(?u)\s+', ' ', text) return [ formataddr(e) for e in getaddresses([text, ]) if valid(e) ] + +def get_payload(msg, decode=False): + if six.PY2: + return msg.get_payload(decode=decode).decode(msg.get_content_charset('utf-8')) + else: + return msg.get_payload(decode=decode) + \ No newline at end of file diff --git a/ietf/utils/management/commands/check_referential_integrity.py b/ietf/utils/management/commands/check_referential_integrity.py index 3b8d7f2a7..8d409afac 100644 --- a/ietf/utils/management/commands/check_referential_integrity.py +++ b/ietf/utils/management/commands/check_referential_integrity.py @@ -1,4 +1,10 @@ # Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import six from tqdm import tqdm @@ -26,18 +32,18 @@ class Command(BaseCommand): debug.pprint('dir(field)') raise if verbosity > 1: - print(" %s -> %s.%s" % (field.name,foreign_model.__module__,foreign_model.__name__), end=' ') + six.print_(" %s -> %s.%s" % (field.name,foreign_model.__module__,foreign_model.__name__), end=' ') used = set(field.model.objects.values_list(field.name,flat=True)) used.discard(None) exists = set(foreign_model.objects.values_list('pk',flat=True)) if verbosity > 1: if used - exists: - print(" ** Bad key values:",list(used - exists)) + six.print_(" ** Bad key values:",list(used - exists)) else: - print(" ok") + six.print_(" ok") else: if used - exists: - print("\n%s.%s.%s -> %s.%s ** Bad key values:" % (model.__module__,model.__name__,field.name,foreign_model.__module__,foreign_model.__name__),list(used - exists)) + six.print_("\n%s.%s.%s -> %s.%s ** Bad key values:" % (model.__module__,model.__name__,field.name,foreign_model.__module__,foreign_model.__name__),list(used - exists)) def check_reverse_field(field): try: @@ -50,33 +56,33 @@ class Command(BaseCommand): foreign_field_name = field.remote_field.name foreign_accessor_name = field.remote_field.get_accessor_name() if verbosity > 1: - print(" %s <- %s -> %s.%s" % (field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), end=' ') + six.print_(" %s <- %s -> %s.%s" % (field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), end=' ') try: used = set(foreign_model.objects.values_list(foreign_field_name, flat=True)) except FieldError: try: used = set(foreign_model.objects.values_list(foreign_accessor_name, flat=True)) except FieldError: - print(" ** Warning: could not find reverse name for %s.%s -> %s.%s" % (field.model.__module__, field.model.__name__, foreign_model.__name__, foreign_field_name), end=' ') + six.print_(" ** Warning: could not find reverse name for %s.%s -> %s.%s" % (field.model.__module__, field.model.__name__, foreign_model.__name__, foreign_field_name), end=' ') used.discard(None) exists = set(field.model.objects.values_list('pk',flat=True)) if verbosity > 1: if used - exists: - print(" ** Bad key values:\n ",list(used - exists)) + six.print_(" ** Bad key values:\n ",list(used - exists)) else: - print(" ok") + six.print_(" ok") else: if used - exists: - print("\n%s.%s <- %s -> %s.%s ** Bad key values:\n " % (field.model.__module__, field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), list(used - exists)) + six.print_("\n%s.%s <- %s -> %s.%s ** Bad key values:\n " % (field.model.__module__, field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), list(used - exists)) for conf in tqdm([ c for c in apps.get_app_configs() if c.name.startswith('ietf.')], desc='apps', disable=verbose): if verbosity > 1: - print("Checking", conf.name) + six.print_("Checking", conf.name) for model in tqdm(list(conf.get_models()), desc='models', disable=verbose): if model._meta.proxy: continue if verbosity > 1: - print(" %s.%s" % (model.__module__,model.__name__)) + six.print_(" %s.%s" % (model.__module__,model.__name__)) for field in [f for f in model._meta.fields if isinstance(f, (ForeignKey, OneToOneField)) ]: check_field(field) for field in [f for f in model._meta.many_to_many ]: diff --git a/ietf/utils/management/commands/coverage_changes.py b/ietf/utils/management/commands/coverage_changes.py index 33e96d5e9..9058fc790 100644 --- a/ietf/utils/management/commands/coverage_changes.py +++ b/ietf/utils/management/commands/coverage_changes.py @@ -1,9 +1,14 @@ # Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals -import os -import json -import codecs import gzip +import io +import json +import os +import six from difflib import ndiff @@ -56,12 +61,12 @@ class Command(BaseCommand): valid_sections = ['template', 'url', 'code'] def read_coverage(self, filename, version=None): - if isinstance(filename, str): + if isinstance(filename, six.string_types): try: if filename.endswith(".gz"): file = gzip.open(filename, "rb") else: - file = codecs.open(filename, "r", encoding="utf-8") + file = io.open(filename, "r", encoding="utf-8") except IOError as e: self.stderr.write("%s" % e) exit(1) diff --git a/ietf/utils/management/commands/create_group_wikis.py b/ietf/utils/management/commands/create_group_wikis.py index c9a007ecf..e648a5985 100644 --- a/ietf/utils/management/commands/create_group_wikis.py +++ b/ietf/utils/management/commands/create_group_wikis.py @@ -1,7 +1,12 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import os import copy +import io import syslog import pkg_resources @@ -132,7 +137,7 @@ class Command(BaseCommand): name = unicode_unquote(name.encode('utf-8')) if os.path.isfile(filename): self.note(" Adding page %s" % name) - with open(filename) as file: + with io.open(filename) as file: text = file.read().decode('utf-8') self.add_wiki_page(env, name, text) diff --git a/ietf/utils/management/commands/dumprelated.py b/ietf/utils/management/commands/dumprelated.py index 61b38441d..60a7ea70b 100644 --- a/ietf/utils/management/commands/dumprelated.py +++ b/ietf/utils/management/commands/dumprelated.py @@ -1,4 +1,10 @@ # Copyright The IETF Trust 2018-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import warnings from collections import OrderedDict @@ -187,7 +193,7 @@ class Command(BaseCommand): if (output and self.stdout.isatty() and options['verbosity'] > 0): progress_output = self.stdout object_count = sum(get_objects(count_only=True)) - stream = open(output, 'w') if output else None + stream = io.open(output, 'w') if output else None try: serializers.serialize( format, get_objects(), indent=indent, diff --git a/ietf/utils/management/commands/import_htpasswd.py b/ietf/utils/management/commands/import_htpasswd.py index 6093dfb08..2a8a2e5b7 100644 --- a/ietf/utils/management/commands/import_htpasswd.py +++ b/ietf/utils/management/commands/import_htpasswd.py @@ -1,3 +1,5 @@ +# Copyright The IETF Trust 2014-2019, All Rights Reserved +import io import sys from textwrap import dedent @@ -6,7 +8,7 @@ from django.contrib.auth.models import User from django.core.management.base import BaseCommand def import_htpasswd_file(filename, verbosity=1, overwrite=False): - with open(filename) as file: + with io.open(filename) as file: for line in file: if not ':' in line: raise ValueError('Found a line without colon separator in the htpassword file %s:'+ diff --git a/ietf/utils/management/commands/makefixture.py b/ietf/utils/management/commands/makefixture.py index 5046eb7ea..fbffb061f 100644 --- a/ietf/utils/management/commands/makefixture.py +++ b/ietf/utils/management/commands/makefixture.py @@ -1,5 +1,10 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- # From https://github.com/ericholscher/django-test-utils/blob/master/test_utils/management/commands/makefixture.py + + +from __future__ import absolute_import, print_function, unicode_literals + """ "Make fixture" command. @@ -39,6 +44,9 @@ python manage.py makefixture --format=xml --indent=4 YourModel[3] AnotherModel a #known issues: #no support for generic relations #no support for one-to-one relations + +import six + from django.core import serializers from django.core.management.base import CommandError from django.core.management.base import LabelCommand @@ -102,7 +110,7 @@ class Command(LabelCommand): objects = [] for model, slice in models: - if isinstance(slice, str) and slice: + if isinstance(slice, six.string_types) and slice: objects.extend(model._default_manager.filter(pk__exact=slice)) elif not slice or type(slice) is list: items = model._default_manager.all() diff --git a/ietf/utils/management/commands/tests.py b/ietf/utils/management/commands/tests.py index fb77a44e1..807141703 100644 --- a/ietf/utils/management/commands/tests.py +++ b/ietf/utils/management/commands/tests.py @@ -1,10 +1,17 @@ # Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os +import six import tempfile from django.core.management import call_command from django.test import TestCase -from io import StringIO +#from io import StringIO import debug # pyflakes:ignore @@ -78,12 +85,12 @@ class CoverageChangeTestCase(TestCase): } """ mfh, master = tempfile.mkstemp(suffix='.json') - with open(master, "w") as file: + with io.open(master, "w") as file: file.write(master_txt) lfh, latest = tempfile.mkstemp(suffix='.json') - with open(latest, "w") as file: + with io.open(latest, "w") as file: file.write(latest_txt) - output = StringIO() + output = six.StringIO() call_command('coverage_changes', master, latest, stdout=output) text = output.getvalue() os.unlink(master) diff --git a/ietf/utils/markup_txt.py b/ietf/utils/markup_txt.py index 7542c439d..6e680fd81 100644 --- a/ietf/utils/markup_txt.py +++ b/ietf/utils/markup_txt.py @@ -1,4 +1,6 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- +# # Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -31,7 +33,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import re +import six # pyflakes:ignore from django.utils.html import escape @@ -39,7 +45,7 @@ from ietf.utils import log from ietf.utils.text import wordwrap def markup(content, width=None): - log.assertion('isinstance(content, str)') + log.assertion('isinstance(content, six.text_type)') # normalize line endings to LF only content = content.replace("\r\n", "\n") content = content.replace("\r", "\n") diff --git a/ietf/utils/pdf.py b/ietf/utils/pdf.py index 5b5ca2371..c4d39601c 100644 --- a/ietf/utils/pdf.py +++ b/ietf/utils/pdf.py @@ -1,10 +1,16 @@ # Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import re def pdf_pages(filename): """Return number of pages in PDF.""" try: - infile = open(filename, "r") + infile = io.open(filename, "r") except IOError: return 0 for line in infile: diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index d3935c924..d146ad75a 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -1,7 +1,6 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- - - +# # Portion Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -35,6 +34,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import, print_function, unicode_literals + +import io import re import os import sys @@ -44,7 +46,6 @@ import pytz import importlib import socket import datetime -import codecs import gzip import unittest import factory.random @@ -228,7 +229,7 @@ def save_test_results(failures, test_labels): # results and avoid re-running tests if we've alread run them with OK # result after the latest code changes: topdir = os.path.dirname(os.path.dirname(settings.BASE_DIR)) - tfile = codecs.open(os.path.join(topdir,".testresult"), "a", encoding='utf-8') + tfile = io.open(os.path.join(topdir,".testresult"), "a", encoding='utf-8') timestr = time.strftime("%Y-%m-%d %H:%M:%S") if failures: tfile.write("%s FAILED (failures=%s)\n" % (timestr, failures)) @@ -266,7 +267,7 @@ class CoverageReporter(Reporter): analysis = self.coverage._analyze(fr) nums = analysis.numbers missing_nums = sorted(analysis.missing) - with open(analysis.filename) as file: + with io.open(analysis.filename) as file: lines = file.read().splitlines() missing_lines = [ lines[l-1] for l in missing_nums ] result["covered"][fr.relative_filename()] = (nums.n_statements, nums.pc_covered/100.0, missing_nums, missing_lines) @@ -526,7 +527,7 @@ class IetfTestRunner(DiscoverRunner): with gzip.open(self.coverage_file, "rb") as file: self.coverage_master = json.load(file) else: - with codecs.open(self.coverage_file, encoding='utf-8') as file: + with io.open(self.coverage_file, encoding='utf-8') as file: self.coverage_master = json.load(file) self.coverage_data = { "time": datetime.datetime.now(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), @@ -613,7 +614,7 @@ class IetfTestRunner(DiscoverRunner): coverage_latest = {} coverage_latest["version"] = "latest" coverage_latest["latest"] = self.coverage_data - with codecs.open(latest_coverage_file, "w", encoding='utf-8') as file: + with open(latest_coverage_file, "w") as file: json.dump(coverage_latest, file, indent=2, sort_keys=True) if self.save_version_coverage: self.coverage_master["version"] = self.save_version_coverage @@ -622,7 +623,7 @@ class IetfTestRunner(DiscoverRunner): with gzip.open(self.coverage_file, "wt", encoding='ascii') as file: json.dump(self.coverage_master, file, sort_keys=True) else: - with codecs.open(self.coverage_file, "w", encoding="utf-8") as file: + with open(self.coverage_file, "w") as file: json.dump(self.coverage_master, file, indent=2, sort_keys=True) super(IetfTestRunner, self).teardown_test_environment(**kwargs) diff --git a/ietf/utils/test_smtpserver.py b/ietf/utils/test_smtpserver.py index 48c3cbc2c..b74797e52 100644 --- a/ietf/utils/test_smtpserver.py +++ b/ietf/utils/test_smtpserver.py @@ -1,8 +1,13 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import smtpd import threading import asyncore +import six import debug # pyflakes:ignore @@ -34,20 +39,26 @@ class SMTPTestChannel(smtpd.SMTPChannel): # mail_options = ['BODY=8BITMIME', 'SMTPUTF8'] def smtp_RCPT(self, arg): - self.rcpt_options = [] - if not self.mailfrom: - self.push('503 Error: need MAIL command') + if (six.PY2 and not self._SMTPChannel__mailfrom) or (six.PY3 and not self.mailfrom): + self.push(str('503 Error: need MAIL command')) return - arg = self._strip_command_keyword('TO:', arg) - address, params = self._getaddr(arg) + if six.PY2: + address = self._SMTPChannel__getaddr('TO:', arg) if arg else None + else: + arg = self._strip_command_keyword('TO:', arg) + address, __ = self._getaddr(arg) if not address: - self.push('501 Syntax: RCPT TO:
') + self.push(str('501 Syntax: RCPT TO:
')) return if "poison" in address: - self.push('550 Error: Not touching that') - return - self.rcpttos.append(address) - self.push('250 Ok') + self.push(str('550 Error: Not touching that')) + return + if six.PY2: + self._SMTPChannel__rcpttos.append(address) + else: + self.rcpt_options = [] + self.rcpttos.append(address) + self.push(str('250 Ok')) class SMTPTestServer(smtpd.SMTPServer): diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index bf1e3ce61..fc5df7ef6 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -1,5 +1,6 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved - +# -*- coding: utf-8 -*- +# # Portion Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -32,28 +33,36 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import os import re import email import html5lib +import six import sys -import urllib.request, urllib.error, urllib.parse + +from six.moves.urllib.parse import unquote from unittest.util import strclass from bs4 import BeautifulSoup import django.test from django.conf import settings +from django.utils.encoding import python_2_unicode_compatible from django.utils.text import slugify import debug # pyflakes:ignore +from ietf.utils.mail import get_payload + real_database_name = settings.DATABASES["default"]["NAME"] def split_url(url): if "?" in url: url, args = url.split("?", 1) - args = dict([ list(map(urllib.parse.unquote,arg.split("=", 1))) for arg in args.split("&") if "=" in arg ]) + args = dict([ list(map(unquote,arg.split("=", 1))) for arg in args.split("&") if "=" in arg ]) else: args = {} return url, args @@ -92,6 +101,7 @@ class ReverseLazyTest(django.test.TestCase): response = self.client.get('/ipr/update/') self.assertRedirects(response, "/ipr/", status_code=301) +@python_2_unicode_compatible class TestCase(django.test.TestCase): """ Does basically the same as django.test.TestCase, but adds asserts for html5 validation. @@ -123,6 +133,8 @@ class TestCase(django.test.TestCase): def tempdir(self, label): slug = slugify(self.__class__.__name__.replace('.','-')) dirname = "tmp-{label}-{slug}-dir".format(**locals()) + if 'VIRTUAL_ENV' in os.environ: + dirname = os.path.join(os.environ['VIRTUAL_ENV'], dirname) path = os.path.abspath(dirname) if not os.path.exists(path): os.mkdir(path) @@ -163,7 +175,8 @@ class TestCase(django.test.TestCase): if subject: mlist = [ m for m in mlist if subject in m["Subject"] ] if text: - mlist = [ m for m in mlist if text in m.get_payload() ] + assert isinstance(text, six.text_type) + mlist = [ m for m in mlist if text in get_payload(m) ] if count and len(mlist) != count: sys.stderr.write("Wrong count in assertMailboxContains(). The complete mailbox contains %s emails:\n\n" % len(mailbox)) for m in mailbox: @@ -175,6 +188,6 @@ class TestCase(django.test.TestCase): self.assertGreater(len(mlist), 0) def __str__(self): - return "%s (%s.%s)" % (self._testMethodName, strclass(self.__class__),self._testMethodName) + return u"%s (%s.%s)" % (self._testMethodName, strclass(self.__class__),self._testMethodName) diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 815869ada..a54758148 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -2,17 +2,20 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import io import os.path -import types import shutil +import six +import types from email.mime.image import MIMEImage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from fnmatch import fnmatch from importlib import import_module -#from .pipe import pipe -#from io import StringIO +from .pipe import pipe from textwrap import dedent from unittest import skipIf from tempfile import mkdtemp @@ -20,7 +23,7 @@ from tempfile import mkdtemp from django.apps import apps from django.contrib.auth.models import User from django.conf import settings -#from django.core.management import call_command +from django.core.management import call_command from django.template import Context from django.template.defaulttags import URLNode from django.template.loader import get_template @@ -29,14 +32,14 @@ from django.urls import reverse as urlreverse import debug # pyflakes:ignore -#from ietf.group.factories import GroupFactory -#from ietf.group.models import Group +from ietf.group.factories import GroupFactory +from ietf.group.models import Group from ietf.person.name import name_parts, unidecode_name from ietf.submit.tests import submission_file from ietf.utils.bower_storage import BowerStorageFinder from ietf.utils.draft import Draft, getmeta from ietf.utils.log import unreachable, assertion -from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox +from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox, get_payload from ietf.utils.test_runner import get_template_paths, set_coverage_checking from ietf.utils.test_utils import TestCase @@ -68,7 +71,7 @@ body self.assertSameEmail(recv['Cc'], 'cc1@example.com, cc2@example.com') self.assertSameEmail(recv['Bcc'], None) self.assertEqual(recv['Subject'], 'subject') - self.assertEqual(recv.get_payload(), 'body\n') + self.assertEqual(get_payload(recv), 'body\n') override = { 'To': 'oto1@example.net, oto2@example.net', @@ -98,7 +101,7 @@ body self.assertSameEmail(recv['Cc'], ', occ2@example.net') self.assertSameEmail(recv['Bcc'], None) self.assertEqual(recv['Subject'], 'osubject') - self.assertEqual(recv.get_payload(), 'body\n') + self.assertEqual(get_payload(recv), 'body\n') extra = {'Fuzz': [ 'bucket' ]} send_mail_preformatted(request=None, preformatted=msg, extra=extra, override={}) @@ -157,12 +160,12 @@ def get_callbacks(urllist): callbacks.update(get_callbacks(entry.url_patterns)) else: if hasattr(entry, '_callback_str'): - callbacks.add(str(entry._callback_str)) + callbacks.add(six.ensure_text(entry._callback_str)) if (hasattr(entry, 'callback') and entry.callback and type(entry.callback) in [types.FunctionType, types.MethodType ]): callbacks.add("%s.%s" % (entry.callback.__module__, entry.callback.__name__)) if hasattr(entry, 'name') and entry.name: - callbacks.add(str(entry.name)) + callbacks.add(six.ensure_text(entry.name)) # There are some entries we don't handle here, mostly clases # (such as Feed subclasses) @@ -275,6 +278,7 @@ class TemplateChecksTestCase(TestCase): r = self.client.get(url) self.assertTemplateUsed(r, '500.html') +@skipIf(six.PY3, "Trac not available for Python3 as of 14 Jul 2019") @skipIf(skip_wiki_glue_testing, skip_message) class TestWikiGlueManagementCommand(TestCase): @@ -283,6 +287,7 @@ class TestWikiGlueManagementCommand(TestCase): # command through command line switches. We have to do it this way because the # management command reads in its own copy of settings.py in its own python # environment, so we can't modify it here. + set_coverage_checking(False) self.wiki_dir_pattern = os.path.abspath('tmp-wiki-dir-root/%s') if not os.path.exists(os.path.dirname(self.wiki_dir_pattern)): os.mkdir(os.path.dirname(self.wiki_dir_pattern)) @@ -293,44 +298,45 @@ class TestWikiGlueManagementCommand(TestCase): def tearDown(self): shutil.rmtree(os.path.dirname(self.wiki_dir_pattern)) shutil.rmtree(os.path.dirname(self.svn_dir_pattern)) + set_coverage_checking(True) -# def test_wiki_create_output(self): -# for type in ['wg','rg','ag','area']: -# GroupFactory(type_id=type) -# groups = Group.objects.filter( -# type__slug__in=['wg','rg','ag','area'], -# state__slug='active' -# ).order_by('acronym') -# out = StringIO() -# err = StringIO() -# call_command('create_group_wikis', stdout=out, stderr=err, verbosity=2, -# wiki_dir_pattern=self.wiki_dir_pattern, -# svn_dir_pattern=self.svn_dir_pattern, -# ) -# command_output = out.getvalue() -# command_errors = err.getvalue() -# self.assertEqual("", command_errors) -# for group in groups: -# self.assertIn("Processing group '%s'" % group.acronym, command_output) -# # Do a bit of verification using trac-admin, too -# admin_code, admin_output, admin_error = pipe( -# 'trac-admin %s permission list' % (self.wiki_dir_pattern % group.acronym)) -# self.assertEqual(admin_code, 0) -# roles = group.role_set.filter(name_id__in=['chair', 'secr', 'ad']) -# for role in roles: -# user = role.email.address.lower() -# self.assertIn("Granting admin permission for %s" % user, command_output) -# self.assertIn(user, admin_output) -# docs = group.document_set.filter(states__slug='active', type_id='draft') -# for doc in docs: -# name = doc.name -# name = name.replace('draft-','') -# name = name.replace(doc.stream_id+'-', '') -# name = name.replace(group.acronym+'-', '') -# self.assertIn("Adding component %s"%name, command_output) -# for page in settings.TRAC_WIKI_PAGES_TEMPLATES: -# self.assertIn("Adding page %s" % os.path.basename(page), command_output) -# self.assertIn("Indexing default repository", command_output) + def test_wiki_create_output(self): + for type in ['wg','rg','ag','area']: + GroupFactory(type_id=type) + groups = Group.objects.filter( + type__slug__in=['wg','rg','ag','area'], + state__slug='active' + ).order_by('acronym') + out = six.StringIO() + err = six.StringIO() + call_command('create_group_wikis', stdout=out, stderr=err, verbosity=2, + wiki_dir_pattern=self.wiki_dir_pattern, + svn_dir_pattern=self.svn_dir_pattern, + ) + command_output = out.getvalue() + command_errors = err.getvalue() + self.assertEqual("", command_errors) + for group in groups: + self.assertIn("Processing group '%s'" % group.acronym, command_output) + # Do a bit of verification using trac-admin, too + admin_code, admin_output, admin_error = pipe( + 'trac-admin %s permission list' % (self.wiki_dir_pattern % group.acronym)) + self.assertEqual(admin_code, 0) + roles = group.role_set.filter(name_id__in=['chair', 'secr', 'ad']) + for role in roles: + user = role.email.address.lower() + self.assertIn("Granting admin permission for %s" % user, command_output) + self.assertIn(user, admin_output) + docs = group.document_set.filter(states__slug='active', type_id='draft') + for doc in docs: + name = doc.name + name = name.replace('draft-','') + name = name.replace(doc.stream_id+'-', '') + name = name.replace(group.acronym+'-', '') + self.assertIn("Adding component %s"%name, command_output) + for page in settings.TRAC_WIKI_PAGES_TEMPLATES: + self.assertIn("Adding page %s" % os.path.basename(page), command_output) + self.assertIn("Indexing default repository", command_output) OMITTED_APPS = [ 'ietf.secr.meetings', @@ -424,7 +430,7 @@ class DraftTests(TestCase): def test_get_meta(self): tempdir = mkdtemp() filename = os.path.join(tempdir,self.draft.source) - with open(filename,'w') as file: + with io.open(filename,'w') as file: file.write(self.draft.text) self.assertEqual(getmeta(filename)['docdeststatus'],'Informational') shutil.rmtree(tempdir) diff --git a/ietf/utils/tests_restapi.py b/ietf/utils/tests_restapi.py index 31f1be70f..b8eb591b6 100644 --- a/ietf/utils/tests_restapi.py +++ b/ietf/utils/tests_restapi.py @@ -1,6 +1,12 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import six +import sys + import debug debug.debug = True @@ -20,12 +26,16 @@ class RestApi(ResourceTestCaseMixin, TestCase): """ # print(' fetching %s' % resource) r = self.api_client.get(resource, format=format) - if format == 'json': - self.assertValidJSONResponse(r) - elif format == 'xml': - self.assertValidXMLResponse(r) - else: - raise Exception("Unknown format found when testing the RestApi: %s" % (format, )) + try: + if format == 'json': + self.assertValidJSONResponse(r) + elif format == 'xml': + self.assertValidXMLResponse(r) + else: + raise Exception("Unknown format found when testing the RestApi: %s" % (format, )) + except Exception: + sys.stderr.write(" * Exception for resource: %s, format: %s\n" % (resource, format)) + raise data = self.deserialize(r) for name in data: if 'list_endpoint' in data[name]: @@ -74,6 +84,6 @@ class RestApi(ResourceTestCaseMixin, TestCase): for doc in doclist: for key in doc: value = doc[key] - if isinstance(value, str) and value.startswith('%s/'%apitop): + if isinstance(value, six.string_types) and value.startswith('%s/'%apitop): self.api_client.get(value, format='json') diff --git a/ietf/utils/text.py b/ietf/utils/text.py index fbcf1470e..3faa978fb 100644 --- a/ietf/utils/text.py +++ b/ietf/utils/text.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + import re +import six import textwrap import unicodedata @@ -56,7 +60,7 @@ def fill(text, width): def wordwrap(text, width=80): """Wraps long lines without loosing the formatting and indentation of short lines""" - if not isinstance(text, (bytes,str)): + if not isinstance(text, six.string_types): return text width = int(width) # ensure we have an int, if this is used as a template filter text = re.sub(" *\r\n", "\n", text) # get rid of DOS line endings @@ -133,7 +137,7 @@ def maybe_split(text, split=True, pos=5000): return text def decode(raw): - assert isinstance(raw, bytes) + assert isinstance(raw, six.binary_type) try: text = raw.decode('utf-8') except UnicodeDecodeError: @@ -145,7 +149,7 @@ def decode(raw): def text_to_dict(t): "Converts text with RFC2822-formatted header fields into a dictionary-like object." # ensure we're handed a unicode parameter - assert isinstance(t, str) + assert isinstance(t, six.text_type) d = {} # Return {} for malformed input if not len(t.lstrip()) == len(t): diff --git a/ietf/utils/urls.py b/ietf/utils/urls.py index 46fe0cc20..a6a3abe9d 100644 --- a/ietf/utils/urls.py +++ b/ietf/utils/urls.py @@ -1,6 +1,12 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import debug # pyflakes:ignore +import six + from inspect import isclass from django.conf.urls import url as django_url @@ -16,7 +22,7 @@ def url(regex, view, kwargs=None, name=None): branch = 'name' elif isinstance(view, (list, tuple)): branch = 'list' - elif isinstance(view, str): + elif isinstance(view, six.string_types): branch = 'string' name = view elif callable(view) and hasattr(view, '__name__'): diff --git a/ietf/virtualenv-manage.py b/ietf/virtualenv-manage.py index dc318c738..6cfe31e06 100755 --- a/ietf/virtualenv-manage.py +++ b/ietf/virtualenv-manage.py @@ -1,6 +1,11 @@ -# Copyright The IETF Trust 2016-2019, All Rights Reserved #!/usr/bin/env python +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import sys @@ -12,7 +17,7 @@ os.chdir(path) # Virtualenv support virtualenv_activation = os.path.join(path, "env", "bin", "activate_this.py") if os.path.exists(virtualenv_activation): - exec(compile(open(virtualenv_activation, "rb").read(), virtualenv_activation, 'exec'), dict(__file__=virtualenv_activation)) + exec(compile(io.open(virtualenv_activation, "rb").read(), virtualenv_activation, 'exec'), dict(__file__=virtualenv_activation)) else: raise RuntimeError("Could not find the expected virtual python environment.") diff --git a/ietf/wsgi.py b/ietf/wsgi.py index 01080e8b3..cacc8ed60 100644 --- a/ietf/wsgi.py +++ b/ietf/wsgi.py @@ -1,4 +1,9 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + """ WSGI configuration for the datatracker. @@ -39,19 +44,20 @@ WSGIPythonEggs /var/www/.python-eggs/ """ +import io import os import sys import syslog path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -syslog.openlog("datatracker", syslog.LOG_PID, syslog.LOG_USER) +syslog.openlog(str("datatracker"), syslog.LOG_PID, syslog.LOG_USER) # Virtualenv support virtualenv_activation = os.path.join(path, "env", "bin", "activate_this.py") if os.path.exists(virtualenv_activation): syslog.syslog("Starting datatracker wsgi with virtualenv %s" % os.path.dirname(os.path.dirname(virtualenv_activation))) - exec(compile(open(virtualenv_activation, "rb").read(), virtualenv_activation, 'exec'), dict(__file__=virtualenv_activation)) + exec(compile(io.open(virtualenv_activation, "rb").read(), virtualenv_activation, 'exec'), dict(__file__=virtualenv_activation)) else: syslog.syslog("Starting datatracker wsgi without virtualenv")