altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Python2/3 compatibility: Changed the use of open() and StringIO to io.open() etc.

Browse source 
- Legacy-Id: 16458
This commit is contained in:
Henrik Levkowetz 2019-07-15 19:14:04 +00:00
parent 4ce6768399
commit 8c6eb3a30a
115 changed files with 1361 additions and 692 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
changelog.py
View file

@ -1,5 +1,9 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import re
import six
from tzparse import tzparse
from datetime import datetime as Datetime
@ -44,7 +48,7 @@ def parse(logfile):
inf_line = r"^ \*\*(.*)\*\* *"
entries = []
if type(logfile) == type(''):
if isinstance(logfile, six.string_types):
logfile = open(logfile)
entry = None
for line in logfile:

22
ietf/api/__init__.py
View file

@ -1,7 +1,14 @@
# Copyright The IETF Trust 2014-2019, All Rights Reserved
import re
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
from urllib.parse import urlencode
import re
import six
from six.moves.urllib.parse import urlencode
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
@ -75,7 +82,7 @@ class TimedeltaField(ApiField):
if value is None:
return None
if isinstance(value, str):
if isinstance(value, six.string_types):
match = TIMEDELTA_REGEX.search(value)
if match:
@ -90,7 +97,7 @@ class TimedeltaField(ApiField):
value = super(TimedeltaField, self).hydrate(bundle)
if value and not hasattr(value, 'seconds'):
if isinstance(value, str):
if isinstance(value, six.string_types):
try:
match = TIMEDELTA_REGEX.search(value)
@ -112,14 +119,17 @@ class ToOneField(tastypie.fields.ToOneField):
def dehydrate(self, bundle, for_list=True):
foreign_obj = None
previous_obj = None
attrib = None
if callable(self.attribute):
previous_obj = bundle.obj
foreign_obj = self.attribute(bundle)
elif isinstance(self.attribute, str):
elif isinstance(self.attribute, six.string_types):
foreign_obj = bundle.obj
for attr in self._attrs:
attrib = attr
previous_obj = foreign_obj
try:
foreign_obj = getattr(foreign_obj, attr, None)
@ -131,7 +141,7 @@ class ToOneField(tastypie.fields.ToOneField):
if callable(self.attribute):
raise ApiFieldError("The related resource for resource %s could not be found." % (previous_obj))
else:
raise ApiFieldError("The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attr))
raise ApiFieldError("The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attrib))
return None
fk_resource = self.get_related_resource(foreign_obj)

8
ietf/api/serializer.py
View file

@ -1,6 +1,12 @@
# Copyright The IETF Trust 2018-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import hashlib
import json
import six
from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist, FieldError
@ -148,7 +154,7 @@ class AdminJsonSerializer(Serializer):
if hasattr(field_value, "_meta"):
self._current[name] = self.expand_related(field_value, name)
else:
self._current[name] = str(field_value)
self._current[name] = six.ensure_text(field_value)
except ObjectDoesNotExist:
pass
except AttributeError:

14
ietf/community/views.py
View file

@ -1,16 +1,22 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import csv
import uuid
import datetime
import json
import six
import uuid
from django.http import HttpResponse, HttpResponseForbidden, HttpResponseRedirect, Http404
from django.shortcuts import get_object_or_404, render
from django.contrib.auth.decorators import login_required
from django.utils.html import strip_tags
import debug # pyflakes:ignore
from ietf.community.models import SearchRule, EmailSubscription
from ietf.community.forms import SearchRuleTypeForm, SearchRuleForm, AddDocumentsForm, SubscriptionForm
from ietf.community.utils import lookup_community_list, can_manage_community_list
@ -174,7 +180,7 @@ def export_to_csv(request, username=None, acronym=None, group_type=None):
response['Content-Disposition'] = 'attachment; filename=%s' % filename
writer = csv.writer(response, dialect=csv.excel, delimiter=',')
writer = csv.writer(response, dialect=csv.excel, delimiter=str(','))
header = [
"Name",
@ -196,7 +202,7 @@ def export_to_csv(request, username=None, acronym=None, group_type=None):
row.append(e.time.strftime("%Y-%m-%d") if e else "")
row.append(strip_tags(doc.friendly_state()))
row.append(doc.group.acronym if doc.group else "")
row.append(str(doc.ad) if doc.ad else "")
row.append(six.ensure_text(doc.ad) if doc.ad else "")
e = doc.latest_event()
row.append(e.time.strftime("%Y-%m-%d") if e else "")
writer.writerow([v.encode("utf-8") for v in row])
@ -221,7 +227,7 @@ def feed(request, username=None, acronym=None, group_type=None):
host = request.get_host()
feed_url = 'https://%s%s' % (host, request.get_full_path())
feed_id = uuid.uuid5(uuid.NAMESPACE_URL, feed_url)
feed_id = uuid.uuid5(uuid.NAMESPACE_URL, str(feed_url))
title = '%s RSS Feed' % clist.long_name()
if significant:
subtitle = 'Significant document changes'

149
ietf/cookies/tests.py
View file

@ -1,6 +1,11 @@
# Copyright The IETF Trust 2015-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
from pyquery import PyQuery
from http.cookies import SimpleCookie
from six.moves.http_cookies import SimpleCookie
from django.urls import reverse as urlreverse
@ -22,7 +27,7 @@ class CookieTests(TestCase):
def test_settings_defaults_from_cookies(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '7', 'expires_soon' : 7, 'left_menu': 'on', })
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '7', str('expires_soon') : 7, str('left_menu'): 'on', })
r = self.client.get(urlreverse("ietf.cookies.views.preferences"))
self.assertEqual(r.status_code, 200)
self.assertListEqual([], list(r.cookies.keys()))
@ -33,7 +38,7 @@ class CookieTests(TestCase):
self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/on"]').contents(), ['On'])
def test_settings_values_from_cookies_garbage(self):
self.client.cookies = SimpleCookie({'full_draft': 'foo', 'new_enough' : 'foo', 'expires_soon' : 'foo', 'left_menu': 'foo', })
self.client.cookies = SimpleCookie({str('full_draft'): 'foo', str('new_enough') : 'foo', str('expires_soon') : 'foo', str('left_menu'): 'foo', })
r = self.client.get(urlreverse("ietf.cookies.views.preferences"))
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
@ -43,7 +48,7 @@ class CookieTests(TestCase):
self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off'])
def test_settings_values_from_cookies_random(self):
self.client.cookies = SimpleCookie({'full_draft': 'zappa', 'new_enough' : '365', 'expires_soon' : '5', 'left_menu': 'zappa', })
self.client.cookies = SimpleCookie({str('full_draft'): 'zappa', str('new_enough') : '365', str('expires_soon') : '5', str('left_menu'): 'zappa', })
r = self.client.get(urlreverse("ietf.cookies.views.preferences"))
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
@ -58,7 +63,7 @@ class CookieTests(TestCase):
# self.assertNotRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon')
def test_settings_values_from_cookies_1(self):
self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '90', 'expires_soon' : 7, 'left_menu': 'off', })
self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '90', str('expires_soon') : 7, str('left_menu'): 'off', })
r = self.client.get(urlreverse("ietf.cookies.views.preferences"))
self.assertEqual(r.status_code, 200)
self.assertListEqual([], list(r.cookies.keys()))
@ -72,7 +77,7 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*7 days')
def test_settings_values_from_cookies_2(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '60', 'expires_soon' : 14, 'left_menu': 'on', })
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '60', str('expires_soon') : 14, str('left_menu'): 'on', })
r = self.client.get(urlreverse("ietf.cookies.views.preferences"))
self.assertEqual(r.status_code, 200)
self.assertListEqual([], list(r.cookies.keys()))
@ -86,7 +91,7 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days')
def test_settings_values_from_cookies_3(self):
self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '30', 'expires_soon' : 21, 'left_menu': 'off'})
self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '30', str('expires_soon') : 21, str('left_menu'): 'off'})
r = self.client.get(urlreverse("ietf.cookies.views.preferences"))
self.assertEqual(r.status_code, 200)
self.assertListEqual([], list(r.cookies.keys()))
@ -100,7 +105,7 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*21 days')
def test_settings_values_from_cookies_4(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '21', 'expires_soon' : 30, 'left_menu': 'on', })
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '21', str('expires_soon') : 30, str('left_menu'): 'on', })
r = self.client.get(urlreverse("ietf.cookies.views.preferences"))
self.assertEqual(r.status_code, 200)
self.assertListEqual([], list(r.cookies.keys()))
@ -114,7 +119,7 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*30 days')
def test_settings_values_from_cookies_5(self):
self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '14', 'expires_soon' : 60, 'left_menu': 'off', })
self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '14', str('expires_soon') : 60, str('left_menu'): 'off', })
r = self.client.get(urlreverse("ietf.cookies.views.preferences"))
self.assertEqual(r.status_code, 200)
self.assertListEqual([], list(r.cookies.keys()))
@ -128,7 +133,7 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*60 days')
def test_settings_values_from_cookies_6(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '7', 'expires_soon' : 90, 'left_menu': 'on', })
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '7', str('expires_soon') : 90, str('left_menu'): 'on', })
r = self.client.get(urlreverse("ietf.cookies.views.preferences"))
self.assertEqual(r.status_code, 200)
self.assertListEqual([], list(r.cookies.keys()))
@ -142,11 +147,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*90 days')
def test_full_draft(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14})
r = self.client.get(urlreverse("ietf.cookies.views.full_draft")) # no value: reset
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['full_draft'].value, '')
self.assertListEqual(['full_draft'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('full_draft')].value, '')
self.assertListEqual([str('full_draft')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days'])
@ -156,21 +161,21 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days')
def test_full_draft_on(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14})
r = self.client.get(urlreverse("ietf.cookies.views.full_draft", kwargs=dict(enabled="on")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['full_draft'].value, 'on')
self.assertListEqual(['full_draft'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('full_draft')].value, 'on')
self.assertListEqual([str('full_draft')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On'])
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*full_draft.*on')
def test_full_draft_off(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14})
r = self.client.get(urlreverse("ietf.cookies.views.full_draft", kwargs=dict(enabled="off")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['full_draft'].value, 'off')
self.assertListEqual(['full_draft'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('full_draft')].value, 'off')
self.assertListEqual([str('full_draft')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off'])
# self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days'])
@ -178,7 +183,7 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*full_draft.*off')
def test_full_draft_foo(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14})
r = self.client.get(urlreverse("ietf.cookies.views.full_draft", kwargs=dict(enabled="foo")))
self.assertEqual(r.status_code, 200)
self.assertListEqual([], list(r.cookies.keys()))
@ -189,11 +194,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*full_draft.*off')
def test_left_menu(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'on', })
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'on', })
r = self.client.get(urlreverse("ietf.cookies.views.left_menu")) # no value: reset
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['left_menu'].value, '')
self.assertListEqual(['left_menu'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('left_menu')].value, '')
self.assertListEqual([str('left_menu')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off'])
self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off'])
@ -201,25 +206,25 @@ class CookieTests(TestCase):
self.assertEqual(q('div a.active[href="/accounts/settings/expires_soon/14"]').contents(), ['14 days'])
def test_left_menu_on(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'off', })
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'off', })
r = self.client.get(urlreverse("ietf.cookies.views.left_menu", kwargs=dict(enabled="on")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['left_menu'].value, 'on')
self.assertListEqual(['left_menu'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('left_menu')].value, 'on')
self.assertListEqual([str('left_menu')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/on"]').contents(), ['On'])
def test_left_menu_off(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'off', })
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'off', })
r = self.client.get(urlreverse("ietf.cookies.views.left_menu", kwargs=dict(enabled="off")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['left_menu'].value, 'off')
self.assertListEqual(['left_menu'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('left_menu')].value, 'off')
self.assertListEqual([str('left_menu')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off'])
def test_left_menu_foo(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'off', })
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'off', })
r = self.client.get(urlreverse("ietf.cookies.views.left_menu", kwargs=dict(enabled="foo")))
self.assertEqual(r.status_code, 200)
self.assertListEqual([], list(r.cookies.keys()))
@ -227,11 +232,11 @@ class CookieTests(TestCase):
self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off'])
def test_new_enough(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14})
r = self.client.get(urlreverse("ietf.cookies.views.new_enough")) # no value: reset
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['new_enough'].value, '')
self.assertListEqual(['new_enough'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('new_enough')].value, '')
self.assertListEqual([str('new_enough')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days'])
@ -241,11 +246,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days')
def test_new_enough_7(self):
self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '14', 'expires_soon' : 21})
self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '14', str('expires_soon') : 21})
r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="7")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['new_enough'].value, '7')
self.assertListEqual(['new_enough'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('new_enough')].value, '7')
self.assertListEqual([str('new_enough')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/7"]').contents(), ['7 days'])
@ -255,11 +260,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*21 days')
def test_new_enough_14(self):
self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '7', 'expires_soon' : 99})
self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '7', str('expires_soon') : 99})
r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="14")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['new_enough'].value, '14')
self.assertListEqual(['new_enough'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('new_enough')].value, '14')
self.assertListEqual([str('new_enough')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days'])
@ -269,11 +274,11 @@ class CookieTests(TestCase):
# self.assertNotRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon')
def test_new_enough_21(self):
self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '14', 'expires_soon' : 90})
self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '14', str('expires_soon') : 90})
r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="21")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['new_enough'].value, '21')
self.assertListEqual(['new_enough'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('new_enough')].value, '21')
self.assertListEqual([str('new_enough')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/21"]').contents(), ['21 days'])
@ -283,11 +288,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*90 days')
def test_new_enough_30(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 7})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 7})
r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="30")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['new_enough'].value, '30')
self.assertListEqual(['new_enough'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('new_enough')].value, '30')
self.assertListEqual([str('new_enough')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/30"]').contents(), ['30 days'])
@ -297,11 +302,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*7 days')
def test_new_enough_60(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14})
r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="60")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['new_enough'].value, '60')
self.assertListEqual(['new_enough'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('new_enough')].value, '60')
self.assertListEqual([str('new_enough')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/60"]').contents(), ['60 days'])
@ -311,11 +316,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days')
def test_new_enough_90(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '22', 'expires_soon' : 60})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '22', str('expires_soon') : 60})
r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="90")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['new_enough'].value, '90')
self.assertListEqual(['new_enough'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('new_enough')].value, '90')
self.assertListEqual([str('new_enough')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/90"]').contents(), ['90 days'])
@ -325,11 +330,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*60 days')
def test_expires_soon(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '14', 'new_enough' : 14})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '14', str('new_enough') : 14})
r = self.client.get(urlreverse("ietf.cookies.views.expires_soon")) # no value: reset
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['expires_soon'].value, '')
self.assertListEqual(['expires_soon'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('expires_soon')].value, '')
self.assertListEqual([str('expires_soon')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days'])
@ -339,11 +344,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*14 days')
def test_expires_soon_7(self):
self.client.cookies = SimpleCookie({'full_draft': 'on', 'expires_soon' : '14', 'new_enough' : 21})
self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('expires_soon') : '14', str('new_enough') : 21})
r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="7")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['expires_soon'].value, '7')
self.assertListEqual(['expires_soon'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('expires_soon')].value, '7')
self.assertListEqual([str('expires_soon')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/21"]').contents(), ['21 days'])
@ -353,11 +358,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*21 days')
def test_expires_soon_14(self):
self.client.cookies = SimpleCookie({'full_draft': 'on', 'expires_soon' : '7', 'new_enough' : 99})
self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('expires_soon') : '7', str('new_enough') : 99})
r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="14")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['expires_soon'].value, '14')
self.assertListEqual(['expires_soon'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('expires_soon')].value, '14')
self.assertListEqual([str('expires_soon')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On'])
self.assertEqual(q('div a.active[href^="/accounts/settings/new_enough/"]').contents(), [])
@ -367,11 +372,11 @@ class CookieTests(TestCase):
# self.assertNotRegexpMatches(r.content, r'ietf-highlight-y.*new_enough')
def test_expires_soon_21(self):
self.client.cookies = SimpleCookie({'full_draft': 'on', 'expires_soon' : '14', 'new_enough' : 90})
self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('expires_soon') : '14', str('new_enough') : 90})
r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="21")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['expires_soon'].value, '21')
self.assertListEqual(['expires_soon'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('expires_soon')].value, '21')
self.assertListEqual([str('expires_soon')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/90"]').contents(), ['90 days'])
@ -381,11 +386,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*90 days')
def test_expires_soon_30(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '14', 'new_enough' : 7})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '14', str('new_enough') : 7})
r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="30")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['expires_soon'].value, '30')
self.assertListEqual(['expires_soon'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('expires_soon')].value, '30')
self.assertListEqual([str('expires_soon')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/7"]').contents(), ['7 days'])
@ -395,11 +400,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*7 days')
def test_expires_soon_60(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '14', 'new_enough' : 14})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '14', str('new_enough') : 14})
r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="60")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['expires_soon'].value, '60')
self.assertListEqual(['expires_soon'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('expires_soon')].value, '60')
self.assertListEqual([str('expires_soon')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days'])
@ -409,11 +414,11 @@ class CookieTests(TestCase):
# self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*14 days')
def test_expires_soon_90(self):
self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '22', 'new_enough' : 60})
self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '22', str('new_enough') : 60})
r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="90")))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.cookies['expires_soon'].value, '90')
self.assertListEqual(['expires_soon'], list(r.cookies.keys()))
self.assertEqual(r.cookies[str('expires_soon')].value, '90')
self.assertListEqual([str('expires_soon')], list(r.cookies.keys()))
q = PyQuery(r.content)
self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off'])
self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/60"]').contents(), ['60 days'])

7
ietf/doc/feeds.py
View file

@ -1,6 +1,11 @@
# Copyright The IETF Trust 2007-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import six
from django.contrib.syndication.views import Feed, FeedDoesNotExist
from django.utils.feedgenerator import Atom1Feed, Rss201rev2Feed
@ -44,7 +49,7 @@ class DocumentChangesFeed(Feed):
return item.time
def item_author_name(self, item):
return str(item.by)
return six.ensure_text(item.by)
def item_link(self, item):
return urlreverse('ietf.doc.views_doc.document_history', kwargs=dict(name=item.doc.canonical_name())) + "#history-%s" % item.pk

10
ietf/doc/fields.py
View file

@ -1,7 +1,11 @@
# Copyright The IETF Trust 2014-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import json
import six
from django.utils.html import escape
from django import forms
@ -53,9 +57,9 @@ class SearchableDocumentsField(forms.CharField):
def prepare_value(self, value):
if not value:
value = ""
if isinstance(value, int):
if isinstance(value, six.integer_types):
value = str(value)
if isinstance(value, str):
if isinstance(value, six.string_types):
items = self.parse_select2_value(value)
# accept both names and pks here
names = [ i for i in items if not i.isdigit() ]
@ -79,7 +83,7 @@ class SearchableDocumentsField(forms.CharField):
"model_name": self.model.__name__.lower()
})
return ",".join(str(o.pk) for o in value)
return ",".join(six.ensure_text(o.pk) for o in value)
def clean(self, value):
value = super(SearchableDocumentsField, self).clean(value)

25
ietf/doc/mails.py
View file

@ -1,16 +1,23 @@
# Copyright The IETF Trust 2010-2019, All Rights Reserved
# -*- coding: utf-8 -*-
# generation of mails
import textwrap, datetime
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import six
import textwrap
from django.template.loader import render_to_string
from django.utils.html import strip_tags
from django.conf import settings
from django.urls import reverse as urlreverse
from django.utils.encoding import force_str
import debug # pyflakes:ignore
from ietf.utils.mail import send_mail, send_mail_text
from ietf.utils.mail import send_mail, send_mail_text, get_payload
from ietf.ipr.utils import iprs_from_docs, related_docs
from ietf.doc.models import WriteupDocEvent, LastCallDocEvent, DocAlias, ConsensusDocEvent
from ietf.doc.utils import needed_ballot_positions
@ -121,7 +128,7 @@ def generate_ballot_writeup(request, doc):
e.doc = doc
e.rev = doc.rev
e.desc = "Ballot writeup was generated"
e.text = str(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana}))
e.text = six.ensure_text(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana}))
# caller is responsible for saving, if necessary
return e
@ -133,7 +140,7 @@ def generate_ballot_rfceditornote(request, doc):
e.doc = doc
e.rev = doc.rev
e.desc = "RFC Editor Note for ballot was generated"
e.text = str(render_to_string("doc/mail/ballot_rfceditornote.txt"))
e.text = six.ensure_text(render_to_string("doc/mail/ballot_rfceditornote.txt"))
e.save()
return e
@ -178,7 +185,7 @@ def generate_last_call_announcement(request, doc):
e.doc = doc
e.rev = doc.rev
e.desc = "Last call announcement was generated"
e.text = str(mail)
e.text = six.ensure_text(mail)
# caller is responsible for saving, if necessary
return e
@ -198,7 +205,7 @@ def generate_approval_mail(request, doc):
e.doc = doc
e.rev = doc.rev
e.desc = "Ballot approval text was generated"
e.text = str(mail)
e.text = six.ensure_text(mail)
# caller is responsible for saving, if necessary
return e
@ -281,7 +288,7 @@ def generate_publication_request(request, doc):
approving_body = "IRSG"
consensus_body = doc.group.acronym.upper()
else:
approving_body = str(doc.stream)
approving_body = six.ensure_text(doc.stream)
consensus_body = approving_body
e = doc.latest_event(WriteupDocEvent, type="changed_rfc_editor_note_text")
@ -383,7 +390,7 @@ def generate_issue_ballot_mail(request, doc, ballot):
def email_iana(request, doc, to, msg, cc=None):
# fix up message and send it with extra info on doc in headers
import email
parsed_msg = email.message_from_string(msg)
parsed_msg = email.message_from_string(force_str(msg))
parsed_msg.set_charset('UTF-8')
extra = extra_automation_headers(doc)
@ -391,7 +398,7 @@ def email_iana(request, doc, to, msg, cc=None):
send_mail_text(request, to,
parsed_msg["From"], parsed_msg["Subject"],
parsed_msg.get_payload(),
get_payload(parsed_msg),
extra=extra,
cc=cc)

10
ietf/doc/management/commands/generate_draft_bibxml_files.py
View file

@ -1,4 +1,10 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import sys
import os
@ -10,14 +16,14 @@ from ietf.doc.models import Document
def write(fn, new):
try:
f = open(fn)
f = io.open(fn)
old = f.read().decode('utf-8')
f.close
except IOError:
old = ""
if old.strip() != new.strip():
sys.stdout.write(os.path.basename(fn)+'\n')
f = open(fn, "wb")
f = io.open(fn, "wb")
f.write(new.encode('utf-8'))
f.close()

5
ietf/doc/migrations/0016_set_document_docalias_fk.py
View file

@ -3,6 +3,9 @@
# Generated by Django 1.11.20 on 2019-05-08 14:04
from __future__ import absolute_import, print_function, unicode_literals
import six
import sys
from tqdm import tqdm
@ -15,7 +18,7 @@ def forward(apps, schema_editor):
n = getattr(o, a+'_id')
if n:
i = nameid[n]
if not isinstance(i, int):
if not isinstance(i, six.integer_types):
raise ValueError("Inappropriate value: %s: nameid[%s]: %s" % (o.__class__.__name__, n, i))
if getattr(o, a+'2_id') != i:
setattr(o, a+'2_id', i)

40
ietf/doc/models.py
View file

@ -1,10 +1,13 @@
# Copyright The IETF Trust 2010-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import logging
import io
import os
import rfc2html
import six
from django.db import models
from django.core import checks
@ -14,6 +17,7 @@ from django.core.validators import URLValidator, RegexValidator
from django.urls import reverse as urlreverse
from django.contrib.contenttypes.models import ContentType
from django.conf import settings
from django.utils.encoding import python_2_unicode_compatible
from django.utils.html import mark_safe
import debug # pyflakes:ignore
@ -33,6 +37,7 @@ from ietf.utils.models import ForeignKey
logger = logging.getLogger('django')
@python_2_unicode_compatible
class StateType(models.Model):
slug = models.CharField(primary_key=True, max_length=30) # draft, draft-iesg, charter, ...
label = models.CharField(max_length=255, help_text="Label that should be used (e.g. in admin) for state drop-down for this type of state") # State, IESG state, WG state, ...
@ -54,6 +59,7 @@ def check_statetype_slugs(app_configs, **kwargs):
))
return errors
@python_2_unicode_compatible
class State(models.Model):
type = ForeignKey(StateType)
slug = models.SlugField()
@ -406,7 +412,7 @@ class DocumentInfo(models.Model):
def relations_that(self, relationship):
"""Return the related-document objects that describe a given relationship targeting self."""
if isinstance(relationship, str):
if isinstance(relationship, six.string_types):
relationship = ( relationship, )
if not isinstance(relationship, tuple):
raise TypeError("Expected a string or tuple, received %s" % type(relationship))
@ -429,7 +435,7 @@ class DocumentInfo(models.Model):
def relations_that_doc(self, relationship):
"""Return the related-document objects that describe a given relationship from self to other documents."""
if isinstance(relationship, str):
if isinstance(relationship, six.string_types):
relationship = ( relationship, )
if not isinstance(relationship, tuple):
raise TypeError("Expected a string or tuple, received %s" % type(relationship))
@ -481,7 +487,7 @@ class DocumentInfo(models.Model):
if ext != '.txt' and os.path.exists(txtpath):
path = txtpath
try:
with open(path, 'rb') as file:
with io.open(path, 'rb') as file:
raw = file.read()
except IOError:
return None
@ -523,6 +529,7 @@ class DocumentInfo(models.Model):
STATUSCHANGE_RELATIONS = ('tops','tois','tohist','toinf','tobcp','toexp')
@python_2_unicode_compatible
class RelatedDocument(models.Model):
source = ForeignKey('Document')
target = ForeignKey('DocAlias')
@ -530,7 +537,7 @@ class RelatedDocument(models.Model):
def action(self):
return self.relationship.name
def __str__(self):
return "%s %s %s" % (self.source.name, self.relationship.name.lower(), self.target.name)
return u"%s %s %s" % (self.source.name, self.relationship.name.lower(), self.target.name)
def is_downref(self):
@ -596,11 +603,12 @@ class DocumentAuthorInfo(models.Model):
abstract = True
ordering = ["document", "order"]
@python_2_unicode_compatible
class DocumentAuthor(DocumentAuthorInfo):
document = ForeignKey('Document')
def __str__(self):
return "%s %s (%s)" % (self.document.name, self.person, self.order)
return u"%s %s (%s)" % (self.document.name, self.person, self.order)
validate_docname = RegexValidator(
@ -609,6 +617,7 @@ validate_docname = RegexValidator(
'invalid'
)
@python_2_unicode_compatible
class Document(DocumentInfo):
name = models.CharField(max_length=255, validators=[validate_docname,], unique=True) # immutable
@ -844,21 +853,24 @@ class DocumentURL(models.Model):
desc = models.CharField(max_length=255, default='', blank=True)
url = models.URLField(max_length=2083) # 2083 is the legal max for URLs
@python_2_unicode_compatible
class RelatedDocHistory(models.Model):
source = ForeignKey('DocHistory')
target = ForeignKey('DocAlias', related_name="reversely_related_document_history_set")
relationship = ForeignKey(DocRelationshipName)
def __str__(self):
return "%s %s %s" % (self.source.doc.name, self.relationship.name.lower(), self.target.name)
return u"%s %s %s" % (self.source.doc.name, self.relationship.name.lower(), self.target.name)
@python_2_unicode_compatible
class DocHistoryAuthor(DocumentAuthorInfo):
# use same naming convention as non-history version to make it a bit
# easier to write generic code
document = ForeignKey('DocHistory', related_name="documentauthor_set")
def __str__(self):
return "%s %s (%s)" % (self.document.doc.name, self.person, self.order)
return u"%s %s (%s)" % (self.document.doc.name, self.person, self.order)
@python_2_unicode_compatible
class DocHistory(DocumentInfo):
doc = ForeignKey(Document, related_name="history_set")
# the name here is used to capture the canonical name at the time
@ -868,7 +880,7 @@ class DocHistory(DocumentInfo):
name = models.CharField(max_length=255)
def __str__(self):
return str(self.doc.name)
return six.ensure_text(self.doc.name)
def canonical_name(self):
if hasattr(self, '_canonical_name'):
@ -903,6 +915,7 @@ class DocHistory(DocumentInfo):
verbose_name = "document history"
verbose_name_plural = "document histories"
@python_2_unicode_compatible
class DocAlias(models.Model):
"""This is used for documents that may appear under multiple names,
and in particular for RFCs, which for continuity still keep the
@ -917,7 +930,7 @@ class DocAlias(models.Model):
return self.docs.first()
def __str__(self):
return "%s-->%s" % (self.name, ','.join([str(d.name) for d in self.docs.all() if isinstance(d, Document) ]))
return u"%s-->%s" % (self.name, ','.join([six.ensure_text(d.name) for d in self.docs.all() if isinstance(d, Document) ]))
document_link = admin_link("document")
class Meta:
verbose_name = "document alias"
@ -1006,6 +1019,7 @@ EVENT_TYPES = [
("downref_approved", "Downref approved"),
]
@python_2_unicode_compatible
class DocEvent(models.Model):
"""An occurrence for a document, used for tracking who, when and what."""
time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened", db_index=True)
@ -1023,7 +1037,7 @@ class DocEvent(models.Model):
return DocHistory.objects.filter(time__lte=self.time,doc__name=self.doc.name).order_by('-time', '-pk').first()
def __str__(self):
return "%s %s by %s at %s" % (self.doc.name, self.get_type_display().lower(), self.by.plain_name(), self.time)
return u"%s %s by %s at %s" % (self.doc.name, self.get_type_display().lower(), self.by.plain_name(), self.time)
def save(self, *args, **kwargs):
super(DocEvent, self).save(*args, **kwargs)
@ -1046,6 +1060,7 @@ class ConsensusDocEvent(DocEvent):
consensus = models.NullBooleanField(default=None)
# IESG events
@python_2_unicode_compatible
class BallotType(models.Model):
doc_type = ForeignKey(DocTypeName, blank=True, null=True)
slug = models.SlugField()
@ -1056,7 +1071,7 @@ class BallotType(models.Model):
positions = models.ManyToManyField(BallotPositionName, blank=True)
def __str__(self):
return "%s: %s" % (self.name, self.doc_type.name)
return u"%s: %s" % (self.name, self.doc_type.name)
class Meta:
ordering = ['order']
@ -1176,6 +1191,7 @@ class SubmissionDocEvent(DocEvent):
submission = ForeignKey(ietf.submit.models.Submission)
# dumping store for removed events
@python_2_unicode_compatible
class DeletedEvent(models.Model):
content_type = ForeignKey(ContentType)
json = models.TextField(help_text="Deleted object in JSON format, with attribute names chosen to be suitable for passing into the relevant create method.")
@ -1183,7 +1199,7 @@ class DeletedEvent(models.Model):
time = models.DateTimeField(default=datetime.datetime.now)
def __str__(self):
return "%s by %s %s" % (self.content_type, self.by, self.time)
return u"%s by %s %s" % (self.content_type, self.by, self.time)
class EditedAuthorsDocEvent(DocEvent):
""" Capture the reasoning or authority for changing a document author list.

2
ietf/doc/resources.py
View file

@ -1,6 +1,8 @@
# Copyright The IETF Trust 2014-2019, All Rights Reserved
# -*- coding: utf-8 -*-
# Autogenerated by the makeresources management command 2015-10-19 12:29 PDT
from ietf.api import ModelResource
from ietf.api import ToOneField
from tastypie.fields import ToManyField, CharField

23
ietf/doc/templatetags/ietf_filters.py
View file

@ -1,8 +1,13 @@
# Copyright The IETF Trust 2007-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import bleach
import datetime
import re
import six
from email.utils import parseaddr
@ -45,24 +50,24 @@ def parse_email_list(value):
Splitting a string of email addresses should return a list:
>>> parse_email_list('joe@example.org, fred@example.com')
>>> six.ensure_str(parse_email_list('joe@example.org, fred@example.com'))
'<a href="mailto:joe@example.org">joe@example.org</a>, <a href="mailto:fred@example.com">fred@example.com</a>'
Parsing a non-string should return the input value, rather than fail:
>>> parse_email_list(['joe@example.org', 'fred@example.com'])
>>> [ six.ensure_str(e) for e in parse_email_list(['joe@example.org', 'fred@example.com']) ]
['joe@example.org', 'fred@example.com']
Null input values should pass through silently:
>>> parse_email_list('')
>>> six.ensure_str(parse_email_list(''))
''
>>> parse_email_list(None)
"""
if value and isinstance(value, (bytes,str)): # testing for 'value' being true isn't necessary; it's a fast-out route
if value and isinstance(value, (six.binary_type, six.text_type)): # testing for 'value' being true isn't necessary; it's a fast-out route
addrs = re.split(", ?", value)
ret = []
for addr in addrs:
@ -96,7 +101,7 @@ def make_one_per_line(value):
"""
Turn a comma-separated list into a carriage-return-seperated list.
>>> make_one_per_line("a, b, c")
>>> six.ensure_str(make_one_per_line("a, b, c"))
'a\\nb\\nc'
Pass through non-strings:
@ -107,7 +112,7 @@ def make_one_per_line(value):
>>> make_one_per_line(None)
"""
if value and isinstance(value, (bytes,str)):
if value and isinstance(value, (six.binary_type, six.text_type)):
return re.sub(", ?", "\n", value)
else:
return value
@ -143,7 +148,7 @@ def sanitize(value):
@register.filter(name='bracket')
def square_brackets(value):
"""Adds square brackets around text."""
if isinstance(value, (bytes,str)):
if isinstance(value, (six.binary_type, six.text_type)):
if value == "":
value = " "
return "[ %s ]" % value
@ -193,7 +198,7 @@ def rfcnospace(string):
@register.filter
def prettystdname(string):
from ietf.doc.utils import prettify_std_name
return prettify_std_name(str(string or ""))
return prettify_std_name(six.ensure_text(string or ""))
@register.filter(name='rfcurl')
def rfclink(string):
@ -336,7 +341,7 @@ def expires_soon(x,request):
@register.filter(name='startswith')
def startswith(x, y):
return str(x).startswith(y)
return six.ensure_text(x).startswith(y)
@register.filter
def has_role(user, role_names):

48
ietf/doc/tests.py
View file

@ -1,19 +1,25 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import os
import shutil
import datetime
import io
import sys
import urllib.parse
import bibtexparser
if sys.version_info[0] == 2 and sys.version_info[1] < 7:
import unittest2 as unittest
else:
import unittest
from six.moves.http_cookies import SimpleCookie
from pyquery import PyQuery
from six.moves.urllib.parse import urlparse, parse_qs
from tempfile import NamedTemporaryFile
from http.cookies import SimpleCookie
from django.urls import reverse as urlreverse
from django.conf import settings
@ -141,71 +147,71 @@ class SearchTests(TestCase):
# exact match
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name)))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
# prefix match
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(draft.name.split("-")[:-1]))))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
# non-prefix match
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(draft.name.split("-")[1:]))))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
# other doctypes than drafts
doc = Document.objects.get(name='charter-ietf-mars')
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name='charter-ietf-ma')))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
doc = Document.objects.filter(name__startswith='conflict-review-').first()
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1]))))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
doc = Document.objects.filter(name__startswith='status-change-').first()
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1]))))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
doc = Document.objects.filter(name__startswith='agenda-').first()
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1]))))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
doc = Document.objects.filter(name__startswith='minutes-').first()
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1]))))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
doc = Document.objects.filter(name__startswith='slides-').first()
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1]))))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name)))
# match with revision
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name + "-" + prev_rev)))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev)))
# match with non-existing revision
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name + "-09")))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
# match with revision and extension
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name + "-" + prev_rev + ".txt")))
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev)))
self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev)))
# no match
r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="draft-ietf-doesnotexist-42")))
self.assertEqual(r.status_code, 302)
parsed = urllib.parse.urlparse(r["Location"])
parsed = urlparse(r["Location"])
self.assertEqual(parsed.path, urlreverse('ietf.doc.views_search.search'))
self.assertEqual(urllib.parse.parse_qs(parsed.query)["name"][0], "draft-ietf-doesnotexist-42")
self.assertEqual(parse_qs(parsed.query)["name"][0], "draft-ietf-doesnotexist-42")
def test_frontpage(self):
r = self.client.get("/")
@ -486,7 +492,7 @@ Man Expires September 22, 2015 [Page 3]
settings.INTERNET_DRAFT_PATH = self.id_dir
self.saved_internet_all_drafts_archive_dir = settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR
settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR = self.id_dir
f = open(os.path.join(self.id_dir, 'draft-ietf-mars-test-01.txt'), 'w')
f = io.open(os.path.join(self.id_dir, 'draft-ietf-mars-test-01.txt'), 'w')
f.write(self.draft_text)
f.close()
@ -528,21 +534,21 @@ Man Expires September 22, 2015 [Page 3]
self.assertNotContains(r, "Show full document text")
self.assertContains(r, "Deimos street")
self.client.cookies = SimpleCookie({'full_draft': 'on'})
self.client.cookies = SimpleCookie({str('full_draft'): str('on')})
r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
self.assertEqual(r.status_code, 200)
self.assertContains(r, "Active Internet-Draft")
self.assertNotContains(r, "Show full document text")
self.assertContains(r, "Deimos street")
self.client.cookies = SimpleCookie({'full_draft': 'off'})
self.client.cookies = SimpleCookie({str('full_draft'): str('off')})
r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
self.assertEqual(r.status_code, 200)
self.assertContains(r, "Active Internet-Draft")
self.assertContains(r, "Show full document text")
self.assertNotContains(r, "Deimos street")
self.client.cookies = SimpleCookie({'full_draft': 'foo'})
self.client.cookies = SimpleCookie({str('full_draft'): str('foo')})
r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
self.assertEqual(r.status_code, 200)
self.assertContains(r, "Active Internet-Draft")
@ -1021,7 +1027,7 @@ class EmailAliasesTests(TestCase):
def setUp(self):
WgDraftFactory(name='draft-ietf-mars-test',group__acronym='mars')
WgDraftFactory(name='draft-ietf-ames-test',group__acronym='ames')
self.doc_alias_file = NamedTemporaryFile(delete=False, encoding='utf-8', mode='w+')
self.doc_alias_file = NamedTemporaryFile(delete=False, mode='w+')
self.doc_alias_file.write("""# Generated by hand at 2015-02-12_16:26:45
virtual.ietf.org anything
draft-ietf-mars-test@ietf.org xfilter-draft-ietf-mars-test

41
ietf/doc/tests_charter.py
View file

@ -1,8 +1,14 @@
# -*- coding: utf-8 -*-
# Copyright The IETF Trust 2011-2019, All Rights Reserved
import os, shutil, datetime
from io import StringIO
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import os
import shutil
from pyquery import PyQuery
from django.conf import settings
@ -21,7 +27,7 @@ from ietf.group.models import Group, GroupMilestone
from ietf.iesg.models import TelechatDate
from ietf.person.models import Person
from ietf.utils.test_utils import TestCase
from ietf.utils.mail import outbox, empty_outbox
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.test_utils import login_testing_unauthorized
class EditCharterTests(TestCase):
@ -35,7 +41,7 @@ class EditCharterTests(TestCase):
shutil.rmtree(self.charter_dir)
def write_charter_file(self, charter):
with open(os.path.join(self.charter_dir, "%s-%s.txt" % (charter.canonical_name(), charter.rev)), "w") as f:
with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (charter.canonical_name(), charter.rev)), "w") as f:
f.write("This is a charter.")
def test_startstop_process(self):
@ -128,24 +134,24 @@ class EditCharterTests(TestCase):
self.assertIn("Internal WG Review", outbox[-3]['Subject'])
self.assertIn("iab@", outbox[-3]['To'])
self.assertIn("iesg@", outbox[-3]['To'])
self.assertIn("A new IETF WG", outbox[-3].get_payload())
body = outbox[-3].get_payload()
for word in ["Chairs", "Ames Man <ameschairman@example.org>",
body = get_payload(outbox[-3])
for word in ["A new IETF WG", "Chairs", "Ames Man <ameschairman@example.org>",
"Secretaries", "Secretary <amessecretary@example.org>",
"Assigned Area Director", "Areað Irector <aread@example.org>",
"Mailing list", "ames-wg@ietf.org",
"Charter", "Milestones"]:
self.assertIn(word, body)
self.assertIn("state changed", outbox[-2]['Subject'].lower())
self.assertIn("iesg-secretary@", outbox[-2]['To'])
body = outbox[-2].get_payload()
body = get_payload(outbox[-2])
for word in ["WG", "Charter", ]:
self.assertIn(word, body)
self.assertIn("State Update Notice", outbox[-1]['Subject'])
self.assertIn("ames-chairs@", outbox[-1]['To'])
body = outbox[-1].get_payload()
body = get_payload(outbox[-1])
for word in ["State changed", "Datatracker URL", ]:
self.assertIn(word, body)
@ -165,7 +171,7 @@ class EditCharterTests(TestCase):
empty_outbox()
r = self.client.post(url, dict(charter_state=str(State.objects.get(used=True,type="charter",slug="intrev").pk), message="test"))
self.assertEqual(r.status_code, 302)
self.assertTrue("A new charter" in outbox[-3].get_payload())
self.assertTrue("A new charter" in get_payload(outbox[-3]))
def test_abandon_bof(self):
charter = CharterFactory(group__state_id='bof',group__type_id='wg')
@ -389,7 +395,7 @@ class EditCharterTests(TestCase):
self.assertEqual(len(q('form input[name=txt]')), 1)
# faulty post
test_file = StringIO("\x10\x11\x12") # post binary file
test_file = io.StringIO("\x10\x11\x12") # post binary file
test_file.name = "unnamed"
r = self.client.post(url, dict(txt=test_file))
@ -401,7 +407,7 @@ class EditCharterTests(TestCase):
latin_1_snippet = b'\xe5' * 10
utf_8_snippet = b'\xc3\xa5' * 10
test_file = StringIO("Windows line\r\nMac line\rUnix line\n" + latin_1_snippet.decode('latin-1'))
test_file = io.StringIO("Windows line\r\nMac line\rUnix line\n" + latin_1_snippet.decode('latin-1'))
test_file.name = "unnamed"
r = self.client.post(url, dict(txt=test_file))
@ -411,9 +417,8 @@ class EditCharterTests(TestCase):
self.assertEqual(charter.rev, next_revision(prev_rev))
self.assertTrue("new_revision" in charter.latest_event().type)
with open(os.path.join(self.charter_dir, charter.canonical_name() + "-" + charter.rev + ".txt")) as f:
self.assertEqual(f.read(),
"Windows line\nMac line\nUnix line\n" + utf_8_snippet.decode('utf_8'))
with io.open(os.path.join(self.charter_dir, charter.canonical_name() + "-" + charter.rev + ".txt")) as f:
self.assertEqual(f.read(), "Windows line\nMac line\nUnix line\n" + utf_8_snippet.decode('utf_8'))
def test_submit_initial_charter(self):
group = GroupFactory(type_id='wg',acronym='mars',list_email='mars-wg@ietf.org')
@ -428,7 +433,7 @@ class EditCharterTests(TestCase):
self.assertEqual(len(q('form input[name=txt]')), 1)
# create charter
test_file = StringIO("Simple test")
test_file = io.StringIO("Simple test")
test_file.name = "unnamed"
r = self.client.post(url, dict(txt=test_file))
@ -658,7 +663,7 @@ class EditCharterTests(TestCase):
#
self.assertTrue("approved" in outbox[0]['Subject'].lower())
self.assertTrue("iesg-secretary" in outbox[0]['To'])
body = outbox[0].get_payload()
body = get_payload(outbox[0])
for word in ["WG", "/wg/ames/about/",
"Charter", "/doc/charter-ietf-ames/", ]:
self.assertIn(word, body)
@ -666,7 +671,7 @@ class EditCharterTests(TestCase):
self.assertTrue("WG Action" in outbox[1]['Subject'])
self.assertTrue("ietf-announce" in outbox[1]['To'])
self.assertTrue("ames-wg@ietf.org" in outbox[1]['Cc'])
body = outbox[1].get_payload()
body = get_payload(outbox[1])
for word in ["Chairs", "Ames Man <ameschairman@example.org>",
"Secretaries", "Secretary <amessecretary@example.org>",
"Assigned Area Director", "Areað Irector <aread@example.org>",

25
ietf/doc/tests_conflict_review.py
View file

@ -1,16 +1,21 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
import debug # pyflakes:ignore
from __future__ import absolute_import, print_function, unicode_literals
import io
import os
import shutil
from pyquery import PyQuery
from io import StringIO
from textwrap import wrap
from django.conf import settings
from django.urls import reverse as urlreverse
import debug # pyflakes:ignore
from ietf.doc.factories import IndividualDraftFactory, ConflictReviewFactory
from ietf.doc.models import Document, DocEvent, NewRevisionDocEvent, BallotPositionDocEvent, TelechatDocEvent, State
from ietf.doc.utils import create_ballot_if_not_open
@ -19,7 +24,7 @@ from ietf.group.models import Person
from ietf.iesg.models import TelechatDate
from ietf.name.models import StreamName
from ietf.utils.test_utils import TestCase
from ietf.utils.mail import outbox, empty_outbox
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.test_utils import login_testing_unauthorized
@ -299,9 +304,9 @@ class ConflictReviewTests(TestCase):
self.assertIn('iana@', outbox[0]['Cc'])
if approve_type == 'appr-noprob':
self.assertIn( 'IESG has no problem', ''.join(wrap(outbox[0].get_payload(), 2**16)))
self.assertIn( 'IESG has no problem', ''.join(wrap(get_payload(outbox[0]), 2**16)))
else:
self.assertIn( 'NOT be published', ''.join(wrap(outbox[0].get_payload(), 2**16)))
self.assertIn( 'NOT be published', ''.join(wrap(get_payload(outbox[0]), 2**16)))
def test_approve_reqnopub(self):
@ -338,7 +343,7 @@ class ConflictReviewSubmitTests(TestCase):
self.assertEqual(r.status_code,302)
doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
self.assertEqual(doc.rev,'00')
with open(path) as f:
with io.open(path) as f:
self.assertEqual(f.read(),"Some initial review text\n")
f.close()
self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc)
@ -352,7 +357,7 @@ class ConflictReviewSubmitTests(TestCase):
# doc.rev is u'00' per the test setup - double-checking that here - if it fails, the breakage is in setUp
self.assertEqual(doc.rev,'00')
path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev))
with open(path,'w') as f:
with io.open(path,'w') as f:
f.write('This is the old proposal.')
f.close()
@ -365,21 +370,21 @@ class ConflictReviewSubmitTests(TestCase):
# faulty posts trying to use file upload
# Copied from wgtracker tests - is this really testing the server code, or is it testing
# how client.post populates Content-Type?
test_file = StringIO("\x10\x11\x12") # post binary file
test_file = io.StringIO("\x10\x11\x12") # post binary file
test_file.name = "unnamed"
r = self.client.post(url, dict(txt=test_file,submit_response="1"))
self.assertEqual(r.status_code, 200)
self.assertContains(r, "does not appear to be a text file")
# sane post uploading a file
test_file = StringIO("This is a new proposal.")
test_file = io.StringIO("This is a new proposal.")
test_file.name = "unnamed"
r = self.client.post(url,dict(txt=test_file,submit_response="1"))
self.assertEqual(r.status_code, 302)
doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission')
self.assertEqual(doc.rev,'01')
path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev))
with open(path) as f:
with io.open(path) as f:
self.assertEqual(f.read(),"This is a new proposal.")
f.close()
self.assertTrue( "submission-01" in doc.latest_event(NewRevisionDocEvent).desc)

23
ietf/doc/tests_draft.py
View file

@ -1,6 +1,9 @@
# Copyright The IETF Trust 2011-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import os
import shutil
import datetime
@ -26,7 +29,7 @@ from ietf.person.models import Person, Email
from ietf.meeting.models import Meeting, MeetingTypeName
from ietf.iesg.models import TelechatDate
from ietf.utils.test_utils import login_testing_unauthorized
from ietf.utils.mail import outbox, empty_outbox
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.test_utils import TestCase
@ -372,7 +375,7 @@ class EditInfoTests(TestCase):
data["telechat_date"] = next_week.isoformat()
r = self.client.post(url,data)
self.assertEqual(r.status_code, 302)
self.assertTrue("may not leave enough time" in outbox[-1].get_payload())
self.assertIn("may not leave enough time", get_payload(outbox[-1]))
def test_start_iesg_process_on_draft(self):
@ -563,7 +566,7 @@ class ExpireIDsTests(TestCase):
settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir
def write_draft_file(self, name, size):
f = open(os.path.join(self.id_dir, name), 'w')
f = io.open(os.path.join(self.id_dir, name), 'w')
f.write("a" * size)
f.close()
@ -1399,9 +1402,9 @@ class ChangeStreamStateTests(TestCase):
self.assertEqual(draft.docevent_set.count() - events_before, 2)
self.assertEqual(len(outbox), mailbox_before + 1)
self.assertTrue("tags changed" in outbox[-1]["Subject"].lower())
self.assertTrue("mars-chairs@ietf.org" in str(outbox[-1]))
self.assertTrue("marsdelegate@example.org" in str(outbox[-1]))
self.assertTrue("plain@example.com" in str(outbox[-1]))
self.assertTrue("mars-chairs@ietf.org" in outbox[-1].as_string())
self.assertTrue("marsdelegate@example.org" in outbox[-1].as_string())
self.assertTrue("plain@example.com" in outbox[-1].as_string())
def test_set_initial_state(self):
role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man')
@ -1436,8 +1439,8 @@ class ChangeStreamStateTests(TestCase):
self.assertTrue(due - datetime.timedelta(days=1) <= reminder[0].due <= due + datetime.timedelta(days=1))
self.assertEqual(len(outbox), 1)
self.assertTrue("state changed" in outbox[0]["Subject"].lower())
self.assertTrue("mars-chairs@ietf.org" in str(outbox[0]))
self.assertTrue("marsdelegate@ietf.org" in str(outbox[0]))
self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string())
self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string())
def test_set_state(self):
role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man')
@ -1481,8 +1484,8 @@ class ChangeStreamStateTests(TestCase):
self.assertTrue(due - datetime.timedelta(days=1) <= reminder[0].due <= due + datetime.timedelta(days=1))
self.assertEqual(len(outbox), 1)
self.assertTrue("state changed" in outbox[0]["Subject"].lower())
self.assertTrue("mars-chairs@ietf.org" in str(outbox[0]))
self.assertTrue("marsdelegate@ietf.org" in str(outbox[0]))
self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string())
self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string())
def test_pubreq_validation(self):
role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man')

14
ietf/doc/tests_material.py
View file

@ -1,10 +1,14 @@
# Copyright The IETF Trust 2014-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import os
import shutil
import datetime
from io import StringIO
import io
from pyquery import PyQuery
import debug # pyflakes:ignore
@ -85,7 +89,7 @@ class GroupMaterialTests(TestCase):
self.assertEqual(r.status_code, 200)
content = "%PDF-1.5\n..."
test_file = StringIO(content)
test_file = io.StringIO(content)
test_file.name = "unnamed.pdf"
# faulty post
@ -110,7 +114,7 @@ class GroupMaterialTests(TestCase):
self.assertEqual(doc.title, "Test File - with fancy title")
self.assertEqual(doc.get_state_slug(), "active")
with open(os.path.join(self.materials_dir, "slides", doc.name + "-" + doc.rev + ".pdf")) as f:
with io.open(os.path.join(self.materials_dir, "slides", doc.name + "-" + doc.rev + ".pdf")) as f:
self.assertEqual(f.read(), content)
# check that posting same name is prevented
@ -165,7 +169,7 @@ class GroupMaterialTests(TestCase):
login_testing_unauthorized(self, "secretary", url)
content = "some text"
test_file = StringIO(content)
test_file = io.StringIO(content)
test_file.name = "unnamed.txt"
# post
@ -179,6 +183,6 @@ class GroupMaterialTests(TestCase):
self.assertEqual(doc.title, "New title")
self.assertEqual(doc.get_state_slug(), "active")
with open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + ".txt")) as f:
with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + ".txt")) as f:
self.assertEqual(f.read(), content)

19
ietf/doc/tests_review.py
View file

@ -1,11 +1,14 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime, os, shutil
import io
import tarfile, tempfile, mailbox
import email.mime.multipart, email.mime.text, email.utils
from io import StringIO
from mock import patch
from requests import Response
@ -98,8 +101,10 @@ class ReviewTests(TestCase):
self.assertEqual(len(outbox),2)
self.assertTrue('reviewteam Early' in outbox[0]['Subject'])
# debug.show("outbox[0]['To']")
self.assertTrue('reviewsecretary@' in outbox[0]['To'])
self.assertTrue('reviewteam3 Early' in outbox[1]['Subject'])
# debug.show("outbox[1]['To']")
self.assertTrue('reviewsecretary3@' in outbox[1]['To'])
# set the reviewteamsetting for the secretary email alias, then do the post again
@ -557,7 +562,7 @@ class ReviewTests(TestCase):
# Test failure to return mailarch results
no_result_path = os.path.join(self.review_dir, "mailarch_no_result.html")
with open(no_result_path, "w") as f:
with io.open(no_result_path, "w") as f:
f.write('Content-Type: text/html\n\n<html><body><div class="xtr"><div class="xtd no-results">No results found</div></div>')
ietf.review.mailarch.construct_query_urls = lambda review_req, query=None: { "query_data_url": "file://" + os.path.abspath(no_result_path) }
@ -614,7 +619,7 @@ class ReviewTests(TestCase):
# complete by uploading file
empty_outbox()
test_file = StringIO("This is a review\nwith two lines")
test_file = io.StringIO("This is a review\nwith two lines")
test_file.name = "unnamed"
r = self.client.post(url, data={
@ -635,7 +640,7 @@ class ReviewTests(TestCase):
self.assertTrue(assignment.review_request.team.acronym.lower() in assignment.review.name)
self.assertTrue(assignment.review_request.doc.rev in assignment.review.name)
with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f:
with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f:
self.assertEqual(f.read(), "This is a review\nwith two lines")
self.assertEqual(len(outbox), 1)
@ -685,7 +690,7 @@ class ReviewTests(TestCase):
self.assertEqual(assignment.state_id, "completed")
self.assertNotEqual(assignment.completed_on, None)
with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f:
with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f:
self.assertEqual(f.read(), "This is a review\nwith two lines")
self.assertEqual(len(outbox), 1)
@ -772,7 +777,7 @@ class ReviewTests(TestCase):
assignment = reload_db_objects(assignment)
self.assertEqual(assignment.state_id, "completed")
with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f:
with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f:
self.assertEqual(f.read(), "This is a review\nwith two lines")
self.assertEqual(len(outbox), 0)
@ -872,7 +877,7 @@ class ReviewTests(TestCase):
event = ReviewAssignmentDocEvent.objects.get(type="closed_review_assignment", review_assignment=assignment)
self.assertEqual(event.time, datetime.datetime(2012, 12, 24, 12, 13, 14))
with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f:
with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f:
self.assertEqual(f.read(), "This is a review\nwith two lines")
self.assertEqual(len(outbox), 0)

10
ietf/doc/tests_status_change.py
View file

@ -1,6 +1,10 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import os
import shutil
@ -421,7 +425,7 @@ class StatusChangeSubmitTests(TestCase):
self.assertEqual(r.status_code,302)
doc = Document.objects.get(name='status-change-imaginary-mid-review')
self.assertEqual(doc.rev,'00')
with open(path) as f:
with io.open(path) as f:
self.assertEqual(f.read(),"Some initial review text\n")
self.assertTrue( "mid-review-00" in doc.latest_event(NewRevisionDocEvent).desc)
@ -434,7 +438,7 @@ class StatusChangeSubmitTests(TestCase):
# doc.rev is u'00' per the test setup - double-checking that here - if it fails, the breakage is in setUp
self.assertEqual(doc.rev,'00')
path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev))
with open(path,'w') as f:
with io.open(path,'w') as f:
f.write('This is the old proposal.')
f.close()
# Put the old proposal into IESG review (exercises ballot tab when looking at an older revision below)
@ -466,7 +470,7 @@ class StatusChangeSubmitTests(TestCase):
doc = Document.objects.get(name='status-change-imaginary-mid-review')
self.assertEqual(doc.rev,'01')
path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev))
with open(path) as f:
with io.open(path) as f:
self.assertEqual(f.read(),"This is a new proposal.")
f.close()
self.assertTrue( "mid-review-01" in doc.latest_event(NewRevisionDocEvent).desc)

26
ietf/doc/utils.py
View file

@ -1,14 +1,20 @@
# Copyright The IETF Trust 2011-2019, All Rights Reserved
# -*- coding: utf-8 -*-
import os
import re
import urllib.request, urllib.parse, urllib.error
import math
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import hashlib
import io
import json
import math
import os
import re
import six
from collections import defaultdict
from six.moves.urllib.parse import quote
from django.conf import settings
from django.contrib import messages
@ -313,7 +319,7 @@ def add_links_in_new_revision_events(doc, events, diff_revisions):
links += ""
if prev != None:
links += ' (<a href="%s?url1=%s&url2=%s">diff from previous</a>)' % (settings.RFCDIFF_BASE_URL, urllib.parse.quote(prev, safe="~"), urllib.parse.quote(diff_url, safe="~"))
links += ' (<a href="%s?url1=%s&url2=%s">diff from previous</a>)' % (settings.RFCDIFF_BASE_URL, quote(prev, safe="~"), quote(diff_url, safe="~"))
# replace the bold filename part
e.desc = re.sub(r"<b>(.+-[0-9][0-9].txt)</b>", links, e.desc)
@ -333,7 +339,7 @@ def add_events_message_info(events):
def get_unicode_document_content(key, filename, codec='utf-8', errors='ignore'):
try:
with open(filename, 'rb') as f:
with io.open(filename, 'rb') as f:
raw_content = f.read().decode(codec,errors)
except IOError:
if settings.DEBUG:
@ -347,7 +353,7 @@ def get_unicode_document_content(key, filename, codec='utf-8', errors='ignore'):
def get_document_content(key, filename, split=True, markup=True):
log.unreachable("2017-12-05")
try:
with open(filename, 'rb') as f:
with io.open(filename, 'rb') as f:
raw_content = f.read()
except IOError:
if settings.DEBUG:
@ -541,7 +547,7 @@ def rebuild_reference_relations(doc,filename=None):
filename=os.path.join(settings.INTERNET_DRAFT_PATH,doc.filename_with_rev())
try:
with open(filename, 'rb') as file:
with io.open(filename, 'rb') as file:
refs = draft.Draft(file.read().decode('utf8'), filename).get_refs()
except IOError as e:
return { 'errors': ["%s :%s" % (e.strerror, filename)] }
@ -661,7 +667,7 @@ def get_initial_notify(doc,extra=None):
receivers = []
if extra:
if isinstance(extra,str):
if isinstance(extra, six.string_types):
extra = extra.split(', ')
receivers.extend(extra)
@ -767,7 +773,7 @@ def get_search_cache_key(params):
from ietf.doc.views_search import SearchForm
fields = set(SearchForm.base_fields) - set(['sort',])
kwargs = dict([ (k,v) for (k,v) in list(params.items()) if k in fields ])
key = "doc:document:search:" + hashlib.sha512(json.dumps(kwargs, sort_keys=True).encode()).hexdigest()
key = "doc:document:search:" + hashlib.sha512(json.dumps(kwargs, sort_keys=True).encode('utf-8')).hexdigest()
return key
def label_wrap(label, items, joiner=',', max=50):

16
ietf/doc/utils_charter.py
View file

@ -1,5 +1,15 @@
# Copyright The IETF Trust 2011-2019, All Rights Reserved
import re, datetime, os, shutil
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import os
import re
import shutil
import six
from django.conf import settings
from django.urls import reverse as urlreverse
@ -53,7 +63,7 @@ def next_approved_revision(rev):
def read_charter_text(doc):
filename = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev))
try:
with open(filename, 'r') as f:
with io.open(filename, 'r') as f:
return f.read()
except IOError:
return "Error: couldn't read charter text"
@ -142,7 +152,7 @@ def generate_ballot_writeup(request, doc):
e.doc = doc
e.rev = doc.rev,
e.desc = "Ballot writeup was generated"
e.text = str(render_to_string("doc/charter/ballot_writeup.txt"))
e.text = six.ensure_text(render_to_string("doc/charter/ballot_writeup.txt"))
# caller is responsible for saving, if necessary
return e

18
ietf/doc/views_charter.py
View file

@ -1,7 +1,15 @@
# Copyright The IETF Trust 2011-2019, All Rights Reserved
# -*- coding: utf-8 -*-
import os, datetime, textwrap, json
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import json
import os
import six
import textwrap
from django.http import HttpResponseRedirect, HttpResponseNotFound, HttpResponseForbidden, Http404
from django.shortcuts import get_object_or_404, redirect, render
@ -413,7 +421,7 @@ def submit(request, name, option=None):
# Save file on disk
filename = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (charter.canonical_name(), charter.rev))
with open(filename, 'w', encoding='utf-8') as destination:
with io.open(filename, 'w', encoding='utf-8') as destination:
if form.cleaned_data['txt']:
destination.write(form.cleaned_data['txt'])
else:
@ -442,7 +450,7 @@ def submit(request, name, option=None):
filename = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (charter_canonical_name, charter_rev))
try:
with open(filename, 'r') as f:
with io.open(filename, 'r') as f:
init["content"] = f.read()
except IOError:
pass
@ -807,8 +815,8 @@ def charter_with_milestones_txt(request, name, rev):
charter_text = ""
try:
with open(os.path.join(settings.CHARTER_PATH, filename), 'r') as f:
charter_text = str(f.read(), errors='ignore')
with io.open(os.path.join(settings.CHARTER_PATH, filename), 'r') as f:
charter_text = six.ensure_text(f.read(), errors='ignore')
except IOError:
charter_text = "Error reading charter text %s" % filename

11
ietf/doc/views_conflict_review.py
View file

@ -1,7 +1,12 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
import datetime, os
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import os
from django import forms
from django.shortcuts import render, get_object_or_404, redirect
@ -159,7 +164,7 @@ class UploadForm(forms.Form):
def save(self, review):
filename = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (review.canonical_name(), review.rev))
with open(filename, 'w', encoding='utf-8') as destination:
with io.open(filename, 'w', encoding='utf-8') as destination:
if self.cleaned_data['txt']:
destination.write(self.cleaned_data['txt'])
else:
@ -223,7 +228,7 @@ def submit(request, name):
else:
filename = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (review.canonical_name(), review.rev))
try:
with open(filename, 'r') as f:
with io.open(filename, 'r') as f:
init["content"] = f.read()
except IOError:
pass

23
ietf/doc/views_doc.py
View file

@ -1,6 +1,6 @@
# Copyright The IETF Trust 2009-2019, All Rights Reserved
# -*- coding: utf-8 -*-
#
# Parts Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
#
@ -33,7 +33,18 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import os, datetime, urllib.request, urllib.parse, urllib.error, json, glob, re
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import glob
import io
import json
import os
import re
import six
from six.moves.urllib.parse import quote
from django.http import HttpResponse, Http404 , HttpResponseForbidden
from django.shortcuts import render, get_object_or_404, redirect
@ -312,7 +323,7 @@ def document_main(request, name, rev=None):
if doc.stream_id == "ietf" and group.type_id == "wg" and group.list_archive:
search_archive = group.list_archive
search_archive = urllib.parse.quote(search_archive, safe="~")
search_archive = quote(search_archive, safe="~")
# conflict reviews
conflict_reviews = [d.document.name for d in doc.related_that("conflrev")]
@ -661,7 +672,7 @@ def check_doc_email_aliases():
pattern = re.compile(r'^expand-(.*?)(\..*?)?@.*? +(.*)$')
good_count = 0
tot_count = 0
with open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file:
with io.open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file:
for line in virtual_file.readlines():
m = pattern.match(line)
tot_count += 1
@ -677,7 +688,7 @@ def get_doc_email_aliases(name):
else:
pattern = re.compile(r'^expand-(.*?)(\..*?)?@.*? +(.*)$')
aliases = []
with open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file:
with io.open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file:
for line in virtual_file.readlines():
m = pattern.match(line)
if m:
@ -1263,7 +1274,7 @@ def add_sessionpresentation(request,name):
if doc.group:
sessions = sorted(sessions,key=lambda x:0 if x.group==doc.group else 1)
session_choices = [(s.pk,str(s)) for s in sessions]
session_choices = [(s.pk, six.ensure_text(s)) for s in sessions]
if request.method == 'POST':
version_form = VersionForm(request.POST,choices=version_choices)

6
ietf/doc/views_material.py
View file

@ -1,7 +1,11 @@
# Copyright The IETF Trust 2014-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
# views for managing group materials (slides, ...)
import io
import os
import re
@ -140,7 +144,7 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None):
f = form.cleaned_data["material"]
file_ext = os.path.splitext(f.name)[1]
with open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + file_ext), 'wb+') as dest:
with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + file_ext), 'wb+') as dest:
for chunk in f.chunks():
dest.write(chunk)

5
ietf/doc/views_review.py
View file

@ -2,6 +2,9 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import os
import datetime
import requests
@ -625,7 +628,7 @@ def complete_review(request, name, assignment_id):
content = form.cleaned_data['review_content']
filename = os.path.join(review.get_file_path(), '{}.txt'.format(review.name, review.rev))
with open(filename, 'w', encoding='utf-8') as destination:
with io.open(filename, 'w', encoding='utf-8') as destination:
destination.write(content)
completion_datetime = datetime.datetime.now()

15
ietf/doc/views_status_change.py
View file

@ -1,7 +1,14 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
import datetime, os, re
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import os
import re
import six
from django import forms
from django.shortcuts import render, get_object_or_404, redirect
@ -124,7 +131,7 @@ class UploadForm(forms.Form):
def save(self, doc):
filename = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev))
with open(filename, 'w', encoding='utf-8') as destination:
with io.open(filename, 'w', encoding='utf-8') as destination:
if self.cleaned_data['txt']:
destination.write(self.cleaned_data['txt'])
else:
@ -188,7 +195,7 @@ def submit(request, name):
else:
filename = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev))
try:
with open(filename, 'r') as f:
with io.open(filename, 'r') as f:
init["content"] = f.read()
except IOError:
pass
@ -634,7 +641,7 @@ def generate_last_call_text(request, doc):
e.doc = doc
e.rev = doc.rev
e.desc = 'Last call announcement was generated'
e.text = str(new_text)
e.text = six.ensure_text(new_text)
e.save()
return e

24
ietf/group/models.py
View file

@ -1,19 +1,22 @@
# Copyright The IETF Trust 2010-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import email.utils
import jsonfield
import os
import re
import six
from urllib.parse import urljoin
from six.moves.urllib.parse import urljoin
from django.conf import settings
from django.core.validators import RegexValidator
from django.db import models
from django.db.models.deletion import CASCADE
from django.dispatch import receiver
from django.utils.encoding import python_2_unicode_compatible
from simple_history.models import HistoricalRecords
@ -27,6 +30,7 @@ from ietf.utils import log
from ietf.utils.models import ForeignKey, OneToOneField
@python_2_unicode_compatible
class GroupInfo(models.Model):
time = models.DateTimeField(default=datetime.datetime.now)
name = models.CharField(max_length=80)
@ -91,7 +95,7 @@ class Group(GroupInfo):
return e[0] if e else None
def has_role(self, user, role_names):
if isinstance(role_names, str) or isinstance(role_names, str):
if isinstance(role_names, six.string_types):
role_names = [role_names]
return user.is_authenticated and self.role_set.filter(name__in=role_names, person__user=user).exists()
@ -247,14 +251,16 @@ class GroupHistory(GroupInfo):
class Meta:
verbose_name_plural="group histories"
@python_2_unicode_compatible
class GroupURL(models.Model):
group = ForeignKey(Group)
name = models.CharField(max_length=255)
url = models.URLField()
def __str__(self):
return "%s (%s)" % (self.url, self.name)
return u"%s (%s)" % (self.url, self.name)
@python_2_unicode_compatible
class GroupMilestoneInfo(models.Model):
group = ForeignKey(Group)
# a group has two sets of milestones, current milestones
@ -281,6 +287,7 @@ class GroupMilestoneHistory(GroupMilestoneInfo):
time = models.DateTimeField()
milestone = ForeignKey(GroupMilestone, related_name="history_set")
@python_2_unicode_compatible
class GroupStateTransitions(models.Model):
"""Captures that a group has overriden the default available
document state transitions for a certain state."""
@ -289,7 +296,7 @@ class GroupStateTransitions(models.Model):
next_states = models.ManyToManyField('doc.State', related_name='previous_groupstatetransitions_states')
def __str__(self):
return '%s "%s" -> %s' % (self.group.acronym, self.state.name, [s.name for s in self.next_states.all()])
return u'%s "%s" -> %s' % (self.group.acronym, self.state.name, [s.name for s in self.next_states.all()])
GROUP_EVENT_CHOICES = [
("changed_state", "Changed state"),
@ -301,6 +308,7 @@ GROUP_EVENT_CHOICES = [
("status_update", "Status update"),
]
@python_2_unicode_compatible
class GroupEvent(models.Model):
"""An occurrence for a group, used for tracking who, when and what."""
group = ForeignKey(Group)
@ -310,7 +318,7 @@ class GroupEvent(models.Model):
desc = models.TextField()
def __str__(self):
return "%s %s at %s" % (self.by.plain_name(), self.get_type_display().lower(), self.time)
return u"%s %s at %s" % (self.by.plain_name(), self.get_type_display().lower(), self.time)
class Meta:
ordering = ['-time', 'id']
@ -321,13 +329,14 @@ class ChangeStateGroupEvent(GroupEvent):
class MilestoneGroupEvent(GroupEvent):
milestone = ForeignKey(GroupMilestone)
@python_2_unicode_compatible
class Role(models.Model):
name = ForeignKey(RoleName)
group = ForeignKey(Group)
person = ForeignKey(Person)
email = ForeignKey(Email, help_text="Email address used by person for this role.")
def __str__(self):
return "%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym or self.group.name)
return u"%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym or self.group.name)
def formatted_ascii_email(self):
return email.utils.formataddr((self.person.plain_ascii(), self.email.address))
@ -338,6 +347,7 @@ class Role(models.Model):
class Meta:
ordering = ['name_id', ]
@python_2_unicode_compatible
class RoleHistory(models.Model):
# RoleHistory doesn't have a time field as it's not supposed to be
# used on its own - there should always be a GroupHistory
@ -348,7 +358,7 @@ class RoleHistory(models.Model):
person = ForeignKey(Person)
email = ForeignKey(Email, help_text="Email address used by person for this role.")
def __str__(self):
return "%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym)
return u"%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym)
class Meta:
verbose_name_plural = "role histories"

12
ietf/group/tests_info.py
View file

@ -1,12 +1,16 @@
# Copyright The IETF Trust 2009-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import os
import shutil
import calendar
import datetime
import io
import bleach
import six
from pyquery import PyQuery
from tempfile import NamedTemporaryFile
@ -115,7 +119,7 @@ class GroupPagesTests(TestCase):
chair = Email.objects.filter(role__group=group, role__name="chair")[0]
with open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f:
with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f:
f.write("This is a charter.")
url = urlreverse('ietf.group.views.wg_summary_area', kwargs=dict(group_type="wg"))
@ -229,7 +233,7 @@ class GroupPagesTests(TestCase):
group = CharterFactory().group
draft = WgDraftFactory(group=group)
with open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f:
with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f:
f.write("This is a charter.")
milestone = GroupMilestone.objects.create(
@ -582,7 +586,7 @@ class GroupEditTests(TestCase):
self.assertTrue(len(q('form .has-error')) > 0)
# edit info
with open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f:
with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f:
f.write("This is a charter.")
area = group.parent
ad = Person.objects.get(name="Areað Irector")
@ -1264,7 +1268,7 @@ class StatusUpdateTests(TestCase):
response = self.client.get(url)
self.assertEqual(response.status_code,200)
q=PyQuery(response.content)
self.assertTrue(bleach.linkify(escape(event.desc)) in str(q('pre')))
self.assertTrue(bleach.linkify(escape(event.desc)) in six.ensure_text(q('pre')))
self.assertFalse(q('a#edit_button'))
self.client.login(username=chair.person.user.username,password='%s+password'%chair.person.user.username)
response = self.client.get(url)

6
ietf/group/tests_review.py
View file

@ -1,8 +1,12 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import debug # pyflakes:ignore
import six
from pyquery import PyQuery
@ -260,7 +264,7 @@ class ReviewTests(TestCase):
q = PyQuery(r.content)
generated_text = q("[name=body]").text()
self.assertTrue(review_req1.doc.name in generated_text)
self.assertTrue(str(Person.objects.get(user__username="marschairman")) in generated_text)
self.assertTrue(six.ensure_text(Person.objects.get(user__username="marschairman")) in generated_text)
empty_outbox()
r = self.client.post(url, {

6
ietf/group/utils.py
View file

@ -1,6 +1,10 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import os
from django.db.models import Q
@ -53,7 +57,7 @@ def get_charter_text(group):
filename = os.path.join(c.get_file_path(), "%s-%s.txt" % (c.canonical_name(), c.rev))
try:
with open(filename) as f:
with io.open(filename) as f:
return f.read()
except IOError:
return 'Error Loading Group Charter'

31
ietf/group/views.py
View file

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright The IETF Trust 2009-2019, All Rights Reserved
#
# Portion Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
#
@ -34,12 +33,18 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import os
import re
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import itertools
import io
import json
import math
import itertools
import datetime
import os
import re
import six
from tempfile import mkstemp
from collections import OrderedDict, defaultdict
from simple_history.utils import update_change_reason
@ -109,7 +114,7 @@ from ietf.doc.models import LastCallDocEvent
from ietf.name.models import ReviewAssignmentStateName
from ietf.utils.mail import send_mail_text, parse_preformatted
from ietf.utils.mail import send_mail_text, parse_preformatted, get_payload
from ietf.ietfauth.utils import user_is_person
from ietf.dbtemplate.models import DBTemplate
@ -204,7 +209,7 @@ def check_group_email_aliases():
pattern = re.compile(r'expand-(.*?)(-\w+)@.*? +(.*)$')
tot_count = 0
good_count = 0
with open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file:
with io.open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file:
for line in virtual_file.readlines():
m = pattern.match(line)
tot_count += 1
@ -630,7 +635,7 @@ def get_group_email_aliases(acronym, group_type):
pattern = re.compile(r'expand-(.*?)(-\w+)@.*? +(.*)$')
aliases = []
with open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file:
with io.open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file:
for line in virtual_file.readlines():
m = pattern.match(line)
if m:
@ -691,7 +696,7 @@ def dependencies(request, acronym, group_type=None, output_type="pdf"):
dothandle, dotname = mkstemp()
os.close(dothandle)
dotfile = open(dotname, "w")
dotfile = io.open(dotname, "w")
dotfile.write(make_dot(group))
dotfile.close()
@ -708,7 +713,7 @@ def dependencies(request, acronym, group_type=None, output_type="pdf"):
pipe("%s -f -l 10 -o %s %s" % (settings.UNFLATTEN_BINARY, unflatname, dotname))
pipe("%s -T%s -o %s %s" % (settings.DOT_BINARY, output_type, outname, unflatname))
outhandle = open(outname, "rb")
outhandle = io.open(outname, "rb")
out = outhandle.read()
outhandle.close()
@ -1443,7 +1448,7 @@ def manage_review_requests(request, acronym, group_type=None, assignment_status=
saving = form_action.startswith("save")
# check for conflicts
review_requests_dict = { str(r.pk): r for r in review_requests }
review_requests_dict = { six.ensure_text(r.pk): r for r in review_requests }
posted_reqs = set(request.POST.getlist("reviewrequest", []))
current_reqs = set(review_requests_dict.keys())
@ -1599,7 +1604,7 @@ def email_open_review_assignments(request, acronym, group_type=None):
(msg,_,_) = parse_preformatted(partial_msg)
body = msg.get_payload()
body = get_payload(msg)
subject = msg['Subject']
form = EmailOpenAssignmentsForm(initial={

10
ietf/idindex/generate_all_id2_txt.py
View file

@ -1,5 +1,7 @@
# Copyright The IETF Trust 2010-2019, All Rights Reserved
#!/usr/bin/env python
# Copyright The IETF Trust 2010-2019, All Rights Reserved
# -*- coding: utf-8 -*-
#
# Portions Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
#
@ -32,11 +34,15 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import, print_function, unicode_literals
import os
import six
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings")
import django
django.setup()
from ietf.idindex.index import all_id2_txt
print(all_id2_txt().encode('utf-8'), end=' ')
six.print_(all_id2_txt().encode('utf-8'), end=' ')

9
ietf/idindex/generate_all_id_txt.py
View file

@ -1,5 +1,6 @@
# Copyright The IETF Trust 2009-2019, All Rights Reserved
#!/usr/bin/env python
# Copyright The IETF Trust 2009-2019, All Rights Reserved
# -*- coding: utf-8 -*-
# Portions Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
#
@ -32,11 +33,15 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import, print_function, unicode_literals
import os
import six
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings")
import django
django.setup()
from ietf.idindex.index import all_id_txt
print(all_id_txt().encode("utf-8"), end=' ')
six.print_(all_id_txt().encode("utf-8"), end=' ')

9
ietf/idindex/generate_id_abstracts_txt.py
View file

@ -1,5 +1,6 @@
# Copyright The IETF Trust 2009-2019, All Rights Reserved
#!/usr/bin/env python
# Copyright The IETF Trust 2009-2019, All Rights Reserved
# -*- coding: utf-8 -*-
# Portions Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
#
@ -32,11 +33,15 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import, print_function, unicode_literals
import os
import six
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings")
import django
django.setup()
from ietf.idindex.index import id_index_txt
print(id_index_txt(with_abstracts=True).encode('utf-8'), end=' ')
six.print_(id_index_txt(with_abstracts=True).encode('utf-8'), end=' ')

9
ietf/idindex/generate_id_index_txt.py
View file

@ -1,5 +1,6 @@
# Copyright The IETF Trust 2009-2019, All Rights Reserved
#!/usr/bin/env python
# Copyright The IETF Trust 2009-2019, All Rights Reserved
# -*- coding: utf-8 -*-
# Portions Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
#
@ -32,11 +33,15 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import absolute_import, print_function, unicode_literals
import os
import six
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings")
import django
django.setup()
from ietf.idindex.index import id_index_txt
print(id_index_txt().encode('utf-8'), end=' ')
six.print_(id_index_txt().encode('utf-8'), end=' ')

16
ietf/idindex/index.py
View file

@ -1,22 +1,26 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
# code to generate plain-text index files that are placed on
# www.ietf.org in the same directory as the I-Ds
import datetime, os
import debug # pyflakes:ignore
import datetime
import os
import pytz
import six
from django.conf import settings
from django.template.loader import render_to_string
from ietf.doc.templatetags.ietf_filters import clean_whitespace
import debug # pyflakes:ignore
from ietf.doc.models import Document, DocEvent, DocumentAuthor, RelatedDocument, DocAlias, State
from ietf.doc.models import LastCallDocEvent, NewRevisionDocEvent
from ietf.doc.models import IESG_SUBSTATE_TAGS
from ietf.doc.templatetags.ietf_filters import clean_whitespace
from ietf.group.models import Group
from ietf.person.models import Person, Email
from ietf.utils import log
@ -191,7 +195,7 @@ def all_id2_txt():
area = d.group.parent.acronym
fields.append(area)
# 9 responsible AD name
fields.append(str(d.ad) if d.ad else "")
fields.append(six.ensure_text(d.ad) if d.ad else "")
# 10
fields.append(d.intended_std_level.name if d.intended_std_level else "")
# 11

17
ietf/idindex/tests.py
View file

@ -1,14 +1,19 @@
# Copyright The IETF Trust 2009-2019, All Rights Reserved
# -*- coding: utf-8 -*-
import os
import datetime
import shutil
import debug # pyflakes:ignore
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import os
import shutil
import six
from django.conf import settings
import debug # pyflakes:ignore
from ietf.doc.factories import WgDraftFactory
from ietf.doc.models import Document, DocAlias, RelatedDocument, State, LastCallDocEvent, NewRevisionDocEvent
from ietf.group.factories import GroupFactory
@ -28,7 +33,7 @@ class IndexTests(TestCase):
shutil.rmtree(self.id_dir)
def write_draft_file(self, name, size):
with open(os.path.join(self.id_dir, name), 'w') as f:
with io.open(os.path.join(self.id_dir, name), 'w') as f:
f.write("a" * size)
def test_all_id_txt(self):
@ -97,7 +102,7 @@ class IndexTests(TestCase):
self.assertEqual(t[6], draft.latest_event(type="new_revision").time.strftime("%Y-%m-%d"))
self.assertEqual(t[7], draft.group.acronym)
self.assertEqual(t[8], draft.group.parent.acronym)
self.assertEqual(t[9], str(draft.ad))
self.assertEqual(t[9], six.ensure_text(draft.ad))
self.assertEqual(t[10], draft.intended_std_level.name)
self.assertEqual(t[11], "")
self.assertEqual(t[12], ".pdf,.txt")

7
ietf/iesg/agenda.py
View file

@ -1,9 +1,12 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
# utilities for constructing agendas for IESG telechats
import codecs
import io
import datetime
from collections import OrderedDict
@ -146,7 +149,7 @@ def fill_in_agenda_administrivia(date, sections):
for s, key, filename in extra_info_files:
try:
with codecs.open(filename, 'r', 'utf-8', 'replace') as f:
with io.open(filename, 'r', encoding='utf-8', errors='replace') as f:
t = f.read().strip()
except IOError:
t = "(Error reading %s)" % filename

12
ietf/iesg/tests.py
View file

@ -1,9 +1,15 @@
# Copyright The IETF Trust 2009-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import os
import shutil
import datetime
import tarfile
from pyquery import PyQuery
from django.conf import settings
@ -448,15 +454,13 @@ class IESGAgendaTests(TestCase):
d1_filename = "%s-%s.txt" % (d1.name, d1.rev)
d2_filename = "%s-%s.txt" % (d2.name, d2.rev)
with open(os.path.join(self.draft_dir, d1_filename), "w") as f:
with io.open(os.path.join(self.draft_dir, d1_filename), "w") as f:
f.write("test content")
url = urlreverse("ietf.iesg.views.telechat_docs_tarfile", kwargs=dict(date=get_agenda_date().isoformat()))
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
import tarfile, io
tar = tarfile.open(None, fileobj=io.BytesIO(r.content))
names = tar.getnames()
self.assertIn(d1_filename, names)

22
ietf/iesg/views.py
View file

@ -1,5 +1,6 @@
# Copyright The IETF Trust 2007-2019, All Rights Reserved
# -*- coding: utf-8 -*-
#
# Portion Copyright (C) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
#
@ -32,16 +33,17 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import os
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import tarfile
import io
import time
import itertools
import json
import debug # pyflakes:ignore
import os
import six
import tarfile
import time
from django import forms
from django.conf import settings
@ -53,6 +55,8 @@ from django.utils.encoding import force_bytes
#from django.views.decorators.cache import cache_page
#from django.views.decorators.vary import vary_on_cookie
import debug # pyflakes:ignore
from ietf.doc.models import Document, State, LastCallDocEvent, ConsensusDocEvent, DocEvent, IESG_BALLOT_ACTIVE_STATES
from ietf.doc.utils import update_telechat, augment_events_with_revision
from ietf.group.models import GroupMilestone, Role
@ -312,9 +316,9 @@ def agenda_documents_txt(request):
row = (
d.computed_telechat_date.isoformat(),
d.name,
str(d.intended_std_level),
six.ensure_text(d.intended_std_level),
"1" if d.stream_id in ("ise", "irtf") else "0",
str(d.area_acronym()).lower(),
six.ensure_text(d.area_acronym()).lower(),
d.ad.plain_name() if d.ad else "None Assigned",
d.rev,
)

7
ietf/ietfauth/htpasswd.py
View file

@ -1,5 +1,10 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import subprocess, hashlib
from django.utils.encoding import force_bytes
@ -11,7 +16,7 @@ def update_htpasswd_file(username, password):
realm = settings.HTDIGEST_REALM
prefix = force_bytes('%s:%s:' % (username, realm))
key = force_bytes(hashlib.md5(prefix + force_bytes(password)).hexdigest())
f = open(pass_file, 'r+b')
f = io.open(pass_file, 'r+b')
pos = f.tell()
line = f.readline()
while line:

25
ietf/ietfauth/tests.py
View file

@ -2,8 +2,11 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import os, shutil, time, datetime
from urllib.parse import urlsplit
from six.moves.urllib.parse import urlsplit
from pyquery import PyQuery
from unittest import skipIf
@ -46,7 +49,7 @@ class IetfAuthTests(TestCase):
self.saved_htpasswd_file = settings.HTPASSWD_FILE
self.htpasswd_dir = self.tempdir('htpasswd')
settings.HTPASSWD_FILE = os.path.join(self.htpasswd_dir, "htpasswd")
open(settings.HTPASSWD_FILE, 'a').close() # create empty file
io.open(settings.HTPASSWD_FILE, 'a').close() # create empty file
self.saved_htdigest_realm = getattr(settings, "HTDIGEST_REALM", None)
settings.HTDIGEST_REALM = "test-realm"
@ -109,11 +112,11 @@ class IetfAuthTests(TestCase):
return confirm_url
def username_in_htpasswd_file(self, username):
with open(settings.HTPASSWD_FILE) as f:
with io.open(settings.HTPASSWD_FILE) as f:
for l in f:
if l.startswith(username + ":"):
return True
with open(settings.HTPASSWD_FILE) as f:
with io.open(settings.HTPASSWD_FILE) as f:
print(f.read())
return False
@ -581,31 +584,30 @@ class IetfAuthTests(TestCase):
self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index'))
for key in person.apikeys.all()[:3]:
url = key.endpoint
# bad method
r = self.client.put(url, {'apikey':key.hash()})
r = self.client.put(key.endpoint, {'apikey':key.hash()})
self.assertEqual(r.status_code, 405)
# missing apikey
r = self.client.post(url, {'dummy':'dummy',})
r = self.client.post(key.endpoint, {'dummy':'dummy',})
self.assertContains(r, 'Missing apikey parameter', status_code=400)
# invalid apikey
r = self.client.post(url, {'apikey':BAD_KEY, 'dummy':'dummy',})
r = self.client.post(key.endpoint, {'apikey':BAD_KEY, 'dummy':'dummy',})
self.assertContains(r, 'Invalid apikey', status_code=400)
# too long since regular login
person.user.last_login = datetime.datetime.now() - datetime.timedelta(days=settings.UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS+1)
person.user.save()
r = self.client.post(url, {'apikey':key.hash(), 'dummy':'dummy',})
r = self.client.post(key.endpoint, {'apikey':key.hash(), 'dummy':'dummy',})
self.assertContains(r, 'Too long since last regular login', status_code=400)
person.user.last_login = datetime.datetime.now()
person.user.save()
# endpoint mismatch
key2 = PersonalApiKey.objects.create(person=person, endpoint='/')
r = self.client.post(url, {'apikey':key2.hash(), 'dummy':'dummy',})
r = self.client.post(key.endpoint, {'apikey':key2.hash(), 'dummy':'dummy',})
self.assertContains(r, 'Apikey endpoint mismatch', status_code=400)
key2.delete()
@ -632,8 +634,7 @@ class IetfAuthTests(TestCase):
time.sleep(2)
for i in range(count):
for key in person.apikeys.all():
url = key.endpoint
self.client.post(url, {'apikey':key.hash(), 'dummy': 'dummy', })
self.client.post(key.endpoint, {'apikey':key.hash(), 'dummy': 'dummy', })
date = str(datetime.date.today())
empty_outbox()

8
ietf/ipr/feeds.py
View file

@ -1,4 +1,10 @@
# Copyright The IETF Trust 2007-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import six
from django.contrib.syndication.views import Feed
from django.utils.feedgenerator import Atom1Feed
@ -22,7 +28,7 @@ class LatestIprDisclosuresFeed(Feed):
return mark_safe(item.title)
def item_description(self, item):
return str(item.title)
return six.ensure_text(item.title)
def item_pubdate(self, item):
return item.time

8
ietf/ipr/fields.py
View file

@ -1,7 +1,11 @@
# Copyright The IETF Trust 2014-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import json
import six
from django.utils.html import escape
from django import forms
@ -48,7 +52,7 @@ class SearchableIprDisclosuresField(forms.CharField):
def prepare_value(self, value):
if not value:
value = ""
if isinstance(value, str):
if isinstance(value, six.string_types):
pks = self.parse_select2_value(value)
# if the user posted a non integer value we need to remove it
for key in pks:
@ -64,7 +68,7 @@ class SearchableIprDisclosuresField(forms.CharField):
# patterns may not have been fully constructed there yet
self.widget.attrs["data-ajax-url"] = urlreverse('ietf.ipr.views.ajax_search')
return ",".join(str(e.pk) for e in value)
return ",".join(six.ensure_text(e.pk) for e in value)
def clean(self, value):
value = super(SearchableIprDisclosuresField, self).clean(value)

8
ietf/ipr/management/commands/process_email.py
View file

@ -1,4 +1,10 @@
# Copyright The IETF Trust 2014-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import sys
from django.core.management.base import BaseCommand, CommandError
@ -20,7 +26,7 @@ class Command(BaseCommand):
if not email:
msg = sys.stdin.read()
else:
msg = open(email, "r").read()
msg = io.open(email, "r").read()
try:
process_response_email(msg)

10
ietf/ipr/tests.py
View file

@ -1,10 +1,14 @@
# Copyright The IETF Trust 2009-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import urllib.request, urllib.parse, urllib.error
from pyquery import PyQuery
from six.moves.urllib.parse import quote
from django.urls import reverse as urlreverse
@ -172,11 +176,11 @@ class IprTests(TestCase):
self.assertContains(r, ipr.title)
# find by doc title
r = self.client.get(url + "?submit=doctitle&doctitle=%s" % urllib.parse.quote(draft.title))
r = self.client.get(url + "?submit=doctitle&doctitle=%s" % quote(draft.title))
self.assertContains(r, ipr.title)
# find by ipr title
r = self.client.get(url + "?submit=iprtitle&iprtitle=%s" % urllib.parse.quote(ipr.title))
r = self.client.get(url + "?submit=iprtitle&iprtitle=%s" % quote(ipr.title))
self.assertContains(r, ipr.title)
def test_feed(self):

8
ietf/ipr/views.py
View file

@ -1,8 +1,12 @@
# Copyright The IETF Trust 2007-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import itertools
import six
from django.conf import settings
from django.contrib import messages
@ -452,7 +456,7 @@ def by_draft_txt(request):
lines = [ "# Machine-readable list of IPR disclosures by draft name" ]
for name, iprs in docipr.items():
lines.append(name + "\t" + "\t".join(str(ipr_id) for ipr_id in sorted(iprs)))
lines.append(name + "\t" + "\t".join(six.ensure_text(ipr_id) for ipr_id in sorted(iprs)))
return HttpResponse("\n".join(lines), content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET)
@ -474,7 +478,7 @@ def by_draft_recursive_txt(request):
lines = [ "# Machine-readable list of IPR disclosures by draft name" ]
for name, iprs in docipr.items():
lines.append(name + "\t" + "\t".join(str(ipr_id) for ipr_id in sorted(iprs)))
lines.append(name + "\t" + "\t".join(six.ensure_text(ipr_id) for ipr_id in sorted(iprs)))
return HttpResponse("\n".join(lines), content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET)

10
ietf/liaisons/fields.py
View file

@ -1,7 +1,11 @@
# Copyright The IETF Trust 2014-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import json
import six
from django.utils.html import escape
from django import forms
@ -47,9 +51,9 @@ class SearchableLiaisonStatementsField(forms.CharField):
def prepare_value(self, value):
if not value:
value = ""
if isinstance(value, int):
if isinstance(value, six.integer_types):
value = str(value)
if isinstance(value, str):
if isinstance(value, six.string_types):
pks = self.parse_select2_value(value)
value = self.model.objects.filter(pk__in=pks)
if isinstance(value, LiaisonStatement):
@ -61,7 +65,7 @@ class SearchableLiaisonStatementsField(forms.CharField):
# patterns may not have been fully constructed there yet
self.widget.attrs["data-ajax-url"] = urlreverse("ietf.liaisons.views.ajax_select2_search_liaison_statements")
return ",".join(str(o.pk) for o in value)
return ",".join(six.ensure_text(o.pk) for o in value)
def clean(self, value):
value = super(SearchableLiaisonStatementsField, self).clean(value)

12
ietf/liaisons/forms.py
View file

@ -1,8 +1,14 @@
# Copyright The IETF Trust 2011-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import datetime, os
import operator
import six
from email.utils import parseaddr
from form_utils.forms import BetterModelForm
@ -198,7 +204,7 @@ class CustomModelMultipleChoiceField(forms.ModelMultipleChoiceField):
if isinstance(value, QuerySet):
return value
if (hasattr(value, '__iter__') and
not isinstance(value, str) and
not isinstance(value, six.text_type) and
not hasattr(value, '_meta')):
return [super(CustomModelMultipleChoiceField, self).prepare_value(v) for v in value]
return super(CustomModelMultipleChoiceField, self).prepare_value(value)
@ -375,7 +381,7 @@ class LiaisonModelForm(BetterModelForm):
if created:
DocAlias.objects.create(name=attach.name).docs.add(attach)
LiaisonStatementAttachment.objects.create(statement=self.instance,document=attach)
attach_file = open(os.path.join(settings.LIAISON_ATTACH_PATH, attach.name + extension), 'wb')
attach_file = io.open(os.path.join(settings.LIAISON_ATTACH_PATH, attach.name + extension), 'wb')
attach_file.write(attached_file.read())
attach_file.close()
@ -446,7 +452,7 @@ class IncomingLiaisonForm(LiaisonModelForm):
self.fields['from_contact'].initial = self.person.role_set.filter(group=queryset[0]).first().email.address
self.fields['from_contact'].widget.attrs['readonly'] = True
self.fields['from_groups'].queryset = queryset
self.fields['from_groups'].widget.submitter = str(self.person)
self.fields['from_groups'].widget.submitter = six.ensure_text(self.person)
# if there's only one possibility make it the default
if len(queryset) == 1:

21
ietf/liaisons/tests.py
View file

@ -1,5 +1,14 @@
# Copyright The IETF Trust 2009-2019, All Rights Reserved
import datetime, os, shutil
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import os
import shutil
import six
import debug # pyflakes:ignore
@ -232,7 +241,7 @@ class ManagementCommandTests(TestCase):
LiaisonStatementFactory(deadline=datetime.date.today()+datetime.timedelta(days=1))
out = StringIO()
out = six.StringIO()
mailbox_before = len(outbox)
call_command('check_liaison_deadlines',stdout=out)
self.assertEqual(len(outbox), mailbox_before + 1)
@ -242,7 +251,7 @@ class ManagementCommandTests(TestCase):
RoleFactory(name_id='liaiman',group__type_id='sdo')
out = StringIO()
out = six.StringIO()
mailbox_before = len(outbox)
call_command('remind_update_sdo_list',stdout=out)
self.assertTrue(len(outbox) > mailbox_before)
@ -432,7 +441,7 @@ class LiaisonManagementTests(TestCase):
self.assertEqual(new_liaison.attachments.count(), attachments_before + 1)
attachment = new_liaison.attachments.order_by("-name")[0]
self.assertEqual(attachment.title, "attachment")
with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f:
with io.open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f:
written_content = f.read()
test_file.seek(0)
@ -736,7 +745,7 @@ class LiaisonManagementTests(TestCase):
self.assertEqual(l.attachments.count(), 1)
attachment = l.attachments.all()[0]
self.assertEqual(attachment.title, "attachment")
with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f:
with io.open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f:
written_content = f.read()
test_file.seek(0)
@ -815,7 +824,7 @@ class LiaisonManagementTests(TestCase):
self.assertEqual(l.attachments.count(), 1)
attachment = l.attachments.all()[0]
self.assertEqual(attachment.title, "attachment")
with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f:
with io.open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f:
written_content = f.read()
test_file.seek(0)

7
ietf/meeting/forms.py
View file

@ -1,8 +1,11 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import os
import codecs
import datetime
from django import forms
@ -287,7 +290,7 @@ class InterimSessionModelForm(forms.ModelForm):
directory = os.path.dirname(path)
if not os.path.exists(directory):
os.makedirs(directory)
with codecs.open(path, "w", encoding='utf-8') as file:
with io.open(path, "w", encoding='utf-8') as file:
file.write(self.cleaned_data['agenda'])

11
ietf/meeting/helpers.py
View file

@ -1,6 +1,11 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import os
import re
from tempfile import mkstemp
@ -211,7 +216,7 @@ def read_session_file(type, num, doc):
# FIXME: uploaded_filename should be replaced with a function call that computes names that are fixed
path = os.path.join(settings.AGENDA_PATH, "%s/%s/%s" % (num, type, doc.uploaded_filename))
if os.path.exists(path):
with open(path) as f:
with io.open(path) as f:
return f.read(), path
else:
return None, path
@ -224,13 +229,13 @@ def convert_draft_to_pdf(doc_name):
outpath = os.path.join(settings.INTERNET_DRAFT_PDF_PATH, doc_name + ".pdf")
try:
infile = open(inpath, "r")
infile = io.open(inpath, "r")
except IOError:
return
t,tempname = mkstemp()
os.close(t)
tempfile = open(tempname, "w")
tempfile = io.open(tempname, "w")
pageend = 0;
newpage = 0;

40
ietf/meeting/models.py
View file

@ -1,11 +1,13 @@
# Copyright The IETF Trust 2007-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
# old meeting models can be found in ../proceedings/models.py
import pytz
import datetime
from urllib.parse import urljoin
import io
from six.moves.urllib.parse import urljoin
import os
import re
import string
@ -19,6 +21,7 @@ from django.conf import settings
# mostly used by json_dict()
#from django.template.defaultfilters import slugify, date as date_format, time as time_format
from django.template.defaultfilters import date as date_format
from django.utils.encoding import python_2_unicode_compatible
from django.utils.text import slugify
from ietf.dbtemplate.models import DBTemplate
@ -54,6 +57,7 @@ def fmt_date(o):
d = datetime_safe.new_date(o)
return d.strftime(DATE_FORMAT)
@python_2_unicode_compatible
class Meeting(models.Model):
# number is either the number for IETF meetings, or some other
# identifier for interim meetings/IESG retreats/liaison summits/...
@ -109,7 +113,7 @@ class Meeting(models.Model):
def __str__(self):
if self.type_id == "ietf":
return "IETF-%s" % (self.number)
return u"IETF-%s" % (self.number)
else:
return self.number
@ -276,7 +280,7 @@ class Meeting(models.Model):
try:
tzfn = os.path.join(settings.TZDATA_ICS_PATH, self.time_zone + ".ics")
if os.path.exists(tzfn):
with open(tzfn) as tzf:
with io.open(tzfn) as tzf:
icstext = tzf.read()
vtimezone = re.search("(?sm)(\nBEGIN:VTIMEZONE.*\nEND:VTIMEZONE\n)", icstext).group(1).strip()
if vtimezone:
@ -310,6 +314,7 @@ class Meeting(models.Model):
# === Rooms, Resources, Floorplans =============================================
@python_2_unicode_compatible
class ResourceAssociation(models.Model):
name = ForeignKey(RoomResourceName)
icon = models.CharField(max_length=64) # icon to be found in /static/img
@ -326,6 +331,7 @@ class ResourceAssociation(models.Model):
res1['resource_id'] = self.pk
return res1
@python_2_unicode_compatible
class Room(models.Model):
meeting = ForeignKey(Meeting)
modified = models.DateTimeField(auto_now=True)
@ -345,7 +351,7 @@ class Room(models.Model):
# end floorplan-related stuff
def __str__(self):
return "%s size: %s" % (self.name, self.capacity)
return u"%s size: %s" % (self.name, self.capacity)
def delete_timeslots(self):
for ts in self.timeslot_set.all():
@ -415,6 +421,7 @@ def floorplan_path(instance, filename):
root, ext = os.path.splitext(filename)
return "%s/floorplan-%s-%s%s" % (settings.FLOORPLAN_MEDIA_DIR, instance.meeting.number, xslugify(instance.name), ext)
@python_2_unicode_compatible
class FloorPlan(models.Model):
name = models.CharField(max_length=255)
short = models.CharField(max_length=3, default='')
@ -427,10 +434,11 @@ class FloorPlan(models.Model):
ordering = ['-id',]
#
def __str__(self):
return 'floorplan-%s-%s' % (self.meeting.number, xslugify(self.name))
return u'floorplan-%s-%s' % (self.meeting.number, xslugify(self.name))
# === Schedules, Sessions, Timeslots and Assignments ===========================
@python_2_unicode_compatible
class TimeSlot(models.Model):
"""
Everything that would appear on the meeting agenda of a meeting is
@ -475,9 +483,9 @@ class TimeSlot(models.Model):
def __str__(self):
location = self.get_location()
if not location:
location = "(no location)"
location = u"(no location)"
return "%s: %s-%s %s, %s" % (self.meeting.number, self.time.strftime("%m-%d %H:%M"), (self.time + self.duration).strftime("%H:%M"), self.name, location)
return u"%s: %s-%s %s, %s" % (self.meeting.number, self.time.strftime("%m-%d %H:%M"), (self.time + self.duration).strftime("%H:%M"), self.name, location)
def end_time(self):
return self.time + self.duration
@ -598,6 +606,7 @@ class TimeSlot(models.Model):
# end of TimeSlot
@python_2_unicode_compatible
class Schedule(models.Model):
"""
Each person may have multiple agendas saved.
@ -617,7 +626,7 @@ class Schedule(models.Model):
# considering copiedFrom = ForeignKey('Schedule', blank=True, null=True)
def __str__(self):
return "%s:%s(%s)" % (self.meeting, self.name, self.owner)
return u"%s:%s(%s)" % (self.meeting, self.name, self.owner)
def base_url(self):
return "/meeting/%s/agenda/%s/%s" % (self.meeting.number, self.owner_email(), self.name)
@ -707,6 +716,7 @@ class Schedule(models.Model):
self.delete()
# to be renamed SchedTimeSessAssignments (stsa)
@python_2_unicode_compatible
class SchedTimeSessAssignment(models.Model):
"""
This model provides an N:M relationship between Session and TimeSlot.
@ -726,7 +736,7 @@ class SchedTimeSessAssignment(models.Model):
ordering = ["timeslot__time", "timeslot__type__slug", "session__group__parent__name", "session__group__acronym", "session__name", ]
def __str__(self):
return "%s [%s<->%s]" % (self.schedule, self.session, self.timeslot)
return u"%s [%s<->%s]" % (self.schedule, self.session, self.timeslot)
@property
def room_name(self):
@ -809,6 +819,7 @@ class SchedTimeSessAssignment(models.Model):
return "-".join(components).lower()
@python_2_unicode_compatible
class Constraint(models.Model):
"""
Specifies a constraint on the scheduling.
@ -829,7 +840,7 @@ class Constraint(models.Model):
active_status = None
def __str__(self):
return "%s %s target=%s person=%s" % (self.source, self.name.name.lower(), self.target, self.person)
return u"%s %s target=%s person=%s" % (self.source, self.name.name.lower(), self.target, self.person)
def brief_display(self):
if self.target and self.person:
@ -857,6 +868,7 @@ class Constraint(models.Model):
return ct1
@python_2_unicode_compatible
class SessionPresentation(models.Model):
session = ForeignKey('Session')
document = ForeignKey(Document)
@ -869,11 +881,12 @@ class SessionPresentation(models.Model):
unique_together = (('session', 'document'),)
def __str__(self):
return "%s -> %s-%s" % (self.session, self.document.name, self.rev)
return u"%s -> %s-%s" % (self.session, self.document.name, self.rev)
constraint_cache_uses = 0
constraint_cache_initials = 0
@python_2_unicode_compatible
class Session(models.Model):
"""Session records that a group should have a session on the
meeting (time and location is stored in a TimeSlot) - if multiple
@ -1113,7 +1126,7 @@ class Session(models.Model):
if doc:
path = os.path.join(settings.AGENDA_PATH, self.meeting.number, "agenda", doc.uploaded_filename)
if os.path.exists(path):
with open(path) as f:
with io.open(path) as f:
return f.read()
else:
return "No agenda file found"
@ -1147,6 +1160,7 @@ class Session(models.Model):
else:
return self.group.acronym
@python_2_unicode_compatible
class ImportantDate(models.Model):
meeting = ForeignKey(Meeting)
date = models.DateField()
@ -1155,7 +1169,7 @@ class ImportantDate(models.Model):
ordering = ["-meeting_id","date", ]
def __str__(self):
return '%s : %s : %s' % ( self.meeting, self.name, self.date )
return u'%s : %s : %s' % ( self.meeting, self.name, self.date )
class SlideSubmission(models.Model):
session = ForeignKey(Session)

6
ietf/meeting/tests_api.py
View file

@ -1,8 +1,12 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
from urllib.parse import urlsplit
from six.moves.urllib.parse import urlsplit
from django.urls import reverse as urlreverse

58
ietf/meeting/tests_views.py
View file

@ -1,17 +1,22 @@
# Copyright The IETF Trust 2009-2019, All Rights Reserved
# -*- coding: utf-8 -*-
import os
import shutil
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import urllib.parse
import io
import os
import random
import shutil
import six
from unittest import skipIf
from mock import patch
from pyquery import PyQuery
from io import StringIO, BytesIO
from bs4 import BeautifulSoup
from six.moves.urllib.parse import urlparse
from django.urls import reverse as urlreverse
from django.conf import settings
@ -68,7 +73,7 @@ class MeetingTests(TestCase):
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(path, "w") as f:
with io.open(path, "w") as f:
f.write(content)
def write_materials_files(self, meeting, session):
@ -381,10 +386,9 @@ class MeetingTests(TestCase):
response = self.client.get(url)
self.assertContains(response, 'test acknowledgements')
@patch('urllib.request.urlopen')
@patch('six.moves.urllib.request.urlopen')
def test_proceedings_attendees(self, mock_urlopen):
mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
mock_urlopen.return_value = six.BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
make_meeting_test_data()
meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96")
finalize(meeting)
@ -394,12 +398,12 @@ class MeetingTests(TestCase):
q = PyQuery(response.content)
self.assertEqual(1,len(q("#id_attendees tbody tr")))
@patch('urllib.request.urlopen')
@patch('six.moves.urllib.request.urlopen')
def test_proceedings_overview(self, mock_urlopen):
'''Test proceedings IETF Overview page.
Note: old meetings aren't supported so need to add a new meeting then test.
'''
mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
mock_urlopen.return_value = six.BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
make_meeting_test_data()
meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96")
finalize(meeting)
@ -476,7 +480,7 @@ class MeetingTests(TestCase):
session.sessionpresentation_set.create(document=doc)
file,_ = submission_file(name=doc.name,format='txt',templatename='test_submission.txt',group=session.group,rev="00")
filename = os.path.join(doc.get_file_path(),file.name)
with open(filename,'w') as draftbits:
with io.open(filename,'w') as draftbits:
draftbits.write(file.getvalue())
url = urlreverse('ietf.meeting.views.session_draft_tarfile', kwargs={'num':session.meeting.number,'acronym':session.group.acronym})
@ -492,7 +496,7 @@ class MeetingTests(TestCase):
session.sessionpresentation_set.create(document=doc)
file,_ = submission_file(name=doc.name,format='txt',templatename='test_submission.txt',group=session.group,rev="00")
filename = os.path.join(doc.get_file_path(),file.name)
with open(filename,'w') as draftbits:
with io.open(filename,'w') as draftbits:
draftbits.write(file.getvalue())
url = urlreverse('ietf.meeting.views.session_draft_pdf', kwargs={'num':session.meeting.number,'acronym':session.group.acronym})
@ -590,7 +594,7 @@ class EditTests(TestCase):
})
self.assertEqual(r.status_code, 302)
# Verify that we actually got redirected to a new place.
self.assertNotEqual(urllib.parse.urlparse(r.url).path, url)
self.assertNotEqual(urlparse(r.url).path, url)
# get
schedule = meeting.get_schedule_by_name("foo")
@ -636,7 +640,7 @@ class EditTests(TestCase):
'saveas': "saveas",
})
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r.url).path, url)
self.assertEqual(urlparse(r.url).path, url)
# TODO: Verify that an error message was in fact returned.
r = self.client.post(url, {
@ -645,7 +649,7 @@ class EditTests(TestCase):
})
# TODO: Verify that an error message was in fact returned.
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r.url).path, url)
self.assertEqual(urlparse(r.url).path, url)
# Non-ASCII alphanumeric characters
r = self.client.post(url, {
@ -654,7 +658,7 @@ class EditTests(TestCase):
})
# TODO: Verify that an error message was in fact returned.
self.assertEqual(r.status_code, 302)
self.assertEqual(urllib.parse.urlparse(r.url).path, url)
self.assertEqual(urlparse(r.url).path, url)
def test_edit_timeslots(self):
@ -1661,7 +1665,7 @@ class IphoneAppJsonTests(TestCase):
self.assertEqual(r.status_code,200)
class FinalizeProceedingsTests(TestCase):
@patch('urllib.request.urlopen')
@patch('six.moves.urllib.request.urlopen')
def test_finalize_proceedings(self, mock_urlopen):
mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]')
make_meeting_test_data()
@ -1711,7 +1715,7 @@ class MaterialsTests(TestCase):
soup = BeautifulSoup(page, 'html.parser')
for a in soup('a'):
href = a.get('href')
path = urllib.parse.urlparse(href).path
path = urlparse(href).path
if (path and path not in seen and path.startswith(top)):
follow(path)
follow(url)
@ -1723,7 +1727,7 @@ class MaterialsTests(TestCase):
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertIn('Upload', str(q("title")))
self.assertIn('Upload', six.ensure_text(q("title")))
self.assertFalse(session.sessionpresentation_set.exists())
test_file = StringIO('%PDF-1.4\n%âãÏÓ\nthis is some text for a test')
test_file.name = "not_really.pdf"
@ -1734,7 +1738,7 @@ class MaterialsTests(TestCase):
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertIn('Revise', str(q("title")))
self.assertIn('Revise', six.ensure_text(q("title")))
test_file = StringIO('%PDF-1.4\n%âãÏÓ\nthis is some different text for a test')
test_file.name = "also_not_really.pdf"
r = self.client.post(url,dict(file=test_file))
@ -1758,7 +1762,7 @@ class MaterialsTests(TestCase):
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertIn('Upload', str(q("title")))
self.assertIn('Upload', six.ensure_text(q("title")))
self.assertFalse(session.sessionpresentation_set.exists())
test_file = StringIO('%PDF-1.4\n%âãÏÓ\nthis is some text for a test')
test_file.name = "not_really.pdf"
@ -1776,7 +1780,7 @@ class MaterialsTests(TestCase):
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertIn('Upload', str(q("title")))
self.assertIn('Upload', six.ensure_text(q("title")))
def test_upload_minutes_agenda(self):
@ -1791,7 +1795,7 @@ class MaterialsTests(TestCase):
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertIn('Upload', str(q("Title")))
self.assertIn('Upload', six.ensure_text(q("Title")))
self.assertFalse(session.sessionpresentation_set.exists())
self.assertFalse(q('form input[type="checkbox"]'))
@ -1845,7 +1849,7 @@ class MaterialsTests(TestCase):
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertIn('Revise', str(q("Title")))
self.assertIn('Revise', six.ensure_text(q("Title")))
test_file = BytesIO(b'this is some different text for a test')
test_file.name = "also_not_really.txt"
r = self.client.post(url,dict(file=test_file,apply_to_all=True))
@ -1879,7 +1883,7 @@ class MaterialsTests(TestCase):
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertIn('Upload', str(q("Title")))
self.assertIn('Upload', six.ensure_text(q("Title")))
self.assertFalse(session.sessionpresentation_set.exists())
self.assertFalse(q('form input[type="checkbox"]'))
@ -1900,7 +1904,7 @@ class MaterialsTests(TestCase):
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertIn('Upload', str(q("title")))
self.assertIn('Upload', six.ensure_text(q("title")))
self.assertFalse(session.sessionpresentation_set.filter(document__type_id=doctype))
test_file = BytesIO(b'this is some text for a test')
test_file.name = "not_really.txt"
@ -1923,7 +1927,7 @@ class MaterialsTests(TestCase):
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertIn('Upload', str(q("title")))
self.assertIn('Upload', six.ensure_text(q("title")))
self.assertFalse(session1.sessionpresentation_set.filter(document__type_id='slides'))
test_file = BytesIO(b'this is not really a slide')
test_file.name = 'not_really.txt'
@ -1951,7 +1955,7 @@ class MaterialsTests(TestCase):
r = self.client.get(url)
self.assertTrue(r.status_code, 200)
q = PyQuery(r.content)
self.assertIn('Revise', str(q("title")))
self.assertIn('Revise', six.ensure_text(q("title")))
test_file = BytesIO(b'new content for the second slide deck')
test_file.name = 'doesnotmatter.txt'
r = self.client.post(url,dict(file=test_file,title='rename the presentation',apply_to_all=False))

12
ietf/meeting/utils.py
View file

@ -1,8 +1,14 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import json
import urllib.request, urllib.error, urllib.parse
import six.moves.urllib.request
from six.moves.urllib.error import HTTPError
from django.conf import settings
from django.template.loader import render_to_string
@ -88,8 +94,8 @@ def create_proceedings_templates(meeting):
# Get meeting attendees from registration system
url = settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number)
try:
attendees = json.load(urllib.request.urlopen(url))
except (ValueError, urllib.error.HTTPError):
attendees = json.load(six.moves.urllib.request.urlopen(url))
except (ValueError, HTTPError):
attendees = []
if attendees:

24
ietf/meeting/views.py
View file

@ -1,18 +1,24 @@
# Copyright The IETF Trust 2007-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import csv
import datetime
import glob
import io
import json
import os
import pytz
import re
import six
import tarfile
import urllib.request, urllib.parse, urllib.error
from calendar import timegm
from collections import OrderedDict, Counter, deque
from six.moves.urllib.parse import unquote
from tempfile import mkstemp
from wsgiref.handlers import format_date_time
@ -199,7 +205,7 @@ def materials_document(request, document, num=None, ext=None):
_, basename = os.path.split(filename)
if not os.path.exists(filename):
raise Http404("File not found: %s" % filename)
with open(filename, 'rb') as file:
with io.open(filename, 'rb') as file:
bytes = file.read()
mtype, chset = get_mime_type(bytes)
@ -518,12 +524,12 @@ def agenda(request, num=None, name=None, base=None, ext=None, owner=None, utc=""
def agenda_csv(schedule, filtered_assignments):
response = HttpResponse(content_type="text/csv; charset=%s"%settings.DEFAULT_CHARSET)
writer = csv.writer(response, delimiter=',', quoting=csv.QUOTE_ALL)
writer = csv.writer(response, delimiter=str(','), quoting=csv.QUOTE_ALL)
headings = ["Date", "Start", "End", "Session", "Room", "Area", "Acronym", "Type", "Description", "Session ID", "Agenda", "Slides"]
def write_row(row):
encoded_row = [v.encode('utf-8') if isinstance(v, str) else v for v in row]
encoded_row = [v.encode('utf-8') if isinstance(v, six.text_type) else v for v in row]
while len(encoded_row) < len(headings):
encoded_row.append(None) # produce empty entries at the end as necessary
@ -681,7 +687,7 @@ def session_draft_tarfile(request, num, acronym):
tarstream = tarfile.open('','w:gz',response)
mfh, mfn = mkstemp()
os.close(mfh)
manifest = open(mfn, "w")
manifest = io.open(mfn, "w")
for doc_name in drafts:
pdf_path = os.path.join(settings.INTERNET_DRAFT_PDF_PATH, doc_name + ".pdf")
@ -709,7 +715,7 @@ def session_draft_pdf(request, num, acronym):
curr_page = 1
pmh, pmn = mkstemp()
os.close(pmh)
pdfmarks = open(pmn, "w")
pdfmarks = io.open(pmn, "w")
pdf_list = ""
for draft in drafts:
@ -730,7 +736,7 @@ def session_draft_pdf(request, num, acronym):
code, out, err = pipe(gs + " -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -sOutputFile=" + pdfn + " " + pdf_list + " " + pmn)
assertion('code == 0')
pdf = open(pdfn,"rb")
pdf = io.open(pdfn,"rb")
pdf_contents = pdf.read()
pdf.close()
@ -873,7 +879,7 @@ def ical_agenda(request, num=None, name=None, acronym=None, session_id=None):
raise Http404
q = request.META.get('QUERY_STRING','') or ""
filter = set(urllib.parse.unquote(q).lower().split(','))
filter = set(unquote(q).lower().split(','))
include = [ i for i in filter if not (i.startswith('-') or i.startswith('~')) ]
include_types = set(["plenary","other"])
exclude = []
@ -1583,7 +1589,7 @@ def propose_session_slides(request, session_id, num):
name = 'slides-%s-%s' % (session.meeting.number, session.docname_token())
name = name + '-' + slugify(title).replace('_', '-')[:128]
filename = '%s-00%s'% (name, ext)
destination = open(os.path.join(settings.SLIDE_STAGING_PATH, filename),'wb+')
destination = io.open(os.path.join(settings.SLIDE_STAGING_PATH, filename),'wb+')
for chunk in file.chunks():
destination.write(chunk)
destination.close()

4
ietf/name/generate_fixtures.py
View file

@ -1,8 +1,10 @@
# Copyright The IETF Trust 2011-2019, All Rights Reserved
#!/usr/bin/python
# simple script for exporting name related base data for the tests
# boiler plate
import io
import os, sys
import django
@ -17,7 +19,7 @@ from django.core.serializers import serialize
def output(name, seq):
try:
f = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/%s.json" % name), 'w')
f = io.open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/%s.json" % name), 'w')
f.write(serialize("json", seq, indent=1))
f.close()
except:

11
ietf/nomcom/forms.py
View file

@ -1,4 +1,11 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import six
from django.conf import settings
from django import forms
from django.urls import reverse
@ -33,12 +40,12 @@ class PositionNomineeField(forms.ChoiceField):
results = []
for position in positions:
accepted_nominees = [np.nominee for np in NomineePosition.objects.filter(position=position,state='accepted').exclude(nominee__duplicated__isnull=False)]
nominees = [('%s_%s' % (position.id, i.id), str(i)) for i in accepted_nominees]
nominees = [('%s_%s' % (position.id, i.id), six.ensure_text(i)) for i in accepted_nominees]
if nominees:
results.append((position.name+" (Accepted)", nominees))
for position in positions:
other_nominees = [np.nominee for np in NomineePosition.objects.filter(position=position).exclude(state='accepted').exclude(nominee__duplicated__isnull=False)]
nominees = [('%s_%s' % (position.id, i.id), str(i)) for i in other_nominees]
nominees = [('%s_%s' % (position.id, i.id), six.ensure_text(i)) for i in other_nominees]
if nominees:
results.append((position.name+" (Declined or Pending)", nominees))
kwargs['choices'] = results

8
ietf/nomcom/management/commands/feedback_email.py
View file

@ -1,4 +1,10 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import sys
from django.core.management.base import BaseCommand, CommandError
@ -31,7 +37,7 @@ class Command(BaseCommand):
if not email:
msg = sys.stdin.read()
else:
msg = open(email, "r").read()
msg = io.open(email, "r").read()
try:
nomcom = NomCom.objects.get(group__acronym__icontains=year,

10
ietf/nomcom/test_data.py
View file

@ -1,4 +1,10 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import tempfile
import os
@ -90,7 +96,7 @@ def check_comments(encryped, plain, privatekey_file):
decrypted_file.close()
encrypted_file.close()
decrypted_comments = open(decrypted_file.name, 'r').read()
decrypted_comments = io.open(decrypted_file.name, 'rb').read().decode('utf-8')
os.unlink(encrypted_file.name)
os.unlink(decrypted_file.name)
@ -112,7 +118,7 @@ def nomcom_test_data():
nomcom_test_cert_file, privatekey_file = generate_cert()
nomcom.public_key.storage = FileSystemStorage(location=settings.NOMCOM_PUBLIC_KEYS_DIR)
nomcom.public_key.save('cert', File(open(nomcom_test_cert_file.name, 'r')))
nomcom.public_key.save('cert', File(io.open(nomcom_test_cert_file.name, 'r')))
# chair and member
create_person(group, "chair", username=CHAIR_USER, email_address='%s%s'%(CHAIR_USER,EMAIL_DOMAIN))

36
ietf/nomcom/tests.py
View file

@ -1,11 +1,17 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
#import tempfile
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import random
import six
import shutil
import urllib.parse
from pyquery import PyQuery
from six.moves.urllib.parse import urlparse
from django.db import IntegrityError
from django.db.models import Max
@ -35,7 +41,7 @@ from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, get_hash
from ietf.person.factories import PersonFactory, EmailFactory
from ietf.person.models import Email, Person
from ietf.stats.models import MeetingRegistration
from ietf.utils.mail import outbox, empty_outbox
from ietf.utils.mail import outbox, empty_outbox, get_payload
from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent
client_test_cert_files = None
@ -334,7 +340,7 @@ class NomcomViewsTest(TestCase):
response = self.client.post(self.private_merge_nominee_url, test_data)
self.assertEqual(response.status_code, 302)
redirect_url = response["Location"]
redirect_path = urllib.parse.urlparse(redirect_url).path
redirect_path = urlparse(redirect_url).path
self.assertEqual(redirect_path, reverse('ietf.nomcom.views.private_index', kwargs={"year": NOMCOM_YEAR}))
response = self.client.get(redirect_url)
@ -407,7 +413,7 @@ class NomcomViewsTest(TestCase):
q = PyQuery(r.content)
reminder_date = '%s-09-30' % self.year
f = open(self.cert_file.name)
f = io.open(self.cert_file.name)
response = self.client.post(self.edit_nomcom_url, {
'public_key': f,
'reminderdates_set-TOTAL_FORMS': q('input[name="reminderdates_set-TOTAL_FORMS"]').val(),
@ -520,7 +526,7 @@ class NomcomViewsTest(TestCase):
self.assertEqual('Nomination receipt', outbox[-1]['Subject'])
self.assertEqual(self.email_from, outbox[-1]['From'])
self.assertIn('plain', outbox[-1]['To'])
self.assertIn('Comments with accents äöå', str(outbox[-1].get_payload(decode=True),"utf-8","replace"))
self.assertIn('Comments with accents äöå', six.ensure_text(outbox[-1].get_payload(decode=True),"utf-8","replace"))
# Nominate the same person for the same position again without asking for confirmation
@ -561,7 +567,7 @@ class NomcomViewsTest(TestCase):
self.assertEqual('Nomination receipt', outbox[-1]['Subject'])
self.assertEqual(self.email_from, outbox[-1]['From'])
self.assertIn('plain', outbox[-1]['To'])
self.assertIn('Comments with accents äöå', str(outbox[-1].get_payload(decode=True),"utf-8","replace"))
self.assertIn('Comments with accents äöå', six.ensure_text(outbox[-1].get_payload(decode=True),"utf-8","replace"))
# Nominate the same person for the same position again without asking for confirmation
@ -621,7 +627,7 @@ class NomcomViewsTest(TestCase):
# save the cert file in tmp
#nomcom.public_key.storage.location = tempfile.gettempdir()
nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r')))
nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r')))
response = self.client.get(nominate_url)
self.assertEqual(response.status_code, 200)
@ -687,7 +693,7 @@ class NomcomViewsTest(TestCase):
# save the cert file in tmp
#nomcom.public_key.storage.location = tempfile.gettempdir()
nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r')))
nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r')))
response = self.client.get(nominate_url)
self.assertEqual(response.status_code, 200)
@ -763,7 +769,7 @@ class NomcomViewsTest(TestCase):
# save the cert file in tmp
#nomcom.public_key.storage.location = tempfile.gettempdir()
nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r')))
nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r')))
response = self.client.get(self.add_questionnaire_url)
self.assertEqual(response.status_code, 200)
@ -801,12 +807,12 @@ class NomcomViewsTest(TestCase):
# We're interested in the confirmation receipt here
self.assertEqual(len(outbox),3)
self.assertEqual('NomCom comment confirmation', outbox[2]['Subject'])
email_body = outbox[2].get_payload()
email_body = get_payload(outbox[2])
self.assertIn(position, email_body)
self.assertNotIn('$', email_body)
self.assertEqual(self.email_from, outbox[-2]['From'])
self.assertIn('plain', outbox[2]['To'])
self.assertIn('Comments with accents äöå', str(outbox[2].get_payload(decode=True),"utf-8","replace"))
self.assertIn('Comments with accents äöå', six.ensure_text(outbox[2].get_payload(decode=True),"utf-8","replace"))
empty_outbox()
self.feedback_view(public=True)
@ -842,7 +848,7 @@ class NomcomViewsTest(TestCase):
# save the cert file in tmp
#nomcom.public_key.storage.location = tempfile.gettempdir()
nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r')))
nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r')))
response = self.client.get(feedback_url)
self.assertEqual(response.status_code, 200)
@ -957,7 +963,7 @@ class FeedbackTest(TestCase):
# save the cert file in tmp
#nomcom.public_key.storage.location = tempfile.gettempdir()
nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r')))
nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r')))
comment_text = 'Plain text. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.'
comments = nomcom.encrypt(comment_text)
@ -979,7 +985,7 @@ class ReminderTest(TestCase):
self.nomcom = get_nomcom_by_year(NOMCOM_YEAR)
self.cert_file, self.privatekey_file = get_cert_files()
#self.nomcom.public_key.storage.location = tempfile.gettempdir()
self.nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r')))
self.nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r')))
gen = Position.objects.get(nomcom=self.nomcom,name='GEN')
rai = Position.objects.get(nomcom=self.nomcom,name='RAI')

26
ietf/nomcom/utils.py
View file

@ -1,8 +1,14 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import hashlib
import os
import re
import six
import tempfile
from email import message_from_string
@ -16,7 +22,7 @@ from django.core.exceptions import ObjectDoesNotExist
from django.urls import reverse
from django.template.loader import render_to_string
from django.shortcuts import get_object_or_404
from django.utils.encoding import smart_str
from django.utils.encoding import force_str
from ietf.dbtemplate.models import DBTemplate
from ietf.person.models import Email, Person
@ -87,7 +93,7 @@ def get_user_email(user):
return user._email_cache
def get_hash_nominee_position(date, nominee_position_id):
return hashlib.md5(('%s%s%s' % (settings.SECRET_KEY, date, nominee_position_id)).encode()).hexdigest()
return hashlib.md5(('%s%s%s' % (settings.SECRET_KEY, date, nominee_position_id)).encode('utf-8')).hexdigest()
def initialize_templates_for_group(group):
@ -160,7 +166,7 @@ def retrieve_nomcom_private_key(request, year):
command = "%s bf -d -in /dev/stdin -k \"%s\" -a"
code, out, error = pipe(command % (settings.OPENSSL_COMMAND,
settings.SECRET_KEY), private_key.encode())
settings.SECRET_KEY), private_key.encode('utf-8'))
if code != 0:
log("openssl error: %s:\n Error %s: %s" %(command, code, error))
return out
@ -172,7 +178,7 @@ def store_nomcom_private_key(request, year, private_key):
else:
command = "%s bf -e -in /dev/stdin -k \"%s\" -a"
code, out, error = pipe(command % (settings.OPENSSL_COMMAND,
settings.SECRET_KEY), private_key.encode())
settings.SECRET_KEY), private_key.encode('utf-8'))
if code != 0:
log("openssl error: %s:\n Error %s: %s" %(command, code, error))
if error:
@ -182,7 +188,7 @@ def store_nomcom_private_key(request, year, private_key):
def validate_private_key(key):
key_file = tempfile.NamedTemporaryFile(delete=False)
key_file.write(key.encode())
key_file.write(key.encode('utf-8'))
key_file.close()
command = "%s rsa -in %s -check -noout"
@ -400,7 +406,7 @@ def getheader(header_text, default="ascii"):
"""Decode the specified header"""
tuples = decode_header(header_text)
header_sections = [ text.decode(charset or default) if isinstance(text, bytes) else text for text, charset in tuples]
header_sections = [ text.decode(charset or default) if isinstance(text, six.binary_type) else text for text, charset in tuples]
return "".join(header_sections)
@ -427,7 +433,7 @@ def get_body(message):
body = []
for part in text_parts:
charset = get_charset(part, get_charset(message))
body.append(str(part.get_payload(decode=True),
body.append(six.ensure_text(part.get_payload(decode=True),
charset,
"replace"))
@ -435,16 +441,14 @@ def get_body(message):
else: # if it is not multipart, the payload will be a string
# representing the message body
body = str(message.get_payload(decode=True),
body = six.ensure_text(message.get_payload(decode=True),
get_charset(message),
"replace")
return body.strip()
def parse_email(text):
if isinstance(text, str):
text = smart_str(text)
msg = message_from_string(text)
msg = message_from_string(force_str(text))
body = get_body(msg)
subject = getheader(msg['Subject'])

12
ietf/person/factories.py
View file

@ -2,12 +2,16 @@
# -*- coding: utf-8 -*-
import os
from __future__ import absolute_import, print_function, unicode_literals
import factory
import faker
import shutil
import random
import faker.config
import os
import random
import shutil
import six
from unidecode import unidecode
from django.conf import settings
@ -54,7 +58,7 @@ class PersonFactory(factory.DjangoModelFactory):
user = factory.SubFactory(UserFactory)
name = factory.LazyAttribute(lambda p: normalize_name('%s %s'%(p.user.first_name, p.user.last_name)))
ascii = factory.LazyAttribute(lambda p: str(unidecode_name(p.name)))
ascii = factory.LazyAttribute(lambda p: six.ensure_text(unidecode_name(p.name)))
class Params:
with_bio = factory.Trait(biography = "\n\n".join(fake.paragraphs()))

12
ietf/person/fields.py
View file

@ -1,8 +1,14 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import json
import six
from collections import Counter
from urllib.parse import urlencode
from six.moves.urllib.parse import urlencode
from django.utils.html import escape
from django import forms
@ -73,7 +79,7 @@ class SearchablePersonsField(forms.CharField):
def prepare_value(self, value):
if not value:
value = ""
if isinstance(value, str):
if isinstance(value, six.string_types):
pks = self.parse_select2_value(value)
if self.model == Person:
value = self.model.objects.filter(pk__in=pks)
@ -162,7 +168,7 @@ class PersonEmailChoiceField(forms.ModelChoiceField):
def label_from_instance(self, email):
if self.label_with == "person":
return str(email.person)
return six.ensure_text(email.person)
elif self.label_with == "email":
return email.address
else:

5
ietf/person/management/commands/deactivate_email_addresses.py
View file

@ -2,7 +2,10 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import flufl.bounce
import io
import mailbox
import sys
@ -67,7 +70,7 @@ class Command(BaseCommand):
self.stderr.write('No person is associated with <%s>\n' % (a, ))
else:
self.stderr.write('Address not found: <%s>\n' % (a, ))
with open('./failed', 'a') as failed:
with io.open('./failed', 'a') as failed:
failed.write(messages[a].as_string(unixfrom=True))
failed.write('\n')

37
ietf/person/models.py
View file

@ -1,22 +1,27 @@
# Copyright The IETF Trust 2010-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import email.utils
import email.header
import six
import uuid
from hashids import Hashids
from urllib.parse import urljoin
from six.moves.urllib.parse import urljoin
from django.conf import settings
from django.core.validators import validate_email
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.core.validators import validate_email
from django.db import models
from django.template.loader import render_to_string
from django.utils.encoding import smart_bytes
from django.utils.encoding import python_2_unicode_compatible, smart_bytes
from django.utils.text import slugify
from simple_history.models import HistoricalRecords
import debug # pyflakes:ignore
@ -30,6 +35,7 @@ from ietf.utils import log
from ietf.utils.models import ForeignKey, OneToOneField
@python_2_unicode_compatible
class Person(models.Model):
history = HistoricalRecords()
user = OneToOneField(User, blank=True, null=True, on_delete=models.SET_NULL)
@ -79,7 +85,11 @@ class Person(models.Model):
def plain_ascii(self):
if not hasattr(self, '_cached_plain_ascii'):
if self.ascii:
ascii = unidecode_name(self.ascii)
if isinstance(self.ascii, six.binary_type):
uname = six.ensure_text(self.ascii)
ascii = unidecode_name(uname)
else:
ascii = unidecode_name(self.ascii)
else:
ascii = unidecode_name(self.name)
prefix, first, middle, last, suffix = name_parts(ascii)
@ -96,7 +106,7 @@ class Person(models.Model):
may be an object or the group acronym."""
if group:
from ietf.group.models import Group
if isinstance(group, str) or isinstance(group, str):
if isinstance(group, six.string_types):
group = Group.objects.get(acronym=group)
e = Email.objects.filter(person=self, role__group=group, role__name=role_name)
else:
@ -225,6 +235,7 @@ class Person(models.Model):
ct1['ascii'] = self.ascii
return ct1
@python_2_unicode_compatible
class Alias(models.Model):
"""This is used for alternative forms of a name. This is the
primary lookup point for names, and should always contain the
@ -252,6 +263,7 @@ class Alias(models.Model):
class Meta:
verbose_name_plural = "Aliases"
@python_2_unicode_compatible
class Email(models.Model):
history = HistoricalRecords()
address = models.CharField(max_length=64, primary_key=True, validators=[validate_email])
@ -323,6 +335,7 @@ PERSON_API_KEY_ENDPOINTS = [
("/api/meeting/session/video/url", "/api/meeting/session/video/url"),
]
@python_2_unicode_compatible
class PersonalApiKey(models.Model):
person = ForeignKey(Person, related_name='apikeys')
endpoint = models.CharField(max_length=128, null=False, blank=False, choices=PERSON_API_KEY_ENDPOINTS)
@ -335,11 +348,8 @@ class PersonalApiKey(models.Model):
@classmethod
def validate_key(cls, s):
import struct, hashlib, base64
try:
key = base64.urlsafe_b64decode(s)
except TypeError:
return None
assert isinstance(s, six.binary_type)
key = base64.urlsafe_b64decode(s)
id, salt, hash = struct.unpack(KEY_STRUCT, key)
k = cls.objects.filter(id=id)
if not k.exists():
@ -372,6 +382,7 @@ PERSON_EVENT_CHOICES = [
("email_address_deactivated", "Email address deactivated"),
]
@python_2_unicode_compatible
class PersonEvent(models.Model):
person = ForeignKey(Person)
time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened")

6
ietf/person/tests.py
View file

@ -2,7 +2,11 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import six
from pyquery import PyQuery
from io import StringIO
from django.urls import reverse as urlreverse
@ -89,7 +93,7 @@ class PersonTests(TestCase):
empty_outbox()
p = PersonFactory(name="Föö Bär")
PersonFactory(name=p.name)
self.assertTrue("possible duplicate" in str(outbox[0]["Subject"]).lower())
self.assertTrue("possible duplicate" in six.ensure_text(outbox[0]["Subject"]).lower())
def test_merge(self):
url = urlreverse("ietf.person.views.merge")

13
ietf/person/utils.py
View file

@ -1,8 +1,13 @@
# Copyright The IETF Trust 2015-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import os
import pprint
import six
import sys
import syslog
@ -50,13 +55,13 @@ def merge_persons(source, target, file=sys.stdout, verbose=False):
objs, opts, user, admin_site, using)
deletable_objects_summary = deletable_objects[1]
if len(deletable_objects_summary) > 1: # should only inlcude one object (Person)
print("Not Deleting Person: {}({})".format(source.ascii,source.pk), file=file)
print("Related objects remain:", file=file)
six.print_("Not Deleting Person: {}({})".format(source.ascii,source.pk), file=file)
six.print_("Related objects remain:", file=file)
pprint.pprint(deletable_objects[1], stream=file)
success = False
else:
success = True
print("Deleting Person: {}({})".format(source.ascii,source.pk), file=file)
six.print_("Deleting Person: {}({})".format(source.ascii,source.pk), file=file)
source.delete()
return success, changes
@ -109,7 +114,7 @@ def move_related_objects(source, target, file, verbose=False):
field_name = related_object.field.name
queryset = getattr(source, accessor).all()
if verbose:
print("Merging {}:{}".format(accessor,queryset.count()),file=file)
six.print_("Merging {}:{}".format(accessor,queryset.count()), file=file)
kwargs = { field_name:target }
queryset.update(**kwargs)

5
ietf/release/views.py
View file

@ -2,6 +2,9 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import os
import re
import json
@ -43,7 +46,7 @@ def get_coverage_data():
with gzip.open(settings.TEST_COVERAGE_MASTER_FILE, "rb") as file:
coverage_data = json.load(file)
else:
with open(settings.TEST_COVERAGE_MASTER_FILE) as file:
with io.open(settings.TEST_COVERAGE_MASTER_FILE) as file:
coverage_data = json.load(file)
cache.set(cache_key, coverage_data, 60*60*24)
return coverage_data

26
ietf/review/mailarch.py
View file

@ -1,11 +1,25 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
# various utilities for working with the mailarch mail archive at
# mailarchive.ietf.org
import datetime, tarfile, mailbox, tempfile, hashlib, base64, email.utils
import urllib.request, urllib.parse, urllib.error
import urllib.request, urllib.error, urllib.parse, contextlib
import debug # pyflakes:ignore
import contextlib
import datetime
import tarfile
import mailbox
import tempfile
import hashlib
import base64
import email.utils
from six.moves.urllib.parse import urlencode
from six.moves.urllib.request import urlopen
import debug # pyflakes:ignore
from pyquery import PyQuery
@ -36,7 +50,7 @@ def construct_query_urls(review_req, query=None):
if not query:
query = review_req.doc.name
encoded_query = "?" + urllib.parse.urlencode({
encoded_query = "?" + urlencode({
"qdr": "c", # custom time frame
"start_date": (datetime.date.today() - datetime.timedelta(days=180)).isoformat(),
"email_list": list_name,
@ -95,7 +109,7 @@ def retrieve_messages(query_data_url):
"""Retrieve and return selected content from mailarch."""
res = []
with contextlib.closing(urllib.request.urlopen(query_data_url, timeout=15)) as fileobj:
with contextlib.closing(urlopen(query_data_url, timeout=15)) as fileobj:
content_type = fileobj.info()["Content-type"]
if not content_type.startswith("application/x-tar"):
if content_type.startswith("text/html"):

11
ietf/review/utils.py
View file

@ -1,9 +1,14 @@
# -*- coding: utf-8 -*-
# Copyright The IETF Trust 2016-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import itertools
import re
import six
import datetime, re, itertools
from collections import defaultdict, namedtuple
from django.db.models import Q, Max, F
@ -959,7 +964,7 @@ def make_assignment_choices(email_queryset, review_req):
if stats:
explanations.append(", ".join(stats))
label = str(e.person)
label = six.ensure_text(e.person)
if explanations:
label = "{}: {}".format(label, "; ".join(explanations))

8
ietf/secr/drafts/tests_views.py
View file

@ -1,5 +1,11 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import os
import shutil
from collections import OrderedDict
@ -130,7 +136,7 @@ class SecrDraftsTestCase(TestCase):
def test_resurrect(self):
draft = WgDraftFactory()
path = os.path.join(self.repository_dir, draft.filename_with_rev())
with open(path, 'w') as file:
with io.open(path, 'w') as file:
file.write('test')
expire_draft(draft)
email_url = urlreverse('ietf.secr.drafts.views.email', kwargs={'id':draft.name}) + "?action=resurrect"

6
ietf/secr/drafts/views.py
View file

@ -1,8 +1,12 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import glob
import io
import os
import shutil
from dateutil.parser import parse
@ -53,7 +57,7 @@ def handle_uploaded_file(f):
'''
Save uploaded draft files to temporary directory
'''
destination = open(os.path.join(settings.IDSUBMIT_MANUAL_STAGING_DIR, f.name), 'wb+')
destination = io.open(os.path.join(settings.IDSUBMIT_MANUAL_STAGING_DIR, f.name), 'wb+')
for chunk in f.chunks():
destination.write(chunk)
destination.close()

43
ietf/secr/meetings/blue_sheets.py
View file

@ -1,5 +1,13 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
from django.conf import settings
from django.utils.encoding import force_bytes
r'''
RTF quick reference (from Word2007RTFSpec9.doc):
@ -17,9 +25,9 @@ RTF quick reference (from Word2007RTFSpec9.doc):
'''
def create_blue_sheets(meeting, groups):
file = open(settings.SECR_BLUE_SHEET_PATH, 'w')
file = io.open(settings.SECR_BLUE_SHEET_PATH, 'wb')
header = '''{\\rtf1\\ansi\\ansicpg1252\\uc1 \\deff0\\deflang1033\\deflangfe1033
header = b'''{\\rtf1\\ansi\\ansicpg1252\\uc1 \\deff0\\deflang1033\\deflangfe1033
{\\fonttbl{\\f0\\froman\\fcharset0\\fprq2{\\*\\panose 02020603050405020304}Times New Roman;}}
{\\colortbl;\\red0\\green0\\blue0;\\red0\\green0\\blue255;\\red0\\green255\\blue255;\\red0\\green255\\blue0;
\\red255\\green0\\blue255;\\red255\\green0\\blue0;\\red255\\green255\\blue0;\\red255\\green255\\blue255;
@ -32,7 +40,7 @@ def create_blue_sheets(meeting, groups):
file.write(header)
for group in groups:
group_header = ''' {\\header \\pard\\plain \\s15\\nowidctlpar\\widctlpar\\tqc\\tx4320\\tqr\\tx8640\\adjustright \\fs20\\cgrid
group_header = b''' {\\header \\pard\\plain \\s15\\nowidctlpar\\widctlpar\\tqc\\tx4320\\tqr\\tx8640\\adjustright \\fs20\\cgrid
{ Mailing List: %s \\tab\\tab Meeting # %s %s (%s) \\par }
\\pard \\s15\\nowidctlpar\\widctlpar\\tqc\\tx4320\\tqr\\tx8640\\adjustright
{\\b\\fs24
@ -60,30 +68,31 @@ def create_blue_sheets(meeting, groups):
\\par }
\\pard \\fi-90\\li90\\nowidctlpar\\widctlpar\\adjustright
{\\fs16
''' % (group.list_email,
meeting.number,
group.acronym,
group.type,
meeting.number,
group.acronym,
group.type,
meeting.number,
group.name,
group.list_email)
''' % (force_bytes(group.list_email),
force_bytes(meeting.number),
force_bytes(group.acronym),
force_bytes(group.type),
force_bytes(meeting.number),
force_bytes(group.acronym),
force_bytes(group.type),
force_bytes(meeting.number),
force_bytes(group.name),
force_bytes(group.list_email),
)
file.write(group_header)
for x in range(1,117):
line = '''\\par %s._________________________________________________ \\tab _____________________________________________________
line = b'''\\par %s._________________________________________________ \\tab _____________________________________________________
\\par
''' % x
''' % force_bytes(x)
file.write(line)
footer = '''}
footer = b'''}
\\pard \\nowidctlpar\\widctlpar\\adjustright
{\\fs16 \\sect }
\\sectd \\pgnrestart\\linex0\\endnhere\\titlepg\\sectdefaultcl
'''
file.write(footer)
file.write('\n}')
file.write(b'\n}')
file.close()

5
ietf/secr/proceedings/proc_utils.py
View file

@ -1,6 +1,9 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
'''
proc_utils.py
@ -10,7 +13,7 @@ import datetime
import os
import re
import subprocess
from urllib.parse import urlencode
from six.moves.urllib.parse import urlencode
import debug # pyflakes:ignore

8
ietf/secr/proceedings/tests.py
View file

@ -1,7 +1,11 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import debug # pyflakes:ignore
import io
import json
import os
import shutil
@ -54,7 +58,7 @@ class VideoRecordingTestCase(TestCase):
def test_get_urls_from_json(self):
path = os.path.join(settings.BASE_DIR, "../test/data/youtube-playlistitems.json")
with open(path) as f:
with io.open(path) as f:
doc = json.load(f)
urls = _get_urls_from_json(doc)
self.assertEqual(len(urls),2)
@ -110,7 +114,7 @@ class RecordingTestCase(TestCase):
path = os.path.join(settings.MEETING_RECORDINGS_DIR,'ietf' + timeslot.meeting.number,filename)
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
with open(path, "w") as f:
with io.open(path, "w") as f:
f.write('dummy')
def get_filename_for_timeslot(self, timeslot):

4
ietf/secr/proceedings/utils.py
View file

@ -1,5 +1,7 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
import glob
import io
import os
from django.conf import settings
@ -34,7 +36,7 @@ def handle_upload_file(file,filename,meeting,subdir, request=None, encoding=None
for f in old_files:
os.remove(f)
destination = open(os.path.join(path,filename), 'wb+')
destination = io.open(os.path.join(path,filename), 'wb+')
if extension in settings.MEETING_VALID_MIME_TYPE_EXTENSIONS['text/html']:
file.open()
text = file.read()

13
ietf/secr/sreq/tests.py
View file

@ -1,6 +1,13 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
from django.urls import reverse
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import six
from django.urls import reverse
import debug # pyflakes:ignore
@ -177,14 +184,14 @@ class SubmitRequestCase(TestCase):
r = self.client.post(url,post_data)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertTrue('Confirm' in str(q("title")))
self.assertTrue('Confirm' in six.ensure_text(q("title")))
# confirm
post_data['submit'] = 'Submit'
r = self.client.post(confirm_url,post_data)
self.assertRedirects(r, reverse('ietf.secr.sreq.views.main'))
self.assertEqual(len(outbox),len_before+1)
notification = outbox[-1]
notification_payload = str(notification.get_payload(decode=True),"utf-8","replace")
notification_payload = six.ensure_text(notification.get_payload(decode=True),"utf-8","replace")
session = Session.objects.get(meeting=meeting,group=group)
self.assertEqual(session.resources.count(),1)
self.assertEqual(session.people_constraints.count(),1)

8
ietf/secr/utils/group.py
View file

@ -1,5 +1,11 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
# Python imports
import io
import os
# Django imports
@ -27,7 +33,7 @@ def get_charter_text(group):
'''
charter = group.charter
path = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (charter.canonical_name(), charter.rev))
f = open(path,'r')
f = io.open(path,'r')
text = f.read()
f.close()

23
ietf/stats/backfill_data.py
View file

@ -1,12 +1,15 @@
#!/usr/bin/env python
# Copyright The IETF Trust 2017-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import sys
import os
import os.path
import argparse
import six
import time
basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
@ -15,7 +18,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py")
if os.path.exists(virtualenv_activation):
exec(compile(open(virtualenv_activation, "rb").read(), virtualenv_activation, 'exec'), dict(__file__=virtualenv_activation))
exec(compile(io.open(virtualenv_activation, "rb").read(), virtualenv_activation, 'exec'), dict(__file__=virtualenv_activation))
import django
django.setup()
@ -43,7 +46,7 @@ if args.document:
docs_qs = docs_qs.filter(docalias__name=args.document)
ts = time.strftime("%Y-%m-%d_%H:%M%z")
logfile = open('backfill-authorstats-%s.log'%ts, 'w')
logfile = io.open('backfill-authorstats-%s.log'%ts, 'w')
print("Writing log to %s" % os.path.abspath(logfile.name))
def say(msg):
@ -53,7 +56,7 @@ def say(msg):
logfile.write(msg)
logfile.write('\n')
def str(text):
def unicode(text):
if text is None:
return text
# order matters here:
@ -83,10 +86,10 @@ for doc in docs_qs.prefetch_related("docalias", "formal_languages", "documentaut
say("Skipping %s, no txt file found at %s" % (doc.name, path))
continue
with open(path, 'rb') as f:
with io.open(path, 'rb') as f:
say("\nProcessing %s" % doc.name)
sys.stdout.flush()
d = Draft(str(f.read()), path)
d = Draft(unicode(f.read()), path)
updated = False
@ -128,10 +131,10 @@ for doc in docs_qs.prefetch_related("docalias", "formal_languages", "documentaut
# it's an extra author - skip those extra authors
seen = set()
for full, _, _, _, _, email, country, company in d.get_author_list():
assert full is None or isinstance(full, str)
assert email is None or isinstance(email, str)
assert country is None or isinstance(country, str)
assert company is None or isinstance(company, str)
assert full is None or isinstance(full, six.text_type)
assert email is None or isinstance(email, six.text_type)
assert country is None or isinstance(country, six.text_type)
assert company is None or isinstance(company, six.text_type)
#full, email, country, company = [ unicode(s) for s in [full, email, country, company, ] ]
if email in seen:
continue

24
ietf/submit/checkers.py
View file

@ -1,14 +1,18 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import os
import re
import sys
from xym import xym
import shutil
import six
import sys
import tempfile
import io
from xym import xym
from django.conf import settings
import debug # pyflakes:ignore
@ -144,19 +148,23 @@ class DraftYangChecker(object):
# This places the yang models as files in workdir
saved_stdout = sys.stdout
saved_stderr = sys.stderr
sys.stdout = io.StringIO()
sys.stderr = io.StringIO()
sys.stdout = six.StringIO()
sys.stderr = six.StringIO()
extractor.extract_yang_model(file.readlines())
model_list = extractor.get_extracted_models(False, True)
out = sys.stdout.getvalue()
err = sys.stderr.getvalue()
sys.stdout = saved_stdout
sys.stderr = saved_stderr
# signature change in xym:
except Exception as exc:
sys.stdout = saved_stdout
sys.stderr = saved_stderr
msg = "Exception when running xym on %s: %s" % (name, exc)
log(msg)
raise
return None, msg, 0, 0, info
finally:
sys.stdout = saved_stdout
sys.stderr = saved_stderr
if not model_list:
# Found no yang models, don't deliver any YangChecker result
return None, "", 0, 0, info
@ -198,7 +206,7 @@ class DraftYangChecker(object):
settings.SUBMIT_YANG_IANA_MODEL_DIR,
])
if os.path.exists(path):
with open(path) as file:
with io.open(path) as file:
text = file.readlines()
# pyang
cmd_template = settings.SUBMIT_PYANG_COMMAND

19
ietf/submit/forms.py
View file

@ -1,11 +1,19 @@
# Copyright The IETF Trust 2011-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import os
import re
import datetime
import email
import pytz
import xml2rfc
import six
import tempfile
import xml2rfc
from email.utils import formataddr
from unidecode import unidecode
@ -13,6 +21,7 @@ from django import forms
from django.conf import settings
from django.utils.html import mark_safe
from django.urls import reverse as urlreverse
from django.utils.encoding import force_str
import debug # pyflakes:ignore
@ -140,7 +149,7 @@ class SubmissionBaseUploadForm(forms.Form):
# over to the xml parser. XXX FIXME: investigate updating
# xml2rfc to be able to work with file handles to in-memory
# files.
with open(tfn, 'wb+') as tf:
with io.open(tfn, 'wb+') as tf:
for chunk in xml_file.chunks():
tf.write(chunk)
os.environ["XML_LIBRARY"] = settings.XML_LIBRARY
@ -185,10 +194,10 @@ class SubmissionBaseUploadForm(forms.Form):
self.revision = None
self.filename = draftname
self.title = self.xmlroot.findtext('front/title').strip()
if type(self.title) is str:
if type(self.title) is six.text_type:
self.title = unidecode(self.title)
self.abstract = (self.xmlroot.findtext('front/abstract') or '').strip()
if type(self.abstract) is str:
if type(self.abstract) is six.text_type:
self.abstract = unidecode(self.abstract)
author_info = self.xmlroot.findall('front/author')
for author in author_info:
@ -508,7 +517,7 @@ class SubmissionEmailForm(forms.Form):
'''Returns a ietf.message.models.Message object'''
self.message_text = self.cleaned_data['message']
try:
message = email.message_from_string(self.message_text)
message = email.message_from_string(force_str(self.message_text))
except Exception as e:
self.add_error('message', e)
return None

8
ietf/submit/management/commands/manualpost_email.py
View file

@ -1,4 +1,10 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import io
import sys
from django.core.management.base import BaseCommand, CommandError
@ -20,7 +26,7 @@ class Command(BaseCommand):
if not email:
msg = sys.stdin.read()
else:
msg = open(email, "r").read()
msg = io.open(email, "r").read()
try:
process_response_email(msg)

54
ietf/submit/tests.py
View file

@ -1,11 +1,16 @@
# Copyright The IETF Trust 2011-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import email
import io
import os
import re
import shutil
import six
import sys
@ -14,6 +19,7 @@ from pyquery import PyQuery
from django.conf import settings
from django.urls import reverse as urlreverse
from django.utils.encoding import force_str
import debug # pyflakes:ignore
@ -40,7 +46,7 @@ from ietf.utils.draft import Draft
def submission_file(name, rev, group, format, templatename, author=None, email=None, title=None, year=None, ascii=True):
# construct appropriate text draft
f = open(os.path.join(settings.BASE_DIR, "submit", templatename))
f = io.open(os.path.join(settings.BASE_DIR, "submit", templatename))
template = f.read()
f.close()
@ -290,16 +296,16 @@ class SubmitTests(TestCase):
self.assertTrue(draft.relations_that_doc("possibly-replaces").first().target, sug_replaced_alias)
self.assertEqual(len(outbox), mailbox_before + 5)
self.assertIn(("I-D Action: %s" % name), outbox[-4]["Subject"])
self.assertIn(author.ascii, str(outbox[-4]))
self.assertIn(author.ascii, six.ensure_text(outbox[-4]))
self.assertIn(("I-D Action: %s" % name), outbox[-3]["Subject"])
self.assertIn(author.ascii, str(outbox[-3]))
self.assertIn(author.ascii, six.ensure_text(outbox[-3]))
self.assertIn("New Version Notification",outbox[-2]["Subject"])
self.assertIn(name, str(outbox[-2]))
self.assertIn("mars", str(outbox[-2]))
self.assertIn(name, six.ensure_text(outbox[-2]))
self.assertIn("mars", six.ensure_text(outbox[-2]))
# Check "Review of suggested possible replacements for..." mail
self.assertIn("review", outbox[-1]["Subject"].lower())
self.assertIn(name, str(outbox[-1]))
self.assertIn(sug_replaced_alias.name, str(outbox[-1]))
self.assertIn(name, six.ensure_text(outbox[-1]))
self.assertIn(sug_replaced_alias.name, six.ensure_text(outbox[-1]))
self.assertIn("ames-chairs@", outbox[-1]["To"].lower())
self.assertIn("mars-chairs@", outbox[-1]["To"].lower())
@ -379,7 +385,7 @@ class SubmitTests(TestCase):
# write the old draft in a file so we can check it's moved away
old_rev = draft.rev
with open(os.path.join(self.repository_dir, "%s-%s.txt" % (name, old_rev)), 'w') as f:
with io.open(os.path.join(self.repository_dir, "%s-%s.txt" % (name, old_rev)), 'w') as f:
f.write("a" * 2000)
old_docevents = list(draft.docevent_set.all())
@ -407,7 +413,7 @@ class SubmitTests(TestCase):
self.assertTrue("unknown-email-" not in confirm_email["To"])
if change_authors:
# Since authors changed, ensure chairs are copied (and that the message says why)
self.assertTrue("chairs have been copied" in str(confirm_email))
self.assertTrue("chairs have been copied" in six.ensure_text(confirm_email))
if group_type in ['wg','rg','ag']:
self.assertTrue("mars-chairs@" in confirm_email["To"].lower())
elif group_type == 'area':
@ -417,7 +423,7 @@ class SubmitTests(TestCase):
if stream_type=='ise':
self.assertTrue("rfc-ise@" in confirm_email["To"].lower())
else:
self.assertNotIn("chairs have been copied", str(confirm_email))
self.assertNotIn("chairs have been copied", six.ensure_text(confirm_email))
self.assertNotIn("mars-chairs@", confirm_email["To"].lower())
confirmation_url = self.extract_confirmation_url(confirm_email)
@ -486,17 +492,17 @@ class SubmitTests(TestCase):
self.assertEqual(len(outbox), mailbox_before + 3)
self.assertTrue(("I-D Action: %s" % name) in outbox[-3]["Subject"])
self.assertTrue(("I-D Action: %s" % name) in draft.message_set.order_by("-time")[0].subject)
self.assertTrue(author.ascii in str(outbox[-3]))
self.assertTrue(author.ascii in six.ensure_text(outbox[-3]))
self.assertTrue("i-d-announce@" in outbox[-3]['To'])
self.assertTrue("New Version Notification" in outbox[-2]["Subject"])
self.assertTrue(name in str(outbox[-2]))
self.assertTrue(name in six.ensure_text(outbox[-2]))
interesting_address = {'ietf':'mars', 'irtf':'irtf-chair', 'iab':'iab-chair', 'ise':'rfc-ise'}[draft.stream_id]
self.assertTrue(interesting_address in str(outbox[-2]))
self.assertTrue(interesting_address in six.ensure_text(outbox[-2]))
if draft.stream_id == 'ietf':
self.assertTrue(draft.ad.role_email("ad").address in str(outbox[-2]))
self.assertTrue(ballot_position.ad.role_email("ad").address in str(outbox[-2]))
self.assertTrue(draft.ad.role_email("ad").address in six.ensure_text(outbox[-2]))
self.assertTrue(ballot_position.ad.role_email("ad").address in six.ensure_text(outbox[-2]))
self.assertTrue("New Version Notification" in outbox[-1]["Subject"])
self.assertTrue(name in str(outbox[-1]))
self.assertTrue(name in six.ensure_text(outbox[-1]))
r = self.client.get(urlreverse('ietf.doc.views_search.recent_drafts'))
self.assertEqual(r.status_code, 200)
self.assertContains(r, draft.name)
@ -556,7 +562,7 @@ class SubmitTests(TestCase):
# both submitter and author get email
self.assertTrue(author.email().address.lower() in confirm_email["To"])
self.assertTrue("submitter@example.com" in confirm_email["To"])
self.assertFalse("chairs have been copied" in str(confirm_email))
self.assertFalse("chairs have been copied" in six.ensure_text(confirm_email))
confirmation_url = self.extract_confirmation_url(outbox[-1])
@ -885,14 +891,14 @@ class SubmitTests(TestCase):
self.assertEqual(Submission.objects.filter(name=name).count(), 1)
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))))
self.assertTrue(name in open(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))).read())
self.assertTrue(name in io.open(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))).read())
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))))
self.assertTrue(name in open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read())
self.assertTrue('<?xml version="1.0" encoding="UTF-8"?>' in open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read())
self.assertTrue(name in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read())
self.assertTrue('<?xml version="1.0" encoding="UTF-8"?>' in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read())
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev))))
self.assertTrue('This is PDF' in open(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev))).read())
self.assertTrue('This is PDF' in io.open(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev))).read())
self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.ps" % (name, rev))))
self.assertTrue('This is PostScript' in open(os.path.join(self.staging_dir, "%s-%s.ps" % (name, rev))).read())
self.assertTrue('This is PostScript' in io.open(os.path.join(self.staging_dir, "%s-%s.ps" % (name, rev))).read())
def test_expire_submissions(self):
s = Submission.objects.create(name="draft-ietf-mars-foo",
@ -1174,7 +1180,7 @@ Please submit my draft at http://test.com/mydraft.txt
Thank you
""".format(datetime.datetime.now().ctime())
message = email.message_from_string(message_string)
message = email.message_from_string(force_str(message_string))
submission, submission_email_event = (
add_submission_email(request=None,
remote_ip ="192.168.0.1",
@ -1257,7 +1263,7 @@ ZSBvZiBsaW5lcyAtIGJ1dCBpdCBjb3VsZCBiZSBhIGRyYWZ0Cg==
--------------090908050800030909090207--
""".format(frm, datetime.datetime.now().ctime())
message = email.message_from_string(message_string)
message = email.message_from_string(force_str(message_string))
submission, submission_email_event = (
add_submission_email(request=None,
remote_ip ="192.168.0.1",

13
ietf/submit/utils.py
View file

@ -1,9 +1,14 @@
# Copyright The IETF Trust 2011-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import datetime
import io
import os
import re
import six # pyflakes:ignore
import xml2rfc
from django.conf import settings
@ -456,7 +461,7 @@ def ensure_person_email_info_exists(name, email, docname):
person = Person()
person.name = name
person.name_from_draft = name
log.assertion('isinstance(person.name, str)')
log.assertion('isinstance(person.name, six.text_type)')
person.ascii = unidecode_name(person.name)
person.save()
else:
@ -605,7 +610,7 @@ def save_files(form):
name = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (form.filename, form.revision, ext))
file_name[ext] = name
with open(name, 'wb+') as destination:
with io.open(name, 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
return file_name
@ -641,7 +646,7 @@ def get_draft_meta(form, saved_files):
# Some meta-information, such as the page-count, can only
# be retrieved from the generated text file. Provide a
# parsed draft object to get at that kind of information.
with open(file_name['txt']) as txt_file:
with io.open(file_name['txt']) as txt_file:
form.parsed_draft = Draft(txt_file.read(), txt_file.name)
else:
@ -667,7 +672,7 @@ def get_draft_meta(form, saved_files):
if s is None:
return ""
if isinstance(s, str):
if isinstance(s, six.text_type):
return s
else:
try:

21
ietf/sync/iana.py
View file

@ -1,13 +1,20 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import base64
import datetime
import email
import json
import re
import urllib.request, urllib.error, urllib.parse
from django.utils.http import urlquote
from six.moves.urllib.request import Request, urlopen
from django.conf import settings
from django.utils.encoding import force_str
from django.utils.http import urlquote
import debug # pyflakes:ignore
@ -23,7 +30,7 @@ from ietf.utils.timezone import local_timezone_to_utc, email_time_to_local_timez
#CHANGES_URL = "https://datatracker.dev.icann.org:8080/data-tracker/changes"
def fetch_protocol_page(url):
f = urllib.request.urlopen(settings.IANA_SYNC_PROTOCOLS_URL)
f = urlopen(settings.IANA_SYNC_PROTOCOLS_URL)
text = f.read()
f.close()
return text
@ -67,12 +74,12 @@ def update_rfc_log_from_protocol_page(rfc_names, rfc_must_published_later_than):
def fetch_changes_json(url, start, end):
url += "?start=%s&end=%s" % (urlquote(local_timezone_to_utc(start).strftime("%Y-%m-%d %H:%M:%S")),
urlquote(local_timezone_to_utc(end).strftime("%Y-%m-%d %H:%M:%S")))
request = urllib.request.Request(url)
request = Request(url)
# HTTP basic auth
username = "ietfsync"
password = settings.IANA_SYNC_PASSWORD
request.add_header("Authorization", "Basic %s" % base64.encodestring("%s:%s" % (username, password)).replace("\n", ""))
f = urllib.request.urlopen(request)
f = urlopen(request)
text = f.read()
f.close()
return text
@ -234,8 +241,8 @@ def strip_version_extension(text):
text = text[:-3]
return text
def parse_review_email(bytes):
msg = email.message_from_bytes(bytes)
def parse_review_email(text):
msg = email.message_from_string(force_str(text))
# doc
doc_name = find_document_name(msg["Subject"]) or ""
doc_name = strip_version_extension(doc_name)

21
ietf/sync/rfceditor.py
View file

@ -1,12 +1,17 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
import re
from __future__ import absolute_import, print_function, unicode_literals
import base64
import datetime
import urllib.request, urllib.parse, urllib.error
import urllib.request, urllib.error, urllib.parse
import re
import socket
import six
from six.moves.urllib.request import Request, urlopen
from six.moves.urllib.parse import urlencode
from xml.dom import pulldom, Node
from django.conf import settings
@ -40,7 +45,7 @@ def get_child_text(parent_node, tag_name):
def fetch_queue_xml(url):
socket.setdefaulttimeout(30)
return urllib.request.urlopen(url)
return urlopen(url)
def parse_queue(response):
"""Parse RFC Editor queue XML into a bunch of tuples + warnings."""
@ -228,7 +233,7 @@ def update_drafts_from_queue(drafts):
def fetch_index_xml(url):
socket.setdefaulttimeout(30)
return urllib.request.urlopen(url)
return urlopen(url)
def parse_index(response):
"""Parse RFC Editor index XML into a bunch of tuples."""
@ -517,7 +522,7 @@ def post_approved_draft(url, name):
the data from the Datatracker and start processing it. Returns
response and error (empty string if no error)."""
request = urllib.request.Request(url)
request = Request(url)
request.add_header("Content-type", "application/x-www-form-urlencoded")
request.add_header("Accept", "text/plain")
# HTTP basic auth
@ -531,7 +536,7 @@ def post_approved_draft(url, name):
log("Posting RFC-Editor notifcation of approved draft '%s' to '%s'" % (name, url))
text = error = ""
try:
f = urllib.request.urlopen(request, data=urllib.parse.urlencode({ 'draft': name }), timeout=20)
f = urlopen(request, data=urlencode({ 'draft': name }), timeout=20)
text = f.read()
status_code = f.getcode()
f.close()
@ -547,6 +552,6 @@ def post_approved_draft(url, name):
# catch everything so we don't leak exceptions, convert them
# into string instead
log("Exception on RFC-Editor notification for draft '%s': '%s'" % (name, e))
error = str(e)
error = six.ensure_text(e)
return text, error

12
ietf/sync/tests.py
View file

@ -1,16 +1,21 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import os
import io
import json
import datetime
import io
import quopri
import shutil
from django.conf import settings
from django.urls import reverse as urlreverse
import debug # pyflakes:ignore
from ietf.doc.factories import WgDraftFactory
from ietf.doc.models import Document, DocAlias, DocEvent, DeletedEvent, DocTagName, RelatedDocument, State, StateDocEvent
from ietf.doc.utils import add_state_change_event
@ -177,7 +182,7 @@ ICANN
rtime = 7*subjects.index(subject) + 5*tags.index(tag) + embedded_names.index(embedded_name)
person=Person.objects.get(user__username="iana")
fromaddr = person.email().formatted_email()
msg = msg_template % dict(person=quopri.encodestring(person.name.encode()),
msg = msg_template % dict(person=quopri.encodestring(person.name.encode('utf-8')),
fromaddr=fromaddr,
draft=draft.name,
rev=draft.rev,
@ -185,7 +190,6 @@ ICANN
rtime=rtime,
subject=subject,
embedded_name=embedded_name,)
doc_name, review_time, by, comment = iana.parse_review_email(msg.encode('utf-8'))
self.assertEqual(doc_name, draft.name)
@ -232,7 +236,7 @@ class RFCSyncTests(TestCase):
settings.INTERNET_DRAFT_ARCHIVE_DIR = self.save_archive_dir
def write_draft_file(self, name, size):
with open(os.path.join(self.id_dir, name), 'w') as f:
with io.open(os.path.join(self.id_dir, name), 'w') as f:
f.write("a" * size)
def test_rfc_index(self):

11
ietf/utils/admin.py
View file

@ -1,4 +1,11 @@
# Copyright The IETF Trust 2011-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import six
from django.contrib import admin
from ietf.utils.models import VersionInfo
@ -9,10 +16,10 @@ def name(obj):
if callable(obj.name):
name = obj.name()
else:
name = str(obj.name)
name = six.ensure_text(obj.name)
if name:
return name
return str(obj)
return six.ensure_text(obj)
def admin_link(field, label=None, ordering="", display=name, suffix=""):
if not label:

21
ietf/utils/draft.py
View file

@ -1,6 +1,11 @@
# Copyright The IETF Trust 2009-2019, All Rights Reserved
#!/usr/bin/python
# Copyright The IETF Trust 2009-2019, All Rights Reserved
# -*- coding: utf-8 -*-
# -*- python -*-
from __future__ import absolute_import, print_function, unicode_literals
"""
NAME
%(program)s - Extract meta-information from an IETF draft.
@ -36,9 +41,11 @@ COPYRIGHT
import datetime
import getopt
import io
import os
import os.path
import re
import six
import stat
import sys
import time
@ -106,7 +113,7 @@ def _err(string):
# ----------------------------------------------------------------------
def _gettext(file):
file = open(file)
file = io.open(file)
text = file.read()
file.close()
@ -128,7 +135,7 @@ def acronym_match(s, l):
class Draft():
def __init__(self, text, source, name_from_source=False):
assert isinstance(text, str)
assert isinstance(text, six.text_type)
self.source = source
self.rawtext = text
self.name_from_source = name_from_source
@ -1203,7 +1210,7 @@ def getmeta(fn):
return
timestamp = time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime(os.stat(filename)[stat.ST_MTIME]))
with open(filename, 'rb') as file:
with io.open(filename, 'rb') as file:
try:
draft = Draft(file.read().decode('utf8'), filename)
except UnicodeDecodeError:
@ -1311,7 +1318,7 @@ def _main(outfile=sys.stdout):
# Option processing
# ----------------------------------------------------------------------
options = ""
for line in re.findall(r"\n +(if|elif) +opt in \[(.+)\]:\s+#(.+)\n", open(sys.argv[0]).read()):
for line in re.findall(r"\n +(if|elif) +opt in \[(.+)\]:\s+#(.+)\n", io.open(sys.argv[0]).read()):
if not options:
options += "OPTIONS\n"
options += " %-16s %s\n" % (line[1].replace('"', ''), line[2])
@ -1357,7 +1364,7 @@ def _main(outfile=sys.stdout):
company_domain = {}
if opt_getauthors:
gadata = open("/www/tools.ietf.org/tools/getauthors/getauthors.data")
gadata = io.open("/www/tools.ietf.org/tools/getauthors/getauthors.data")
for line in gadata:
if line.startswith("company:"):
try:
@ -1376,7 +1383,7 @@ def _main(outfile=sys.stdout):
import gzip
file = gzip.open(file)
else:
file = open(file)
file = io.open(file)
basename = os.path.basename(file.name)
if basename.startswith("draft-"):

10
ietf/utils/fields.py
View file

@ -1,7 +1,12 @@
# Copyright The IETF Trust 2012-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import re
import datetime
import six
import re
import debug # pyflakes:ignore
@ -18,7 +23,7 @@ class MultiEmailField(forms.Field):
if not value:
return []
if isinstance(value, str):
if isinstance(value, six.string_types):
values = value.split(',')
return [ x.strip() for x in values if x.strip() ]
else:
@ -26,7 +31,6 @@ class MultiEmailField(forms.Field):
def validate(self, value):
"Check if value consists only of valid emails."
# Use the parent's handling of required fields, etc.
super(MultiEmailField, self).validate(value)

9
ietf/utils/html.py
View file

@ -1,12 +1,17 @@
# Copyright The IETF Trust 2010-2019, All Rights Reserved
# -*- coding: utf-8 -*-
# Taken from http://code.google.com/p/soclone/source/browse/trunk/soclone/utils/html.py
"""Utilities for working with HTML."""
from __future__ import absolute_import, print_function, unicode_literals
import bleach
import copy
import lxml.etree
import lxml.html
import lxml.html.clean
import six
import debug # pyflakes:ignore
@ -54,7 +59,7 @@ class Cleaner(lxml.html.clean.Cleaner):
# Copied from lxml 4.2.0 and modified to insert charset meta:
def clean_html(self, html):
result_type = type(html)
if isinstance(html, str):
if isinstance(html, six.string_types):
doc = lxml.html.fromstring(html)
else:
doc = copy.deepcopy(html)

19
ietf/utils/log.py
View file

@ -1,9 +1,14 @@
# Copyright The IETF Trust 2007-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import sys
import logging
import inspect
import os.path
import six
import traceback
try:
@ -28,23 +33,27 @@ def getclass(frame):
def getcaller():
parent, pfile, pline, pfunction, lines, index = inspect.stack()[2]
moduleinfo = inspect.getmoduleinfo(pfile)
pmodule = moduleinfo[0] if moduleinfo else None
pmodule = inspect.getmodulename(pfile)
pclass = getclass(parent)
return (pmodule, pclass, pfunction, pfile, pline)
def log(msg):
def log(msg, e=None):
"Uses syslog by preference. Logs the given calling point and message."
global logfunc
def _flushfunc():
pass
_logfunc = logfunc
if settings.SERVER_MODE == 'test':
return
## Comment in when debugging for instance test smtp server failures:
# if e:
# _logfunc = debug.say
# _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition)
# else:
return
elif settings.DEBUG == True:
_logfunc = debug.say
_flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition)
if isinstance(msg, str):
if isinstance(msg, six.text_type):
msg = msg.encode('unicode_escape')
try:
mod, cls, func, file, line = getcaller()

46
ietf/utils/mail.py
View file

@ -1,9 +1,14 @@
# Copyright The IETF Trust 2007-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import copy
import datetime
import logging
import re
import six
import smtplib
import sys
import textwrap
@ -25,6 +30,7 @@ from django.core.exceptions import ImproperlyConfigured, ValidationError
from django.core.validators import validate_email
from django.template.loader import render_to_string
from django.template import Context,RequestContext
from django.utils.encoding import force_text, force_str, force_bytes
import debug # pyflakes:ignore
@ -115,12 +121,12 @@ def send_smtp(msg, bcc=None):
# advertise the AUTH capability.
server.ehlo()
server.login(settings.EMAIL_HOST_USER, settings.EMAIL_HOST_PASSWORD)
unhandled = server.sendmail(frm, to, msg.as_bytes())
unhandled = server.sendmail(frm, to, force_bytes(msg.as_string()))
if unhandled != {}:
raise SMTPSomeRefusedRecipients(message="%d addresses were refused"%len(unhandled),original_msg=msg,refusals=unhandled)
except Exception as e:
# need to improve log message
log("Exception while trying to send email from '%s' to %s subject '%s'" % (frm, to, msg.get('Subject', '[no subject]')))
log("Exception while trying to send email from '%s' to %s subject '%s'" % (frm, to, msg.get('Subject', '[no subject]')), e=e)
if isinstance(e, smtplib.SMTPException):
e.original_msg=msg
raise
@ -131,7 +137,7 @@ def send_smtp(msg, bcc=None):
server.quit()
except smtplib.SMTPServerDisconnected:
pass
subj = msg.get('Subject', '[no subject]')
subj = force_text(msg.get('Subject', '[no subject]'))
log("sent email from '%s' to %s id %s subject '%s'" % (frm, to, msg.get('Message-ID', ''), subj))
def copy_email(msg, to, toUser=False, originalBcc=None):
@ -180,7 +186,7 @@ def send_mail(request, to, frm, subject, template, context, *args, **kwargs):
return send_mail_text(request, to, frm, subject, txt, *args, **kwargs)
def encode_message(txt):
assert isinstance(txt, str)
assert isinstance(txt, six.text_type)
return MIMEText(txt.encode('utf-8'), 'plain', 'UTF-8')
def send_mail_text(request, to, frm, subject, txt, cc=None, extra=None, toUser=False, bcc=None, copy=True):
@ -219,7 +225,13 @@ def formataddr(addrtuple):
address field. Does what's needed, and returns a string value suitable for
use in a To: or Cc: email header field.
"""
return simple_formataddr(addrtuple, charset='utf-8')
if six.PY2:
name, addr = addrtuple
if name and not isascii(name):
name = str(Header(name, 'utf-8'))
return simple_formataddr((name, addr))
else:
return simple_formataddr(addrtuple)
def parseaddr(addr):
"""
@ -230,7 +242,7 @@ def parseaddr(addr):
"""
addr = ''.join( [ ( s.decode(m) if m else s.decode()) if isinstance(s, bytes) else s for (s,m) in decode_header(addr) ] )
addr = ''.join( [ ( s.decode(m) if m else s.decode()) if isinstance(s, six.binary_type) else s for (s,m) in decode_header(addr) ] )
name, addr = simple_parseaddr(addr)
return name, addr
@ -269,7 +281,7 @@ def condition_message(to, frm, subject, msg, cc, extra):
if name:
to_hdr.append('"%s"' % name)
to_hdr.append("<%s>," % addr)
to_str = to_hdr.encode()
to_str = to_hdr.encode('utf-8')
if to_str and to_str[-1] == ',':
to_str=to_str[:-1]
# It's important to use this string, and not assign the Header object.
@ -352,7 +364,7 @@ def send_mail_mime(request, to, frm, subject, msg, cc=None, extra=None, toUser=F
def parse_preformatted(preformatted, extra={}, override={}):
"""Parse preformatted string containing mail with From:, To:, ...,"""
msg = message_from_string(preformatted)
msg = message_from_string(force_str(preformatted))
msg.set_charset('UTF-8')
for k, v in override.items():
@ -408,7 +420,7 @@ def send_mail_preformatted(request, preformatted, extra={}, override={}):
extra headers as needed)."""
(msg, extra, bcc) = parse_preformatted(preformatted, extra, override)
txt = msg.get_payload()
txt = get_payload(msg)
send_mail_text(request, msg['To'], msg["From"], msg["Subject"], txt, extra=extra, bcc=bcc)
return msg
@ -440,10 +452,10 @@ def exception_components(e):
def log_smtp_exception(e):
(extype, value, tb) = exception_components(e)
log("SMTP Exception: %s : %s" % (extype,value))
log("SMTP Exception: %s : %s" % (extype,value), e)
if isinstance(e,SMTPSomeRefusedRecipients):
log(" SomeRefused: %s"%(e.summary_refusals()))
log(" Traceback: %s" % tb)
log(" SomeRefused: %s"%(e.summary_refusals()), e)
log(" Traceback: %s" % tb, e)
return (extype, value, tb)
def build_warning_message(request, e):
@ -540,9 +552,17 @@ def get_email_addresses_from_text(text):
validate_email(addr)
return True
except ValidationError:
logger.error(f'Bad data: get_email_addresses_from_text() got an invalid email address tuple: {email}, in "{text}".')
logger.error('Bad data: get_email_addresses_from_text() got an '
'invalid email address tuple: {email}, in "{text}".'.format(email=email, text=text))
return False
# whitespace normalization -- getaddresses doesn't do this
text = re.sub(r'(?u)\s+', ' ', text)
return [ formataddr(e) for e in getaddresses([text, ]) if valid(e) ]
def get_payload(msg, decode=False):
if six.PY2:
return msg.get_payload(decode=decode).decode(msg.get_content_charset('utf-8'))
else:
return msg.get_payload(decode=decode)

28
ietf/utils/management/commands/check_referential_integrity.py
View file

@ -1,4 +1,10 @@
# Copyright The IETF Trust 2015-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import six
from tqdm import tqdm
@ -26,18 +32,18 @@ class Command(BaseCommand):
debug.pprint('dir(field)')
raise
if verbosity > 1:
print(" %s -> %s.%s" % (field.name,foreign_model.__module__,foreign_model.__name__), end=' ')
six.print_(" %s -> %s.%s" % (field.name,foreign_model.__module__,foreign_model.__name__), end=' ')
used = set(field.model.objects.values_list(field.name,flat=True))
used.discard(None)
exists = set(foreign_model.objects.values_list('pk',flat=True))
if verbosity > 1:
if used - exists:
print(" ** Bad key values:",list(used - exists))
six.print_(" ** Bad key values:",list(used - exists))
else:
print(" ok")
six.print_(" ok")
else:
if used - exists:
print("\n%s.%s.%s -> %s.%s ** Bad key values:" % (model.__module__,model.__name__,field.name,foreign_model.__module__,foreign_model.__name__),list(used - exists))
six.print_("\n%s.%s.%s -> %s.%s ** Bad key values:" % (model.__module__,model.__name__,field.name,foreign_model.__module__,foreign_model.__name__),list(used - exists))
def check_reverse_field(field):
try:
@ -50,33 +56,33 @@ class Command(BaseCommand):
foreign_field_name = field.remote_field.name
foreign_accessor_name = field.remote_field.get_accessor_name()
if verbosity > 1:
print(" %s <- %s -> %s.%s" % (field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), end=' ')
six.print_(" %s <- %s -> %s.%s" % (field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), end=' ')
try:
used = set(foreign_model.objects.values_list(foreign_field_name, flat=True))
except FieldError:
try:
used = set(foreign_model.objects.values_list(foreign_accessor_name, flat=True))
except FieldError:
print(" ** Warning: could not find reverse name for %s.%s -> %s.%s" % (field.model.__module__, field.model.__name__, foreign_model.__name__, foreign_field_name), end=' ')
six.print_(" ** Warning: could not find reverse name for %s.%s -> %s.%s" % (field.model.__module__, field.model.__name__, foreign_model.__name__, foreign_field_name), end=' ')
used.discard(None)
exists = set(field.model.objects.values_list('pk',flat=True))
if verbosity > 1:
if used - exists:
print(" ** Bad key values:\n ",list(used - exists))
six.print_(" ** Bad key values:\n ",list(used - exists))
else:
print(" ok")
six.print_(" ok")
else:
if used - exists:
print("\n%s.%s <- %s -> %s.%s ** Bad key values:\n " % (field.model.__module__, field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), list(used - exists))
six.print_("\n%s.%s <- %s -> %s.%s ** Bad key values:\n " % (field.model.__module__, field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), list(used - exists))
for conf in tqdm([ c for c in apps.get_app_configs() if c.name.startswith('ietf.')], desc='apps', disable=verbose):
if verbosity > 1:
print("Checking", conf.name)
six.print_("Checking", conf.name)
for model in tqdm(list(conf.get_models()), desc='models', disable=verbose):
if model._meta.proxy:
continue
if verbosity > 1:
print(" %s.%s" % (model.__module__,model.__name__))
six.print_(" %s.%s" % (model.__module__,model.__name__))
for field in [f for f in model._meta.fields if isinstance(f, (ForeignKey, OneToOneField)) ]:
check_field(field)
for field in [f for f in model._meta.many_to_many ]:

15
ietf/utils/management/commands/coverage_changes.py
View file

@ -1,9 +1,14 @@
# Copyright The IETF Trust 2015-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import os
import json
import codecs
import gzip
import io
import json
import os
import six
from difflib import ndiff
@ -56,12 +61,12 @@ class Command(BaseCommand):
valid_sections = ['template', 'url', 'code']
def read_coverage(self, filename, version=None):
if isinstance(filename, str):
if isinstance(filename, six.string_types):
try:
if filename.endswith(".gz"):
file = gzip.open(filename, "rb")
else:
file = codecs.open(filename, "r", encoding="utf-8")
file = io.open(filename, "r", encoding="utf-8")
except IOError as e:
self.stderr.write("%s" % e)
exit(1)

7
ietf/utils/management/commands/create_group_wikis.py
View file

@ -1,7 +1,12 @@
# Copyright The IETF Trust 2016-2019, All Rights Reserved
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import os
import copy
import io
import syslog
import pkg_resources
@ -132,7 +137,7 @@ class Command(BaseCommand):
name = unicode_unquote(name.encode('utf-8'))
if os.path.isfile(filename):
self.note(" Adding page %s" % name)
with open(filename) as file:
with io.open(filename) as file:
text = file.read().decode('utf-8')
self.add_wiki_page(env, name, text)

Some files were not shown because too many files have changed in this diff Show more