diff --git a/bin/check-copyright b/bin/check-copyright index 2ae4ae208..0e6c0a36c 100755 --- a/bin/check-copyright +++ b/bin/check-copyright @@ -1,5 +1,5 @@ -#!/usr/bin/env python -# -*- python -*- +#!/usr/bin/env python3.7 +# -*- mode: python; coding: utf-8 -*- # Copyright The IETF Trust 2019, All Rights Reserved """ NAME @@ -10,12 +10,18 @@ SYNOPSIS DESCRIPTION Given a list of files or filename wildcard patterns, check all for - an IETF Trust copyright notice with the current year. + an IETF Trust copyright notice with the current year. Optionally + generate a diff on standard out which can be used by 'patch'. + + An invocation similar to the following can be particularly useful with + a set of changed version-controlled files, as it will fix up the + Copyright statements of any python files with pending changes: + + $ check-copyright -p $(svn st | cut -c 9- | grep '\.py$' ) | patch -p0 + %(options)s -FILES - AUTHOR Written by Henrik Levkowetz, @@ -43,10 +49,12 @@ import pytz import tzparse import debug -version = "0.10" +version = "1.0.0" program = os.path.basename(sys.argv[0]) progdir = os.path.dirname(sys.argv[0]) +debug.debug = True + # ---------------------------------------------------------------------- # Parse options @@ -63,8 +71,8 @@ if len(sys.argv) < 1: sys.exit(1) try: - opts, files = getopt.gnu_getopt(sys.argv[1:], "hvV", ["help", "version", "verbose",]) -except Exception, e: + opts, files = getopt.gnu_getopt(sys.argv[1:], "hC:pvV", ["help", "copyright=", "patch", "version", "verbose",]) +except Exception as e: print( "%s: %s" % (program, e)) sys.exit(1) @@ -73,16 +81,22 @@ except Exception, e: # set default values, if any opt_verbose = 0 +opt_patch = False +opt_copyright = "Copyright The IETF Trust {years}, All Rights Reserved" # handle individual options for opt, value in opts: if opt in ["-h", "--help"]: # Output this help, then exit print( __doc__ % locals() ) sys.exit(1) + elif opt in ["-p", "--patch"]: # Generate patch output rather than error messages + opt_patch = True + elif opt in ["-C", "--copyright"]: # Copyright line pattern using {years} for years + opt_copyright = value elif opt in ["-V", "--version"]: # Output version information, then exit print( program, version ) sys.exit(0) - elif opt in ["-v", "--verbose"]: # Output version information, then exit + elif opt in ["-v", "--verbose"]: # Be more verbose opt_verbose += 1 # ---------------------------------------------------------------------- @@ -107,7 +121,7 @@ def pipe(cmd, inp=None): args = shlex.split(cmd) bufsize = 4096 stdin = PIPE if inp else None - pipe = Popen(args, stdin=stdin, stdout=PIPE, stderr=PIPE, bufsize=bufsize) + pipe = Popen(args, stdin=stdin, stdout=PIPE, stderr=PIPE, bufsize=bufsize, encoding='utf-8', universal_newlines=True) out, err = pipe.communicate(inp) code = pipe.returncode if code != 0: @@ -156,9 +170,6 @@ import json cwd = os.getcwd() -if cwd.split(os.path.sep)[-1] != 'trunk': - die("Expected to run this operation in trunk, but the current\ndirectory is '%s'" % cwd) - # Get current initinfo from cache and svn cachefn = os.path.join(os.environ.get('HOME', '.'), '.initinfo') @@ -177,24 +188,67 @@ write_cache = False loginfo_format = r'^r[0-9]+ \| [^@]+@[^@]+ \| \d\d\d\d-\d\d-\d\d ' year = time.strftime('%Y') +copyright_re = "(?i)"+opt_copyright.format(years=r"(\d+-)?\d+") for path in files: - note("Checking path %s" % path) - if not path in initinfo: - initinfo.update(get_first_commit(path)) - write_cache = True - date = initinfo[path]['date'] - init = date[:4] - copyright = "(?i)Copyright The IETF Trust (%s-)?%s, All Rights Reserved" % (init, year) - with open(path) as file: - chunk = file.read(4000) - if os.path.basename(path) == '__init__.py' and len(chunk)==0: + try: + if not os.path.exists(path): + note("File does not exist: %s" % path) continue - if not re.search(copyright, chunk): - sys.stdout.write("%s(1): Error: Missing or bad copyright. " % path) - if year == init: - print(" Expected: Copyright The IETF Trust %s, All Rights Reserved" % year) - else: - print(" Expected: Copyright The IETF Trust %s-%s, All Rights Reserved" % (init, year)) + note("Checking path %s" % path) + if not path in initinfo: + initinfo.update(get_first_commit(path)) + write_cache = True + date = initinfo[path]['date'] + init = date[:4] + + copyright_year_re = "(?i)"+opt_copyright.format(years=r"({init}-)?{year}".format(init=init, year=year)) + with open(path) as file: + try: + chunk = file.read(4000) + except UnicodeDecodeError as e: + sys.stderr.write(f'Error when reading {file.name}: {e}\n') + raise + if os.path.basename(path) == '__init__.py' and len(chunk)==0: + continue + if not re.search(copyright_year_re, chunk): + if year == init: + copyright = opt_copyright.format(years=year) + else: + copyright = opt_copyright.format(years=f"{init}-{year}") + if opt_patch: + print(f"--- {file.name}\t(original)") + print(f"+++ {file.name}\t(modified)") + if not re.search(copyright_re, chunk): + # Simple case, just insert copyright at the top + print( "@@ -1,3 +1,4 @@") + print(f"+# {copyright}") + for i, line in list(enumerate(chunk.splitlines()))[:3]: + print(f" {line}") + else: + # Find old copyright, then emit preceding lines, + # change, and following lines. + pos = None + for i, line in enumerate(chunk.splitlines(), start=1): + if re.search(copyright_re, line): + pos = i + break + if not pos: + raise RuntimeError("Unexpected state: Expected a copyright line, but found none") + print(f"@@ -1,{pos+3} +1,{pos+3} @@") + for i, line in list(enumerate(chunk.splitlines(), start=1))[:pos+3]: + if i == pos: + print(f"-{line}") + print(f"+# {copyright}") + else: + print(f" {line}") + else: + sys.stderr.write(f"{path}(1): Error: Missing or bad copyright. Expected: {copyright}") + except Exception: + if write_cache: + cache = initinfo + with open(cachefn, "w") as file: + json.dump(cache, file, indent=2, sort_keys=True) + raise if write_cache: cache = initinfo diff --git a/bin/test-crawl b/bin/test-crawl index 0a8d17977..6226d6837 100755 --- a/bin/test-crawl +++ b/bin/test-crawl @@ -1,4 +1,5 @@ #!/usr/bin/env python +# Copyright The IETF Trust 2013-2019, All Rights Reserved import os, sys, re, datetime, argparse, traceback, json, subprocess import html5lib @@ -62,6 +63,7 @@ import debug # pyflakes:ignore from ietf.name.models import DocTypeName from ietf.utils.html import unescape +from ietf.utils.test_utils import unicontent # --- Constants --- @@ -387,7 +389,7 @@ if __name__ == "__main__": if ctype == "text/html": try: if args.follow and not skip_extract_from(url): - for u in extract_html_urls(r.content): + for u in extract_html_urls(unicontent(r)): if u not in visited and u not in urls: urls[u] = url referrers[u] = url @@ -403,7 +405,7 @@ if __name__ == "__main__": elif ctype == "application/json": try: if args.follow: - for u in extract_tastypie_urls(r.content): + for u in extract_tastypie_urls(unicontent(r)): if u not in visited and u not in urls: urls[u] = url referrers[u] = url diff --git a/changelog.py b/changelog.py index 27980b393..bf27661de 100644 --- a/changelog.py +++ b/changelog.py @@ -1,4 +1,9 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals import re +import six from tzparse import tzparse from datetime import datetime as Datetime @@ -38,12 +43,12 @@ class ChangeLogEntry: title = "" def parse(logfile): - ver_line = "^(\w+) \((\S+)\) (\S+;)? (?:urgency=(\S+))?$" - sig_line = "^ -- ([^<]+) <([^>]+)> (.*?) *$" + ver_line = r"^(\w+) \((\S+)\) (\S+;)? (?:urgency=(\S+))?$" + sig_line = r"^ -- ([^<]+) <([^>]+)> (.*?) *$" inf_line = r"^ \*\*(.*)\*\* *" entries = [] - if type(logfile) == type(''): + if isinstance(logfile, six.string_types): logfile = open(logfile) entry = None for line in logfile: @@ -66,5 +71,5 @@ def parse(logfile): elif entry: entry.logentry += line else: - print "Unexpected line: '%s'" % line + print("Unexpected line: '%s'" % line) return entries diff --git a/debug.py b/debug.py index dfd465a5a..393681f7f 100644 --- a/debug.py +++ b/debug.py @@ -1,3 +1,4 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved import os import sys import time as timeutils diff --git a/docker/install-extras b/docker/install-extras index fc81638b5..5ebbfa853 100755 --- a/docker/install-extras +++ b/docker/install-extras @@ -5,3 +5,4 @@ su - -c "apt-get update \ && apt-get install -qy graphviz ghostscript apache2-utils \ && apt-get clean \ && rm -rf /var/lib/apt/lists/*" + \ No newline at end of file diff --git a/form_utils/admin.py b/form_utils/admin.py deleted file mode 100644 index b074bdb99..000000000 --- a/form_utils/admin.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.contrib import admin -from django import forms - -from form_utils.fields import ClearableFileField - -class ClearableFileFieldsAdmin(admin.ModelAdmin): - def formfield_for_dbfield(self, db_field, **kwargs): - field = super(ClearableFileFieldsAdmin, self).formfield_for_dbfield( - db_field, **kwargs) - if isinstance(field, forms.FileField): - field = ClearableFileField(field) - return field diff --git a/form_utils/fields.py b/form_utils/fields.py deleted file mode 100644 index d1d00ecf7..000000000 --- a/form_utils/fields.py +++ /dev/null @@ -1,51 +0,0 @@ -from django import forms - -from form_utils.widgets import ClearableFileInput - -class FakeEmptyFieldFile(object): - """ - A fake FieldFile that will convice a FileField model field to - actually replace an existing file name with an empty string. - - FileField.save_form_data only overwrites its instance data if the - incoming form data evaluates to True in a boolean context (because - an empty file input is assumed to mean "no change"). We want to be - able to clear it without requiring the use of a model FileField - subclass (keeping things at the form level only). In order to do - this we need our form field to return a value that evaluates to - True in a boolean context, but to the empty string when coerced to - unicode. This object fulfills that requirement. - - It also needs the _committed attribute to satisfy the test in - FileField.pre_save. - - This is, of course, hacky and fragile, and depends on internal - knowledge of the FileField and FieldFile classes. But it will - serve until Django FileFields acquire a native ability to be - cleared (ticket 7048). - - """ - def __unicode__(self): - return u'' - _committed = True - -class ClearableFileField(forms.MultiValueField): - default_file_field_class = forms.FileField - widget = ClearableFileInput - - def __init__(self, file_field=None, template=None, *args, **kwargs): - file_field = file_field or self.default_file_field_class(*args, - **kwargs) - fields = (file_field, forms.BooleanField(required=False)) - kwargs['required'] = file_field.required - kwargs['widget'] = self.widget(file_widget=file_field.widget, - template=template) - super(ClearableFileField, self).__init__(fields, *args, **kwargs) - - def compress(self, data_list): - if data_list[1] and not data_list[0]: - return FakeEmptyFieldFile() - return data_list[0] - -class ClearableImageField(ClearableFileField): - default_file_field_class = forms.ImageField diff --git a/form_utils/forms.py b/form_utils/forms.py deleted file mode 100644 index 1d83d1d4a..000000000 --- a/form_utils/forms.py +++ /dev/null @@ -1,278 +0,0 @@ -""" -forms for django-form-utils - -Time-stamp: <2010-04-28 02:57:16 carljm forms.py> - -""" -from copy import deepcopy - -from django import forms -from django.forms.utils import flatatt, ErrorDict -from django.utils.safestring import mark_safe - -class Fieldset(object): - """ - An iterable Fieldset with a legend and a set of BoundFields. - - """ - def __init__(self, form, name, boundfields, legend='', classes='', description=''): - self.form = form - self.boundfields = boundfields - if legend is None: legend = name - self.legend = legend and mark_safe(legend) - self.classes = classes - self.description = mark_safe(description) - self.name = name - - - def _errors(self): - return ErrorDict(((k, v) for (k, v) in self.form.errors.iteritems() - if k in [f.name for f in self.boundfields])) - errors = property(_errors) - - def __iter__(self): - for bf in self.boundfields: - yield _mark_row_attrs(bf, self.form) - - def __repr__(self): - return "%s('%s', %s, legend='%s', classes='%s', description='%s')" % ( - self.__class__.__name__, self.name, - [f.name for f in self.boundfields], self.legend, self.classes, self.description) - -class FieldsetCollection(object): - def __init__(self, form, fieldsets): - self.form = form - self.fieldsets = fieldsets - self._cached_fieldsets = [] - - def __len__(self): - return len(self.fieldsets) or 1 - - def __iter__(self): - if not self._cached_fieldsets: - self._gather_fieldsets() - for field in self._cached_fieldsets: - yield field - - def __getitem__(self, key): - if not self._cached_fieldsets: - self._gather_fieldsets() - for field in self._cached_fieldsets: - if field.name == key: - return field - raise KeyError - - def _gather_fieldsets(self): - if not self.fieldsets: - self.fieldsets = (('main', {'fields': self.form.fields.keys(), - 'legend': ''}),) - for name, options in self.fieldsets: - try: - field_names = [n for n in options['fields'] - if n in self.form.fields] - except KeyError: - raise ValueError("Fieldset definition must include 'fields' option." ) - boundfields = [forms.forms.BoundField(self.form, self.form.fields[n], n) - for n in field_names] - self._cached_fieldsets.append(Fieldset(self.form, name, - boundfields, options.get('legend', None), - ' '.join(options.get('classes', ())), - options.get('description', ''))) - -def _get_meta_attr(attrs, attr, default): - try: - ret = getattr(attrs['Meta'], attr) - except (KeyError, AttributeError): - ret = default - return ret - -def _set_meta_attr(attrs, attr, value): - try: - setattr(attrs['Meta'], attr, value) - return True - except KeyError: - return False - -def get_fieldsets(bases, attrs): - """ - Get the fieldsets definition from the inner Meta class. - - """ - fieldsets = _get_meta_attr(attrs, 'fieldsets', None) - if fieldsets is None: - #grab the fieldsets from the first base class that has them - for base in bases: - fieldsets = getattr(base, 'base_fieldsets', None) - if fieldsets is not None: - break - fieldsets = fieldsets or [] - return fieldsets - -def get_fields_from_fieldsets(fieldsets): - """ - Get a list of all fields included in a fieldsets definition. - - """ - fields = [] - try: - for name, options in fieldsets: - fields.extend(options['fields']) - except (TypeError, KeyError): - raise ValueError('"fieldsets" must be an iterable of two-tuples, ' - 'and the second tuple must be a dictionary ' - 'with a "fields" key') - return fields - -def get_row_attrs(bases, attrs): - """ - Get the row_attrs definition from the inner Meta class. - - """ - return _get_meta_attr(attrs, 'row_attrs', {}) - -def _mark_row_attrs(bf, form): - row_attrs = deepcopy(form._row_attrs.get(bf.name, {})) - if bf.field.required: - req_class = 'required' - else: - req_class = 'optional' - if 'class' in row_attrs: - row_attrs['class'] = row_attrs['class'] + ' ' + req_class - else: - row_attrs['class'] = req_class - bf.row_attrs = mark_safe(flatatt(row_attrs)) - return bf - -class BetterFormBaseMetaclass(type): - def __new__(cls, name, bases, attrs): - attrs['base_fieldsets'] = get_fieldsets(bases, attrs) - fields = get_fields_from_fieldsets(attrs['base_fieldsets']) - if (_get_meta_attr(attrs, 'fields', None) is None and - _get_meta_attr(attrs, 'exclude', None) is None): - _set_meta_attr(attrs, 'fields', fields) - attrs['base_row_attrs'] = get_row_attrs(bases, attrs) - new_class = super(BetterFormBaseMetaclass, - cls).__new__(cls, name, bases, attrs) - return new_class - -class BetterFormMetaclass(BetterFormBaseMetaclass, - forms.forms.DeclarativeFieldsMetaclass): - pass - -class BetterModelFormMetaclass(BetterFormBaseMetaclass, - forms.models.ModelFormMetaclass): - pass - -class BetterBaseForm(object): - """ - ``BetterForm`` and ``BetterModelForm`` are subclasses of Form - and ModelForm that allow for declarative definition of fieldsets - and row_attrs in an inner Meta class. - - The row_attrs declaration is a dictionary mapping field names to - dictionaries of attribute/value pairs. The attribute/value - dictionaries will be flattened into HTML-style attribute/values - (i.e. {'style': 'display: none'} will become ``style="display: - none"``), and will be available as the ``row_attrs`` attribute of - the ``BoundField``. Also, a CSS class of "required" or "optional" - will automatically be added to the row_attrs of each - ``BoundField``, depending on whether the field is required. - - There is no automatic inheritance of ``row_attrs``. - - The fieldsets declaration is a list of two-tuples very similar to - the ``fieldsets`` option on a ModelAdmin class in - ``django.contrib.admin``. - - The first item in each two-tuple is a name for the fieldset, and - the second is a dictionary of fieldset options. - - Valid fieldset options in the dictionary include: - - ``fields`` (required): A tuple of field names to display in this - fieldset. - - ``classes``: A list of extra CSS classes to apply to the fieldset. - - ``legend``: This value, if present, will be the contents of a ``legend`` - tag to open the fieldset. - - ``description``: A string of optional extra text to be displayed - under the ``legend`` of the fieldset. - - When iterated over, the ``fieldsets`` attribute of a - ``BetterForm`` (or ``BetterModelForm``) yields ``Fieldset``s. - Each ``Fieldset`` has a ``name`` attribute, a ``legend`` - attribute, , a ``classes`` attribute (the ``classes`` tuple - collapsed into a space-separated string), and a description - attribute, and when iterated over yields its ``BoundField``s. - - Subclasses of a ``BetterForm`` will inherit their parent's - fieldsets unless they define their own. - - A ``BetterForm`` or ``BetterModelForm`` can still be iterated over - directly to yield all of its ``BoundField``s, regardless of - fieldsets. - - """ - def __init__(self, *args, **kwargs): - self._fieldsets = deepcopy(self.base_fieldsets) - self._row_attrs = deepcopy(self.base_row_attrs) - self._fieldset_collection = None - super(BetterBaseForm, self).__init__(*args, **kwargs) - - @property - def fieldsets(self): - if not self._fieldset_collection: - self._fieldset_collection = FieldsetCollection(self, - self._fieldsets) - return self._fieldset_collection - - def __iter__(self): - for bf in super(BetterBaseForm, self).__iter__(): - yield _mark_row_attrs(bf, self) - - def __getitem__(self, name): - bf = super(BetterBaseForm, self).__getitem__(name) - return _mark_row_attrs(bf, self) - -class BetterForm(BetterBaseForm, forms.Form): - __metaclass__ = BetterFormMetaclass - __doc__ = BetterBaseForm.__doc__ - -class BetterModelForm(BetterBaseForm, forms.ModelForm): - __metaclass__ = BetterModelFormMetaclass - __doc__ = BetterBaseForm.__doc__ - - -class BasePreviewForm (object): - """ - Mixin to add preview functionality to a form. If the form is submitted with - the following k/v pair in its ``data`` dictionary: - - 'submit': 'preview' (value string is case insensitive) - - Then ``PreviewForm.preview`` will be marked ``True`` and the form will - be marked invalid (though this invalidation will not put an error in - its ``errors`` dictionary). - - """ - def __init__(self, *args, **kwargs): - super(BasePreviewForm, self).__init__(*args, **kwargs) - self.preview = self.check_preview(kwargs.get('data', None)) - - def check_preview(self, data): - if data and data.get('submit', '').lower() == u'preview': - return True - return False - - def is_valid(self, *args, **kwargs): - if self.preview: - return False - return super(BasePreviewForm, self).is_valid() - -class PreviewModelForm(BasePreviewForm, BetterModelForm): - pass - -class PreviewForm(BasePreviewForm, BetterForm): - pass diff --git a/form_utils/media/form_utils/js/autoresize.js b/form_utils/media/form_utils/js/autoresize.js deleted file mode 100644 index 8c269b866..000000000 --- a/form_utils/media/form_utils/js/autoresize.js +++ /dev/null @@ -1,3 +0,0 @@ -$(document).ready(function() { - $('textarea.autoresize').autogrow(); - }); diff --git a/form_utils/media/form_utils/js/jquery.autogrow.js b/form_utils/media/form_utils/js/jquery.autogrow.js deleted file mode 100644 index aeae46545..000000000 --- a/form_utils/media/form_utils/js/jquery.autogrow.js +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Auto Expanding Text Area (1.2.2) - * by Chrys Bader (www.chrysbader.com) - * chrysb@gmail.com - * - * Special thanks to: - * Jake Chapa - jake@hybridstudio.com - * John Resig - jeresig@gmail.com - * - * Copyright (c) 2008 Chrys Bader (www.chrysbader.com) - * Licensed under the GPL (GPL-LICENSE.txt) license. - * - * - * NOTE: This script requires jQuery to work. Download jQuery at www.jquery.com - * - */ - -(function(jQuery) { - - var self = null; - - jQuery.fn.autogrow = function(o) - { - return this.each(function() { - new jQuery.autogrow(this, o); - }); - }; - - - /** - * The autogrow object. - * - * @constructor - * @name jQuery.autogrow - * @param Object e The textarea to create the autogrow for. - * @param Hash o A set of key/value pairs to set as configuration properties. - * @cat Plugins/autogrow - */ - - jQuery.autogrow = function (e, o) - { - this.options = o || {}; - this.dummy = null; - this.interval = null; - this.line_height = this.options.lineHeight || parseInt(jQuery(e).css('line-height')); - this.min_height = this.options.minHeight || parseInt(jQuery(e).css('min-height')); - this.max_height = this.options.maxHeight || parseInt(jQuery(e).css('max-height'));; - this.textarea = jQuery(e); - - if(this.line_height == NaN) - this.line_height = 0; - - // Only one textarea activated at a time, the one being used - this.init(); - }; - - jQuery.autogrow.fn = jQuery.autogrow.prototype = { - autogrow: '1.2.2' - }; - - jQuery.autogrow.fn.extend = jQuery.autogrow.extend = jQuery.extend; - - jQuery.autogrow.fn.extend({ - - init: function() { - var self = this; - this.textarea.css({overflow: 'hidden', display: 'block'}); - this.textarea.bind('focus', function() { self.startExpand() } ).bind('blur', function() { self.stopExpand() }); - this.checkExpand(); - }, - - startExpand: function() { - var self = this; - this.interval = window.setInterval(function() {self.checkExpand()}, 400); - }, - - stopExpand: function() { - clearInterval(this.interval); - }, - - checkExpand: function() { - - if (this.dummy == null) - { - this.dummy = jQuery('
'); - this.dummy.css({ - 'font-size' : this.textarea.css('font-size'), - 'font-family': this.textarea.css('font-family'), - 'width' : this.textarea.css('width'), - 'padding' : this.textarea.css('padding'), - 'line-height': this.line_height + 'px', - 'overflow-x' : 'hidden', - 'position' : 'absolute', - 'top' : 0, - 'left' : -9999 - }).appendTo('body'); - } - - // Strip HTML tags - var html = this.textarea.val().replace(/(<|>)/g, ''); - - // IE is different, as per usual - if ($.browser.msie) - { - html = html.replace(/\n/g, '
new'); - } - else - { - html = html.replace(/\n/g, '
new'); - } - - if (this.dummy.html() != html) - { - this.dummy.html(html); - - if (this.max_height > 0 && (this.dummy.height() + this.line_height > this.max_height)) - { - this.textarea.css('overflow-y', 'auto'); - } - else - { - this.textarea.css('overflow-y', 'hidden'); - if (this.textarea.height() < this.dummy.height() + this.line_height || (this.dummy.height() < this.textarea.height())) - { - this.textarea.animate({height: (this.dummy.height() + this.line_height) + 'px'}, 100); - } - } - } - } - - }); -})(jQuery); \ No newline at end of file diff --git a/form_utils/models.py b/form_utils/models.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/form_utils/settings.py b/form_utils/settings.py deleted file mode 100644 index a8462e62f..000000000 --- a/form_utils/settings.py +++ /dev/null @@ -1,12 +0,0 @@ -import posixpath - -from django.conf import settings - -JQUERY_URL = getattr( - settings, 'JQUERY_URL', - 'http://ajax.googleapis.com/ajax/libs/jquery/1.4/jquery.min.js') - -if not ((':' in JQUERY_URL) or (JQUERY_URL.startswith('/'))): - JQUERY_URL = posixpath.join(settings.MEDIA_URL, JQUERY_URL) - -FORM_UTILS_MEDIA_URL = getattr(settings, 'FORM_UTILS_MEDIA_URL', settings.MEDIA_URL) diff --git a/form_utils/templates/form_utils/better_form.html b/form_utils/templates/form_utils/better_form.html deleted file mode 100644 index d25e429b1..000000000 --- a/form_utils/templates/form_utils/better_form.html +++ /dev/null @@ -1,16 +0,0 @@ -{% extends "form_utils/form.html" %} - -{% block fields %} -{% for fieldset in form.fieldsets %} -
-{% if fieldset.legend %} -{{ fieldset.legend }} -{% endif %} - -
-{% endfor %} -{% endblock %} diff --git a/form_utils/templates/form_utils/fields_as_lis.html b/form_utils/templates/form_utils/fields_as_lis.html deleted file mode 100644 index 99c79719f..000000000 --- a/form_utils/templates/form_utils/fields_as_lis.html +++ /dev/null @@ -1,11 +0,0 @@ -{% for field in fields %} -{% if field.is_hidden %} -{{ field }} -{% else %} - - {{ field.errors }} - {{ field.label_tag }} - {{ field }} - -{% endif %} -{% endfor %} diff --git a/form_utils/templates/form_utils/form.html b/form_utils/templates/form_utils/form.html deleted file mode 100644 index 4b235d533..000000000 --- a/form_utils/templates/form_utils/form.html +++ /dev/null @@ -1,13 +0,0 @@ -{% block errors %} -{% if form.non_field_errors %}{{ form.non_field_errors }}{% endif %} -{% endblock %} - -{% block fields %} -
-
    -{% with form as fields %} -{% include "form_utils/fields_as_lis.html" %} -{% endwith %} -
-
-{% endblock %} diff --git a/form_utils/templatetags/CVS/Entries b/form_utils/templatetags/CVS/Entries deleted file mode 100644 index 6cfab49ff..000000000 --- a/form_utils/templatetags/CVS/Entries +++ /dev/null @@ -1,3 +0,0 @@ -/__init__.py/1.1/Fri Jan 28 21:08:54 2011// -/form_utils_tags.py/1.1/Fri Jan 28 21:08:54 2011// -D diff --git a/form_utils/templatetags/CVS/Repository b/form_utils/templatetags/CVS/Repository deleted file mode 100644 index da7c7547a..000000000 --- a/form_utils/templatetags/CVS/Repository +++ /dev/null @@ -1 +0,0 @@ -ietfsec/form_utils/templatetags diff --git a/form_utils/templatetags/CVS/Root b/form_utils/templatetags/CVS/Root deleted file mode 100644 index 85344fb37..000000000 --- a/form_utils/templatetags/CVS/Root +++ /dev/null @@ -1 +0,0 @@ -/a/cvs diff --git a/form_utils/templatetags/__init__.py b/form_utils/templatetags/__init__.py deleted file mode 100644 index f5d2536b7..000000000 --- a/form_utils/templatetags/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -__init__.py for django-form-utils - templatetags - -Time-stamp: <2008-10-13 12:14:37 carljm __init__.py> - -""" diff --git a/form_utils/templatetags/form_utils_tags.py b/form_utils/templatetags/form_utils_tags.py deleted file mode 100644 index 4be8c8f0d..000000000 --- a/form_utils/templatetags/form_utils_tags.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -templatetags for django-form-utils - -Time-stamp: <2009-03-26 12:32:08 carljm form_utils_tags.py> - -""" -from django import template - -from form_utils.forms import BetterForm, BetterModelForm -from form_utils.utils import select_template_from_string - -register = template.Library() - -@register.filter -def render(form, template_name=None): - """ - Renders a ``django.forms.Form`` or - ``form_utils.forms.BetterForm`` instance using a template. - - The template name(s) may be passed in as the argument to the - filter (use commas to separate multiple template names for - template selection). - - If not provided, the default template name is - ``form_utils/form.html``. - - If the form object to be rendered is an instance of - ``form_utils.forms.BetterForm`` or - ``form_utils.forms.BetterModelForm``, the template - ``form_utils/better_form.html`` will be used instead if present. - - """ - default = 'form_utils/form.html' - if isinstance(form, (BetterForm, BetterModelForm)): - default = ','.join(['form_utils/better_form.html', default]) - tpl = select_template_from_string(template_name or default) - - return tpl.render(template.Context({'form': form})) - - - - diff --git a/form_utils/utils.py b/form_utils/utils.py deleted file mode 100644 index 79fd76bce..000000000 --- a/form_utils/utils.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -utility functions for django-form-utils - -Time-stamp: <2009-03-26 12:32:41 carljm utils.py> - -""" -from django.template import loader - -def select_template_from_string(arg): - """ - Select a template from a string, which can include multiple - template paths separated by commas. - - """ - if ',' in arg: - tpl = loader.select_template( - [tn.strip() for tn in arg.split(',')]) - else: - tpl = loader.get_template(arg) - return tpl diff --git a/form_utils/widgets.py b/form_utils/widgets.py deleted file mode 100644 index 92bf9ca72..000000000 --- a/form_utils/widgets.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -widgets for django-form-utils - -parts of this code taken from http://www.djangosnippets.org/snippets/934/ - - thanks baumer1122 - -""" -import os -import posixpath - -from django import forms -from django.conf import settings -from django.utils.functional import curry -from django.utils.safestring import mark_safe -from django.core.files.uploadedfile import SimpleUploadedFile as UploadedFile - -from form_utils.settings import JQUERY_URL, FORM_UTILS_MEDIA_URL - -try: - from sorl.thumbnail.main import DjangoThumbnail - def thumbnail(image_path, width, height): - t = DjangoThumbnail(relative_source=image_path, requested_size=(width,height)) - return u'%s' % (t.absolute_url, image_path) -except ImportError: - def thumbnail(image_path, width, height): - absolute_url = posixpath.join(settings.MEDIA_URL, image_path) - return u'%s' % (absolute_url, image_path) - -class ImageWidget(forms.FileInput): - template = '%(input)s
%(image)s' - - def __init__(self, attrs=None, template=None, width=200, height=200): - if template is not None: - self.template = template - self.width = width - self.height = height - super(ImageWidget, self).__init__(attrs) - - def render(self, name, value, attrs=None): - input_html = super(forms.FileInput, self).render(name, value, attrs) - if hasattr(value, 'width') and hasattr(value, 'height'): - image_html = thumbnail(value.name, self.width, self.height) - output = self.template % {'input': input_html, - 'image': image_html} - else: - output = input_html - return mark_safe(output) - -class ClearableFileInput(forms.MultiWidget): - default_file_widget_class = forms.FileInput - template = '%(input)s Clear: %(checkbox)s' - - def __init__(self, file_widget=None, - attrs=None, template=None): - if template is not None: - self.template = template - file_widget = file_widget or self.default_file_widget_class() - super(ClearableFileInput, self).__init__( - widgets=[file_widget, forms.CheckboxInput()], - attrs=attrs) - - def render(self, name, value, attrs=None): - if isinstance(value, list): - self.value = value[0] - else: - self.value = value - return super(ClearableFileInput, self).render(name, value, attrs) - - def decompress(self, value): - # the clear checkbox is never initially checked - return [value, None] - - def format_output(self, rendered_widgets): - if self.value: - return self.template % {'input': rendered_widgets[0], - 'checkbox': rendered_widgets[1]} - return rendered_widgets[0] - -root = lambda path: posixpath.join(FORM_UTILS_MEDIA_URL, path) - -class AutoResizeTextarea(forms.Textarea): - """ - A Textarea widget that automatically resizes to accomodate its contents. - - """ - class Media: - - js = (JQUERY_URL, - root('form_utils/js/jquery.autogrow.js'), - root('form_utils/js/autoresize.js')) - - def __init__(self, *args, **kwargs): - attrs = kwargs.setdefault('attrs', {}) - try: - attrs['class'] = "%s autoresize" % (attrs['class'],) - except KeyError: - attrs['class'] = 'autoresize' - attrs.setdefault('cols', 80) - attrs.setdefault('rows', 5) - super(AutoResizeTextarea, self).__init__(*args, **kwargs) - -class InlineAutoResizeTextarea(AutoResizeTextarea): - def __init__(self, *args, **kwargs): - attrs = kwargs.setdefault('attrs', {}) - try: - attrs['class'] = "%s inline" % (attrs['class'],) - except KeyError: - attrs['class'] = 'inline' - attrs.setdefault('cols', 40) - attrs.setdefault('rows', 2) - super(InlineAutoResizeTextarea, self).__init__(*args, **kwargs) - diff --git a/ietf/__init__.py b/ietf/__init__.py index d42f8797c..6a4a01a7a 100644 --- a/ietf/__init__.py +++ b/ietf/__init__.py @@ -1,7 +1,10 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved # -*- coding: utf-8 -*- -import checks # pyflakes:ignore + +from __future__ import absolute_import, print_function, unicode_literals + +from . import checks # pyflakes:ignore # Don't add patch number here: __version__ = "6.98.5.dev0" diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index ba8ad4c91..ff0b57bd3 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -1,7 +1,14 @@ +# Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime import re import six -import datetime -from urllib import urlencode + +from six.moves.urllib.parse import urlencode from django.conf import settings from django.core.exceptions import ObjectDoesNotExist @@ -65,7 +72,7 @@ class ModelResource(tastypie.resources.ModelResource): return "%s:%s:%s:%s" % (self._meta.api_name, self._meta.resource_name, ':'.join(args), smooshed) -TIMEDELTA_REGEX = re.compile('^(?P\d+d)?\s?(?P\d+h)?\s?(?P\d+m)?\s?(?P\d+s?)$') +TIMEDELTA_REGEX = re.compile(r'^(?P\d+d)?\s?(?P\d+h)?\s?(?P\d+m)?\s?(?P\d+s?)$') class TimedeltaField(ApiField): dehydrated_type = 'timedelta' @@ -112,6 +119,8 @@ class ToOneField(tastypie.fields.ToOneField): def dehydrate(self, bundle, for_list=True): foreign_obj = None + previous_obj = None + attrib = None if callable(self.attribute): previous_obj = bundle.obj @@ -120,6 +129,7 @@ class ToOneField(tastypie.fields.ToOneField): foreign_obj = bundle.obj for attr in self._attrs: + attrib = attr previous_obj = foreign_obj try: foreign_obj = getattr(foreign_obj, attr, None) @@ -129,9 +139,9 @@ class ToOneField(tastypie.fields.ToOneField): if not foreign_obj: if not self.null: if callable(self.attribute): - raise ApiFieldError(u"The related resource for resource %s could not be found." % (previous_obj)) + raise ApiFieldError("The related resource for resource %s could not be found." % (previous_obj)) else: - raise ApiFieldError(u"The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attr)) + raise ApiFieldError("The model '%r' has an empty attribute '%s' and doesn't allow a null value." % (previous_obj, attrib)) return None fk_resource = self.get_related_resource(foreign_obj) diff --git a/ietf/api/management/commands/makeresources.py b/ietf/api/management/commands/makeresources.py index c564f36a2..54830b5c1 100644 --- a/ietf/api/management/commands/makeresources.py +++ b/ietf/api/management/commands/makeresources.py @@ -1,10 +1,12 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import print_function +from __future__ import absolute_import, print_function, unicode_literals import os import datetime import collections +import io + from importlib import import_module import debug # pyflakes:ignore @@ -16,7 +18,11 @@ from django.template import Template, Context from tastypie.resources import ModelResource -resource_head_template = """# Autogenerated by the makeresources management command {{date}} +resource_head_template = """# Copyright The IETF Trust {{date}}, All Rights Reserved +# -*- coding: utf-8 -*- +# Generated by the makeresources management command {{date}} + + from tastypie.resources import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore @@ -79,7 +85,7 @@ class Command(AppCommand): if missing_resources: print("Updating resources.py for %s" % app.name) - with open(resource_file_path, "a") as rfile: + with io.open(resource_file_path, "a") as rfile: info = dict( app=app.name, app_label=app.label, @@ -164,7 +170,7 @@ class Command(AppCommand): fields=model._meta.fields, m2m_fields=model._meta.many_to_many, name=model_name, - imports=[ v for k,v in imports.items() ], + imports=[ v for k,v in list(imports.items()) ], foreign_keys=foreign_keys, m2m_keys=m2m_keys, resource_name=resource_name, @@ -184,7 +190,7 @@ class Command(AppCommand): while len(new_models) > 0: list_len = len(new_models) #debug.show('len(new_models)') - keys = new_models.keys() + keys = list(new_models.keys()) for model_name in keys: internal_fk_count = 0 for fk in new_models[model_name]["foreign_keys"]+new_models[model_name]["m2m_keys"]: @@ -207,7 +213,7 @@ class Command(AppCommand): internal_fk_count_limit += 1 else: print("Failed also with partial ordering, writing resource classes without ordering") - new_model_list = [ v for k,v in new_models.items() ] + new_model_list = [ v for k,v in list(new_models.items()) ] break if rfile.tell() == 0: diff --git a/ietf/api/serializer.py b/ietf/api/serializer.py index 11a61f1d3..98e276efe 100644 --- a/ietf/api/serializer.py +++ b/ietf/api/serializer.py @@ -1,5 +1,12 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import hashlib import json +import six from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist, FieldError @@ -27,9 +34,9 @@ def filter_from_queryargs(request): def is_ascii(s): return all(ord(c) < 128 for c in s) # limit parameter keys to ascii. - params = dict( (k,v) for (k,v) in request.GET.items() if is_ascii(k) ) - filter = fix_ranges(dict([(k,params[k]) for k in params.keys() if not k.startswith("not__")])) - exclude = fix_ranges(dict([(k[5:],params[k]) for k in params.keys() if k.startswith("not__")])) + params = dict( (k,v) for (k,v) in list(request.GET.items()) if is_ascii(k) ) + filter = fix_ranges(dict([(k,params[k]) for k in list(params.keys()) if not k.startswith("not__")])) + exclude = fix_ranges(dict([(k[5:],params[k]) for k in list(params.keys()) if k.startswith("not__")])) return filter, exclude def unique_obj_name(obj): @@ -89,7 +96,7 @@ class AdminJsonSerializer(Serializer): use_natural_keys = False def serialize(self, queryset, **options): - qi = options.get('query_info', '') + qi = options.get('query_info', '').encode('utf-8') if len(list(queryset)) == 1: obj = queryset[0] key = 'json:%s:%s' % (hashlib.md5(qi).hexdigest(), unique_obj_name(obj)) @@ -147,7 +154,7 @@ class AdminJsonSerializer(Serializer): if hasattr(field_value, "_meta"): self._current[name] = self.expand_related(field_value, name) else: - self._current[name] = unicode(field_value) + self._current[name] = six.text_type(field_value) except ObjectDoesNotExist: pass except AttributeError: @@ -224,7 +231,7 @@ class JsonExportMixin(object): def json_view(self, request, filter={}, expand=[]): qfilter, exclude = filter_from_queryargs(request) - for k in qfilter.keys(): + for k in list(qfilter.keys()): if k.startswith("_"): del qfilter[k] qfilter.update(filter) @@ -244,7 +251,7 @@ class JsonExportMixin(object): try: qs = self.get_queryset().filter(**filter).exclude(**exclude) except (FieldError, ValueError) as e: - return HttpResponse(json.dumps({u"error": str(e)}, sort_keys=True, indent=3), content_type=content_type) + return HttpResponse(json.dumps({"error": str(e)}, sort_keys=True, indent=3), content_type=content_type) try: if expand: qs = qs.select_related() @@ -252,7 +259,7 @@ class JsonExportMixin(object): items = [(getattr(o, key), serializer.serialize([o], expand=expand, query_info=query_info) ) for o in qs ] qd = dict( ( k, json.loads(v)[0] ) for k,v in items ) except (FieldError, ValueError) as e: - return HttpResponse(json.dumps({u"error": str(e)}, sort_keys=True, indent=3), content_type=content_type) + return HttpResponse(json.dumps({"error": str(e)}, sort_keys=True, indent=3), content_type=content_type) text = json.dumps({smart_text(self.model._meta): qd}, sort_keys=True, indent=3) return HttpResponse(text, content_type=content_type) diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 64f7b0625..accc96989 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1,6 +1,9 @@ -# Copyright The IETF Trust 2015-2018, All Rights Reserved +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals -import json import os import sys @@ -194,7 +197,7 @@ class TastypieApiTestCase(ResourceTestCaseMixin, TestCase): client = Client(Accept='application/json') r = client.get("/api/v1/") self.assertValidJSONResponse(r) - resource_list = json.loads(r.content) + resource_list = r.json() for name in self.apps: if not name in self.apps: @@ -207,19 +210,19 @@ class TastypieApiTestCase(ResourceTestCaseMixin, TestCase): def test_all_model_resources_exist(self): client = Client(Accept='application/json') r = client.get("/api/v1") - top = json.loads(r.content) + top = r.json() for name in self.apps: app_name = self.apps[name] app = import_module(app_name) self.assertEqual("/api/v1/%s/"%name, top[name]["list_endpoint"]) r = client.get(top[name]["list_endpoint"]) self.assertValidJSONResponse(r) - app_resources = json.loads(r.content) + app_resources = r.json() # model_list = apps.get_app_config(name).get_models() for model in model_list: - if not model._meta.model_name in app_resources.keys(): + if not model._meta.model_name in list(app_resources.keys()): #print("There doesn't seem to be any resource for model %s.models.%s"%(app.__name__,model.__name__,)) - self.assertIn(model._meta.model_name, app_resources.keys(), + self.assertIn(model._meta.model_name, list(app_resources.keys()), "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,)) diff --git a/ietf/api/views.py b/ietf/api/views.py index 027a6a8c1..2376d23e4 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -1,7 +1,6 @@ -# Copyright The IETF Trust 2017, All Rights Reserved +# Copyright The IETF Trust 2017-2019, All Rights Reserved # -*- coding: utf-8 -*- - -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals from jwcrypto.jwk import JWK diff --git a/ietf/checks.py b/ietf/checks.py index ae8e3b611..1fdde2dc1 100644 --- a/ietf/checks.py +++ b/ietf/checks.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import os import patch import sys @@ -367,7 +373,7 @@ def maybe_patch_library(app_configs, **kwargs): patch_path = os.path.join(cwd, patch_file) patch_set = patch.fromfile(patch_path) if patch_set: - if not patch_set.apply(root=library_path): + if not patch_set.apply(root=library_path.encode('utf-8')): errors.append(checks.Warning( "Could not apply patch from file '%s'"%patch_file, hint=("Make sure that the patch file contains a unified diff and has valid file paths\n\n" diff --git a/ietf/community/admin.py b/ietf/community/admin.py index e366f48df..fd7889f74 100644 --- a/ietf/community/admin.py +++ b/ietf/community/admin.py @@ -1,24 +1,24 @@ # Copyright The IETF Trust 2017-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals from django.contrib import admin from ietf.community.models import CommunityList, SearchRule, EmailSubscription class CommunityListAdmin(admin.ModelAdmin): - list_display = [u'id', 'user', 'group'] + list_display = ['id', 'user', 'group'] raw_id_fields = ['user', 'group', 'added_docs'] admin.site.register(CommunityList, CommunityListAdmin) class SearchRuleAdmin(admin.ModelAdmin): - list_display = [u'id', 'community_list', 'rule_type', 'state', 'group', 'person', 'text'] + list_display = ['id', 'community_list', 'rule_type', 'state', 'group', 'person', 'text'] raw_id_fields = ['community_list', 'state', 'group', 'person', 'name_contains_index'] search_fields = ['person__name', 'group__acronym', 'text', ] admin.site.register(SearchRule, SearchRuleAdmin) class EmailSubscriptionAdmin(admin.ModelAdmin): - list_display = [u'id', 'community_list', 'email', 'notify_on'] + list_display = ['id', 'community_list', 'email', 'notify_on'] raw_id_fields = ['community_list', 'email'] admin.site.register(EmailSubscription, EmailSubscriptionAdmin) diff --git a/ietf/community/forms.py b/ietf/community/forms.py index d764d2307..1677c950c 100644 --- a/ietf/community/forms.py +++ b/ietf/community/forms.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django import forms from django.db.models import Q @@ -82,9 +88,9 @@ class SearchRuleForm(forms.ModelForm): if 'group' in self.fields: self.fields['group'].queryset = self.fields['group'].queryset.filter(state="active").order_by("acronym") - self.fields['group'].choices = [(g.pk, u"%s - %s" % (g.acronym, g.name)) for g in self.fields['group'].queryset] + self.fields['group'].choices = [(g.pk, "%s - %s" % (g.acronym, g.name)) for g in self.fields['group'].queryset] - for name, f in self.fields.iteritems(): + for name, f in self.fields.items(): f.required = True def clean_text(self): diff --git a/ietf/community/migrations/0001_initial.py b/ietf/community/migrations/0001_initial.py index 44ddb6200..784093123 100644 --- a/ietf/community/migrations/0001_initial.py +++ b/ietf/community/migrations/0001_initial.py @@ -1,6 +1,7 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/community/migrations/0002_auto_20180220_1052.py b/ietf/community/migrations/0002_auto_20180220_1052.py index 2e3f85b17..93a0ebd5e 100644 --- a/ietf/community/migrations/0002_auto_20180220_1052.py +++ b/ietf/community/migrations/0002_auto_20180220_1052.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.conf import settings from django.db import migrations, models diff --git a/ietf/community/migrations/0003_add_communitylist_docs2_m2m.py b/ietf/community/migrations/0003_add_communitylist_docs2_m2m.py index 6ab7a26c7..34c620fbe 100644 --- a/ietf/community/migrations/0003_add_communitylist_docs2_m2m.py +++ b/ietf/community/migrations/0003_add_communitylist_docs2_m2m.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-21 14:23 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/community/migrations/0004_set_document_m2m_keys.py b/ietf/community/migrations/0004_set_document_m2m_keys.py index 2c2db4d13..095ef5243 100644 --- a/ietf/community/migrations/0004_set_document_m2m_keys.py +++ b/ietf/community/migrations/0004_set_document_m2m_keys.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-21 14:27 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys @@ -21,7 +23,7 @@ def forward(apps, schema_editor): # Document id fixup ------------------------------------------------------------ objs = Document.objects.in_bulk() - nameid = { o.name: o.id for id, o in objs.iteritems() } + nameid = { o.name: o.id for id, o in objs.items() } sys.stderr.write('\n') diff --git a/ietf/community/migrations/0005_1_del_docs_m2m_table.py b/ietf/community/migrations/0005_1_del_docs_m2m_table.py index 68afc2e9f..0ff9ace87 100644 --- a/ietf/community/migrations/0005_1_del_docs_m2m_table.py +++ b/ietf/community/migrations/0005_1_del_docs_m2m_table.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-22 08:15 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/community/migrations/0005_2_add_docs_m2m_table.py b/ietf/community/migrations/0005_2_add_docs_m2m_table.py index 9e0438702..8847eadc4 100644 --- a/ietf/community/migrations/0005_2_add_docs_m2m_table.py +++ b/ietf/community/migrations/0005_2_add_docs_m2m_table.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-22 08:15 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/community/migrations/0006_copy_docs_m2m_table.py b/ietf/community/migrations/0006_copy_docs_m2m_table.py index 7d0f76803..9a9b40a7b 100644 --- a/ietf/community/migrations/0006_copy_docs_m2m_table.py +++ b/ietf/community/migrations/0006_copy_docs_m2m_table.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-27 05:56 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/community/migrations/0007_remove_docs2_m2m.py b/ietf/community/migrations/0007_remove_docs2_m2m.py index 9687aa3ad..dce7e45cc 100644 --- a/ietf/community/migrations/0007_remove_docs2_m2m.py +++ b/ietf/community/migrations/0007_remove_docs2_m2m.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-30 03:06 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/community/models.py b/ietf/community/models.py index ec8e10f6e..a5c98f8e5 100644 --- a/ietf/community/models.py +++ b/ietf/community/models.py @@ -1,13 +1,21 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.contrib.auth.models import User from django.db import models from django.db.models import signals from django.urls import reverse as urlreverse +from django.utils.encoding import python_2_unicode_compatible from ietf.doc.models import Document, DocEvent, State from ietf.group.models import Group from ietf.person.models import Person, Email from ietf.utils.models import ForeignKey +@python_2_unicode_compatible class CommunityList(models.Model): user = ForeignKey(User, blank=True, null=True) group = ForeignKey(Group, blank=True, null=True) @@ -21,7 +29,7 @@ class CommunityList(models.Model): else: return 'ID list' - def __unicode__(self): + def __str__(self): return self.long_name() def get_absolute_url(self): @@ -33,6 +41,7 @@ class CommunityList(models.Model): return "" +@python_2_unicode_compatible class SearchRule(models.Model): # these types define the UI for setting up the rule, and also # helps when interpreting the rule and matching documents @@ -75,9 +84,10 @@ class SearchRule(models.Model): # when new documents are submitted name_contains_index = models.ManyToManyField(Document) - def __unicode__(self): + def __str__(self): return "%s %s %s/%s/%s/%s" % (self.community_list, self.rule_type, self.state, self.group, self.person, self.text) +@python_2_unicode_compatible class EmailSubscription(models.Model): community_list = ForeignKey(CommunityList) email = ForeignKey(Email) @@ -88,8 +98,8 @@ class EmailSubscription(models.Model): ] notify_on = models.CharField(max_length=30, choices=NOTIFICATION_CHOICES, default="all") - def __unicode__(self): - return u"%s to %s (%s changes)" % (self.email, self.community_list, self.notify_on) + def __str__(self): + return "%s to %s (%s changes)" % (self.email, self.community_list, self.notify_on) def notify_events(sender, instance, **kwargs): diff --git a/ietf/community/resources.py b/ietf/community/resources.py index eea118ed7..2d87f3ce9 100644 --- a/ietf/community/resources.py +++ b/ietf/community/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2014-11-13 23:53 + + from ietf.api import ModelResource from tastypie.fields import ToOneField, ToManyField from tastypie.constants import ALL, ALL_WITH_RELATIONS diff --git a/ietf/community/tests.py b/ietf/community/tests.py index e622ef56a..5d50b0151 100644 --- a/ietf/community/tests.py +++ b/ietf/community/tests.py @@ -1,7 +1,8 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- -import json + +from __future__ import absolute_import, print_function, unicode_literals from pyquery import PyQuery @@ -19,7 +20,7 @@ from ietf.group.utils import setup_default_community_list_for_group from ietf.doc.models import State from ietf.doc.utils import add_state_change_event from ietf.person.models import Person, Email -from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent +from ietf.utils.test_utils import login_testing_unauthorized, TestCase from ietf.utils.mail import outbox from ietf.doc.factories import WgDraftFactory from ietf.group.factories import GroupFactory, RoleFactory @@ -97,7 +98,7 @@ class CommunityListTests(TestCase): ) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.name in unicontent(r)) + self.assertContains(r, draft.name) def test_manage_personal_list(self): PersonFactory(user__username='plain') @@ -119,7 +120,7 @@ class CommunityListTests(TestCase): # document shows up on GET r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.name in unicontent(r)) + self.assertContains(r, draft.name) # remove document r = self.client.post(url, { "action": "remove_document", "document": draft.name }) @@ -226,7 +227,7 @@ class CommunityListTests(TestCase): # track r = self.client.post(url, HTTP_X_REQUESTED_WITH='XMLHttpRequest') self.assertEqual(r.status_code, 200) - self.assertEqual(json.loads(r.content)["success"], True) + self.assertEqual(r.json()["success"], True) clist = CommunityList.objects.get(user__username="plain") self.assertEqual(list(clist.added_docs.all()), [draft]) @@ -234,7 +235,7 @@ class CommunityListTests(TestCase): url = urlreverse(ietf.community.views.untrack_document, kwargs={ "username": "plain", "name": draft.name }) r = self.client.post(url, HTTP_X_REQUESTED_WITH='XMLHttpRequest') self.assertEqual(r.status_code, 200) - self.assertEqual(json.loads(r.content)["success"], True) + self.assertEqual(r.json()["success"], True) clist = CommunityList.objects.get(user__username="plain") self.assertEqual(list(clist.added_docs.all()), []) @@ -261,7 +262,7 @@ class CommunityListTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) # this is a simple-minded test, we don't actually check the fields - self.assertTrue(draft.name in unicontent(r)) + self.assertContains(r, draft.name) def test_csv_for_group(self): draft = WgDraftFactory() @@ -296,12 +297,12 @@ class CommunityListTests(TestCase): ) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.name in unicontent(r)) + self.assertContains(r, draft.name) # only significant r = self.client.get(url + "?significant=1") self.assertEqual(r.status_code, 200) - self.assertTrue('' not in unicontent(r)) + self.assertNotContains(r, '') def test_feed_for_group(self): draft = WgDraftFactory() diff --git a/ietf/community/utils.py b/ietf/community/utils.py index 357a48366..2d7ed5ef5 100644 --- a/ietf/community/utils.py +++ b/ietf/community/utils.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import re from django.db.models import Q diff --git a/ietf/community/views.py b/ietf/community/views.py index 4572e5e34..09a12ab93 100644 --- a/ietf/community/views.py +++ b/ietf/community/views.py @@ -1,16 +1,22 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import csv -import uuid import datetime import json +import six +import uuid from django.http import HttpResponse, HttpResponseForbidden, HttpResponseRedirect, Http404 from django.shortcuts import get_object_or_404, render from django.contrib.auth.decorators import login_required from django.utils.html import strip_tags +import debug # pyflakes:ignore + from ietf.community.models import SearchRule, EmailSubscription from ietf.community.forms import SearchRuleTypeForm, SearchRuleForm, AddDocumentsForm, SubscriptionForm from ietf.community.utils import lookup_community_list, can_manage_community_list @@ -135,7 +141,7 @@ def track_document(request, name, username=None, acronym=None): clist.added_docs.add(doc) if request.is_ajax(): - return HttpResponse(json.dumps({ 'success': True }), content_type='text/plain') + return HttpResponse(json.dumps({ 'success': True }), content_type='application/json') else: return HttpResponseRedirect(clist.get_absolute_url()) @@ -155,7 +161,7 @@ def untrack_document(request, name, username=None, acronym=None): clist.added_docs.remove(doc) if request.is_ajax(): - return HttpResponse(json.dumps({ 'success': True }), content_type='text/plain') + return HttpResponse(json.dumps({ 'success': True }), content_type='application/json') else: return HttpResponseRedirect(clist.get_absolute_url()) @@ -176,7 +182,7 @@ def export_to_csv(request, username=None, acronym=None, group_type=None): response['Content-Disposition'] = 'attachment; filename=%s' % filename - writer = csv.writer(response, dialect=csv.excel, delimiter=',') + writer = csv.writer(response, dialect=csv.excel, delimiter=str(',')) header = [ "Name", @@ -198,7 +204,7 @@ def export_to_csv(request, username=None, acronym=None, group_type=None): row.append(e.time.strftime("%Y-%m-%d") if e else "") row.append(strip_tags(doc.friendly_state())) row.append(doc.group.acronym if doc.group else "") - row.append(unicode(doc.ad) if doc.ad else "") + row.append(six.text_type(doc.ad) if doc.ad else "") e = doc.latest_event() row.append(e.time.strftime("%Y-%m-%d") if e else "") writer.writerow([v.encode("utf-8") for v in row]) @@ -223,8 +229,8 @@ def feed(request, username=None, acronym=None, group_type=None): host = request.get_host() feed_url = 'https://%s%s' % (host, request.get_full_path()) - feed_id = uuid.uuid5(uuid.NAMESPACE_URL, feed_url.encode('utf-8')) - title = u'%s RSS Feed' % clist.long_name() + feed_id = uuid.uuid5(uuid.NAMESPACE_URL, str(feed_url)) + title = '%s RSS Feed' % clist.long_name() if significant: subtitle = 'Significant document changes' else: @@ -235,7 +241,7 @@ def feed(request, username=None, acronym=None, group_type=None): 'entries': events[:50], 'title': title, 'subtitle': subtitle, - 'id': feed_id.get_urn(), + 'id': feed_id.urn, 'updated': datetime.datetime.now(), }, content_type='text/xml') diff --git a/ietf/cookies/__init__.py b/ietf/cookies/__init__.py index 306c9879b..f17c21d38 100644 --- a/ietf/cookies/__init__.py +++ b/ietf/cookies/__init__.py @@ -1,6 +1,9 @@ -# Copyright The IETF Trust 2010, All Rights Reserved +# Copyright The IETF Trust 2010-2019, All Rights Reserved # coding: latin-1 + +from __future__ import absolute_import, print_function, unicode_literals + from types import ModuleType # These people will be sent a stack trace if there's an uncaught exception in @@ -9,7 +12,7 @@ DEBUG_EMAILS = [ ('Tero Kivinen', 'kivinen@iki.fi'), ] -for k in locals().keys(): +for k in list(locals().keys()): m = locals()[k] if isinstance(m, ModuleType): if hasattr(m, "DEBUG_EMAILS"): diff --git a/ietf/cookies/tests.py b/ietf/cookies/tests.py index dd02080c0..302cab100 100644 --- a/ietf/cookies/tests.py +++ b/ietf/cookies/tests.py @@ -1,5 +1,11 @@ +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from pyquery import PyQuery -from Cookie import SimpleCookie +from six.moves.http_cookies import SimpleCookie from django.urls import reverse as urlreverse @@ -12,7 +18,7 @@ class CookieTests(TestCase): def test_settings_defaults(self): r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) - self.assertListEqual([], r.cookies.keys()) + self.assertListEqual([], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -21,10 +27,10 @@ class CookieTests(TestCase): def test_settings_defaults_from_cookies(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '7', 'expires_soon' : 7, 'left_menu': 'on', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '7', str('expires_soon') : 7, str('left_menu'): 'on', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) - self.assertListEqual([], r.cookies.keys()) + self.assertListEqual([], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/7"]').contents(), ['7 days']) @@ -32,7 +38,7 @@ class CookieTests(TestCase): self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/on"]').contents(), ['On']) def test_settings_values_from_cookies_garbage(self): - self.client.cookies = SimpleCookie({'full_draft': 'foo', 'new_enough' : 'foo', 'expires_soon' : 'foo', 'left_menu': 'foo', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'foo', str('new_enough') : 'foo', str('expires_soon') : 'foo', str('left_menu'): 'foo', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -42,7 +48,7 @@ class CookieTests(TestCase): self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off']) def test_settings_values_from_cookies_random(self): - self.client.cookies = SimpleCookie({'full_draft': 'zappa', 'new_enough' : '365', 'expires_soon' : '5', 'left_menu': 'zappa', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'zappa', str('new_enough') : '365', str('expires_soon') : '5', str('left_menu'): 'zappa', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -57,10 +63,10 @@ class CookieTests(TestCase): # self.assertNotRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon') def test_settings_values_from_cookies_1(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '90', 'expires_soon' : 7, 'left_menu': 'off', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '90', str('expires_soon') : 7, str('left_menu'): 'off', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) - self.assertListEqual([], r.cookies.keys()) + self.assertListEqual([], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/90"]').contents(), ['90 days']) @@ -71,10 +77,10 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*7 days') def test_settings_values_from_cookies_2(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '60', 'expires_soon' : 14, 'left_menu': 'on', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '60', str('expires_soon') : 14, str('left_menu'): 'on', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) - self.assertListEqual([], r.cookies.keys()) + self.assertListEqual([], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/60"]').contents(), ['60 days']) @@ -85,10 +91,10 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days') def test_settings_values_from_cookies_3(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '30', 'expires_soon' : 21, 'left_menu': 'off'}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '30', str('expires_soon') : 21, str('left_menu'): 'off'}) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) - self.assertListEqual([], r.cookies.keys()) + self.assertListEqual([], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/30"]').contents(), ['30 days']) @@ -99,10 +105,10 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*21 days') def test_settings_values_from_cookies_4(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '21', 'expires_soon' : 30, 'left_menu': 'on', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '21', str('expires_soon') : 30, str('left_menu'): 'on', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) - self.assertListEqual([], r.cookies.keys()) + self.assertListEqual([], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/21"]').contents(), ['21 days']) @@ -113,10 +119,10 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*30 days') def test_settings_values_from_cookies_5(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '14', 'expires_soon' : 60, 'left_menu': 'off', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '14', str('expires_soon') : 60, str('left_menu'): 'off', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) - self.assertListEqual([], r.cookies.keys()) + self.assertListEqual([], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -127,10 +133,10 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*60 days') def test_settings_values_from_cookies_6(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '7', 'expires_soon' : 90, 'left_menu': 'on', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '7', str('expires_soon') : 90, str('left_menu'): 'on', }) r = self.client.get(urlreverse("ietf.cookies.views.preferences")) self.assertEqual(r.status_code, 200) - self.assertListEqual([], r.cookies.keys()) + self.assertListEqual([], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/7"]').contents(), ['7 days']) @@ -141,11 +147,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*90 days') def test_full_draft(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.full_draft")) # no value: reset self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['full_draft'].value, '') - self.assertListEqual(['full_draft'], r.cookies.keys()) + self.assertEqual(r.cookies[str('full_draft')].value, '') + self.assertListEqual([str('full_draft')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -155,21 +161,21 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days') def test_full_draft_on(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.full_draft", kwargs=dict(enabled="on"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['full_draft'].value, 'on') - self.assertListEqual(['full_draft'], r.cookies.keys()) + self.assertEqual(r.cookies[str('full_draft')].value, 'on') + self.assertListEqual([str('full_draft')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*full_draft.*on') def test_full_draft_off(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.full_draft", kwargs=dict(enabled="off"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['full_draft'].value, 'off') - self.assertListEqual(['full_draft'], r.cookies.keys()) + self.assertEqual(r.cookies[str('full_draft')].value, 'off') + self.assertListEqual([str('full_draft')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) # self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -177,10 +183,10 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*full_draft.*off') def test_full_draft_foo(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.full_draft", kwargs=dict(enabled="foo"))) self.assertEqual(r.status_code, 200) - self.assertListEqual([], r.cookies.keys()) + self.assertListEqual([], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) # self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -188,11 +194,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*full_draft.*off') def test_left_menu(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'on', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'on', }) r = self.client.get(urlreverse("ietf.cookies.views.left_menu")) # no value: reset self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['left_menu'].value, '') - self.assertListEqual(['left_menu'], r.cookies.keys()) + self.assertEqual(r.cookies[str('left_menu')].value, '') + self.assertListEqual([str('left_menu')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off']) @@ -200,37 +206,37 @@ class CookieTests(TestCase): self.assertEqual(q('div a.active[href="/accounts/settings/expires_soon/14"]').contents(), ['14 days']) def test_left_menu_on(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'off', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'off', }) r = self.client.get(urlreverse("ietf.cookies.views.left_menu", kwargs=dict(enabled="on"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['left_menu'].value, 'on') - self.assertListEqual(['left_menu'], r.cookies.keys()) + self.assertEqual(r.cookies[str('left_menu')].value, 'on') + self.assertListEqual([str('left_menu')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/on"]').contents(), ['On']) def test_left_menu_off(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'off', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'off', }) r = self.client.get(urlreverse("ietf.cookies.views.left_menu", kwargs=dict(enabled="off"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['left_menu'].value, 'off') - self.assertListEqual(['left_menu'], r.cookies.keys()) + self.assertEqual(r.cookies[str('left_menu')].value, 'off') + self.assertListEqual([str('left_menu')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off']) def test_left_menu_foo(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14, 'left_menu': 'off', }) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14, str('left_menu'): 'off', }) r = self.client.get(urlreverse("ietf.cookies.views.left_menu", kwargs=dict(enabled="foo"))) self.assertEqual(r.status_code, 200) - self.assertListEqual([], r.cookies.keys()) + self.assertListEqual([], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/left_menu/off"]').contents(), ['Off']) def test_new_enough(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough")) # no value: reset self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '') - self.assertListEqual(['new_enough'], r.cookies.keys()) + self.assertEqual(r.cookies[str('new_enough')].value, '') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -240,11 +246,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days') def test_new_enough_7(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '14', 'expires_soon' : 21}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '14', str('expires_soon') : 21}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="7"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '7') - self.assertListEqual(['new_enough'], r.cookies.keys()) + self.assertEqual(r.cookies[str('new_enough')].value, '7') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/7"]').contents(), ['7 days']) @@ -254,11 +260,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*21 days') def test_new_enough_14(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '7', 'expires_soon' : 99}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '7', str('expires_soon') : 99}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="14"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '14') - self.assertListEqual(['new_enough'], r.cookies.keys()) + self.assertEqual(r.cookies[str('new_enough')].value, '14') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -268,11 +274,11 @@ class CookieTests(TestCase): # self.assertNotRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon') def test_new_enough_21(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'new_enough' : '14', 'expires_soon' : 90}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('new_enough') : '14', str('expires_soon') : 90}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="21"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '21') - self.assertListEqual(['new_enough'], r.cookies.keys()) + self.assertEqual(r.cookies[str('new_enough')].value, '21') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/21"]').contents(), ['21 days']) @@ -282,11 +288,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*90 days') def test_new_enough_30(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 7}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 7}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="30"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '30') - self.assertListEqual(['new_enough'], r.cookies.keys()) + self.assertEqual(r.cookies[str('new_enough')].value, '30') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/30"]').contents(), ['30 days']) @@ -296,11 +302,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*7 days') def test_new_enough_60(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '14', 'expires_soon' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '14', str('expires_soon') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="60"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '60') - self.assertListEqual(['new_enough'], r.cookies.keys()) + self.assertEqual(r.cookies[str('new_enough')].value, '60') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/60"]').contents(), ['60 days']) @@ -310,11 +316,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*14 days') def test_new_enough_90(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'new_enough' : '22', 'expires_soon' : 60}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('new_enough') : '22', str('expires_soon') : 60}) r = self.client.get(urlreverse("ietf.cookies.views.new_enough", kwargs=dict(days="90"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['new_enough'].value, '90') - self.assertListEqual(['new_enough'], r.cookies.keys()) + self.assertEqual(r.cookies[str('new_enough')].value, '90') + self.assertListEqual([str('new_enough')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/90"]').contents(), ['90 days']) @@ -324,11 +330,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*expires_soon.*60 days') def test_expires_soon(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '14', 'new_enough' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '14', str('new_enough') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon")) # no value: reset self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '') - self.assertListEqual(['expires_soon'], r.cookies.keys()) + self.assertEqual(r.cookies[str('expires_soon')].value, '') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -338,11 +344,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*14 days') def test_expires_soon_7(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'expires_soon' : '14', 'new_enough' : 21}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('expires_soon') : '14', str('new_enough') : 21}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="7"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '7') - self.assertListEqual(['expires_soon'], r.cookies.keys()) + self.assertEqual(r.cookies[str('expires_soon')].value, '7') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/21"]').contents(), ['21 days']) @@ -352,11 +358,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*21 days') def test_expires_soon_14(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'expires_soon' : '7', 'new_enough' : 99}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('expires_soon') : '7', str('new_enough') : 99}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="14"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '14') - self.assertListEqual(['expires_soon'], r.cookies.keys()) + self.assertEqual(r.cookies[str('expires_soon')].value, '14') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href^="/accounts/settings/new_enough/"]').contents(), []) @@ -366,11 +372,11 @@ class CookieTests(TestCase): # self.assertNotRegexpMatches(r.content, r'ietf-highlight-y.*new_enough') def test_expires_soon_21(self): - self.client.cookies = SimpleCookie({'full_draft': 'on', 'expires_soon' : '14', 'new_enough' : 90}) + self.client.cookies = SimpleCookie({str('full_draft'): 'on', str('expires_soon') : '14', str('new_enough') : 90}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="21"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '21') - self.assertListEqual(['expires_soon'], r.cookies.keys()) + self.assertEqual(r.cookies[str('expires_soon')].value, '21') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/on"]').contents(), ['On']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/90"]').contents(), ['90 days']) @@ -380,11 +386,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*90 days') def test_expires_soon_30(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '14', 'new_enough' : 7}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '14', str('new_enough') : 7}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="30"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '30') - self.assertListEqual(['expires_soon'], r.cookies.keys()) + self.assertEqual(r.cookies[str('expires_soon')].value, '30') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/7"]').contents(), ['7 days']) @@ -394,11 +400,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*7 days') def test_expires_soon_60(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '14', 'new_enough' : 14}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '14', str('new_enough') : 14}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="60"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '60') - self.assertListEqual(['expires_soon'], r.cookies.keys()) + self.assertEqual(r.cookies[str('expires_soon')].value, '60') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/14"]').contents(), ['14 days']) @@ -408,11 +414,11 @@ class CookieTests(TestCase): # self.assertRegexpMatches(r.content, r'ietf-highlight-y.*new_enough.*14 days') def test_expires_soon_90(self): - self.client.cookies = SimpleCookie({'full_draft': 'off', 'expires_soon' : '22', 'new_enough' : 60}) + self.client.cookies = SimpleCookie({str('full_draft'): 'off', str('expires_soon') : '22', str('new_enough') : 60}) r = self.client.get(urlreverse("ietf.cookies.views.expires_soon", kwargs=dict(days="90"))) self.assertEqual(r.status_code, 200) - self.assertEqual(r.cookies['expires_soon'].value, '90') - self.assertListEqual(['expires_soon'], r.cookies.keys()) + self.assertEqual(r.cookies[str('expires_soon')].value, '90') + self.assertListEqual([str('expires_soon')], list(r.cookies.keys())) q = PyQuery(r.content) self.assertEqual(q('div a.active[href="/accounts/settings/full_draft/off"]').contents(), ['Off']) self.assertEqual(q('div a.active[href="/accounts/settings/new_enough/60"]').contents(), ['60 days']) diff --git a/ietf/cookies/views.py b/ietf/cookies/views.py index 6bc5774d9..58268dc61 100644 --- a/ietf/cookies/views.py +++ b/ietf/cookies/views.py @@ -1,4 +1,8 @@ -# Copyright The IETF Trust 2010, All Rights Reserved +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from django.shortcuts import render from django.conf import settings @@ -10,7 +14,7 @@ def preferences(request, **kwargs): new_cookies = {} del_cookies = [] preferences['defaults'] = settings.USER_PREFERENCE_DEFAULTS - for key in settings.USER_PREFERENCE_DEFAULTS.keys(): + for key in list(settings.USER_PREFERENCE_DEFAULTS.keys()): if key in kwargs: if kwargs[key] == None: del_cookies += [key] diff --git a/ietf/dbtemplate/forms.py b/ietf/dbtemplate/forms.py index e26fbae65..987d61b60 100644 --- a/ietf/dbtemplate/forms.py +++ b/ietf/dbtemplate/forms.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django import forms from django.core.exceptions import ValidationError from django.template import Context @@ -20,7 +26,7 @@ class DBTemplateForm(forms.ModelForm): PlainTemplate(content).render(Context({})) else: raise ValidationError("Unexpected DBTemplate.type.slug: %s" % self.type.slug) - except Exception, e: + except Exception as e: raise ValidationError(e) return content diff --git a/ietf/dbtemplate/migrations/0001_initial.py b/ietf/dbtemplate/migrations/0001_initial.py index cc8a64a0f..dd4944906 100644 --- a/ietf/dbtemplate/migrations/0001_initial.py +++ b/ietf/dbtemplate/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/dbtemplate/migrations/0002_auto_20180220_1052.py b/ietf/dbtemplate/migrations/0002_auto_20180220_1052.py index e6926053c..84f026f7a 100644 --- a/ietf/dbtemplate/migrations/0002_auto_20180220_1052.py +++ b/ietf/dbtemplate/migrations/0002_auto_20180220_1052.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations import django.db.models.deletion diff --git a/ietf/dbtemplate/migrations/0003_adjust_review_templates.py b/ietf/dbtemplate/migrations/0003_adjust_review_templates.py index 5a5ffc48b..80699981c 100644 --- a/ietf/dbtemplate/migrations/0003_adjust_review_templates.py +++ b/ietf/dbtemplate/migrations/0003_adjust_review_templates.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-03-05 11:39 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/dbtemplate/migrations/0004_adjust_assignment_email_summary_templates.py b/ietf/dbtemplate/migrations/0004_adjust_assignment_email_summary_templates.py index 0c4463a74..9aa7493fe 100644 --- a/ietf/dbtemplate/migrations/0004_adjust_assignment_email_summary_templates.py +++ b/ietf/dbtemplate/migrations/0004_adjust_assignment_email_summary_templates.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-03-13 13:41 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/dbtemplate/models.py b/ietf/dbtemplate/models.py index 29efd07d5..0678236c5 100644 --- a/ietf/dbtemplate/models.py +++ b/ietf/dbtemplate/models.py @@ -1,11 +1,11 @@ # -*- coding: utf-8 -*- # Copyright The IETF Trust 2012-2019, All Rights Reserved -from __future__ import unicode_literals, print_function - +from __future__ import absolute_import, print_function, unicode_literals from django.db import models from django.core.exceptions import ValidationError from django.template import Context +from django.utils.encoding import python_2_unicode_compatible from ietf.group.models import Group from ietf.name.models import DBTemplateTypeName @@ -19,6 +19,7 @@ TEMPLATE_TYPES = ( ) +@python_2_unicode_compatible class DBTemplate(models.Model): path = models.CharField( max_length=255, unique=True, blank=False, null=False, ) title = models.CharField( max_length=255, blank=False, null=False, ) @@ -27,7 +28,7 @@ class DBTemplate(models.Model): content = models.TextField( blank=False, null=False, ) group = ForeignKey( Group, blank=True, null=True, ) - def __unicode__(self): + def __str__(self): return self.title def clean(self): @@ -41,6 +42,6 @@ class DBTemplate(models.Model): PlainTemplate(self.content).render(Context({})) else: raise ValidationError("Unexpected DBTemplate.type.slug: %s" % self.type.slug) - except Exception, e: + except Exception as e: raise ValidationError(e) diff --git a/ietf/dbtemplate/resources.py b/ietf/dbtemplate/resources.py index 73e11a9de..3db6e4e11 100644 --- a/ietf/dbtemplate/resources.py +++ b/ietf/dbtemplate/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2014-11-13 23:53 + + from ietf.api import ModelResource from tastypie.fields import ToOneField from tastypie.constants import ALL, ALL_WITH_RELATIONS diff --git a/ietf/dbtemplate/template.py b/ietf/dbtemplate/template.py index 32882ca2c..1b207458a 100644 --- a/ietf/dbtemplate/template.py +++ b/ietf/dbtemplate/template.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import os import string from docutils.core import publish_string @@ -7,7 +13,7 @@ import debug # pyflakes:ignore from django.template.loaders.base import Loader as BaseLoader from django.template.base import Template as DjangoTemplate, TemplateEncodingError from django.template.exceptions import TemplateDoesNotExist -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_text from ietf.dbtemplate.models import DBTemplate @@ -20,7 +26,7 @@ class Template(object): def __init__(self, template_string, origin=None, name=''): try: - template_string = smart_unicode(template_string) + template_string = smart_text(template_string) except UnicodeDecodeError: raise TemplateEncodingError("Templates can only be constructed from unicode or UTF-8 strings.") self.template_string = string.Template(template_string) @@ -54,7 +60,7 @@ class RSTTemplate(PlainTemplate): 'template': RST_TEMPLATE, 'halt_level': 2, }) - except SystemMessage, e: + except SystemMessage as e: e.message = e.message.replace(':', 'line ') args = list(e.args) args[0] = args[0].replace(':', 'line ') diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index 44b0a8224..73dfb22e3 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -1,10 +1,13 @@ # Copyright The IETF Trust 2010-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + from django.contrib import admin from django import forms -from models import (StateType, State, RelatedDocument, DocumentAuthor, Document, RelatedDocHistory, +from .models import (StateType, State, RelatedDocument, DocumentAuthor, Document, RelatedDocHistory, DocHistoryAuthor, DocHistory, DocAlias, DocReminder, DocEvent, NewRevisionDocEvent, StateDocEvent, ConsensusDocEvent, BallotType, BallotDocEvent, WriteupDocEvent, LastCallDocEvent, TelechatDocEvent, BallotPositionDocEvent, ReviewRequestDocEvent, InitialReviewDocEvent, @@ -155,7 +158,7 @@ admin.site.register(EditedAuthorsDocEvent, DocEventAdmin) class DeletedEventAdmin(admin.ModelAdmin): - list_display = [u'id', 'content_type', 'json', 'by', 'time'] + list_display = ['id', 'content_type', 'json', 'by', 'time'] list_filter = ['time'] raw_id_fields = ['content_type', 'by'] admin.site.register(DeletedEvent, DeletedEventAdmin) diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py index a99e62bb4..5e1add9cd 100644 --- a/ietf/doc/expire.py +++ b/ietf/doc/expire.py @@ -1,5 +1,10 @@ +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- # expiry of Internet Drafts + +from __future__ import absolute_import, print_function, unicode_literals + from django.conf import settings import datetime, os, shutil, glob, re @@ -92,7 +97,7 @@ def send_expire_warning_for_draft(doc): request = None if to or cc: send_mail(request, to, frm, - u"Expiration impending: %s" % doc.file_tag(), + "Expiration impending: %s" % doc.file_tag(), "doc/draft/expire_warning_email.txt", dict(doc=doc, state=state, @@ -112,7 +117,7 @@ def send_expire_notice_for_draft(doc): (to,cc) = gather_address_lists('doc_expired',doc=doc) send_mail(request, to, "I-D Expiring System ", - u"I-D was expired %s" % doc.file_tag(), + "I-D was expired %s" % doc.file_tag(), "doc/draft/id_expired_email.txt", dict(doc=doc, state=state, @@ -167,7 +172,7 @@ def clean_up_draft_files(): cut_off = datetime.date.today() pattern = os.path.join(settings.INTERNET_DRAFT_PATH, "draft-*.*") - filename_re = re.compile('^(.*)-(\d\d)$') + filename_re = re.compile(r'^(.*)-(\d\d)$') def splitext(fn): """ diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index ad46c66f2..af707eda3 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import debug # pyflakes:ignore import factory import datetime diff --git a/ietf/doc/feeds.py b/ietf/doc/feeds.py index 2c7f6791f..3fb50898d 100644 --- a/ietf/doc/feeds.py +++ b/ietf/doc/feeds.py @@ -1,6 +1,11 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import datetime +import six from django.contrib.syndication.views import Feed, FeedDoesNotExist from django.utils.feedgenerator import Atom1Feed, Rss201rev2Feed @@ -22,8 +27,8 @@ class DocumentChangesFeed(Feed): return "Changes for %s" % obj.display_name() def link(self, obj): - if obj is None: - raise FeedDoesNotExist + if obj is None: + raise FeedDoesNotExist return urlreverse('ietf.doc.views_doc.document_history', kwargs=dict(name=obj.canonical_name())) def subtitle(self, obj): @@ -32,19 +37,19 @@ class DocumentChangesFeed(Feed): def items(self, obj): events = obj.docevent_set.all().order_by("-time","-id") augment_events_with_revision(obj, events) - return events + return events def item_title(self, item): - return u"[%s] %s [rev. %s]" % (item.by, truncatewords(strip_tags(item.desc), 15), item.rev) + return "[%s] %s [rev. %s]" % (item.by, truncatewords(strip_tags(item.desc), 15), item.rev) def item_description(self, item): return truncatewords_html(format_textarea(item.desc), 20) def item_pubdate(self, item): - return item.time + return item.time def item_author_name(self, item): - return unicode(item.by) + return six.text_type(item.by) def item_link(self, item): return urlreverse('ietf.doc.views_doc.document_history', kwargs=dict(name=item.doc.canonical_name())) + "#history-%s" % item.pk @@ -62,12 +67,12 @@ class InLastCallFeed(Feed): d.lc_event = d.latest_event(LastCallDocEvent, type="sent_last_call") docs = [d for d in docs if d.lc_event] - docs.sort(key=lambda d: d.lc_event.expires) + docs.sort(key=lambda d: d.lc_event.expires) - return docs + return docs def item_title(self, item): - return u"%s (%s - %s)" % (item.name, + return "%s (%s - %s)" % (item.name, datefilter(item.lc_event.time, "F j"), datefilter(item.lc_event.expires, "F j, Y")) diff --git a/ietf/doc/fields.py b/ietf/doc/fields.py index a797917dc..9c358949f 100644 --- a/ietf/doc/fields.py +++ b/ietf/doc/fields.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import json +import six from django.utils.html import escape from django import forms @@ -53,9 +57,9 @@ class SearchableDocumentsField(forms.CharField): def prepare_value(self, value): if not value: value = "" - if isinstance(value, (int, long)): + if isinstance(value, six.integer_types): value = str(value) - if isinstance(value, basestring): + if isinstance(value, six.string_types): items = self.parse_select2_value(value) # accept both names and pks here names = [ i for i in items if not i.isdigit() ] @@ -79,7 +83,7 @@ class SearchableDocumentsField(forms.CharField): "model_name": self.model.__name__.lower() }) - return u",".join(unicode(o.pk) for o in value) + return ",".join(six.text_type(o.pk) for o in value) def clean(self, value): value = super(SearchableDocumentsField, self).clean(value) @@ -90,10 +94,10 @@ class SearchableDocumentsField(forms.CharField): found_pks = [ str(o.pk) for o in objs ] failed_pks = [ x for x in pks if x not in found_pks ] if failed_pks: - raise forms.ValidationError(u"Could not recognize the following documents: {names}. You can only input documents already registered in the Datatracker.".format(names=", ".join(failed_pks))) + raise forms.ValidationError("Could not recognize the following documents: {names}. You can only input documents already registered in the Datatracker.".format(names=", ".join(failed_pks))) if self.max_entries != None and len(objs) > self.max_entries: - raise forms.ValidationError(u"You can select at most %s entries." % self.max_entries) + raise forms.ValidationError("You can select at most %s entries." % self.max_entries) return objs diff --git a/ietf/doc/forms.py b/ietf/doc/forms.py index c007d6998..1dd23f474 100644 --- a/ietf/doc/forms.py +++ b/ietf/doc/forms.py @@ -1,6 +1,9 @@ -# Copyright The IETF Trust 2017, All Rights Reserved +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals -from __future__ import unicode_literals import datetime import debug #pyflakes:ignore diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py index a4a8fa6d7..7ff511903 100644 --- a/ietf/doc/mails.py +++ b/ietf/doc/mails.py @@ -1,15 +1,23 @@ +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- # generation of mails -import textwrap, datetime + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import six +import textwrap from django.template.loader import render_to_string from django.utils.html import strip_tags from django.conf import settings from django.urls import reverse as urlreverse +from django.utils.encoding import force_str, force_text import debug # pyflakes:ignore -from ietf.utils.mail import send_mail, send_mail_text +from ietf.utils.mail import send_mail, send_mail_text, get_payload from ietf.ipr.utils import iprs_from_docs, related_docs from ietf.doc.models import WriteupDocEvent, LastCallDocEvent, DocAlias, ConsensusDocEvent from ietf.doc.utils import needed_ballot_positions @@ -32,15 +40,15 @@ def email_state_changed(request, doc, text, mailtrigger_id=None): cc=cc) def email_ad_approved_doc(request, doc, text): - to = "iesg@iesg.org" - bcc = "iesg-secretary@ietf.org" - frm = request.user.person.formatted_email() - send_mail(request, to, frm, - "Approved: %s" % doc.filename_with_rev(), - "doc/mail/ad_approval_email.txt", - dict(text=text, - docname=doc.filename_with_rev()), - bcc=bcc) + to = "iesg@iesg.org" + bcc = "iesg-secretary@ietf.org" + frm = request.user.person.formatted_email() + send_mail(request, to, frm, + "Approved: %s" % doc.filename_with_rev(), + "doc/mail/ad_approval_email.txt", + dict(text=text, + docname=doc.filename_with_rev()), + bcc=bcc) def email_stream_changed(request, doc, old_stream, new_stream, text=""): """Email the change text to the notify group and to the stream chairs""" @@ -55,7 +63,7 @@ def email_stream_changed(request, doc, old_stream, new_stream, text=""): return if not text: - text = u"Stream changed to %s from %s" % (new_stream, old_stream) + text = "Stream changed to %s from %s" % (new_stream, old_stream) text = strip_tags(text) send_mail(request, to, None, @@ -119,8 +127,8 @@ def generate_ballot_writeup(request, doc): e.by = request.user.person e.doc = doc e.rev = doc.rev - e.desc = u"Ballot writeup was generated" - e.text = unicode(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana})) + e.desc = "Ballot writeup was generated" + e.text = force_text(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana})) # caller is responsible for saving, if necessary return e @@ -131,8 +139,8 @@ def generate_ballot_rfceditornote(request, doc): e.by = request.user.person e.doc = doc e.rev = doc.rev - e.desc = u"RFC Editor Note for ballot was generated" - e.text = unicode(render_to_string("doc/mail/ballot_rfceditornote.txt")) + e.desc = "RFC Editor Note for ballot was generated" + e.text = force_text(render_to_string("doc/mail/ballot_rfceditornote.txt")) e.save() return e @@ -176,8 +184,8 @@ def generate_last_call_announcement(request, doc): e.by = request.user.person e.doc = doc e.rev = doc.rev - e.desc = u"Last call announcement was generated" - e.text = unicode(mail) + e.desc = "Last call announcement was generated" + e.text = force_text(mail) # caller is responsible for saving, if necessary return e @@ -196,8 +204,8 @@ def generate_approval_mail(request, doc): e.by = request.user.person e.doc = doc e.rev = doc.rev - e.desc = u"Ballot approval text was generated" - e.text = unicode(mail) + e.desc = "Ballot approval text was generated" + e.text = force_text(mail) # caller is responsible for saving, if necessary return e @@ -280,7 +288,7 @@ def generate_publication_request(request, doc): approving_body = "IRSG" consensus_body = doc.group.acronym.upper() else: - approving_body = str(doc.stream) + approving_body = six.text_type(doc.stream) consensus_body = approving_body e = doc.latest_event(WriteupDocEvent, type="changed_rfc_editor_note_text") @@ -374,7 +382,7 @@ def generate_issue_ballot_mail(request, doc, ballot): last_call_has_expired=last_call_has_expired, needed_ballot_positions= needed_ballot_positions(doc, - doc.active_ballot().active_ad_positions().values() + list(doc.active_ballot().active_ad_positions().values()) ), ) ) @@ -382,7 +390,7 @@ def generate_issue_ballot_mail(request, doc, ballot): def email_iana(request, doc, to, msg, cc=None): # fix up message and send it with extra info on doc in headers import email - parsed_msg = email.message_from_string(msg.encode("utf-8")) + parsed_msg = email.message_from_string(force_str(msg)) parsed_msg.set_charset('UTF-8') extra = extra_automation_headers(doc) @@ -390,7 +398,7 @@ def email_iana(request, doc, to, msg, cc=None): send_mail_text(request, to, parsed_msg["From"], parsed_msg["Subject"], - parsed_msg.get_payload().decode(str(parsed_msg.get_charset())), + get_payload(parsed_msg), extra=extra, cc=cc) @@ -451,7 +459,7 @@ def email_adopted(request, doc, prev_state, new_state, by, comment=""): state_type = (prev_state or new_state).type send_mail(request, to, settings.DEFAULT_FROM_EMAIL, - u'The %s %s has placed %s in state "%s"' % + 'The %s %s has placed %s in state "%s"' % (doc.group.acronym.upper(),doc.group.type_id.upper(), doc.name, new_state or "None"), 'doc/mail/doc_adopted_email.txt', dict(doc=doc, @@ -469,7 +477,7 @@ def email_stream_state_changed(request, doc, prev_state, new_state, by, comment= state_type = (prev_state or new_state).type send_mail(request, to, settings.DEFAULT_FROM_EMAIL, - u"%s changed for %s" % (state_type.label, doc.name), + "%s changed for %s" % (state_type.label, doc.name), 'doc/mail/stream_state_changed_email.txt', dict(doc=doc, url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(), @@ -485,7 +493,7 @@ def email_stream_tags_changed(request, doc, added_tags, removed_tags, by, commen (to, cc) = gather_address_lists('doc_stream_state_edited',doc=doc) send_mail(request, to, settings.DEFAULT_FROM_EMAIL, - u"Tags changed for %s" % doc.name, + "Tags changed for %s" % doc.name, 'doc/mail/stream_tags_changed_email.txt', dict(doc=doc, url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(), diff --git a/ietf/doc/management/commands/generate_draft_bibxml_files.py b/ietf/doc/management/commands/generate_draft_bibxml_files.py index 2057cf67f..78a052927 100644 --- a/ietf/doc/management/commands/generate_draft_bibxml_files.py +++ b/ietf/doc/management/commands/generate_draft_bibxml_files.py @@ -1,3 +1,10 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import sys import os @@ -9,19 +16,19 @@ from ietf.doc.models import Document def write(fn, new): try: - f = open(fn) + f = io.open(fn) old = f.read().decode('utf-8') f.close except IOError: old = "" if old.strip() != new.strip(): sys.stdout.write(os.path.basename(fn)+'\n') - f = open(fn, "wb") + f = io.open(fn, "wb") f.write(new.encode('utf-8')) f.close() class Command(BaseCommand): - help = (u'Generate draft bibxml files, for xml2rfc references') + help = ('Generate draft bibxml files, for xml2rfc references') def handle(self, *args, **options): documents = Document.objects.filter(type__slug='draft') diff --git a/ietf/doc/migrations/0001_initial.py b/ietf/doc/migrations/0001_initial.py index ba3edeef3..6ec0f547a 100644 --- a/ietf/doc/migrations/0001_initial.py +++ b/ietf/doc/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import datetime import django.core.validators diff --git a/ietf/doc/migrations/0002_auto_20180220_1052.py b/ietf/doc/migrations/0002_auto_20180220_1052.py index 6cce3e312..78b706cf5 100644 --- a/ietf/doc/migrations/0002_auto_20180220_1052.py +++ b/ietf/doc/migrations/0002_auto_20180220_1052.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/doc/migrations/0003_auto_20180401_1231.py b/ietf/doc/migrations/0003_auto_20180401_1231.py index f277b40df..f6f6144bc 100644 --- a/ietf/doc/migrations/0003_auto_20180401_1231.py +++ b/ietf/doc/migrations/0003_auto_20180401_1231.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.11 on 2018-04-01 12:31 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models @@ -14,6 +17,6 @@ class Migration(migrations.Migration): operations = [ migrations.AddIndex( model_name='docevent', - index=models.Index(fields=[b'type', b'doc'], name='doc_doceven_type_43e53e_idx'), + index=models.Index(fields=['type', 'doc'], name='doc_doceven_type_43e53e_idx'), ), ] diff --git a/ietf/doc/migrations/0004_add_draft_stream_replaced_states.py b/ietf/doc/migrations/0004_add_draft_stream_replaced_states.py index 155596f50..5ab95997f 100644 --- a/ietf/doc/migrations/0004_add_draft_stream_replaced_states.py +++ b/ietf/doc/migrations/0004_add_draft_stream_replaced_states.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.13 on 2018-05-03 11:50 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/doc/migrations/0005_fix_replaced_iab_irtf_stream_docs.py b/ietf/doc/migrations/0005_fix_replaced_iab_irtf_stream_docs.py index 3b339094f..da03c5fa1 100644 --- a/ietf/doc/migrations/0005_fix_replaced_iab_irtf_stream_docs.py +++ b/ietf/doc/migrations/0005_fix_replaced_iab_irtf_stream_docs.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.13 on 2018-05-03 12:16 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/doc/migrations/0006_ballotpositiondocevent_send_email.py b/ietf/doc/migrations/0006_ballotpositiondocevent_send_email.py index fbae0042a..44ea6295f 100644 --- a/ietf/doc/migrations/0006_ballotpositiondocevent_send_email.py +++ b/ietf/doc/migrations/0006_ballotpositiondocevent_send_email.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.15 on 2018-10-03 06:39 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/doc/migrations/0007_idexists.py b/ietf/doc/migrations/0007_idexists.py index edba831de..3731661ae 100644 --- a/ietf/doc/migrations/0007_idexists.py +++ b/ietf/doc/migrations/0007_idexists.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-04 10:56 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from tqdm import tqdm diff --git a/ietf/doc/migrations/0008_add_uploaded_filename.py b/ietf/doc/migrations/0008_add_uploaded_filename.py index 769139f42..82e528b8d 100644 --- a/ietf/doc/migrations/0008_add_uploaded_filename.py +++ b/ietf/doc/migrations/0008_add_uploaded_filename.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.17 on 2018-12-28 13:11 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/doc/migrations/0009_move_non_url_externalurls_to_uploaded_filename.py b/ietf/doc/migrations/0009_move_non_url_externalurls_to_uploaded_filename.py index f8a6fd083..95ef47d11 100644 --- a/ietf/doc/migrations/0009_move_non_url_externalurls_to_uploaded_filename.py +++ b/ietf/doc/migrations/0009_move_non_url_externalurls_to_uploaded_filename.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.17 on 2018-12-28 13:33 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations from django.db.models import F diff --git a/ietf/doc/migrations/0010_auto_20190225_1302.py b/ietf/doc/migrations/0010_auto_20190225_1302.py index 74c3b3bd4..1d581c59b 100644 --- a/ietf/doc/migrations/0010_auto_20190225_1302.py +++ b/ietf/doc/migrations/0010_auto_20190225_1302.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-02-25 13:02 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/doc/migrations/0011_reviewassignmentdocevent.py b/ietf/doc/migrations/0011_reviewassignmentdocevent.py index 63df9a962..0482dd967 100644 --- a/ietf/doc/migrations/0011_reviewassignmentdocevent.py +++ b/ietf/doc/migrations/0011_reviewassignmentdocevent.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.18 on 2019-01-11 11:22 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/doc/migrations/0012_add_event_type_closed_review_assignment.py b/ietf/doc/migrations/0012_add_event_type_closed_review_assignment.py index 7b1e55970..813943da5 100644 --- a/ietf/doc/migrations/0012_add_event_type_closed_review_assignment.py +++ b/ietf/doc/migrations/0012_add_event_type_closed_review_assignment.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-01 04:43 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/doc/migrations/0013_add_document_docalias_id.py b/ietf/doc/migrations/0013_add_document_docalias_id.py index e86c4270b..09d3f1a87 100644 --- a/ietf/doc/migrations/0013_add_document_docalias_id.py +++ b/ietf/doc/migrations/0013_add_document_docalias_id.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-08 08:41 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/doc/migrations/0014_set_document_docalias_id.py b/ietf/doc/migrations/0014_set_document_docalias_id.py index 06ff9b89e..785b82bf2 100644 --- a/ietf/doc/migrations/0014_set_document_docalias_id.py +++ b/ietf/doc/migrations/0014_set_document_docalias_id.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-08 08:42 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys diff --git a/ietf/doc/migrations/0015_1_add_fk_to_document_id.py b/ietf/doc/migrations/0015_1_add_fk_to_document_id.py index a60d9864f..1efabe53d 100644 --- a/ietf/doc/migrations/0015_1_add_fk_to_document_id.py +++ b/ietf/doc/migrations/0015_1_add_fk_to_document_id.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-08 10:29 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import django.core.validators from django.db import migrations, models diff --git a/ietf/doc/migrations/0015_2_add_doc_document_m2m_fields.py b/ietf/doc/migrations/0015_2_add_doc_document_m2m_fields.py index 5e62dbf26..2bd8a7ae1 100644 --- a/ietf/doc/migrations/0015_2_add_doc_document_m2m_fields.py +++ b/ietf/doc/migrations/0015_2_add_doc_document_m2m_fields.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-28 12:42 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys, time diff --git a/ietf/doc/migrations/0016_set_document_docalias_fk.py b/ietf/doc/migrations/0016_set_document_docalias_fk.py index ac1376b76..e931f1306 100644 --- a/ietf/doc/migrations/0016_set_document_docalias_fk.py +++ b/ietf/doc/migrations/0016_set_document_docalias_fk.py @@ -1,8 +1,11 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-08 14:04 -from __future__ import unicode_literals + +from __future__ import absolute_import, print_function, unicode_literals + +import six import sys from tqdm import tqdm @@ -15,7 +18,7 @@ def forward(apps, schema_editor): n = getattr(o, a+'_id') if n: i = nameid[n] - if not isinstance(i, (int, long)): + if not isinstance(i, six.integer_types): raise ValueError("Inappropriate value: %s: nameid[%s]: %s" % (o.__class__.__name__, n, i)) if getattr(o, a+'2_id') != i: setattr(o, a+'2_id', i) @@ -44,7 +47,7 @@ def forward(apps, schema_editor): # Document id fixup ------------------------------------------------------------ objs = Document.objects.in_bulk() - nameid = { o.name: o.id for id, o in objs.iteritems() } + nameid = { o.name: o.id for id, o in objs.items() } sys.stderr.write('\n') @@ -78,7 +81,7 @@ def forward(apps, schema_editor): sys.stderr.write('\n') objs = DocAlias.objects.in_bulk() - nameid = { o.name: o.id for id, o in objs.iteritems() } + nameid = { o.name: o.id for id, o in objs.items() } sys.stderr.write('Setting DocAlias FKs:\n') diff --git a/ietf/doc/migrations/0017_make_document_id_primary_key.py b/ietf/doc/migrations/0017_make_document_id_primary_key.py index 851b2dbeb..3bbaa8176 100644 --- a/ietf/doc/migrations/0017_make_document_id_primary_key.py +++ b/ietf/doc/migrations/0017_make_document_id_primary_key.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-09 05:46 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/doc/migrations/0018_remove_old_document_field.py b/ietf/doc/migrations/0018_remove_old_document_field.py index 146317da2..19a8a4065 100644 --- a/ietf/doc/migrations/0018_remove_old_document_field.py +++ b/ietf/doc/migrations/0018_remove_old_document_field.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-20 09:53 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion @@ -57,7 +59,7 @@ class Migration(migrations.Migration): ), migrations.AddIndex( model_name='docevent', - index=models.Index(fields=[b'type', b'doc2'], name='doc_doceven_type_ac7748_idx'), + index=models.Index(fields=['type', 'doc2'], name='doc_doceven_type_ac7748_idx'), ), # The following 9 migrations are related to the m2m fields on Document # Remove the intermediary model field pointing to Document.name diff --git a/ietf/doc/migrations/0019_rename_field_document2.py b/ietf/doc/migrations/0019_rename_field_document2.py index 5b4504d01..1d7cc0b21 100644 --- a/ietf/doc/migrations/0019_rename_field_document2.py +++ b/ietf/doc/migrations/0019_rename_field_document2.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-21 05:31 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models @@ -63,7 +65,7 @@ class Migration(migrations.Migration): ), migrations.AddIndex( model_name='docevent', - index=models.Index(fields=[b'type', b'doc'], name='doc_doceven_type_43e53e_idx'), + index=models.Index(fields=['type', 'doc'], name='doc_doceven_type_43e53e_idx'), ), # Add back the m2m field we removed in 0018_... migrations.AddField( diff --git a/ietf/doc/migrations/0020_copy_docs_m2m_table.py b/ietf/doc/migrations/0020_copy_docs_m2m_table.py index 4a6555c71..1bc395c33 100644 --- a/ietf/doc/migrations/0020_copy_docs_m2m_table.py +++ b/ietf/doc/migrations/0020_copy_docs_m2m_table.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-21 05:31 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys, time diff --git a/ietf/doc/migrations/0021_remove_docs2_m2m.py b/ietf/doc/migrations/0021_remove_docs2_m2m.py index c11464cdc..469523732 100644 --- a/ietf/doc/migrations/0021_remove_docs2_m2m.py +++ b/ietf/doc/migrations/0021_remove_docs2_m2m.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-30 03:36 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/doc/migrations/0022_document_primary_key_cleanup.py b/ietf/doc/migrations/0022_document_primary_key_cleanup.py index 23d74f13a..5b666907c 100644 --- a/ietf/doc/migrations/0022_document_primary_key_cleanup.py +++ b/ietf/doc/migrations/0022_document_primary_key_cleanup.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-06-10 03:47 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/doc/migrations/0023_one_to_many_docalias.py b/ietf/doc/migrations/0023_one_to_many_docalias.py index 90b88bcd8..cf7f54756 100644 --- a/ietf/doc/migrations/0023_one_to_many_docalias.py +++ b/ietf/doc/migrations/0023_one_to_many_docalias.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-06-10 04:36 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys diff --git a/ietf/doc/models.py b/ietf/doc/models.py index fdc015bc2..eaa5d9c91 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1,8 +1,10 @@ -# Copyright The IETF Trust 2007-2019, All Rights Reserved +# Copyright The IETF Trust 2010-2019, All Rights Reserved # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals import datetime import logging +import io import os import rfc2html import six @@ -15,6 +17,7 @@ from django.core.validators import URLValidator, RegexValidator from django.urls import reverse as urlreverse from django.contrib.contenttypes.models import ContentType from django.conf import settings +from django.utils.encoding import python_2_unicode_compatible, force_text from django.utils.html import mark_safe import debug # pyflakes:ignore @@ -34,11 +37,12 @@ from ietf.utils.models import ForeignKey logger = logging.getLogger('django') +@python_2_unicode_compatible class StateType(models.Model): slug = models.CharField(primary_key=True, max_length=30) # draft, draft-iesg, charter, ... label = models.CharField(max_length=255, help_text="Label that should be used (e.g. in admin) for state drop-down for this type of state") # State, IESG state, WG state, ... - def __unicode__(self): + def __str__(self): return self.slug @checks.register('db-consistency') @@ -55,6 +59,7 @@ def check_statetype_slugs(app_configs, **kwargs): )) return errors +@python_2_unicode_compatible class State(models.Model): type = ForeignKey(StateType) slug = models.SlugField() @@ -65,7 +70,7 @@ class State(models.Model): next_states = models.ManyToManyField('State', related_name="previous_states", blank=True) - def __unicode__(self): + def __str__(self): return self.name class Meta: @@ -372,7 +377,7 @@ class DocumentInfo(models.Model): return self.rfc_number() def author_list(self): - return u", ".join(author.email_id for author in self.documentauthor_set.all() if author.email_id) + return ", ".join(author.email_id for author in self.documentauthor_set.all() if author.email_id) def authors(self): return [ a.person for a in self.documentauthor_set.all() ] @@ -407,7 +412,7 @@ class DocumentInfo(models.Model): def relations_that(self, relationship): """Return the related-document objects that describe a given relationship targeting self.""" - if isinstance(relationship, str): + if isinstance(relationship, six.string_types): relationship = ( relationship, ) if not isinstance(relationship, tuple): raise TypeError("Expected a string or tuple, received %s" % type(relationship)) @@ -482,7 +487,7 @@ class DocumentInfo(models.Model): if ext != '.txt' and os.path.exists(txtpath): path = txtpath try: - with open(path, 'rb') as file: + with io.open(path, 'rb') as file: raw = file.read() except IOError: return None @@ -524,13 +529,14 @@ class DocumentInfo(models.Model): STATUSCHANGE_RELATIONS = ('tops','tois','tohist','toinf','tobcp','toexp') +@python_2_unicode_compatible class RelatedDocument(models.Model): source = ForeignKey('Document') target = ForeignKey('DocAlias') relationship = ForeignKey(DocRelationshipName) def action(self): return self.relationship.name - def __unicode__(self): + def __str__(self): return u"%s %s %s" % (self.source.name, self.relationship.name.lower(), self.target.name) def is_downref(self): @@ -597,10 +603,11 @@ class DocumentAuthorInfo(models.Model): abstract = True ordering = ["document", "order"] +@python_2_unicode_compatible class DocumentAuthor(DocumentAuthorInfo): document = ForeignKey('Document') - def __unicode__(self): + def __str__(self): return u"%s %s (%s)" % (self.document.name, self.person, self.order) @@ -610,10 +617,11 @@ validate_docname = RegexValidator( 'invalid' ) +@python_2_unicode_compatible class Document(DocumentInfo): name = models.CharField(max_length=255, validators=[validate_docname,], unique=True) # immutable - def __unicode__(self): + def __str__(self): return self.name def get_absolute_url(self): @@ -641,10 +649,10 @@ class Document(DocumentInfo): return self._cached_absolute_url def file_tag(self): - return u"<%s>" % self.filename_with_rev() + return "<%s>" % self.filename_with_rev() def filename_with_rev(self): - return u"%s-%s.txt" % (self.name, self.rev) + return "%s-%s.txt" % (self.name, self.rev) def latest_event(self, *args, **filter_args): """Get latest event of optional Python type and with filter @@ -845,21 +853,24 @@ class DocumentURL(models.Model): desc = models.CharField(max_length=255, default='', blank=True) url = models.URLField(max_length=2083) # 2083 is the legal max for URLs +@python_2_unicode_compatible class RelatedDocHistory(models.Model): source = ForeignKey('DocHistory') target = ForeignKey('DocAlias', related_name="reversely_related_document_history_set") relationship = ForeignKey(DocRelationshipName) - def __unicode__(self): + def __str__(self): return u"%s %s %s" % (self.source.doc.name, self.relationship.name.lower(), self.target.name) +@python_2_unicode_compatible class DocHistoryAuthor(DocumentAuthorInfo): # use same naming convention as non-history version to make it a bit # easier to write generic code document = ForeignKey('DocHistory', related_name="documentauthor_set") - def __unicode__(self): + def __str__(self): return u"%s %s (%s)" % (self.document.doc.name, self.person, self.order) +@python_2_unicode_compatible class DocHistory(DocumentInfo): doc = ForeignKey(Document, related_name="history_set") # the name here is used to capture the canonical name at the time @@ -868,8 +879,8 @@ class DocHistory(DocumentInfo): # property name = models.CharField(max_length=255) - def __unicode__(self): - return unicode(self.doc.name) + def __str__(self): + return force_text(self.doc.name) def canonical_name(self): if hasattr(self, '_canonical_name'): @@ -904,6 +915,7 @@ class DocHistory(DocumentInfo): verbose_name = "document history" verbose_name_plural = "document histories" +@python_2_unicode_compatible class DocAlias(models.Model): """This is used for documents that may appear under multiple names, and in particular for RFCs, which for continuity still keep the @@ -917,8 +929,8 @@ class DocAlias(models.Model): def document(self): return self.docs.first() - def __unicode__(self): - return "%s-->%s" % (self.name, ','.join([unicode(d.name) for d in self.docs.all() if isinstance(d, Document) ])) + def __str__(self): + return u"%s-->%s" % (self.name, ','.join([force_text(d.name) for d in self.docs.all() if isinstance(d, Document) ])) document_link = admin_link("document") class Meta: verbose_name = "document alias" @@ -1007,6 +1019,7 @@ EVENT_TYPES = [ ("downref_approved", "Downref approved"), ] +@python_2_unicode_compatible class DocEvent(models.Model): """An occurrence for a document, used for tracking who, when and what.""" time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened", db_index=True) @@ -1023,7 +1036,7 @@ class DocEvent(models.Model): def get_dochistory(self): return DocHistory.objects.filter(time__lte=self.time,doc__name=self.doc.name).order_by('-time', '-pk').first() - def __unicode__(self): + def __str__(self): return u"%s %s by %s at %s" % (self.doc.name, self.get_type_display().lower(), self.by.plain_name(), self.time) def save(self, *args, **kwargs): @@ -1047,6 +1060,7 @@ class ConsensusDocEvent(DocEvent): consensus = models.NullBooleanField(default=None) # IESG events +@python_2_unicode_compatible class BallotType(models.Model): doc_type = ForeignKey(DocTypeName, blank=True, null=True) slug = models.SlugField() @@ -1056,7 +1070,7 @@ class BallotType(models.Model): order = models.IntegerField(default=0) positions = models.ManyToManyField(BallotPositionName, blank=True) - def __unicode__(self): + def __str__(self): return u"%s: %s" % (self.name, self.doc_type.name) class Meta: @@ -1177,13 +1191,14 @@ class SubmissionDocEvent(DocEvent): submission = ForeignKey(ietf.submit.models.Submission) # dumping store for removed events +@python_2_unicode_compatible class DeletedEvent(models.Model): content_type = ForeignKey(ContentType) json = models.TextField(help_text="Deleted object in JSON format, with attribute names chosen to be suitable for passing into the relevant create method.") by = ForeignKey(Person) time = models.DateTimeField(default=datetime.datetime.now) - def __unicode__(self): + def __str__(self): return u"%s by %s %s" % (self.content_type, self.by, self.time) class EditedAuthorsDocEvent(DocEvent): diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index 08f5489fc..b80d2b409 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -1,6 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2015-10-19 12:29 PDT + + from ietf.api import ModelResource from ietf.api import ToOneField from tastypie.fields import ToManyField, CharField diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 66f0e82ac..53428c359 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -1,10 +1,13 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import bleach import datetime import re - -import types +import six from email.utils import parseaddr @@ -14,6 +17,7 @@ from django.utils.html import escape from django.template.defaultfilters import truncatewords_html, linebreaksbr, stringfilter, striptags from django.utils.safestring import mark_safe, SafeData from django.utils.html import strip_tags +from django.utils.encoding import force_text import debug # pyflakes:ignore @@ -47,24 +51,24 @@ def parse_email_list(value): Splitting a string of email addresses should return a list: - >>> unicode(parse_email_list('joe@example.org, fred@example.com')) - u'joe@example.org, fred@example.com' + >>> six.ensure_str(parse_email_list('joe@example.org, fred@example.com')) + 'joe@example.org, fred@example.com' Parsing a non-string should return the input value, rather than fail: - >>> parse_email_list(['joe@example.org', 'fred@example.com']) + >>> [ six.ensure_str(e) for e in parse_email_list(['joe@example.org', 'fred@example.com']) ] ['joe@example.org', 'fred@example.com'] Null input values should pass through silently: - >>> parse_email_list('') + >>> six.ensure_str(parse_email_list('')) '' >>> parse_email_list(None) """ - if value and isinstance(value, (types.StringType,types.UnicodeType)): # testing for 'value' being true isn't necessary; it's a fast-out route + if value and isinstance(value, (six.binary_type, six.text_type)): # testing for 'value' being true isn't necessary; it's a fast-out route addrs = re.split(", ?", value) ret = [] for addr in addrs: @@ -88,7 +92,7 @@ def strip_email(value): @register.filter(name='fix_angle_quotes') def fix_angle_quotes(value): if "<" in value: - value = re.sub("<([\w\-\.]+@[\w\-\.]+)>", "<\1>", value) + value = re.sub(r"<([\w\-\.]+@[\w\-\.]+)>", "<\1>", value) return value # there's an "ahref -> a href" in GEN_UTIL @@ -98,7 +102,7 @@ def make_one_per_line(value): """ Turn a comma-separated list into a carriage-return-seperated list. - >>> make_one_per_line("a, b, c") + >>> six.ensure_str(make_one_per_line("a, b, c")) 'a\\nb\\nc' Pass through non-strings: @@ -109,7 +113,7 @@ def make_one_per_line(value): >>> make_one_per_line(None) """ - if value and isinstance(value, (types.StringType,types.UnicodeType)): + if value and isinstance(value, (six.binary_type, six.text_type)): return re.sub(", ?", "\n", value) else: return value @@ -145,9 +149,9 @@ def sanitize(value): @register.filter(name='bracket') def square_brackets(value): """Adds square brackets around text.""" - if isinstance(value, (types.StringType,types.UnicodeType)): - if value == "": - value = " " + if isinstance(value, (six.binary_type, six.text_type)): + if value == "": + value = " " return "[ %s ]" % value elif value > 0: return "[ X ]" @@ -195,7 +199,7 @@ def rfcnospace(string): @register.filter def prettystdname(string): from ietf.doc.utils import prettify_std_name - return prettify_std_name(unicode(string or "")) + return prettify_std_name(force_text(string or "")) @register.filter(name='rfcurl') def rfclink(string): @@ -213,13 +217,13 @@ def urlize_ietf_docs(string, autoescape=None): """ if autoescape and not isinstance(string, SafeData): string = escape(string) - string = re.sub("(?)(RFC ?)0{0,3}(\d+)", "\\1\\2", string) - string = re.sub("(?)(BCP ?)0{0,3}(\d+)", "\\1\\2", string) - string = re.sub("(?)(STD ?)0{0,3}(\d+)", "\\1\\2", string) - string = re.sub("(?)(FYI ?)0{0,3}(\d+)", "\\1\\2", string) - string = re.sub("(?)(draft-[-0-9a-zA-Z._+]+)", "\\1", string) - string = re.sub("(?)(conflict-review-[-0-9a-zA-Z._+]+)", "\\1", string) - string = re.sub("(?)(status-change-[-0-9a-zA-Z._+]+)", "\\1", string) + string = re.sub(r"(?)(RFC ?)0{0,3}(\d+)", "\\1\\2", string) + string = re.sub(r"(?)(BCP ?)0{0,3}(\d+)", "\\1\\2", string) + string = re.sub(r"(?)(STD ?)0{0,3}(\d+)", "\\1\\2", string) + string = re.sub(r"(?)(FYI ?)0{0,3}(\d+)", "\\1\\2", string) + string = re.sub(r"(?)(draft-[-0-9a-zA-Z._+]+)", "\\1", string) + string = re.sub(r"(?)(conflict-review-[-0-9a-zA-Z._+]+)", "\\1", string) + string = re.sub(r"(?)(status-change-[-0-9a-zA-Z._+]+)", "\\1", string) return mark_safe(string) urlize_ietf_docs = stringfilter(urlize_ietf_docs) @@ -338,7 +342,7 @@ def expires_soon(x,request): @register.filter(name='startswith') def startswith(x, y): - return unicode(x).startswith(y) + return six.text_type(x).startswith(y) @register.filter def has_role(user, role_names): @@ -377,14 +381,14 @@ def format_snippet(text, trunc_words=25): full = keep_spacing(collapsebr(linebreaksbr(mark_safe(sanitize_fragment(text))))) snippet = truncatewords_html(full, trunc_words) if snippet != full: - return mark_safe(u'
%s
' % (snippet, full)) + return mark_safe('
%s
' % (snippet, full)) return full @register.simple_tag def doc_edit_button(url_name, *args, **kwargs): """Given URL name/args/kwargs, looks up the URL just like "url" tag and returns a properly formatted button for the document material tables.""" from django.urls import reverse as urlreverse - return mark_safe(u'Edit' % (urlreverse(url_name, args=args, kwargs=kwargs))) + return mark_safe('Edit' % (urlreverse(url_name, args=args, kwargs=kwargs))) @register.filter def textify(text): @@ -419,7 +423,7 @@ if __name__ == "__main__": _test() @register.filter -def plural(text, seq, arg=u's'): +def plural(text, seq, arg='s'): "Similar to pluralize, but looks at the text, too" from django.template.defaultfilters import pluralize if text.endswith('s'): @@ -461,8 +465,8 @@ def capfirst_allcaps(text): """Like capfirst, except it doesn't lowercase words in ALL CAPS.""" result = text i = False - for token in re.split("(\W+)", striptags(text)): - if not re.match("^[A-Z]+$", token): + for token in re.split(r"(\W+)", striptags(text)): + if not re.match(r"^[A-Z]+$", token): if not i: result = result.replace(token, token.capitalize()) i = True @@ -474,8 +478,8 @@ def capfirst_allcaps(text): def lower_allcaps(text): """Like lower, except it doesn't lowercase words in ALL CAPS.""" result = text - for token in re.split("(\W+)", striptags(text)): - if not re.match("^[A-Z]+$", token): + for token in re.split(r"(\W+)", striptags(text)): + if not re.match(r"^[A-Z]+$", token): result = result.replace(token, token.lower()) return result @@ -505,9 +509,9 @@ def nbsp(value): @register.filter() def comma_separated_list(seq, end_word="and"): if len(seq) < 2: - return u"".join(seq) + return "".join(seq) else: - return u", ".join(seq[:-1]) + u" %s %s"%(end_word, seq[-1]) + return ", ".join(seq[:-1]) + " %s %s"%(end_word, seq[-1]) @register.filter() def zaptmp(s): @@ -515,7 +519,7 @@ def zaptmp(s): @register.filter() def rfcbis(s): - m = re.search('^.*-rfc(\d+)-?bis(-.*)?$', s) + m = re.search(r'^.*-rfc(\d+)-?bis(-.*)?$', s) return None if m is None else 'rfc' + m.group(1) @register.filter diff --git a/ietf/doc/templatetags/managed_groups.py b/ietf/doc/templatetags/managed_groups.py index 225d892d4..53a16d3a2 100644 --- a/ietf/doc/templatetags/managed_groups.py +++ b/ietf/doc/templatetags/managed_groups.py @@ -1,6 +1,8 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django import template diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index dfe565cc6..bfac87497 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -1,20 +1,25 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import os import shutil import datetime -import json +import io import sys -import urlparse import bibtexparser + if sys.version_info[0] == 2 and sys.version_info[1] < 7: import unittest2 as unittest else: import unittest + +from six.moves.http_cookies import SimpleCookie from pyquery import PyQuery +from six.moves.urllib.parse import urlparse, parse_qs from tempfile import NamedTemporaryFile -from Cookie import SimpleCookie from django.urls import reverse as urlreverse from django.conf import settings @@ -56,72 +61,72 @@ class SearchTests(TestCase): # no match r = self.client.get(base_url + "?activedrafts=on&name=thisisnotadocumentname") self.assertEqual(r.status_code, 200) - self.assertTrue("no documents match" in str(r.content).lower()) + self.assertContains(r, "No documents match") r = self.client.get(base_url + "?rfcs=on&name=xyzzy") self.assertEqual(r.status_code, 200) - self.assertTrue("no documents match" in unicontent(r).lower()) + self.assertContains(r, "No documents match") r = self.client.get(base_url + "?olddrafts=on&name=bar") self.assertEqual(r.status_code, 200) - self.assertTrue("no documents match" in unicontent(r).lower()) + self.assertContains(r, "No documents match") r = self.client.get(base_url + "?olddrafts=on&name=foo") self.assertEqual(r.status_code, 200) - self.assertTrue("draft-foo-mars-test" in unicontent(r).lower()) + self.assertContains(r, "draft-foo-mars-test") # find by rfc/active/inactive draft.set_state(State.objects.get(type="draft", slug="rfc")) r = self.client.get(base_url + "?rfcs=on&name=%s" % draft.name) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) draft.set_state(State.objects.get(type="draft", slug="active")) r = self.client.get(base_url + "?activedrafts=on&name=%s" % draft.name) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) draft.set_state(State.objects.get(type="draft", slug="expired")) r = self.client.get(base_url + "?olddrafts=on&name=%s" % draft.name) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) draft.set_state(State.objects.get(type="draft", slug="active")) # find by title r = self.client.get(base_url + "?activedrafts=on&name=%s" % draft.title.split()[0]) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) # find by author r = self.client.get(base_url + "?activedrafts=on&by=author&author=%s" % draft.documentauthor_set.first().person.name_parts()[1]) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) # find by group r = self.client.get(base_url + "?activedrafts=on&by=group&group=%s" % draft.group.acronym) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) # find by area r = self.client.get(base_url + "?activedrafts=on&by=area&area=%s" % draft.group.parent_id) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) # find by area r = self.client.get(base_url + "?activedrafts=on&by=area&area=%s" % draft.group.parent_id) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) # find by AD r = self.client.get(base_url + "?activedrafts=on&by=ad&ad=%s" % draft.ad_id) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) # find by IESG state r = self.client.get(base_url + "?activedrafts=on&by=state&state=%s&substate=" % draft.get_state("draft-iesg").pk) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) def test_search_for_name(self): draft = WgDraftFactory(name='draft-ietf-mars-test',group=GroupFactory(acronym='mars',parent=Group.objects.get(acronym='farfut')),authors=[PersonFactory()],ad=PersonFactory()) @@ -129,9 +134,9 @@ class SearchTests(TestCase): CharterFactory(group=draft.group,name='charter-ietf-mars') DocumentFactory(type_id='conflrev',name='conflict-review-imaginary-irtf-submission') DocumentFactory(type_id='statchg',name='status-change-imaginary-mid-review') - DocumentFactory(type_id='agenda',name='agenda-42-mars') - DocumentFactory(type_id='minutes',name='minutes-42-mars') - DocumentFactory(type_id='slides',name='slides-42-mars') + DocumentFactory(type_id='agenda',name='agenda-72-mars') + DocumentFactory(type_id='minutes',name='minutes-72-mars') + DocumentFactory(type_id='slides',name='slides-72-mars') draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="changed_document", by=Person.objects.get(user__username="secretary"), desc="Test")]) @@ -142,76 +147,76 @@ class SearchTests(TestCase): # exact match r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) # prefix match r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(draft.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) # non-prefix match r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(draft.name.split("-")[1:])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) # other doctypes than drafts doc = Document.objects.get(name='charter-ietf-mars') r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name='charter-ietf-ma'))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) doc = Document.objects.filter(name__startswith='conflict-review-').first() r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) doc = Document.objects.filter(name__startswith='status-change-').first() r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) doc = Document.objects.filter(name__startswith='agenda-').first() r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) doc = Document.objects.filter(name__startswith='minutes-').first() r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) doc = Document.objects.filter(name__startswith='slides-').first() r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="-".join(doc.name.split("-")[:-1])))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) # match with revision r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name + "-" + prev_rev))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev))) # match with non-existing revision r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name + "-09"))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) # match with revision and extension r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name=draft.name + "-" + prev_rev + ".txt"))) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev))) + self.assertEqual(urlparse(r["Location"]).path, urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name, rev=prev_rev))) # no match r = self.client.get(urlreverse('ietf.doc.views_search.search_for_name', kwargs=dict(name="draft-ietf-doesnotexist-42"))) self.assertEqual(r.status_code, 302) - parsed = urlparse.urlparse(r["Location"]) + parsed = urlparse(r["Location"]) self.assertEqual(parsed.path, urlreverse('ietf.doc.views_search.search')) - self.assertEqual(urlparse.parse_qs(parsed.query)["name"][0], "draft-ietf-doesnotexist-42") + self.assertEqual(parse_qs(parsed.query)["name"][0], "draft-ietf-doesnotexist-42") def test_frontpage(self): r = self.client.get("/") self.assertEqual(r.status_code, 200) - self.assertTrue("Document Search" in unicontent(r)) + self.assertContains(r, "Document Search") def test_docs_for_ad(self): ad = PersonFactory() @@ -229,13 +234,11 @@ class SearchTests(TestCase): r = self.client.get(urlreverse('ietf.doc.views_search.docs_for_ad', kwargs=dict(name=ad.full_name_as_key()))) self.assertEqual(r.status_code, 200) - response_content = unicontent(r) - #debug.show('response_content') - self.assertTrue(draft.name in response_content) - self.assertTrue(rfc.canonical_name() in response_content) - self.assertTrue(conflrev.name in response_content) - self.assertTrue(statchg.name in response_content) - self.assertTrue(charter.name in response_content) + self.assertContains(r, draft.name) + self.assertContains(r, rfc.canonical_name()) + self.assertContains(r, conflrev.name) + self.assertContains(r, statchg.name) + self.assertContains(r, charter.name) def test_drafts_in_last_call(self): @@ -243,7 +246,7 @@ class SearchTests(TestCase): draft.set_state(State.objects.get(type="draft-iesg", slug="lc")) r = self.client.get(urlreverse('ietf.doc.views_search.drafts_in_last_call')) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) def test_in_iesg_process(self): doc_in_process = IndividualDraftFactory() @@ -251,8 +254,8 @@ class SearchTests(TestCase): doc_not_in_process = IndividualDraftFactory() r = self.client.get(urlreverse('ietf.doc.views_search.drafts_in_iesg_process')) self.assertEqual(r.status_code, 200) - self.assertTrue(doc_in_process.title in unicontent(r)) - self.assertFalse(doc_not_in_process.title in unicontent(r)) + self.assertContains(r, doc_in_process.title) + self.assertNotContains(r, doc_not_in_process.title) def test_indexes(self): draft = IndividualDraftFactory() @@ -260,12 +263,12 @@ class SearchTests(TestCase): r = self.client.get(urlreverse('ietf.doc.views_search.index_all_drafts')) self.assertEqual(r.status_code, 200) - self.assertIn(draft.name, unicontent(r)) - self.assertIn(rfc.canonical_name().upper(),unicontent(r)) + self.assertContains(r, draft.name) + self.assertContains(r, rfc.canonical_name().upper()) r = self.client.get(urlreverse('ietf.doc.views_search.index_active_drafts')) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, draft.title) def test_ajax_search_docs(self): draft = IndividualDraftFactory() @@ -277,7 +280,7 @@ class SearchTests(TestCase): }) r = self.client.get(url, dict(q=draft.name)) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) + data = r.json() self.assertEqual(data[0]["id"], draft.pk) # DocAlias @@ -290,7 +293,7 @@ class SearchTests(TestCase): r = self.client.get(url, dict(q=doc_alias.name)) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) + data = r.json() self.assertEqual(data[0]["id"], doc_alias.pk) def test_recent_drafts(self): @@ -489,7 +492,7 @@ Man Expires September 22, 2015 [Page 3] settings.INTERNET_DRAFT_PATH = self.id_dir self.saved_internet_all_drafts_archive_dir = settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR = self.id_dir - f = open(os.path.join(self.id_dir, 'draft-ietf-mars-test-01.txt'), 'w') + f = io.open(os.path.join(self.id_dir, 'draft-ietf-mars-test-01.txt'), 'w') f.write(self.draft_text) f.close() @@ -509,53 +512,53 @@ Man Expires September 22, 2015 [Page 3] r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 200) - self.assertTrue("Active Internet-Draft" in unicontent(r)) - self.assertTrue("Show full document text" in unicontent(r)) - self.assertFalse("Deimos street" in unicontent(r)) + self.assertContains(r, "Active Internet-Draft") + self.assertContains(r, "Show full document text") + self.assertNotContains(r, "Deimos street") r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)) + "?include_text=0") self.assertEqual(r.status_code, 200) - self.assertTrue("Active Internet-Draft" in unicontent(r)) - self.assertFalse("Show full document text" in unicontent(r)) - self.assertTrue("Deimos street" in unicontent(r)) + self.assertContains(r, "Active Internet-Draft") + self.assertNotContains(r, "Show full document text") + self.assertContains(r, "Deimos street") r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)) + "?include_text=foo") self.assertEqual(r.status_code, 200) - self.assertTrue("Active Internet-Draft" in unicontent(r)) - self.assertFalse("Show full document text" in unicontent(r)) - self.assertTrue("Deimos street" in unicontent(r)) + self.assertContains(r, "Active Internet-Draft") + self.assertNotContains(r, "Show full document text") + self.assertContains(r, "Deimos street") r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)) + "?include_text=1") self.assertEqual(r.status_code, 200) - self.assertTrue("Active Internet-Draft" in unicontent(r)) - self.assertFalse("Show full document text" in unicontent(r)) - self.assertTrue("Deimos street" in unicontent(r)) + self.assertContains(r, "Active Internet-Draft") + self.assertNotContains(r, "Show full document text") + self.assertContains(r, "Deimos street") - self.client.cookies = SimpleCookie({'full_draft': 'on'}) + self.client.cookies = SimpleCookie({str('full_draft'): str('on')}) r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 200) - self.assertTrue("Active Internet-Draft" in unicontent(r)) - self.assertFalse("Show full document text" in unicontent(r)) - self.assertTrue("Deimos street" in unicontent(r)) + self.assertContains(r, "Active Internet-Draft") + self.assertNotContains(r, "Show full document text") + self.assertContains(r, "Deimos street") - self.client.cookies = SimpleCookie({'full_draft': 'off'}) + self.client.cookies = SimpleCookie({str('full_draft'): str('off')}) r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 200) - self.assertTrue("Active Internet-Draft" in unicontent(r)) - self.assertTrue("Show full document text" in unicontent(r)) - self.assertFalse("Deimos street" in unicontent(r)) + self.assertContains(r, "Active Internet-Draft") + self.assertContains(r, "Show full document text") + self.assertNotContains(r, "Deimos street") - self.client.cookies = SimpleCookie({'full_draft': 'foo'}) + self.client.cookies = SimpleCookie({str('full_draft'): str('foo')}) r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 200) - self.assertTrue("Active Internet-Draft" in unicontent(r)) - self.assertTrue("Show full document text" in unicontent(r)) - self.assertFalse("Deimos street" in unicontent(r)) + self.assertContains(r, "Active Internet-Draft") + self.assertContains(r, "Show full document text") + self.assertNotContains(r, "Deimos street") r = self.client.get(urlreverse("ietf.doc.views_doc.document_html", kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 200) - self.assertTrue("Versions:" in unicontent(r)) - self.assertTrue("Deimos street" in unicontent(r)) + self.assertContains(r, "Versions:") + self.assertContains(r, "Deimos street") q = PyQuery(r.content) self.assertEqual(len(q('.rfcmarkup pre')), 4) self.assertEqual(len(q('.rfcmarkup span.h1')), 2) @@ -569,7 +572,7 @@ Man Expires September 22, 2015 [Page 3] r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 200) - self.assertTrue("Expired Internet-Draft" in unicontent(r)) + self.assertContains(r, "Expired Internet-Draft") # replaced draft draft.set_state(State.objects.get(type="draft", slug="repl")) @@ -588,8 +591,8 @@ Man Expires September 22, 2015 [Page 3] r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name))) self.assertEqual(r.status_code, 200) - self.assertTrue("Replaced Internet-Draft" in unicontent(r)) - self.assertTrue(replacement.name in unicontent(r)) + self.assertContains(r, "Replaced Internet-Draft") + self.assertContains(r, replacement.name) rel.delete() # draft published as RFC @@ -610,8 +613,8 @@ Man Expires September 22, 2015 [Page 3] r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=rfc_alias.name))) self.assertEqual(r.status_code, 200) - self.assertTrue("RFC 123456" in unicontent(r)) - self.assertTrue(draft.name in unicontent(r)) + self.assertContains(r, "RFC 123456") + self.assertContains(r, draft.name) # naked RFC - also wierd that we test a PS from the ISE rfc = IndividualDraftFactory( @@ -621,7 +624,7 @@ Man Expires September 22, 2015 [Page 3] std_level_id="ps") r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=rfc.name))) self.assertEqual(r.status_code, 200) - self.assertTrue("RFC 1234567" in unicontent(r)) + self.assertContains(r, "RFC 1234567") # unknown draft r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name="draft-xyz123"))) @@ -631,9 +634,9 @@ Man Expires September 22, 2015 [Page 3] IndividualDraftFactory(name='draft-imaginary-independent-submission') ConflictReviewFactory(name='conflict-review-imaginary-irtf-submission') CharterFactory(name='charter-ietf-mars') - DocumentFactory(type_id='agenda',name='agenda-42-mars') - DocumentFactory(type_id='minutes',name='minutes-42-mars') - DocumentFactory(type_id='slides',name='slides-42-mars-1-active') + DocumentFactory(type_id='agenda',name='agenda-72-mars') + DocumentFactory(type_id='minutes',name='minutes-72-mars') + DocumentFactory(type_id='slides',name='slides-72-mars-1-active') statchg = DocumentFactory(type_id='statchg',name='status-change-imaginary-mid-review') statchg.set_state(State.objects.get(type_id='statchg',slug='adrev')) @@ -642,12 +645,12 @@ Man Expires September 22, 2015 [Page 3] "conflict-review-imaginary-irtf-submission", "status-change-imaginary-mid-review", "charter-ietf-mars", - "agenda-42-mars", - "minutes-42-mars", - "slides-42-mars-1-active", + "agenda-72-mars", + "minutes-72-mars", + "slides-72-mars-1-active", # TODO: add - #"bluesheets-42-mars-1", - #"recording-42-mars-1-00", + #"bluesheets-72-mars-1", + #"recording-72-mars-1-00", ]: doc = Document.objects.get(name=docname) # give it some history @@ -658,14 +661,14 @@ Man Expires September 22, 2015 [Page 3] r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) self.assertEqual(r.status_code, 200) - self.assertTrue("%s-01"%docname in unicontent(r)) + self.assertContains(r, "%s-01"%docname) r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name,rev="01"))) self.assertEqual(r.status_code, 302) r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name,rev="00"))) self.assertEqual(r.status_code, 200) - self.assertTrue("%s-00"%docname in unicontent(r)) + self.assertContains(r, "%s-00"%docname) class DocTestCase(TestCase): def test_document_charter(self): @@ -680,7 +683,7 @@ class DocTestCase(TestCase): self.assertEqual(r.status_code, 200) def test_document_material(self): - MeetingFactory(type_id='ietf',number='42') + MeetingFactory(type_id='ietf',number='72') mars = GroupFactory(type_id='wg',acronym='mars') marschairman = PersonFactory(user__username='marschairman') mars.role_set.create(name_id='chair',person=marschairman,email=marschairman.email()) @@ -694,8 +697,8 @@ class DocTestCase(TestCase): doc.set_state(State.objects.get(type="slides", slug="active")) session = Session.objects.create( - name = "session-42-mars-1", - meeting = Meeting.objects.get(number='42'), + name = "session-72-mars-1", + meeting = Meeting.objects.get(number='72'), group = Group.objects.get(acronym='mars'), status = SessionStatusName.objects.create(slug='scheduled', name='Scheduled'), modified = datetime.datetime.now(), @@ -730,12 +733,12 @@ class DocTestCase(TestCase): r = self.client.get(urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name))) self.assertEqual(r.status_code, 200) - self.assertTrue(pos.comment in unicontent(r)) + self.assertContains(r, pos.comment) # test with ballot_id r = self.client.get(urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name, ballot_id=ballot.pk))) self.assertEqual(r.status_code, 200) - self.assertTrue(pos.comment in unicontent(r)) + self.assertContains(r, pos.comment) # test popup too while we're at it r = self.client.get(urlreverse("ietf.doc.views_doc.ballot_popup", kwargs=dict(name=doc.name, ballot_id=ballot.pk))) @@ -748,7 +751,7 @@ class DocTestCase(TestCase): doc.save_with_history([e]) r = self.client.get(urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name))) self.assertEqual(r.status_code, 200) - self.assertTrue( '(%s for -%s)' % (pos.comment_time.strftime('%Y-%m-%d'), oldrev) in unicontent(r)) + self.assertContains(r, '(%s for -%s)' % (pos.comment_time.strftime('%Y-%m-%d'), oldrev)) def test_document_ballot_needed_positions(self): # draft @@ -758,10 +761,10 @@ class DocTestCase(TestCase): create_ballot_if_not_open(None, doc, ad, 'approve') r = self.client.get(urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name))) - self.assertTrue('more YES or NO' in unicontent(r)) + self.assertContains(r, 'more YES or NO') Document.objects.filter(pk=doc.pk).update(intended_std_level='inf') r = self.client.get(urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name))) - self.assertFalse('more YES or NO' in unicontent(r)) + self.assertNotContains(r, 'more YES or NO') # status change DocAlias.objects.create(name='rfc9998').docs.add(IndividualDraftFactory()) @@ -772,29 +775,28 @@ class DocTestCase(TestCase): r = self.client.post(urlreverse('ietf.doc.views_status_change.change_state',kwargs=dict(name=doc.name)),dict(new_state=iesgeval_pk)) self.assertEqual(r.status_code, 302) r = self.client.get(r._headers["location"][1]) - self.assertTrue(">IESG Evaluation<" in unicontent(r)) + self.assertContains(r, ">IESG Evaluation<") doc.relateddocument_set.create(target=DocAlias.objects.get(name='rfc9998'),relationship_id='tohist') r = self.client.get(urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name))) - self.assertFalse('Needs a YES' in unicontent(r)) - self.assertFalse('more YES or NO' in unicontent(r)) + self.assertNotContains(r, 'Needs a YES') + self.assertNotContains(r, 'more YES or NO') doc.relateddocument_set.create(target=DocAlias.objects.get(name='rfc9999'),relationship_id='tois') r = self.client.get(urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name))) - self.assertTrue('more YES or NO' in unicontent(r)) + self.assertContains(r, 'more YES or NO') def test_document_json(self): doc = IndividualDraftFactory() r = self.client.get(urlreverse("ietf.doc.views_doc.document_json", kwargs=dict(name=doc.name))) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) + data = r.json() self.assertEqual(doc.name, data['name']) self.assertEqual(doc.pages,data['pages']) def test_writeup(self): - doc = IndividualDraftFactory(states = [('draft','active'),('draft-iesg','iesg-eva')], -) + doc = IndividualDraftFactory(states = [('draft','active'),('draft-iesg','iesg-eva')],) appr = WriteupDocEvent.objects.create( doc=doc, @@ -823,9 +825,9 @@ class DocTestCase(TestCase): url = urlreverse('ietf.doc.views_doc.document_writeup', kwargs=dict(name=doc.name)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(appr.text in unicontent(r)) - self.assertTrue(notes.text in unicontent(r)) - self.assertTrue(rfced_note.text in r.content) + self.assertContains(r, appr.text) + self.assertContains(r, notes.text) + self.assertContains(r, rfced_note.text) def test_history(self): doc = IndividualDraftFactory() @@ -840,7 +842,7 @@ class DocTestCase(TestCase): url = urlreverse('ietf.doc.views_doc.document_history', kwargs=dict(name=doc.name)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(e.desc in unicontent(r)) + self.assertContains(r, e.desc) def test_document_feed(self): doc = IndividualDraftFactory() @@ -854,7 +856,7 @@ class DocTestCase(TestCase): r = self.client.get("/feed/document-changes/%s/" % doc.name) self.assertEqual(r.status_code, 200) - self.assertTrue(e.desc in unicontent(r)) + self.assertContains(r, e.desc) def test_last_call_feed(self): doc = IndividualDraftFactory() @@ -871,7 +873,7 @@ class DocTestCase(TestCase): r = self.client.get("/feed/last-call/") self.assertEqual(r.status_code, 200) - self.assertTrue(doc.name in unicontent(r)) + self.assertContains(r, doc.name) def test_rfc_feed(self): WgRfcFactory() @@ -884,7 +886,7 @@ class DocTestCase(TestCase): url = urlreverse('ietf.doc.views_help.state_help', kwargs=dict(type="draft-iesg")) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(State.objects.get(type="draft-iesg", slug="lc").name in unicontent(r)) + self.assertContains(r, State.objects.get(type="draft-iesg", slug="lc").name) def test_document_nonietf_pubreq_button(self): doc = IndividualDraftFactory() @@ -892,17 +894,17 @@ class DocTestCase(TestCase): self.client.login(username='iab-chair', password='iab-chair+password') r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) self.assertEqual(r.status_code, 200) - self.assertNotIn("Request publication", unicontent(r)) + self.assertNotContains(r, "Request publication") Document.objects.filter(pk=doc.pk).update(stream='iab') r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) self.assertEqual(r.status_code, 200) - self.assertIn("Request publication", unicontent(r)) + self.assertContains(r, "Request publication") doc.states.add(State.objects.get(type_id='draft-stream-iab',slug='rfc-edit')) r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=doc.name))) self.assertEqual(r.status_code, 200) - self.assertNotIn("Request publication", unicontent(r)) + self.assertNotContains(r, "Request publication") def test_document_bibtex(self): @@ -918,12 +920,12 @@ class DocTestCase(TestCase): # url = urlreverse('ietf.doc.views_doc.document_bibtex', kwargs=dict(name=rfc.name)) r = self.client.get(url) - entry = bibtexparser.loads(r.content).get_entry_dict()["rfc%s"%num] - self.assertEqual(entry['series'], u'Request for Comments') + entry = bibtexparser.loads(unicontent(r)).get_entry_dict()["rfc%s"%num] + self.assertEqual(entry['series'], 'Request for Comments') self.assertEqual(entry['number'], num) - self.assertEqual(entry['doi'], u'10.17487/RFC%s'%num) - self.assertEqual(entry['year'], u'2010') - self.assertEqual(entry['month'], u'oct') + self.assertEqual(entry['doi'], '10.17487/RFC%s'%num) + self.assertEqual(entry['year'], '2010') + self.assertEqual(entry['month'], 'oct') # self.assertNotIn('day', entry) @@ -931,28 +933,28 @@ class DocTestCase(TestCase): stream_id = 'rse', states = [('draft','rfc'),('draft-iesg','pub')], std_level_id = 'ind', - time = datetime.datetime(1990,04,01), + time = datetime.datetime(1990,0o4,0o1), ) num = april1.rfc_number() DocEventFactory.create(doc=april1, type='published_rfc', time = '1990-04-01') # url = urlreverse('ietf.doc.views_doc.document_bibtex', kwargs=dict(name=april1.name)) r = self.client.get(url) - entry = bibtexparser.loads(r.content).get_entry_dict()['rfc%s'%num] - self.assertEqual(entry['series'], u'Request for Comments') + entry = bibtexparser.loads(unicontent(r)).get_entry_dict()['rfc%s'%num] + self.assertEqual(entry['series'], 'Request for Comments') self.assertEqual(entry['number'], num) - self.assertEqual(entry['doi'], u'10.17487/RFC%s'%num) - self.assertEqual(entry['year'], u'1990') - self.assertEqual(entry['month'], u'apr') - self.assertEqual(entry['day'], u'1') + self.assertEqual(entry['doi'], '10.17487/RFC%s'%num) + self.assertEqual(entry['year'], '1990') + self.assertEqual(entry['month'], 'apr') + self.assertEqual(entry['day'], '1') draft = IndividualDraftFactory.create() - docname = u'%s-%s' % (draft.name, draft.rev) + docname = '%s-%s' % (draft.name, draft.rev) bibname = docname[6:] # drop the 'draft-' prefix url = urlreverse('ietf.doc.views_doc.document_bibtex', kwargs=dict(name=draft.name)) r = self.client.get(url) - entry = bibtexparser.loads(r.content).get_entry_dict()[bibname] - self.assertEqual(entry['note'], u'Work in Progress') + entry = bibtexparser.loads(unicontent(r)).get_entry_dict()[bibname] + self.assertEqual(entry['note'], 'Work in Progress') self.assertEqual(entry['number'], docname) self.assertEqual(entry['year'], str(draft.pub_date().year)) self.assertEqual(entry['month'], draft.pub_date().strftime('%b').lower()) @@ -969,7 +971,7 @@ class AddCommentTestCase(TestCase): # normal get r = self.client.get(url) self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) + q = PyQuery(unicontent(r)) self.assertEqual(len(q('form textarea[name=comment]')), 1) # request resurrect @@ -983,9 +985,9 @@ class AddCommentTestCase(TestCase): self.assertEqual("This is a test.", draft.latest_event().desc) self.assertEqual("added_comment", draft.latest_event().type) self.assertEqual(len(outbox), mailbox_before + 1) - self.assertTrue("Comment added" in outbox[-1]['Subject']) - self.assertTrue(draft.name in outbox[-1]['Subject']) - self.assertTrue('draft-ietf-mars-test@' in outbox[-1]['To']) + self.assertIn("Comment added", outbox[-1]['Subject']) + self.assertIn(draft.name, outbox[-1]['Subject']) + self.assertIn('draft-ietf-mars-test@', outbox[-1]['To']) # Make sure we can also do it as IANA self.client.login(username="iana", password="iana+password") @@ -993,7 +995,7 @@ class AddCommentTestCase(TestCase): # normal get r = self.client.get(url) self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) + q = PyQuery(unicontent(r)) self.assertEqual(len(q('form textarea[name=comment]')), 1) @@ -1012,12 +1014,12 @@ class ReferencesTest(TestCase): RelatedDocument.objects.get_or_create(source=doc1,target=doc2,relationship=DocRelationshipName.objects.get(slug='refnorm')) url = urlreverse('ietf.doc.views_doc.document_references', kwargs=dict(name=doc1.name)) r = self.client.get(url) - self.assertEquals(r.status_code, 200) - self.assertTrue(doc2.name in unicontent(r)) + self.assertEqual(r.status_code, 200) + self.assertContains(r, doc2.name) url = urlreverse('ietf.doc.views_doc.document_referenced_by', kwargs=dict(name=doc2.name)) r = self.client.get(url) - self.assertEquals(r.status_code, 200) - self.assertTrue(doc1.name in unicontent(r)) + self.assertEqual(r.status_code, 200) + self.assertContains(r, doc1.name) class EmailAliasesTests(TestCase): @@ -1025,7 +1027,7 @@ class EmailAliasesTests(TestCase): def setUp(self): WgDraftFactory(name='draft-ietf-mars-test',group__acronym='mars') WgDraftFactory(name='draft-ietf-ames-test',group__acronym='ames') - self.doc_alias_file = NamedTemporaryFile(delete=False) + self.doc_alias_file = NamedTemporaryFile(delete=False, mode='w+') self.doc_alias_file.write("""# Generated by hand at 2015-02-12_16:26:45 virtual.ietf.org anything draft-ietf-mars-test@ietf.org xfilter-draft-ietf-mars-test @@ -1071,8 +1073,8 @@ expand-draft-ietf-ames-test.all@virtual.ietf.org ames-author@example.ames, ames url = urlreverse('ietf.doc.views_doc.document_email', kwargs=dict(name="draft-ietf-mars-test")) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue('draft-ietf-mars-test.all@ietf.org' in unicontent(r)) - self.assertTrue('ballot_saved' in unicontent(r)) + self.assertContains(r, 'draft-ietf-mars-test.all@ietf.org') + self.assertContains(r, 'ballot_saved') class DocumentMeetingTests(TestCase): @@ -1266,17 +1268,17 @@ class ChartTests(ResourceTestCaseMixin, TestCase): # No qurey arguments; expect an empty json object r = self.client.get(conf_url) self.assertValidJSONResponse(r) - self.assertEqual(r.content, '{}') + self.assertEqual(unicontent(r), '{}') # No match r = self.client.get(conf_url + '?activedrafts=on&name=thisisnotadocumentname') self.assertValidJSONResponse(r) - d = json.loads(r.content) + d = r.json() self.assertEqual(d['chart']['type'], settings.CHART_TYPE_COLUMN_OPTIONS['chart']['type']) r = self.client.get(conf_url + '?activedrafts=on&name=%s'%doc.name[6:12]) self.assertValidJSONResponse(r) - d = json.loads(r.content) + d = r.json() self.assertEqual(d['chart']['type'], settings.CHART_TYPE_COLUMN_OPTIONS['chart']['type']) self.assertEqual(len(d['series'][0]['data']), 0) @@ -1288,17 +1290,17 @@ class ChartTests(ResourceTestCaseMixin, TestCase): # No qurey arguments; expect an empty json list r = self.client.get(data_url) self.assertValidJSONResponse(r) - self.assertEqual(r.content, '[]') + self.assertEqual(unicontent(r), '[]') # No match r = self.client.get(data_url + '?activedrafts=on&name=thisisnotadocumentname') self.assertValidJSONResponse(r) - d = json.loads(r.content) - self.assertEqual(r.content, '[]') + d = r.json() + self.assertEqual(unicontent(r), '[]') r = self.client.get(data_url + '?activedrafts=on&name=%s'%doc.name[6:12]) self.assertValidJSONResponse(r) - d = json.loads(r.content) + d = r.json() self.assertEqual(len(d), 1) self.assertEqual(len(d[0]), 2) @@ -1322,7 +1324,7 @@ class ChartTests(ResourceTestCaseMixin, TestCase): r = self.client.get(conf_url) self.assertValidJSONResponse(r) - d = json.loads(r.content) + d = r.json() self.assertEqual(d['chart']['type'], settings.CHART_TYPE_COLUMN_OPTIONS['chart']['type']) self.assertEqual("New draft revisions over time for %s" % person.name, d['title']['text']) @@ -1330,7 +1332,7 @@ class ChartTests(ResourceTestCaseMixin, TestCase): r = self.client.get(data_url) self.assertValidJSONResponse(r) - d = json.loads(r.content) + d = r.json() self.assertEqual(len(d), 1) self.assertEqual(len(d[0]), 2) diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py index de43c58d1..ca1d9c1f0 100644 --- a/ietf/doc/tests_ballot.py +++ b/ietf/doc/tests_ballot.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + import datetime from pyquery import PyQuery @@ -19,8 +20,8 @@ from ietf.name.models import BallotPositionName from ietf.iesg.models import TelechatDate from ietf.person.models import Person, PersonalApiKey from ietf.person.factories import PersonFactory -from ietf.utils.test_utils import TestCase, unicontent, login_testing_unauthorized -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.test_utils import TestCase, login_testing_unauthorized +from ietf.utils.mail import outbox, empty_outbox, get_payload from ietf.utils.text import unwrap @@ -112,8 +113,7 @@ class EditPositionTests(TestCase): discuss=" This is a discussion test. \n ", comment=" This is a test. \n ") ) - self.assertEqual(r.content, "Done") - self.assertEqual(r.status_code, 200) + self.assertContains(r, "Done") pos = draft.latest_event(BallotPositionDocEvent, ad=ad) self.assertEqual(pos.pos.slug, "discuss") @@ -172,7 +172,7 @@ class EditPositionTests(TestCase): self.assertEqual(len(outbox), mailbox_before + 1) m = outbox[-1] self.assertIn('COMMENT', m['Subject']) - self.assertIn('New comment', m.get_payload()) + self.assertIn('New comment', get_payload(m)) def test_edit_position_as_secretary(self): @@ -363,7 +363,7 @@ class BallotWriteupsTests(TestCase): q = PyQuery(r.content) self.assertEqual(len(q('textarea[name=ballot_writeup]')), 1) self.assertTrue(q('[type=submit]:contains("Save")')) - self.assertTrue("IANA does not" in unicontent(r)) + self.assertContains(r, "IANA does not") # save r = self.client.post(url, dict( @@ -393,8 +393,8 @@ class BallotWriteupsTests(TestCase): q = PyQuery(r.content) self.assertEqual(len(q('textarea[name=rfc_editor_note]')), 1) self.assertTrue(q('[type=submit]:contains("Save")')) - self.assertTrue("" in r.content) - self.assertTrue("This is a note for the RFC Editor" in r.content) + self.assertContains(r, "") + self.assertContains(r, "This is a note for the RFC Editor") # save with a note empty_outbox() @@ -540,8 +540,8 @@ class BallotWriteupsTests(TestCase): e.by = Person.objects.get(name="(System)") e.doc = draft e.rev = draft.rev - e.desc = u"Ballot approval text was generated" - e.text = u"Test approval text." + e.desc = "Ballot approval text was generated" + e.text = "Test approval text." e.save() events.append(e) @@ -550,8 +550,8 @@ class BallotWriteupsTests(TestCase): e.by = Person.objects.get(name="(System)") e.doc = draft e.rev = draft.rev - e.desc = u"Ballot writeup was generated" - e.text = u"Test ballot writeup text." + e.desc = "Ballot writeup was generated" + e.text = "Test ballot writeup text." e.save() events.append(e) @@ -560,8 +560,8 @@ class BallotWriteupsTests(TestCase): e.by = Person.objects.get(name="(System)") e.doc = draft e.rev = draft.rev - e.desc = u"RFC Editor Note for ballot was generated" - e.text = u"Test note to the RFC Editor text." + e.desc = "RFC Editor Note for ballot was generated" + e.text = "Test note to the RFC Editor text." e.save() events.append(e) @@ -588,7 +588,7 @@ class BallotWriteupsTests(TestCase): # RFC Editor Notes for documents in the IRTF Stream e = DocEvent(doc=draft, rev=draft.rev, by=Person.objects.get(name="(System)"), type='changed_stream') - e.desc = u"Changed stream to %s" % 'irtf' + e.desc = "Changed stream to %s" % 'irtf' e.save() draft.stream_id = 'irtf' @@ -603,7 +603,7 @@ class BallotWriteupsTests(TestCase): # RFC Editor Notes for documents in the IAB Stream e = DocEvent(doc=draft, rev=draft.rev, by=Person.objects.get(name="(System)"), type='changed_stream') - e.desc = u"Changed stream to %s" % 'ise' + e.desc = "Changed stream to %s" % 'ise' e.save() draft.stream_id = 'ise' @@ -733,22 +733,19 @@ class ApproveBallotTests(TestCase): # Only Secretariat can use this URL login_testing_unauthorized(self, "ad", url) r = self.client.get(url) - self.assertEqual(r.status_code, 403) - self.assertTrue("Restricted to role Secretariat" in r.content) + self.assertContains(r, "Restricted to role Secretariat", status_code=403) # There are no downrefs, the page should say so login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) - self.assertEqual(r.status_code, 200) - self.assertTrue("No downward references for" in r.content) + self.assertContains(r, "No downward references for") # Add a downref, the page should ask if it should be added to the registry rel = draft.relateddocument_set.create(target=rfc.docalias.get(name='rfc6666'),relationship_id='refnorm') d = [rdoc for rdoc in draft.relateddocument_set.all() if rel.is_approved_downref()] original_len = len(d) r = self.client.get(url) - self.assertEqual(r.status_code, 200) - self.assertTrue("normatively references rfc6666" in r.content) + self.assertContains(r, "normatively references rfc6666") # POST with the downref checked r = self.client.post(url, dict(checkboxes=rel.pk)) @@ -794,7 +791,7 @@ class MakeLastCallTests(TestCase): self.assertTrue("ietf-announce@" in outbox[-2]['To']) for prefix in ['draft-ietf-mars-test','mars-chairs','aread']: self.assertTrue(prefix+"@" in outbox[-2]['Cc']) - self.assertIn("The following IPR Declarations",outbox[-2].get_payload()) + self.assertIn("The following IPR Declarations", get_payload(outbox[-2])) self.assertTrue("Last Call" in outbox[-1]['Subject']) self.assertTrue("drafts-lastcall@icann.org" in outbox[-1]['To']) diff --git a/ietf/doc/tests_charter.py b/ietf/doc/tests_charter.py index 30f08dd4b..c44cd01eb 100644 --- a/ietf/doc/tests_charter.py +++ b/ietf/doc/tests_charter.py @@ -1,8 +1,14 @@ # -*- coding: utf-8 -*- -# Copyright The IETF Trust 2011, All Rights Reserved +# Copyright The IETF Trust 2011-2019, All Rights Reserved + + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os +import shutil -import os, shutil, datetime -from StringIO import StringIO from pyquery import PyQuery from django.conf import settings @@ -20,8 +26,8 @@ from ietf.group.factories import RoleFactory, GroupFactory from ietf.group.models import Group, GroupMilestone from ietf.iesg.models import TelechatDate from ietf.person.models import Person -from ietf.utils.test_utils import TestCase, unicontent -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.test_utils import TestCase +from ietf.utils.mail import outbox, empty_outbox, get_payload from ietf.utils.test_utils import login_testing_unauthorized class EditCharterTests(TestCase): @@ -35,7 +41,7 @@ class EditCharterTests(TestCase): shutil.rmtree(self.charter_dir) def write_charter_file(self, charter): - with open(os.path.join(self.charter_dir, "%s-%s.txt" % (charter.canonical_name(), charter.rev)), "w") as f: + with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (charter.canonical_name(), charter.rev)), "w") as f: f.write("This is a charter.") def test_startstop_process(self): @@ -70,8 +76,8 @@ class EditCharterTests(TestCase): ames = GroupFactory(acronym='ames',state_id='proposed',list_email='ames-wg@ietf.org',parent=area) RoleFactory(name_id='ad',group=ames,person=Person.objects.get(user__username='ad')) - RoleFactory(name_id='chair',group=ames,person__name=u'Ames Man',person__user__email='ameschairman@example.org') - RoleFactory(name_id='secr',group=ames,person__name=u'Secretary',person__user__email='amessecretary@example.org') + RoleFactory(name_id='chair',group=ames,person__name='Ames Man',person__user__email='ameschairman@example.org') + RoleFactory(name_id='secr',group=ames,person__name='Secretary',person__user__email='amessecretary@example.org') CharterFactory(group=ames) mars = GroupFactory(acronym='mars',parent=area) @@ -128,24 +134,24 @@ class EditCharterTests(TestCase): self.assertIn("Internal WG Review", outbox[-3]['Subject']) self.assertIn("iab@", outbox[-3]['To']) self.assertIn("iesg@", outbox[-3]['To']) - self.assertIn("A new IETF WG", outbox[-3].get_payload()) - body = outbox[-3].get_payload() - for word in ["Chairs", "Ames Man ", + body = get_payload(outbox[-3]) + for word in ["A new IETF WG", "Chairs", "Ames Man ", "Secretaries", "Secretary ", "Assigned Area Director", "Areað Irector ", "Mailing list", "ames-wg@ietf.org", "Charter", "Milestones"]: + self.assertIn(word, body) self.assertIn("state changed", outbox[-2]['Subject'].lower()) self.assertIn("iesg-secretary@", outbox[-2]['To']) - body = outbox[-2].get_payload() + body = get_payload(outbox[-2]) for word in ["WG", "Charter", ]: self.assertIn(word, body) self.assertIn("State Update Notice", outbox[-1]['Subject']) self.assertIn("ames-chairs@", outbox[-1]['To']) - body = outbox[-1].get_payload() + body = get_payload(outbox[-1]) for word in ["State changed", "Datatracker URL", ]: self.assertIn(word, body) @@ -165,7 +171,7 @@ class EditCharterTests(TestCase): empty_outbox() r = self.client.post(url, dict(charter_state=str(State.objects.get(used=True,type="charter",slug="intrev").pk), message="test")) self.assertEqual(r.status_code, 302) - self.assertTrue("A new charter" in outbox[-3].get_payload()) + self.assertTrue("A new charter" in get_payload(outbox[-3])) def test_abandon_bof(self): charter = CharterFactory(group__state_id='bof',group__type_id='wg') @@ -389,19 +395,19 @@ class EditCharterTests(TestCase): self.assertEqual(len(q('form input[name=txt]')), 1) # faulty post - test_file = StringIO("\x10\x11\x12") # post binary file + test_file = io.StringIO("\x10\x11\x12") # post binary file test_file.name = "unnamed" r = self.client.post(url, dict(txt=test_file)) self.assertEqual(r.status_code, 200) - self.assertTrue("does not appear to be a text file" in unicontent(r)) + self.assertContains(r, "does not appear to be a text file") # post prev_rev = charter.rev - latin_1_snippet = '\xe5' * 10 - utf_8_snippet = '\xc3\xa5' * 10 - test_file = StringIO("Windows line\r\nMac line\rUnix line\n" + latin_1_snippet) + latin_1_snippet = b'\xe5' * 10 + utf_8_snippet = b'\xc3\xa5' * 10 + test_file = io.StringIO("Windows line\r\nMac line\rUnix line\n" + latin_1_snippet.decode('latin-1')) test_file.name = "unnamed" r = self.client.post(url, dict(txt=test_file)) @@ -411,9 +417,8 @@ class EditCharterTests(TestCase): self.assertEqual(charter.rev, next_revision(prev_rev)) self.assertTrue("new_revision" in charter.latest_event().type) - with open(os.path.join(self.charter_dir, charter.canonical_name() + "-" + charter.rev + ".txt")) as f: - self.assertEqual(f.read(), - "Windows line\nMac line\nUnix line\n" + utf_8_snippet) + with io.open(os.path.join(self.charter_dir, charter.canonical_name() + "-" + charter.rev + ".txt")) as f: + self.assertEqual(f.read(), "Windows line\nMac line\nUnix line\n" + utf_8_snippet.decode('utf_8')) def test_submit_initial_charter(self): group = GroupFactory(type_id='wg',acronym='mars',list_email='mars-wg@ietf.org') @@ -428,7 +433,7 @@ class EditCharterTests(TestCase): self.assertEqual(len(q('form input[name=txt]')), 1) # create charter - test_file = StringIO("Simple test") + test_file = io.StringIO("Simple test") test_file.name = "unnamed" r = self.client.post(url, dict(txt=test_file)) @@ -591,8 +596,8 @@ class EditCharterTests(TestCase): RoleFactory(name_id='ad',group=area,person=Person.objects.get(user__username='ad')) charter = CharterFactory(group__acronym='ames',group__list_email='ames-wg@ietf.org',group__parent=area,group__state_id='bof') group = charter.group - RoleFactory(name_id='chair',group=group,person__name=u'Ames Man',person__user__email='ameschairman@example.org') - RoleFactory(name_id='secr',group=group,person__name=u'Secretary',person__user__email='amessecretary@example.org') + RoleFactory(name_id='chair',group=group,person__name='Ames Man',person__user__email='ameschairman@example.org') + RoleFactory(name_id='secr',group=group,person__name='Secretary',person__user__email='amessecretary@example.org') url = urlreverse('ietf.doc.views_charter.approve', kwargs=dict(name=charter.name)) login_testing_unauthorized(self, "secretary", url) @@ -658,7 +663,7 @@ class EditCharterTests(TestCase): # self.assertTrue("approved" in outbox[0]['Subject'].lower()) self.assertTrue("iesg-secretary" in outbox[0]['To']) - body = outbox[0].get_payload() + body = get_payload(outbox[0]) for word in ["WG", "/wg/ames/about/", "Charter", "/doc/charter-ietf-ames/", ]: self.assertIn(word, body) @@ -666,7 +671,7 @@ class EditCharterTests(TestCase): self.assertTrue("WG Action" in outbox[1]['Subject']) self.assertTrue("ietf-announce" in outbox[1]['To']) self.assertTrue("ames-wg@ietf.org" in outbox[1]['Cc']) - body = outbox[1].get_payload() + body = get_payload(outbox[1]) for word in ["Chairs", "Ames Man ", "Secretaries", "Secretary ", "Assigned Area Director", "Areað Irector ", @@ -696,7 +701,7 @@ class EditCharterTests(TestCase): url = urlreverse('ietf.doc.views_charter.charter_with_milestones_txt', kwargs=dict(name=charter.name, rev=charter.rev)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(m.desc in unicontent(r)) + self.assertContains(r, m.desc) def test_chartering_from_bof(self): ad_role = RoleFactory(group__type_id='area',name_id='ad') diff --git a/ietf/doc/tests_conflict_review.py b/ietf/doc/tests_conflict_review.py index fc0c2aa27..eb1c83371 100644 --- a/ietf/doc/tests_conflict_review.py +++ b/ietf/doc/tests_conflict_review.py @@ -1,15 +1,21 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- -import debug # pyflakes:ignore + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import shutil from pyquery import PyQuery -from StringIO import StringIO from textwrap import wrap from django.conf import settings from django.urls import reverse as urlreverse +import debug # pyflakes:ignore + from ietf.doc.factories import IndividualDraftFactory, ConflictReviewFactory from ietf.doc.models import Document, DocEvent, NewRevisionDocEvent, BallotPositionDocEvent, TelechatDocEvent, State from ietf.doc.utils import create_ballot_if_not_open @@ -17,8 +23,8 @@ from ietf.doc.views_conflict_review import default_approval_text from ietf.group.models import Person from ietf.iesg.models import TelechatDate from ietf.name.models import StreamName -from ietf.utils.test_utils import TestCase, unicontent -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.test_utils import TestCase +from ietf.utils.mail import outbox, empty_outbox, get_payload from ietf.utils.test_utils import login_testing_unauthorized @@ -63,9 +69,9 @@ class ConflictReviewTests(TestCase): self.assertEqual(r.status_code, 302) review_doc = Document.objects.get(name='conflict-review-imaginary-independent-submission') self.assertEqual(review_doc.get_state('conflrev').slug,'needshep') - self.assertEqual(review_doc.rev,u'00') - self.assertEqual(review_doc.ad.name,u'Areað Irector') - self.assertEqual(review_doc.notify,u'ipu@ietf.org') + self.assertEqual(review_doc.rev,'00') + self.assertEqual(review_doc.ad.name,'Areað Irector') + self.assertEqual(review_doc.notify,'ipu@ietf.org') doc = Document.objects.get(name='draft-imaginary-independent-submission') self.assertTrue(doc in [x.target.document for x in review_doc.relateddocument_set.filter(relationship__slug='conflrev')]) @@ -87,34 +93,34 @@ class ConflictReviewTests(TestCase): # can't start conflict reviews on documents not in a stream r = self.client.get(url) - self.assertEquals(r.status_code, 404) + self.assertEqual(r.status_code, 404) # can't start conflict reviews on documents in some other stream doc.stream = StreamName.objects.get(slug='irtf') doc.save_with_history([DocEvent.objects.create(doc=doc, rev=doc.rev, type="changed_stream", by=Person.objects.get(user__username="secretary"), desc="Test")]) r = self.client.get(url) - self.assertEquals(r.status_code, 404) + self.assertEqual(r.status_code, 404) # successful get doc.stream = StreamName.objects.get(slug='ise') doc.save_with_history([DocEvent.objects.create(doc=doc, rev=doc.rev, type="changed_stream", by=Person.objects.get(user__username="secretary"), desc="Test")]) r = self.client.get(url) - self.assertEquals(r.status_code, 200) + self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEquals(len(q('form input[name=notify]')),1) - self.assertEquals(len(q('form select[name=ad]')),0) + self.assertEqual(len(q('form input[name=notify]')),1) + self.assertEqual(len(q('form select[name=ad]')),0) # successfully starts a review, and notifies the secretariat messages_before = len(outbox) r = self.client.post(url,dict(notify='ipu@ietf.org')) - self.assertEquals(r.status_code, 302) + self.assertEqual(r.status_code, 302) review_doc = Document.objects.get(name='conflict-review-imaginary-independent-submission') - self.assertEquals(review_doc.get_state('conflrev').slug,'needshep') - self.assertEquals(review_doc.rev,u'00') - self.assertEquals(review_doc.telechat_date(),None) - self.assertEquals(review_doc.ad.name,u'Ietf Chair') - self.assertEquals(review_doc.notify,u'ipu@ietf.org') + self.assertEqual(review_doc.get_state('conflrev').slug,'needshep') + self.assertEqual(review_doc.rev,'00') + self.assertEqual(review_doc.telechat_date(),None) + self.assertEqual(review_doc.ad.name,'Ietf Chair') + self.assertEqual(review_doc.notify,'ipu@ietf.org') doc = Document.objects.get(name='draft-imaginary-independent-submission') self.assertTrue(doc in [x.target.document for x in review_doc.relateddocument_set.filter(relationship__slug='conflrev')]) @@ -264,7 +270,7 @@ class ConflictReviewTests(TestCase): def approve_test_helper(self,approve_type): doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission') - url = urlreverse('ietf.doc.views_conflict_review.approve',kwargs=dict(name=doc.name)) + url = urlreverse('ietf.doc.views_conflict_review.approve_conflict_review',kwargs=dict(name=doc.name)) login_testing_unauthorized(self, "secretary", url) @@ -278,9 +284,9 @@ class ConflictReviewTests(TestCase): q = PyQuery(r.content) self.assertEqual(len(q('[type=submit]:contains("Send announcement")')), 1) if approve_type == 'appr-noprob': - self.assertIn( 'IESG has no problem', ''.join(wrap(r.content,2**16))) + self.assertContains(r, 'IESG has no problem') else: - self.assertIn( 'NOT be published', ''.join(wrap(r.content,2**16))) + self.assertContains(r, 'NOT be published') # submit empty_outbox() @@ -296,12 +302,13 @@ class ConflictReviewTests(TestCase): self.assertIn('irtf-chair', outbox[0]['To']) self.assertIn('ietf-announce@', outbox[0]['Cc']) self.assertIn('iana@', outbox[0]['Cc']) + if approve_type == 'appr-noprob': - self.assertIn( 'IESG has no problem', ''.join(wrap(unicode(outbox[0]),2**16))) + self.assertIn( 'IESG has no problem', ''.join(wrap(get_payload(outbox[0]), 2**16))) else: - self.assertIn( 'NOT be published', ''.join(wrap(unicode(outbox[0]),2**16))) - - + self.assertIn( 'NOT be published', ''.join(wrap(get_payload(outbox[0]), 2**16))) + + def test_approve_reqnopub(self): self.approve_test_helper('appr-reqnopub') @@ -330,13 +337,13 @@ class ConflictReviewSubmitTests(TestCase): # sane post using textbox path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - self.assertEqual(doc.rev,u'00') + self.assertEqual(doc.rev,'00') self.assertFalse(os.path.exists(path)) r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1")) self.assertEqual(r.status_code,302) doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission') - self.assertEqual(doc.rev,u'00') - with open(path) as f: + self.assertEqual(doc.rev,'00') + with io.open(path) as f: self.assertEqual(f.read(),"Some initial review text\n") f.close() self.assertTrue( "submission-00" in doc.latest_event(NewRevisionDocEvent).desc) @@ -348,9 +355,9 @@ class ConflictReviewSubmitTests(TestCase): # A little additional setup # doc.rev is u'00' per the test setup - double-checking that here - if it fails, the breakage is in setUp - self.assertEqual(doc.rev,u'00') + self.assertEqual(doc.rev,'00') path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - with open(path,'w') as f: + with io.open(path,'w') as f: f.write('This is the old proposal.') f.close() @@ -363,21 +370,21 @@ class ConflictReviewSubmitTests(TestCase): # faulty posts trying to use file upload # Copied from wgtracker tests - is this really testing the server code, or is it testing # how client.post populates Content-Type? - test_file = StringIO("\x10\x11\x12") # post binary file + test_file = io.StringIO("\x10\x11\x12") # post binary file test_file.name = "unnamed" r = self.client.post(url, dict(txt=test_file,submit_response="1")) self.assertEqual(r.status_code, 200) - self.assertTrue("does not appear to be a text file" in unicontent(r)) + self.assertContains(r, "does not appear to be a text file") # sane post uploading a file - test_file = StringIO("This is a new proposal.") + test_file = io.StringIO("This is a new proposal.") test_file.name = "unnamed" r = self.client.post(url,dict(txt=test_file,submit_response="1")) self.assertEqual(r.status_code, 302) doc = Document.objects.get(name='conflict-review-imaginary-irtf-submission') - self.assertEqual(doc.rev,u'01') + self.assertEqual(doc.rev,'01') path = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - with open(path) as f: + with io.open(path) as f: self.assertEqual(f.read(),"This is a new proposal.") f.close() self.assertTrue( "submission-01" in doc.latest_event(NewRevisionDocEvent).desc) diff --git a/ietf/doc/tests_downref.py b/ietf/doc/tests_downref.py index d0f4b43fc..714e2d965 100644 --- a/ietf/doc/tests_downref.py +++ b/ietf/doc/tests_downref.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2017-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals + +from __future__ import absolute_import, print_function, unicode_literals + from django.urls import reverse as urlreverse from pyquery import PyQuery @@ -12,7 +14,7 @@ from ietf.doc.factories import WgDraftFactory, WgRfcFactory from ietf.doc.models import RelatedDocument, State from ietf.person.factories import PersonFactory from ietf.utils.test_utils import TestCase -from ietf.utils.test_utils import login_testing_unauthorized, unicontent +from ietf.utils.test_utils import login_testing_unauthorized class Downref(TestCase): @@ -32,26 +34,20 @@ class Downref(TestCase): # normal - get the table without the "Add downref" button self.client.login(username="plain", password="plain+password") r = self.client.get(url) - self.assertEqual(r.status_code, 200) - content = unicontent(r) - self.assertTrue('

Downref registry

' in content) - self.assertFalse('Add downref' in content) + self.assertContains(r, '

Downref registry

') + self.assertNotContains(r, 'Add downref') # secretariat - get the table with the "Add downref" button self.client.login(username='secretary', password='secretary+password') r = self.client.get(url) - self.assertEqual(r.status_code, 200) - content = unicontent(r) - self.assertTrue('

Downref registry

' in content) - self.assertTrue('Add downref' in content) + self.assertContains(r, '

Downref registry

') + self.assertContains(r, 'Add downref') # area director - get the table with the "Add downref" button self.client.login(username='ad', password='ad+password') r = self.client.get(url) - self.assertEqual(r.status_code, 200) - content = unicontent(r) - self.assertTrue('

Downref registry

' in content) - self.assertTrue('Add downref' in content) + self.assertContains(r, '

Downref registry

') + self.assertContains(r, 'Add downref') def test_downref_registry_add(self): url = urlreverse('ietf.doc.views_downref.downref_registry_add') @@ -60,42 +56,32 @@ class Downref(TestCase): # secretariat - get the form to add entries to the registry self.client.login(username='secretary', password='secretary+password') r = self.client.get(url) - self.assertEqual(r.status_code, 200) - content = unicontent(r) - self.assertTrue('

Add entry to the downref registry

' in content) - self.assertTrue('Save downref' in content) + self.assertContains(r, '

Add entry to the downref registry

') + self.assertContains(r, 'Save downref') # area director - get the form to add entries to the registry self.client.login(username='ad', password='ad+password') r = self.client.get(url) - self.assertEqual(r.status_code, 200) - content = unicontent(r) - self.assertTrue('

Add entry to the downref registry

' in content) - self.assertTrue('Save downref' in content) + self.assertContains(r, '

Add entry to the downref registry

') + self.assertContains(r, 'Save downref') # error - already in the downref registry r = self.client.post(url, dict(rfc=self.rfcalias.pk, drafts=(self.doc.pk, ))) - self.assertEqual(r.status_code, 200) - content = unicontent(r) - self.assertTrue('Downref is already in the registry' in content) + self.assertContains(r, 'Downref is already in the registry') # error - source is not in an approved state r = self.client.get(url) self.assertEqual(r.status_code, 200) r = self.client.post(url, dict(rfc=self.rfcalias.pk, drafts=(self.draft.pk, ))) - self.assertEqual(r.status_code, 200) - content = unicontent(r) - self.assertTrue('Draft is not yet approved' in content) + self.assertContains(r, 'Draft is not yet approved') # error - the target is not a normative reference of the source self.draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="pub")) r = self.client.get(url) self.assertEqual(r.status_code, 200) r = self.client.post(url, dict(rfc=self.rfcalias.pk, drafts=(self.draft.pk, ))) - self.assertEqual(r.status_code, 200) - content = unicontent(r) - self.assertTrue('There does not seem to be a normative reference to RFC' in content) - self.assertTrue('Save downref anyway' in content) + self.assertContains(r, 'There does not seem to be a normative reference to RFC') + self.assertContains(r, 'Save downref anyway') # normal - approve the document so the downref is now okay RelatedDocument.objects.create(source=self.draft, target=self.rfcalias, relationship_id='refnorm') @@ -108,9 +94,7 @@ class Downref(TestCase): self.assertEqual(r.status_code, 302) newurl = urlreverse('ietf.doc.views_downref.downref_registry') r = self.client.get(newurl) - self.assertEqual(r.status_code, 200) - content = unicontent(r) - self.assertTrue('%s from %s"% (draft.intended_std_level_id, 'bcp') + e.desc = "Intended Status changed to %s from %s"% (draft.intended_std_level_id, 'bcp') e.save() draft.intended_std_level_id = 'bcp' @@ -467,7 +470,7 @@ class EditInfoTests(TestCase): self.assertEqual(r.status_code, 403) # BCPs must have a consensus e = DocEvent(doc=draft, rev=draft.rev, by=Person.objects.get(name="(System)"), type='changed_document') - e.desc = u"Intended Status changed to %s from %s"% (draft.intended_std_level_id, 'inf') + e.desc = "Intended Status changed to %s from %s"% (draft.intended_std_level_id, 'inf') e.save() draft.intended_std_level_id = 'inf' @@ -563,7 +566,7 @@ class ExpireIDsTests(TestCase): settings.INTERNET_DRAFT_ARCHIVE_DIR = self.saved_archive_dir def write_draft_file(self, name, size): - f = open(os.path.join(self.id_dir, name), 'w') + f = io.open(os.path.join(self.id_dir, name), 'w') f.write("a" * size) f.close() @@ -774,7 +777,7 @@ class ExpireLastCallTests(TestCase): class IndividualInfoFormsTests(TestCase): def setUp(self): - doc = WgDraftFactory(group__acronym='mars',shepherd=PersonFactory(user__username='plain',name=u'Plain Man').email_set.first()) + doc = WgDraftFactory(group__acronym='mars',shepherd=PersonFactory(user__username='plain',name='Plain Man').email_set.first()) self.docname = doc.name def test_doc_change_stream(self): @@ -1056,7 +1059,7 @@ class IndividualInfoFormsTests(TestCase): self.assertTrue(doc.latest_event(WriteupDocEvent,type="changed_protocol_writeup").text.startswith('here is a new writeup')) # file upload - test_file = StringIO.StringIO("This is a different writeup.") + test_file = io.StringIO("This is a different writeup.") test_file.name = "unnamed" r = self.client.post(url,dict(txt=test_file,submit_response="1")) self.assertEqual(r.status_code, 302) @@ -1362,7 +1365,7 @@ class AdoptDraftTests(TestCase): class ChangeStreamStateTests(TestCase): def test_set_tags(self): - role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name=u'WG Cháir Man') + role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') RoleFactory(name_id='delegate',group=role.group,person__user__email='marsdelegate@example.org') draft = WgDraftFactory(group=role.group,shepherd=PersonFactory(user__username='plain',user__email='plain@example.com').email_set.first()) draft.tags.set(DocTagName.objects.filter(slug="w-expert")) @@ -1399,12 +1402,12 @@ class ChangeStreamStateTests(TestCase): self.assertEqual(draft.docevent_set.count() - events_before, 2) self.assertEqual(len(outbox), mailbox_before + 1) self.assertTrue("tags changed" in outbox[-1]["Subject"].lower()) - self.assertTrue("mars-chairs@ietf.org" in unicode(outbox[-1])) - self.assertTrue("marsdelegate@example.org" in unicode(outbox[-1])) - self.assertTrue("plain@example.com" in unicode(outbox[-1])) + self.assertTrue("mars-chairs@ietf.org" in outbox[-1].as_string()) + self.assertTrue("marsdelegate@example.org" in outbox[-1].as_string()) + self.assertTrue("plain@example.com" in outbox[-1].as_string()) def test_set_initial_state(self): - role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name=u'WG Cháir Man') + role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') RoleFactory(name_id='delegate',group=role.group,person__user__email='marsdelegate@ietf.org') draft = WgDraftFactory(group=role.group) draft.states.all().delete() @@ -1436,11 +1439,11 @@ class ChangeStreamStateTests(TestCase): self.assertTrue(due - datetime.timedelta(days=1) <= reminder[0].due <= due + datetime.timedelta(days=1)) self.assertEqual(len(outbox), 1) self.assertTrue("state changed" in outbox[0]["Subject"].lower()) - self.assertTrue("mars-chairs@ietf.org" in unicode(outbox[0])) - self.assertTrue("marsdelegate@ietf.org" in unicode(outbox[0])) + self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string()) + self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string()) def test_set_state(self): - role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name=u'WG Cháir Man') + role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') RoleFactory(name_id='delegate',group=role.group,person__user__email='marsdelegate@ietf.org') draft = WgDraftFactory(group=role.group) @@ -1481,11 +1484,11 @@ class ChangeStreamStateTests(TestCase): self.assertTrue(due - datetime.timedelta(days=1) <= reminder[0].due <= due + datetime.timedelta(days=1)) self.assertEqual(len(outbox), 1) self.assertTrue("state changed" in outbox[0]["Subject"].lower()) - self.assertTrue("mars-chairs@ietf.org" in unicode(outbox[0])) - self.assertTrue("marsdelegate@ietf.org" in unicode(outbox[0])) + self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string()) + self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string()) def test_pubreq_validation(self): - role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name=u'WG Cháir Man') + role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') RoleFactory(name_id='delegate',group=role.group,person__user__email='marsdelegate@ietf.org') draft = WgDraftFactory(group=role.group) @@ -1509,7 +1512,7 @@ class ChangeStreamStateTests(TestCase): class ChangeReplacesTests(TestCase): def setUp(self): - role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name=u'WG Cháir Man') + role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') RoleFactory(name_id='delegate',group=role.group,person__user__email='marsdelegate@ietf.org') #draft = WgDraftFactory(group=role.group) @@ -1520,7 +1523,7 @@ class ChangeReplacesTests(TestCase): title="Base A", group=mars_wg, ) - p = PersonFactory(name=u"basea_author") + p = PersonFactory(name="basea_author") e = Email.objects.create(address="basea_author@example.com", person=p, origin=p.user.username) self.basea.documentauthor_set.create(person=p, email=e, order=1) @@ -1530,7 +1533,7 @@ class ChangeReplacesTests(TestCase): group=mars_wg, expires = datetime.datetime.now() - datetime.timedelta(days = 365 - settings.INTERNET_DRAFT_DAYS_TO_EXPIRE), ) - p = PersonFactory(name=u"baseb_author") + p = PersonFactory(name="baseb_author") e = Email.objects.create(address="baseb_author@example.com", person=p, origin=p.user.username) self.baseb.documentauthor_set.create(person=p, email=e, order=1) @@ -1539,7 +1542,7 @@ class ChangeReplacesTests(TestCase): title="Replace Base A", group=mars_wg, ) - p = PersonFactory(name=u"replacea_author") + p = PersonFactory(name="replacea_author") e = Email.objects.create(address="replacea_author@example.com", person=p, origin=p.user.username) self.replacea.documentauthor_set.create(person=p, email=e, order=1) @@ -1548,7 +1551,7 @@ class ChangeReplacesTests(TestCase): title="Replace Base A and Base B", group=mars_wg, ) - p = PersonFactory(name=u"replaceboth_author") + p = PersonFactory(name="replaceboth_author") e = Email.objects.create(address="replaceboth_author@example.com", person=p, origin=p.user.username) self.replaceboth.documentauthor_set.create(person=p, email=e, order=1) @@ -1627,15 +1630,15 @@ class ChangeReplacesTests(TestCase): login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) - self.assertEquals(r.status_code, 200) + self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEquals(len(q('form[name=review-suggested-replaces]')), 1) + self.assertEqual(len(q('form[name=review-suggested-replaces]')), 1) r = self.client.post(url, dict(replaces=[replaced.pk])) - self.assertEquals(r.status_code, 302) + self.assertEqual(r.status_code, 302) self.assertTrue(not self.replacea.related_that_doc("possibly-replaces")) self.assertEqual(len(self.replacea.related_that_doc("replaces")), 1) - self.assertEquals(Document.objects.get(pk=self.basea.pk).get_state().slug, 'repl') + self.assertEqual(Document.objects.get(pk=self.basea.pk).get_state().slug, 'repl') class MoreReplacesTests(TestCase): diff --git a/ietf/doc/tests_material.py b/ietf/doc/tests_material.py index c2d857d70..47c71f90d 100644 --- a/ietf/doc/tests_material.py +++ b/ietf/doc/tests_material.py @@ -1,10 +1,14 @@ -# Copyright The IETF Trust 2011-2019, All Rights Reserved +# Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import os import shutil import datetime -from StringIO import StringIO +import io + from pyquery import PyQuery import debug # pyflakes:ignore @@ -19,7 +23,7 @@ from ietf.meeting.factories import MeetingFactory from ietf.meeting.models import Meeting, Session, SessionPresentation from ietf.name.models import SessionStatusName from ietf.person.models import Person -from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent +from ietf.utils.test_utils import TestCase, login_testing_unauthorized class GroupMaterialTests(TestCase): @@ -68,7 +72,7 @@ class GroupMaterialTests(TestCase): # normal get r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue("Slides" in unicontent(r)) + self.assertContains(r, "Slides") url = urlreverse('ietf.doc.views_material.choose_material_type', kwargs=dict(acronym='mars')) r = self.client.get(url) @@ -85,7 +89,7 @@ class GroupMaterialTests(TestCase): self.assertEqual(r.status_code, 200) content = "%PDF-1.5\n..." - test_file = StringIO(content) + test_file = io.StringIO(content) test_file.name = "unnamed.pdf" # faulty post @@ -110,7 +114,7 @@ class GroupMaterialTests(TestCase): self.assertEqual(doc.title, "Test File - with fancy title") self.assertEqual(doc.get_state_slug(), "active") - with open(os.path.join(self.materials_dir, "slides", doc.name + "-" + doc.rev + ".pdf")) as f: + with io.open(os.path.join(self.materials_dir, "slides", doc.name + "-" + doc.rev + ".pdf")) as f: self.assertEqual(f.read(), content) # check that posting same name is prevented @@ -165,7 +169,7 @@ class GroupMaterialTests(TestCase): login_testing_unauthorized(self, "secretary", url) content = "some text" - test_file = StringIO(content) + test_file = io.StringIO(content) test_file.name = "unnamed.txt" # post @@ -179,6 +183,6 @@ class GroupMaterialTests(TestCase): self.assertEqual(doc.title, "New title") self.assertEqual(doc.get_state_slug(), "active") - with open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + ".txt")) as f: + with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + ".txt")) as f: self.assertEqual(f.read(), content) diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index 40fe8da5f..de3ef3eeb 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -1,11 +1,14 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- -import datetime, os, shutil, json + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime, os, shutil +import io import tarfile, tempfile, mailbox import email.mime.multipart, email.mime.text, email.utils -from StringIO import StringIO from mock import patch from requests import Response @@ -33,7 +36,7 @@ from ietf.review.utils import reviewer_rotation_list, possibly_advance_next_revi from ietf.utils.test_utils import TestCase from ietf.utils.test_data import create_person -from ietf.utils.test_utils import login_testing_unauthorized, unicontent, reload_db_objects +from ietf.utils.test_utils import login_testing_unauthorized, reload_db_objects from ietf.utils.mail import outbox, empty_outbox, parseaddr, on_behalf_of from ietf.person.factories import PersonFactory @@ -100,6 +103,8 @@ class ReviewTests(TestCase): self.assertTrue('reviewteam Early' in outbox[0]['Subject']) self.assertTrue('reviewsecretary@' in outbox[0]['To']) self.assertTrue('reviewteam3 Early' in outbox[1]['Subject']) + if not 'reviewsecretary3@' in outbox[1]['To']: + print(outbox[1].as_string()) self.assertTrue('reviewsecretary3@' in outbox[1]['To']) # set the reviewteamsetting for the secretary email alias, then do the post again @@ -152,9 +157,7 @@ class ReviewTests(TestCase): url = urlreverse('ietf.doc.views_doc.document_main', kwargs={ "name": doc.name }) r = self.client.get(url) - self.assertEqual(r.status_code, 200) - content = unicontent(r) - self.assertTrue("{} Review".format(review_req.type.name) in content) + self.assertContains(r, "{} Review".format(review_req.type.name)) def test_review_request(self): doc = WgDraftFactory(group__acronym='mars',rev='01') @@ -166,9 +169,8 @@ class ReviewTests(TestCase): url = urlreverse('ietf.doc.views_review.review_request', kwargs={ "name": doc.name, "request_id": review_req.pk }) r = self.client.get(url) - self.assertEqual(r.status_code, 200) - self.assertIn(review_req.team.acronym, unicontent(r)) - self.assertIn(review_req.team.name, unicontent(r)) + self.assertContains(r, review_req.team.acronym) + self.assertContains(r, review_req.team.name) url = urlreverse('ietf.doc.views_review.review_request_forced_login', kwargs={ "name": doc.name, "request_id": review_req.pk }) r = self.client.get(url) @@ -193,7 +195,7 @@ class ReviewTests(TestCase): self.client.login(username="reviewsecretary", password="reviewsecretary+password") r = self.client.get(req_url) self.assertEqual(r.status_code, 200) - self.assertTrue(close_url in unicontent(r)) + self.assertContains(r, close_url) self.client.logout() # get close page @@ -311,8 +313,8 @@ class ReviewTests(TestCase): def test_assign_reviewer(self): doc = WgDraftFactory(pages=2) review_team = ReviewTeamFactory(acronym="reviewteam", name="Review Team", type_id="review", list_email="reviewteam@ietf.org", parent=Group.objects.get(acronym="farfut")) - rev_role = RoleFactory(group=review_team,person__user__username='reviewer',person__user__email='reviewer@example.com',person__name=u'Some Reviewer',name_id='reviewer') - RoleFactory(group=review_team,person__user__username='marschairman',person__name=u'WG Cháir Man',name_id='reviewer') + rev_role = RoleFactory(group=review_team,person__user__username='reviewer',person__user__email='reviewer@example.com',person__name='Some Reviewer',name_id='reviewer') + RoleFactory(group=review_team,person__user__username='marschairman',person__name='WG Cháir Man',name_id='reviewer') RoleFactory(group=review_team,person__user__username='reviewsecretary',person__user__email='reviewsecretary@example.com',name_id='secr') ReviewerSettings.objects.create(team=review_team, person=rev_role.person, min_interval=14, skip_next=0) @@ -353,7 +355,7 @@ class ReviewTests(TestCase): reviewer_settings.save() # Need one more person in review team one so we can test incrementing skip_count without immediately decrementing it - another_reviewer = PersonFactory.create(name = u"Extra TestReviewer") # needs to be lexically greater than the existing one + another_reviewer = PersonFactory.create(name = "Extra TestReviewer") # needs to be lexically greater than the existing one another_reviewer.role_set.create(name_id='reviewer', email=another_reviewer.email(), group=review_req.team) UnavailablePeriod.objects.create( @@ -381,7 +383,7 @@ class ReviewTests(TestCase): self.client.login(username="reviewsecretary", password="reviewsecretary+password") r = self.client.get(req_url) self.assertEqual(r.status_code, 200) - self.assertTrue(assign_url in unicontent(r)) + self.assertContains(r, assign_url) self.client.logout() # get assign page @@ -455,14 +457,14 @@ class ReviewTests(TestCase): self.client.login(username="reviewsecretary", password="reviewsecretary+password") r = self.client.get(req_url) self.assertEqual(r.status_code, 200) - self.assertTrue(reject_url in unicontent(r)) + self.assertContains(r, reject_url) self.client.logout() # get reject page login_testing_unauthorized(self, "reviewsecretary", reject_url) r = self.client.get(reject_url) self.assertEqual(r.status_code, 200) - self.assertTrue(unicode(assignment.reviewer.person) in unicontent(r)) + self.assertContains(r, str(assignment.reviewer.person)) # reject empty_outbox() @@ -539,7 +541,7 @@ class ReviewTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) - messages = json.loads(r.content)["messages"] + messages = r.json()["messages"] self.assertEqual(len(messages), 2) today = datetime.date.today() @@ -560,7 +562,7 @@ class ReviewTests(TestCase): # Test failure to return mailarch results no_result_path = os.path.join(self.review_dir, "mailarch_no_result.html") - with open(no_result_path, "w") as f: + with io.open(no_result_path, "w") as f: f.write('Content-Type: text/html\n\n
No results found
') ietf.review.mailarch.construct_query_urls = lambda review_req, query=None: { "query_data_url": "file://" + os.path.abspath(no_result_path) } @@ -568,7 +570,7 @@ class ReviewTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) - result = json.loads(r.content) + result = r.json() self.assertNotIn('messages', result) self.assertIn('No results found', result['error']) @@ -617,7 +619,7 @@ class ReviewTests(TestCase): # complete by uploading file empty_outbox() - test_file = StringIO("This is a review\nwith two lines") + test_file = io.StringIO("This is a review\nwith two lines") test_file.name = "unnamed" r = self.client.post(url, data={ @@ -638,7 +640,7 @@ class ReviewTests(TestCase): self.assertTrue(assignment.review_request.team.acronym.lower() in assignment.review.name) self.assertTrue(assignment.review_request.doc.rev in assignment.review.name) - with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: + with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: self.assertEqual(f.read(), "This is a review\nwith two lines") self.assertEqual(len(outbox), 1) @@ -662,10 +664,8 @@ class ReviewTests(TestCase): # check the review document page url = urlreverse('ietf.doc.views_doc.document_main', kwargs={ "name": assignment.review.name }) r = self.client.get(url) - self.assertEqual(r.status_code, 200) - content = unicontent(r) - self.assertIn("{} Review".format(assignment.review_request.type.name), content) - self.assertIn("This is a review", content) + self.assertContains(r, "{} Review".format(assignment.review_request.type.name)) + self.assertContains(r, "This is a review") def test_complete_review_enter_content(self): @@ -690,7 +690,7 @@ class ReviewTests(TestCase): self.assertEqual(assignment.state_id, "completed") self.assertNotEqual(assignment.completed_on, None) - with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: + with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: self.assertEqual(f.read(), "This is a review\nwith two lines") self.assertEqual(len(outbox), 1) @@ -753,7 +753,7 @@ class ReviewTests(TestCase): # Mock up the url response for the request.get() call to retrieve the mailing list url response = Response() response.status_code = 200 - response._content = "This is a review\nwith two lines" + response._content = b"This is a review\nwith two lines" mock.return_value = response # Run the test @@ -768,7 +768,7 @@ class ReviewTests(TestCase): "state": ReviewAssignmentStateName.objects.get(slug="completed").pk, "reviewed_rev": assignment.review_request.doc.rev, "review_submission": "link", - "review_content": response.content, + "review_content": response.content.decode(), "review_url": "http://example.com/testreview/", "review_file": "", }) @@ -777,7 +777,7 @@ class ReviewTests(TestCase): assignment = reload_db_objects(assignment) self.assertEqual(assignment.state_id, "completed") - with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: + with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: self.assertEqual(f.read(), "This is a review\nwith two lines") self.assertEqual(len(outbox), 0) @@ -877,7 +877,7 @@ class ReviewTests(TestCase): event = ReviewAssignmentDocEvent.objects.get(type="closed_review_assignment", review_assignment=assignment) self.assertEqual(event.time, datetime.datetime(2012, 12, 24, 12, 13, 14)) - with open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: + with io.open(os.path.join(self.review_subdir, assignment.review.name + ".txt")) as f: self.assertEqual(f.read(), "This is a review\nwith two lines") self.assertEqual(len(outbox), 0) diff --git a/ietf/doc/tests_status_change.py b/ietf/doc/tests_status_change.py index 36d3415bb..01dd42457 100644 --- a/ietf/doc/tests_status_change.py +++ b/ietf/doc/tests_status_change.py @@ -1,13 +1,17 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import shutil import debug # pyflakes:ignore from pyquery import PyQuery -from StringIO import StringIO +from io import StringIO from textwrap import wrap from django.conf import settings @@ -20,7 +24,7 @@ from ietf.doc.utils import create_ballot_if_not_open from ietf.doc.views_status_change import default_approval_text from ietf.group.models import Person from ietf.iesg.models import TelechatDate -from ietf.utils.test_utils import TestCase, unicontent +from ietf.utils.test_utils import TestCase from ietf.utils.mail import outbox from ietf.utils.test_utils import login_testing_unauthorized @@ -73,9 +77,9 @@ class StatusChangeTests(TestCase): self.assertEqual(r.status_code, 302) status_change = Document.objects.get(name='status-change-imaginary-new') self.assertEqual(status_change.get_state('statchg').slug,'adrev') - self.assertEqual(status_change.rev,u'00') - self.assertEqual(status_change.ad.name,u'Areað Irector') - self.assertEqual(status_change.notify,u'ipu@ietf.org') + self.assertEqual(status_change.rev,'00') + self.assertEqual(status_change.ad.name,'Areað Irector') + self.assertEqual(status_change.notify,'ipu@ietf.org') self.assertTrue(status_change.relateddocument_set.filter(relationship__slug='tois',target__docs__name='draft-ietf-random-thing')) def test_change_state(self): @@ -112,10 +116,10 @@ class StatusChangeTests(TestCase): doc.save_with_history([DocEvent.objects.create(doc=doc, rev=doc.rev, type="changed_document", by=Person.objects.get(user__username="secretary"), desc="Test")]) lc_req_pk = str(State.objects.get(slug='lc-req',type__slug='statchg').pk) r = self.client.post(url,dict(new_state=lc_req_pk)) - self.assertEquals(r.status_code, 200) + self.assertEqual(r.status_code, 200) doc = Document.objects.get(name='status-change-imaginary-mid-review') - self.assertEquals(doc.get_state('statchg').slug,'lc-req') - self.assertEquals(len(outbox), messages_before + 1) + self.assertEqual(doc.get_state('statchg').slug,'lc-req') + self.assertEqual(len(outbox), messages_before + 1) self.assertTrue('Last Call:' in outbox[-1]['Subject']) # successful change to IESG Evaluation @@ -171,15 +175,15 @@ class StatusChangeTests(TestCase): # normal get r = self.client.get(url) - self.assertEquals(r.status_code, 200) + self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEquals(len(q('input[name=title]')),1) + self.assertEqual(len(q('input[name=title]')),1) # change title r = self.client.post(url,dict(title='New title')) - self.assertEquals(r.status_code,302) + self.assertEqual(r.status_code,302) doc = Document.objects.get(name='status-change-imaginary-mid-review') - self.assertEquals(doc.title,'New title') + self.assertEqual(doc.title,'New title') self.assertTrue(doc.latest_event(DocEvent,type="added_comment").desc.startswith('Title changed')) def test_edit_ad(self): @@ -265,8 +269,8 @@ class StatusChangeTests(TestCase): q = PyQuery(r.content) self.assertEqual(len(q('form.edit-last-call-text')),1) - self.assertTrue( 'RFC9999 from Proposed Standard to Internet Standard' in ''.join(wrap(r.content,2**16))) - self.assertTrue( 'RFC9998 from Informational to Historic' in ''.join(wrap(r.content,2**16))) + self.assertContains(r, 'RFC9999 from Proposed Standard to Internet Standard') + self.assertContains(r, 'RFC9998 from Informational to Historic') # save r = self.client.post(url,dict(last_call_text="Bogus last call text",save_last_call_text="1")) @@ -278,17 +282,17 @@ class StatusChangeTests(TestCase): # reset r = self.client.post(url,dict(regenerate_last_call_text="1")) self.assertEqual(r.status_code,200) - self.assertTrue( 'RFC9999 from Proposed Standard to Internet Standard' in ''.join(wrap(r.content,2**16))) - self.assertTrue( 'RFC9998 from Informational to Historic' in ''.join(wrap(r.content,2**16))) + self.assertContains(r, 'RFC9999 from Proposed Standard to Internet Standard') + self.assertContains(r, 'RFC9998 from Informational to Historic') # request last call messages_before = len(outbox) r = self.client.post(url,dict(last_call_text='stuff',send_last_call_request='Save+and+Request+Last+Call')) self.assertEqual(r.status_code,200) - self.assertTrue( 'Last call requested' in ''.join(wrap(r.content,2**16))) + self.assertContains(r, 'Last call requested') self.assertEqual(len(outbox), messages_before + 1) self.assertTrue('Last Call:' in outbox[-1]['Subject']) - self.assertTrue('Last Call Request has been submitted' in ''.join(wrap(unicode(outbox[-1]),2**16))) + self.assertTrue('Last Call Request has been submitted' in ''.join(wrap(outbox[-1].as_string(), width=2**16))) def test_approve(self): @@ -310,8 +314,8 @@ class StatusChangeTests(TestCase): self.assertEqual(len(q('[type=submit]:contains("Send announcement")')), 1) # There should be two messages to edit self.assertEqual(q('input#id_form-TOTAL_FORMS').val(),'2') - self.assertTrue( '(rfc9999) to Internet Standard' in ''.join(wrap(r.content,2**16))) - self.assertTrue( '(rfc9998) to Historic' in ''.join(wrap(r.content,2**16))) + self.assertContains(r, '(rfc9999) to Internet Standard') + self.assertContains(r, '(rfc9998) to Historic') # submit messages_before = len(outbox) @@ -328,10 +332,10 @@ class StatusChangeTests(TestCase): self.assertTrue('Action:' in outbox[-1]['Subject']) self.assertTrue('ietf-announce' in outbox[-1]['To']) self.assertTrue('rfc-editor' in outbox[-1]['Cc']) - self.assertTrue('(rfc9998) to Historic' in ''.join(wrap(unicode(outbox[-1])+unicode(outbox[-2]),2**16))) - self.assertTrue('(rfc9999) to Internet Standard' in ''.join(wrap(unicode(outbox[-1])+unicode(outbox[-2]),2**16))) + self.assertTrue('(rfc9998) to Historic' in ''.join(wrap(outbox[-1].as_string()+outbox[-2].as_string(), 2**16))) + self.assertTrue('(rfc9999) to Internet Standard' in ''.join(wrap(outbox[-1].as_string()+outbox[-2].as_string(),2**16))) - self.assertTrue(doc.latest_event(DocEvent,type="added_comment").desc.startswith('The following approval message was sent')) + self.assertTrue(doc.latest_event(DocEvent,type="added_comment").desc.startswith('The following approval message was sent')) def test_edit_relations(self): doc = Document.objects.get(name='status-change-imaginary-mid-review') @@ -415,13 +419,13 @@ class StatusChangeSubmitTests(TestCase): # sane post using textbox path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - self.assertEqual(doc.rev,u'00') + self.assertEqual(doc.rev,'00') self.assertFalse(os.path.exists(path)) r = self.client.post(url,dict(content="Some initial review text\n",submit_response="1")) self.assertEqual(r.status_code,302) doc = Document.objects.get(name='status-change-imaginary-mid-review') - self.assertEqual(doc.rev,u'00') - with open(path) as f: + self.assertEqual(doc.rev,'00') + with io.open(path) as f: self.assertEqual(f.read(),"Some initial review text\n") self.assertTrue( "mid-review-00" in doc.latest_event(NewRevisionDocEvent).desc) @@ -432,9 +436,9 @@ class StatusChangeSubmitTests(TestCase): # A little additional setup # doc.rev is u'00' per the test setup - double-checking that here - if it fails, the breakage is in setUp - self.assertEqual(doc.rev,u'00') + self.assertEqual(doc.rev,'00') path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - with open(path,'w') as f: + with io.open(path,'w') as f: f.write('This is the old proposal.') f.close() # Put the old proposal into IESG review (exercises ballot tab when looking at an older revision below) @@ -456,7 +460,7 @@ class StatusChangeSubmitTests(TestCase): test_file.name = "unnamed" r = self.client.post(url, dict(txt=test_file,submit_response="1")) self.assertEqual(r.status_code, 200) - self.assertTrue("does not appear to be a text file" in unicontent(r)) + self.assertContains(r, "does not appear to be a text file") # sane post uploading a file test_file = StringIO("This is a new proposal.") @@ -464,9 +468,9 @@ class StatusChangeSubmitTests(TestCase): r = self.client.post(url,dict(txt=test_file,submit_response="1")) self.assertEqual(r.status_code, 302) doc = Document.objects.get(name='status-change-imaginary-mid-review') - self.assertEqual(doc.rev,u'01') + self.assertEqual(doc.rev,'01') path = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - with open(path) as f: + with io.open(path) as f: self.assertEqual(f.read(),"This is a new proposal.") f.close() self.assertTrue( "mid-review-01" in doc.latest_event(NewRevisionDocEvent).desc) @@ -481,7 +485,7 @@ class StatusChangeSubmitTests(TestCase): url = urlreverse('ietf.doc.views_doc.document_main',kwargs=dict(name=doc.name,rev='00')) r = self.client.get(url) self.assertEqual(r.status_code,200) - self.assertTrue("This is the old proposal." in unicontent(r)) + self.assertContains(r, "This is the old proposal.") def setUp(self): DocumentFactory(type_id='statchg',name='status-change-imaginary-mid-review',notify='notify@example.org') diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py index 4e5220646..2de7d2806 100644 --- a/ietf/doc/urls.py +++ b/ietf/doc/urls.py @@ -1,3 +1,5 @@ +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -30,6 +32,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + from django.conf.urls import include from django.views.generic import RedirectView from django.conf import settings @@ -47,8 +52,8 @@ urlpatterns = [ url(r'^$', views_search.search), url(r'^search/?$', views_search.search), url(r'^in-last-call/?$', views_search.drafts_in_last_call), - url(r'^ad/(?P[^/]+)/?$(?u)', views_search.docs_for_ad), - url(r'^ad2/(?P[\w.-]+)/$(?u)', RedirectView.as_view(url='/doc/ad/%(name)s/', permanent=True)), + url(r'^ad/(?P[^/]+)/?$', views_search.docs_for_ad), + url(r'^ad2/(?P[\w.-]+)/$', RedirectView.as_view(url='/doc/ad/%(name)s/', permanent=True)), url(r'^rfc-status-changes/?$', views_status_change.rfc_status_changes), url(r'^start-rfc-status-change/(?:%(name)s/)?$' % settings.URL_REGEXPS, views_status_change.start_rfc_status_change), url(r'^iesg/?$', views_search.drafts_in_iesg_process), diff --git a/ietf/doc/urls_conflict_review.py b/ietf/doc/urls_conflict_review.py index 037e3bb32..3e244d852 100644 --- a/ietf/doc/urls_conflict_review.py +++ b/ietf/doc/urls_conflict_review.py @@ -1,3 +1,8 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from ietf.doc import views_conflict_review, views_doc from ietf.utils.urls import url @@ -6,7 +11,7 @@ urlpatterns = [ url(r'^state/$', views_conflict_review.change_state), url(r'^submit/$', views_conflict_review.submit), url(r'^ad/$', views_conflict_review.edit_ad), - url(r'^approve/$', views_conflict_review.approve), + url(r'^approve/$', views_conflict_review.approve_conflict_review), url(r'^start_conflict_review/$', views_conflict_review.start_review), url(r'^telechat/$', views_doc.telechat_date, name='ietf.doc.views_doc.telechat_date;conflict-review'), url(r'^notices/$', views_doc.edit_notify, name='ietf.doc.views_doc.edit_notify;conflict-review'), diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index d38d00e43..91a2e79be 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -1,14 +1,20 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- -import os -import re -import urllib -import math + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import hashlib +import io import json +import math +import os +import re +import six + from collections import defaultdict +from six.moves.urllib.parse import quote from django.conf import settings from django.contrib import messages @@ -56,7 +62,7 @@ def save_document_in_history(doc): def transfer_fields(obj, HistModel): mfields = get_model_fields_as_dict(item) # map doc -> dochist - for k, v in mfields.iteritems(): + for k, v in mfields.items(): if v == doc: mfields[k] = dochist HistModel.objects.create(**mfields) @@ -209,7 +215,7 @@ def create_ballot(request, doc, by, ballot_slug, time=None): else: e = BallotDocEvent(type="created_ballot", by=by, doc=doc, rev=doc.rev) e.ballot_type = BallotType.objects.get(doc_type=doc.type, slug=ballot_slug) - e.desc = u'Created "%s" ballot' % e.ballot_type.name + e.desc = 'Created "%s" ballot' % e.ballot_type.name e.save() def create_ballot_if_not_open(request, doc, by, ballot_slug, time=None): @@ -220,7 +226,7 @@ def create_ballot_if_not_open(request, doc, by, ballot_slug, time=None): else: e = BallotDocEvent(type="created_ballot", by=by, doc=doc, rev=doc.rev) e.ballot_type = ballot_type - e.desc = u'Created "%s" ballot' % e.ballot_type.name + e.desc = 'Created "%s" ballot' % e.ballot_type.name e.save() return e else: @@ -313,7 +319,7 @@ def add_links_in_new_revision_events(doc, events, diff_revisions): links += "" if prev != None: - links += ' (
diff from previous)' % (settings.RFCDIFF_BASE_URL, urllib.quote(prev, safe="~"), urllib.quote(diff_url, safe="~")) + links += ' (diff from previous)' % (settings.RFCDIFF_BASE_URL, quote(prev, safe="~"), quote(diff_url, safe="~")) # replace the bold filename part e.desc = re.sub(r"(.+-[0-9][0-9].txt)", links, e.desc) @@ -333,7 +339,7 @@ def add_events_message_info(events): def get_unicode_document_content(key, filename, codec='utf-8', errors='ignore'): try: - with open(filename, 'rb') as f: + with io.open(filename, 'rb') as f: raw_content = f.read().decode(codec,errors) except IOError: if settings.DEBUG: @@ -347,7 +353,7 @@ def get_unicode_document_content(key, filename, codec='utf-8', errors='ignore'): def get_document_content(key, filename, split=True, markup=True): log.unreachable("2017-12-05") try: - with open(filename, 'rb') as f: + with io.open(filename, 'rb') as f: raw_content = f.read() except IOError: if settings.DEBUG: @@ -363,7 +369,7 @@ def get_document_content(key, filename, split=True, markup=True): return text.decode(raw_content) def tags_suffix(tags): - return (u"::" + u"::".join(t.name for t in tags)) if tags else u"" + return ("::" + "::".join(t.name for t in tags)) if tags else "" def add_state_change_event(doc, by, prev_state, new_state, prev_tags=[], new_tags=[], timestamp=None): """Add doc event to explain that state change just happened.""" @@ -541,7 +547,7 @@ def rebuild_reference_relations(doc,filename=None): filename=os.path.join(settings.INTERNET_DRAFT_PATH,doc.filename_with_rev()) try: - with open(filename, 'rb') as file: + with io.open(filename, 'rb') as file: refs = draft.Draft(file.read().decode('utf8'), filename).get_refs() except IOError as e: return { 'errors': ["%s :%s" % (e.strerror, filename)] } @@ -551,7 +557,7 @@ def rebuild_reference_relations(doc,filename=None): warnings = [] errors = [] unfound = set() - for ( ref, refType ) in refs.iteritems(): + for ( ref, refType ) in refs.items(): refdoc = DocAlias.objects.filter( name=ref ) count = refdoc.count() if count == 0: @@ -587,9 +593,9 @@ def set_replaces_for_document(request, doc, new_replaces, by, email_subject, com events = [] e = DocEvent(doc=doc, rev=doc.rev, by=by, type='changed_document') - new_replaces_names = u", ".join(d.name for d in new_replaces) or u"None" - old_replaces_names = u", ".join(d.name for d in old_replaces) or u"None" - e.desc = u"This document now replaces %s instead of %s" % (new_replaces_names, old_replaces_names) + new_replaces_names = ", ".join(d.name for d in new_replaces) or "None" + old_replaces_names = ", ".join(d.name for d in old_replaces) or "None" + e.desc = "This document now replaces %s instead of %s" % (new_replaces_names, old_replaces_names) e.save() events.append(e) @@ -661,7 +667,7 @@ def get_initial_notify(doc,extra=None): receivers = [] if extra: - if isinstance(extra,basestring): + if isinstance(extra, six.string_types): extra = extra.split(', ') receivers.extend(extra) @@ -759,15 +765,15 @@ def make_rev_history(doc): } if hasattr(e, 'newrevisiondocevent') and doc.history_set.filter(rev=e.newrevisiondocevent.rev).exists(): history[url]['pages'] = doc.history_set.filter(rev=e.newrevisiondocevent.rev).first().pages - history = history.values() + history = list(history.values()) return sorted(history, key=lambda x: x['published']) def get_search_cache_key(params): from ietf.doc.views_search import SearchForm fields = set(SearchForm.base_fields) - set(['sort',]) - kwargs = dict([ (k,v) for (k,v) in params.items() if k in fields ]) - key = "doc:document:search:" + hashlib.sha512(json.dumps(kwargs, sort_keys=True)).hexdigest() + kwargs = dict([ (k,v) for (k,v) in list(params.items()) if k in fields ]) + key = "doc:document:search:" + hashlib.sha512(json.dumps(kwargs, sort_keys=True).encode('utf-8')).hexdigest() return key def label_wrap(label, items, joiner=',', max=50): @@ -817,21 +823,21 @@ def build_doc_meta_block(doc, path): ipr_url = "%s?submit=draft&id=%s" % (urlreverse('ietf.ipr.views.search'), name) for i, line in enumerate(lines): # add draft links - line = re.sub(r'\b(draft-[-a-z0-9]+)\b', '\g<1>'%(path, ), line) + line = re.sub(r'\b(draft-[-a-z0-9]+)\b', r'\g<1>'%(path, ), line) # add rfcXXXX to RFC links - line = re.sub(r' (rfc[0-9]+)\b', ' \g<1>'%(path, ), line) + line = re.sub(r' (rfc[0-9]+)\b', r' \g<1>'%(path, ), line) # add XXXX to RFC links - line = re.sub(r' ([0-9]{3,5})\b', ' \g<1>'%(path, ), line) + line = re.sub(r' ([0-9]{3,5})\b', r' \g<1>'%(path, ), line) # add draft revision links - line = re.sub(r' ([0-9]{2})\b', ' \g<1>'%(path, name, ), line) + line = re.sub(r' ([0-9]{2})\b', r' \g<1>'%(path, name, ), line) if rfcnum: # add errata link - line = re.sub(r'Errata exist', 'Errata exist'%(errata_url, ), line) + line = re.sub(r'Errata exist', r'Errata exist'%(errata_url, ), line) if is_hst or not rfcnum: # make current draft rev bold - line = re.sub(r'>(%s)<'%rev, '>\g<1><', line) - line = re.sub(r'IPR declarations', 'IPR declarations'%(ipr_url, ), line) - line = line.replace(r'[txt]', '[txt]' % doc.href()) + line = re.sub(r'>(%s)<'%rev, r'>\g<1><', line) + line = re.sub(r'IPR declarations', r'IPR declarations'%(ipr_url, ), line) + line = line.replace(r'[txt]', r'[txt]' % doc.href()) lines[i] = line return lines # diff --git a/ietf/doc/utils_charter.py b/ietf/doc/utils_charter.py index 731f234c9..7b44ad3ff 100644 --- a/ietf/doc/utils_charter.py +++ b/ietf/doc/utils_charter.py @@ -1,9 +1,19 @@ -import re, datetime, os, shutil +# Copyright The IETF Trust 2011-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os +import re +import shutil from django.conf import settings from django.urls import reverse as urlreverse from django.template.loader import render_to_string -from django.utils.encoding import smart_text +from django.utils.encoding import smart_text, force_text import debug # pyflakes:ignore @@ -52,7 +62,7 @@ def next_approved_revision(rev): def read_charter_text(doc): filename = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) try: - with open(filename, 'r') as f: + with io.open(filename, 'r') as f: return f.read() except IOError: return "Error: couldn't read charter text" @@ -140,8 +150,8 @@ def generate_ballot_writeup(request, doc): e.by = request.user.person e.doc = doc e.rev = doc.rev, - e.desc = u"Ballot writeup was generated" - e.text = unicode(render_to_string("doc/charter/ballot_writeup.txt")) + e.desc = "Ballot writeup was generated" + e.text = force_text(render_to_string("doc/charter/ballot_writeup.txt")) # caller is responsible for saving, if necessary return e diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index ae25bef31..14227f4a8 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import debug # pyflakes:ignore @@ -16,9 +19,9 @@ def wrap_value(v): def fill_in_telechat_date(docs, doc_dict=None, doc_ids=None): if doc_dict is None: doc_dict = dict((d.pk, d) for d in docs) - doc_ids = doc_dict.keys() + doc_ids = list(doc_dict.keys()) if doc_ids is None: - doc_ids = doc_dict.keys() + doc_ids = list(doc_dict.keys()) seen = set() for e in TelechatDocEvent.objects.filter(doc__id__in=doc_ids, type="scheduled_for_telechat").order_by('-time'): @@ -36,7 +39,7 @@ def fill_in_document_sessions(docs, doc_dict, doc_ids): # get presentations presentations = SessionPresentation.objects.filter(session_id__in=[ s.id for s in sessions ]) session_list = [ (p.document_id, p.session) for p in presentations ] - for d in doc_dict.values(): + for d in list(doc_dict.values()): d.sessions = [] for (i, s) in session_list: if i in doc_ids: @@ -48,7 +51,7 @@ def fill_in_document_table_attributes(docs, have_telechat_date=False): # TODO - this function evolved from something that assumed it was handling only drafts. It still has places where it assumes all docs are drafts where that is not a correct assumption doc_dict = dict((d.pk, d) for d in docs) - doc_ids = doc_dict.keys() + doc_ids = list(doc_dict.keys()) rfc_aliases = dict([ (a.document.id, a.name) for a in DocAlias.objects.filter(name__startswith="rfc", docs__id__in=doc_ids) ]) @@ -103,7 +106,7 @@ def fill_in_document_table_attributes(docs, have_telechat_date=False): d.expirable = expirable_draft(d) if d.get_state_slug() != "rfc": - d.milestones = [ m for (t, m) in sorted(((m.time, m) for m in d.groupmilestone_set.all() if m.state_id == "active")) ] + d.milestones = [ m for (t, s, v, m) in sorted(((m.time, m.state.slug, m.desc, m) for m in d.groupmilestone_set.all() if m.state_id == "active")) ] d.reviewed_by_teams = sorted(set(r.team.acronym for r in d.reviewrequest_set.filter(state__in=["assigned","accepted","part-completed","completed"]).distinct().select_related('team'))) e = d.latest_event_cache.get('started_iesg_process', None) @@ -112,7 +115,7 @@ def fill_in_document_table_attributes(docs, have_telechat_date=False): # RFCs # errata - erratas = set(Document.objects.filter(tags="errata", name__in=rfc_aliases.keys()).distinct().values_list("name", flat=True)) + erratas = set(Document.objects.filter(tags="errata", name__in=list(rfc_aliases.keys())).distinct().values_list("name", flat=True)) for d in docs: d.has_errata = d.name in erratas @@ -122,7 +125,7 @@ def fill_in_document_table_attributes(docs, have_telechat_date=False): d.obsoleted_by_list = [] d.updated_by_list = [] - xed_by = RelatedDocument.objects.filter(target__name__in=rfc_aliases.values(), + xed_by = RelatedDocument.objects.filter(target__name__in=list(rfc_aliases.values()), relationship__in=("obs", "updates")).select_related('target') rel_rfc_aliases = dict([ (a.document.id, a.name) for a in DocAlias.objects.filter(name__startswith="rfc", docs__id__in=[rel.source_id for rel in xed_by]) ]) for rel in xed_by: @@ -163,13 +166,16 @@ def prepare_document_table(request, docs, query=None, max_results=200): # sort def generate_sort_key(d): + def num(i): + # sortable representation of number as string + return ('%09d' % int(i)) + res = [] rfc_num = d.rfc_number() - if d.type_id == "draft": - res.append(["Active", "Expired", "Replaced", "Withdrawn", "RFC"].index(d.search_heading.split()[0])) + res.append(num(["Active", "Expired", "Replaced", "Withdrawn", "RFC"].index(d.search_heading.split()[0]))) else: res.append(d.type_id); res.append("-"); @@ -182,14 +188,14 @@ def prepare_document_table(request, docs, query=None, max_results=200): res.append(str(d.latest_revision_date)) elif sort_key == "status": if rfc_num != None: - res.append(int(rfc_num)) + res.append(num(rfc_num)) else: - res.append(d.get_state().order if d.get_state() else None) + res.append(num(d.get_state().order) if d.get_state() else None) elif sort_key == "ipr": res.append(len(d.ipr())) elif sort_key == "ad": if rfc_num != None: - res.append(int(rfc_num)) + res.append(num(rfc_num)) elif d.get_state_slug() == "active": if d.get_state("draft-iesg"): res.append(d.get_state("draft-iesg").order) @@ -197,7 +203,7 @@ def prepare_document_table(request, docs, query=None, max_results=200): res.append(0) else: if rfc_num != None: - res.append(int(rfc_num)) + res.append(num(rfc_num)) else: res.append(d.canonical_name()) diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index da6db8dfe..78e0d64f7 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -1,6 +1,11 @@ +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- # ballot management (voting, commenting, writeups, ...) for Area # Directors and Secretariat + +from __future__ import absolute_import, print_function, unicode_literals + import datetime, json from django import forms @@ -162,17 +167,17 @@ def save_position(form, doc, ballot, ad, login=None, send_email=False): # figure out a description if not old_pos and pos.pos.slug != "norecord": - pos.desc = u"[Ballot Position Update] New position, %s, has been recorded for %s" % (pos.pos.name, pos.ad.plain_name()) + pos.desc = "[Ballot Position Update] New position, %s, has been recorded for %s" % (pos.pos.name, pos.ad.plain_name()) elif old_pos and pos.pos != old_pos.pos: pos.desc = "[Ballot Position Update] Position for %s has been changed to %s from %s" % (pos.ad.plain_name(), pos.pos.name, old_pos.pos.name) if not pos.desc and changes: - pos.desc = u"Ballot %s text updated for %s" % (u" and ".join(changes), ad.plain_name()) + pos.desc = "Ballot %s text updated for %s" % (" and ".join(changes), ad.plain_name()) # only add new event if we actually got a change if pos.desc: if login != ad: - pos.desc += u" by %s" % login.plain_name() + pos.desc += " by %s" % login.plain_name() pos.save() @@ -362,7 +367,7 @@ def send_ballot_comment(request, name, ballot_id): if extra_cc: cc.extend(extra_cc) - send_mail_text(request, addrs.to, frm, subject, body, cc=u", ".join(cc)) + send_mail_text(request, addrs.to, frm, subject, body, cc=", ".join(cc)) return HttpResponseRedirect(return_to_url) diff --git a/ietf/doc/views_charter.py b/ietf/doc/views_charter.py index 2f6f36e19..bf827aaf7 100644 --- a/ietf/doc/views_charter.py +++ b/ietf/doc/views_charter.py @@ -1,7 +1,14 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- -import os, datetime, textwrap, json + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import json +import os +import textwrap from django.http import HttpResponseRedirect, HttpResponseNotFound, HttpResponseForbidden, Http404 from django.shortcuts import get_object_or_404, redirect, render @@ -11,6 +18,7 @@ from django.utils.safestring import mark_safe from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required +from django.utils.encoding import force_text import debug # pyflakes:ignore @@ -413,11 +421,11 @@ def submit(request, name, option=None): # Save file on disk filename = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (charter.canonical_name(), charter.rev)) - with open(filename, 'wb') as destination: + with io.open(filename, 'w', encoding='utf-8') as destination: if form.cleaned_data['txt']: destination.write(form.cleaned_data['txt']) else: - destination.write(form.cleaned_data['content'].encode("utf-8")) + destination.write(form.cleaned_data['content']) if option in ['initcharter','recharter'] and charter.ad == None: charter.ad = getattr(group.ad_role(),'person',None) @@ -442,7 +450,7 @@ def submit(request, name, option=None): filename = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (charter_canonical_name, charter_rev)) try: - with open(filename, 'r') as f: + with io.open(filename, 'r') as f: init["content"] = f.read() except IOError: pass @@ -807,8 +815,8 @@ def charter_with_milestones_txt(request, name, rev): charter_text = "" try: - with open(os.path.join(settings.CHARTER_PATH, filename), 'r') as f: - charter_text = unicode(f.read(), errors='ignore') + with io.open(os.path.join(settings.CHARTER_PATH, filename), 'r') as f: + charter_text = force_text(f.read(), errors='ignore') except IOError: charter_text = "Error reading charter text %s" % filename diff --git a/ietf/doc/views_conflict_review.py b/ietf/doc/views_conflict_review.py index 03d06235b..04ed5675c 100644 --- a/ietf/doc/views_conflict_review.py +++ b/ietf/doc/views_conflict_review.py @@ -1,7 +1,12 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- -import datetime, os + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os from django import forms from django.shortcuts import render, get_object_or_404, redirect @@ -159,7 +164,7 @@ class UploadForm(forms.Form): def save(self, review): filename = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (review.canonical_name(), review.rev)) - with open(filename, 'wb') as destination: + with io.open(filename, 'w', encoding='utf-8') as destination: if self.cleaned_data['txt']: destination.write(self.cleaned_data['txt']) else: @@ -223,7 +228,7 @@ def submit(request, name): else: filename = os.path.join(settings.CONFLICT_REVIEW_PATH, '%s-%s.txt' % (review.canonical_name(), review.rev)) try: - with open(filename, 'r') as f: + with io.open(filename, 'r') as f: init["content"] = f.read() except IOError: pass @@ -301,7 +306,7 @@ class AnnouncementForm(forms.Form): announcement_text = forms.CharField(widget=forms.Textarea, label="IETF Conflict Review Announcement", help_text="Edit the announcement message.", required=True, strip=False) @role_required("Secretariat") -def approve(request, name): +def approve_conflict_review(request, name): """Approve this conflict review, setting the appropriate state and send the announcement to the right parties.""" review = get_object_or_404(Document, type="conflrev", name=name) @@ -468,7 +473,7 @@ def start_review_as_secretariat(request, name): notify_addresses = build_notify_addresses(doc_to_review) init = { "ad" : Role.objects.filter(group__acronym='ietf',name='chair')[0].person.id, - "notify" : u', '.join(notify_addresses), + "notify" : ', '.join(notify_addresses), } form = StartReviewForm(initial=init) @@ -502,7 +507,7 @@ def start_review_as_stream_owner(request, name): notify_addresses = build_notify_addresses(doc_to_review) init = { - "notify" : u', '.join(notify_addresses), + "notify" : ', '.join(notify_addresses), } form = SimpleStartReviewForm(initial=init) diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 343d6e1f4..d10323ce5 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1,6 +1,6 @@ -# Copyright The IETF Trust 2016-2019, All Rights Reserved +# Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- - +# # Parts Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -33,7 +33,18 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os, datetime, urllib, json, glob, re + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import glob +import io +import json +import os +import re +import six + +from six.moves.urllib.parse import quote from django.http import HttpResponse, Http404 , HttpResponseForbidden from django.shortcuts import render, get_object_or_404, redirect @@ -251,7 +262,7 @@ def document_main(request, name, rev=None): if iesg_state and iesg_state.slug in IESG_BALLOT_ACTIVE_STATES: active_ballot = doc.active_ballot() if active_ballot: - ballot_summary = needed_ballot_positions(doc, active_ballot.active_ad_positions().values()) + ballot_summary = needed_ballot_positions(doc, list(active_ballot.active_ad_positions().values())) # submission submission = "" @@ -312,7 +323,7 @@ def document_main(request, name, rev=None): if doc.stream_id == "ietf" and group.type_id == "wg" and group.list_archive: search_archive = group.list_archive - search_archive = urllib.quote(search_archive, safe="~") + search_archive = quote(search_archive, safe="~") # conflict reviews conflict_reviews = [d.document.name for d in doc.related_that("conflrev")] @@ -458,7 +469,7 @@ def document_main(request, name, rev=None): if doc.get_state_slug() in ("intrev", "iesgrev"): active_ballot = doc.active_ballot() if active_ballot: - ballot_summary = needed_ballot_positions(doc, active_ballot.active_ad_positions().values()) + ballot_summary = needed_ballot_positions(doc, list(active_ballot.active_ad_positions().values())) else: ballot_summary = "No active ballot found." @@ -493,14 +504,14 @@ def document_main(request, name, rev=None): if doc.rev == "00" and not os.path.isfile(pathname): # This could move to a template - content = u"A conflict review response has not yet been proposed." + content = "A conflict review response has not yet been proposed." else: content = doc.text_or_error() # pyflakes:ignore content = markup_txt.markup(content) ballot_summary = None if doc.get_state_slug() in ("iesgeval") and doc.active_ballot(): - ballot_summary = needed_ballot_positions(doc, doc.active_ballot().active_ad_positions().values()) + ballot_summary = needed_ballot_positions(doc, list(doc.active_ballot().active_ad_positions().values())) return render(request, "doc/document_conflict_review.html", dict(doc=doc, @@ -521,13 +532,13 @@ def document_main(request, name, rev=None): if doc.rev == "00" and not os.path.isfile(pathname): # This could move to a template - content = u"Status change text has not yet been proposed." + content = "Status change text has not yet been proposed." else: content = doc.text_or_error() # pyflakes:ignore ballot_summary = None if doc.get_state_slug() in ("iesgeval"): - ballot_summary = needed_ballot_positions(doc, doc.active_ballot().active_ad_positions().values()) + ballot_summary = needed_ballot_positions(doc, list(doc.active_ballot().active_ad_positions().values())) if isinstance(doc,Document): sorted_relations=doc.relateddocument_set.all().order_by('relationship__name') @@ -623,7 +634,7 @@ def document_main(request, name, rev=None): def document_html(request, name, rev=None): if name.startswith('rfc0'): name = "rfc" + name[3:].lstrip('0') - if name.startswith('review-') and re.search('-\d\d\d\d-\d\d$', name): + if name.startswith('review-') and re.search(r'-\d\d\d\d-\d\d$', name): name = "%s-%s" % (name, rev) if rev and not name.startswith('charter-') and re.search('[0-9]{1,2}-[0-9]{2}', rev): name = "%s-%s" % (name, rev[:-3]) @@ -658,10 +669,10 @@ def document_html(request, name, rev=None): return render(request, "doc/document_html.html", {"doc":doc, "top":top, "navbar_mode":"navbar-static-top", }) def check_doc_email_aliases(): - pattern = re.compile('^expand-(.*?)(\..*?)?@.*? +(.*)$') + pattern = re.compile(r'^expand-(.*?)(\..*?)?@.*? +(.*)$') good_count = 0 tot_count = 0 - with open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file: + with io.open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file: for line in virtual_file.readlines(): m = pattern.match(line) tot_count += 1 @@ -673,11 +684,11 @@ def check_doc_email_aliases(): def get_doc_email_aliases(name): if name: - pattern = re.compile('^expand-(%s)(\..*?)?@.*? +(.*)$'%name) + pattern = re.compile(r'^expand-(%s)(\..*?)?@.*? +(.*)$'%name) else: - pattern = re.compile('^expand-(.*?)(\..*?)?@.*? +(.*)$') + pattern = re.compile(r'^expand-(.*?)(\..*?)?@.*? +(.*)$') aliases = [] - with open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file: + with io.open(settings.DRAFT_VIRTUAL_PATH,"r") as virtual_file: for line in virtual_file.readlines(): m = pattern.match(line) if m: @@ -1263,7 +1274,7 @@ def add_sessionpresentation(request,name): if doc.group: sessions = sorted(sessions,key=lambda x:0 if x.group==doc.group else 1) - session_choices = [(s.pk,unicode(s)) for s in sessions] + session_choices = [(s.pk, six.text_type(s)) for s in sessions] if request.method == 'POST': version_form = VersionForm(request.POST,choices=version_choices) diff --git a/ietf/doc/views_downref.py b/ietf/doc/views_downref.py index 91cf88253..4336566f8 100644 --- a/ietf/doc/views_downref.py +++ b/ietf/doc/views_downref.py @@ -1,6 +1,9 @@ -# Copyright The IETF Trust 2017, All Rights Reserved +# Copyright The IETF Trust 2017-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals -from __future__ import unicode_literals from django.urls import reverse as urlreverse from django.http import HttpResponseRedirect diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index fc586a452..3a4be72c5 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2010-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + # changing state and metadata on Internet Drafts import datetime @@ -65,10 +68,10 @@ class ChangeStateForm(forms.Form): prev_tag = prev_tag[0] if prev_tag else None if state == prev and tag == prev_tag: - self._errors['comment'] = ErrorList([u'State not changed. Comments entered will be lost with no state change. Please go back and use the Add Comment feature on the history tab to add comments without changing state.']) + self._errors['comment'] = ErrorList(['State not changed. Comments entered will be lost with no state change. Please go back and use the Add Comment feature on the history tab to add comments without changing state.']) if state != '(None)' and state.slug == 'idexists' and tag: - self._errors['substate'] = ErrorList([u'Clear substate before setting the document to the idexists state.']) + self._errors['substate'] = ErrorList(['Clear substate before setting the document to the idexists state.']) return retclean @@ -134,7 +137,7 @@ def change_state(request, name): email_state_changed(request, doc, msg,'doc_state_edited') if new_state.slug == "approved" and new_tags == [] and has_role(request.user, "Area Director"): - email_ad_approved_doc(request, doc, comment) + email_ad_approved_doc(request, doc, comment) if prev_state and prev_state.slug in ("ann", "rfcqueue") and new_state.slug not in ("rfcqueue", "pub"): email_pulled_from_rfc_queue(request, doc, comment, prev_state, new_state) @@ -268,7 +271,7 @@ def change_stream(request, name): events = [] e = DocEvent(doc=doc, rev=doc.rev, by=login, type='changed_document') - e.desc = u"Stream changed to %s from %s"% (new_stream, old_stream or "None") + e.desc = "Stream changed to %s from %s"% (new_stream, old_stream or "None") e.save() events.append(e) @@ -281,7 +284,7 @@ def change_stream(request, name): doc.save_with_history(events) - msg = u"\n".join(e.desc for e in events) + msg = "\n".join(e.desc for e in events) email_stream_changed(request, doc, old_stream, new_stream, msg) @@ -437,7 +440,7 @@ def change_intention(request, name): events = [] e = DocEvent(doc=doc, rev=doc.rev, by=login, type='changed_document') - e.desc = u"Intended Status changed to %s from %s"% (new_level,old_level) + e.desc = "Intended Status changed to %s from %s"% (new_level,old_level) e.save() events.append(e) @@ -459,7 +462,7 @@ def change_intention(request, name): doc.save_with_history(events) - msg = u"\n".join(e.desc for e in events) + msg = "\n".join(e.desc for e in events) email_intended_status_changed(request, doc, msg) @@ -578,7 +581,7 @@ def to_iesg(request,name): doc.notify = notify changes.append("State Change Notice email list changed to %s" % doc.notify) - # Get the last available writeup + # Get the last available writeup previous_writeup = doc.latest_event(WriteupDocEvent,type="changed_protocol_writeup") if previous_writeup != None: changes.append(previous_writeup.text) @@ -719,9 +722,9 @@ def edit_info(request, name): if r["area"] != doc.group: if r["area"].type_id == "area": - changes.append(u"Assigned to %s" % r["area"].name) + changes.append("Assigned to %s" % r["area"].name) else: - changes.append(u"No longer assigned to any area") + changes.append("No longer assigned to any area") doc.group = r["area"] for c in changes: @@ -904,7 +907,7 @@ def edit_shepherd_writeup(request, name): writeup = form.cleaned_data['content'] e = WriteupDocEvent(doc=doc, rev=doc.rev, by=login, type="changed_protocol_writeup") - # Add the shepherd writeup to description if the document is in submitted for publication state + # Add the shepherd writeup to description if the document is in submitted for publication state stream_state = doc.get_state("draft-stream-%s" % doc.stream_id) iesg_state = doc.get_state("draft-iesg") if (iesg_state or (stream_state and stream_state.slug=='sub-pub')): @@ -1160,9 +1163,9 @@ def edit_document_urls(request, name): res = [] for u in urls: if u.desc: - res.append(u"%s %s (%s)" % (u.tag.slug, u.url, u.desc.strip('()'))) + res.append("%s %s (%s)" % (u.tag.slug, u.url, u.desc.strip('()'))) else: - res.append(u"%s %s" % (u.tag.slug, u.url)) + res.append("%s %s" % (u.tag.slug, u.url)) return fs.join(res) doc = get_object_or_404(Document, name=name) @@ -1376,9 +1379,9 @@ def adopt_draft(request, name): # stream if doc.stream != new_stream: e = DocEvent(type="changed_stream", doc=doc, rev=doc.rev, by=by) - e.desc = u"Changed stream to %s" % new_stream.name + e.desc = "Changed stream to %s" % new_stream.name if doc.stream: - e.desc += u" from %s" % doc.stream.name + e.desc += " from %s" % doc.stream.name e.save() events.append(e) old_stream = doc.stream @@ -1389,7 +1392,7 @@ def adopt_draft(request, name): # group if group != doc.group: e = DocEvent(type="changed_group", doc=doc, rev=doc.rev, by=by) - e.desc = u"Changed group to %s (%s)" % (group.name, group.acronym.upper()) + e.desc = "Changed group to %s (%s)" % (group.name, group.acronym.upper()) if doc.group.type_id != "individ": e.desc += " from %s (%s)" % (doc.group.name, doc.group.acronym.upper()) e.save() @@ -1456,12 +1459,12 @@ def release_draft(request, name): events = [] if doc.stream.slug == 'ise' or doc.group.type_id != 'individ': - existing_tags = set(doc.tags.all()) + existing_tags = list(doc.tags.all()) if existing_tags: doc.tags.clear() e = DocEvent(type="changed_document", doc=doc, rev=doc.rev, by=by) l = [] - l.append(u"Tag%s %s cleared." % (pluralize(existing_tags), ", ".join(t.name for t in existing_tags))) + l.append("Tag%s %s cleared." % (pluralize(existing_tags), ", ".join(t.name for t in existing_tags))) e.desc = " ".join(l) e.save() events.append(e) @@ -1487,7 +1490,7 @@ def release_draft(request, name): if doc.stream: e = DocEvent(type="changed_stream", doc=doc, rev=doc.rev, by=by) - e.desc = u"Changed stream to None from %s" % doc.stream.name + e.desc = "Changed stream to None from %s" % doc.stream.name e.save() events.append(e) old_stream = doc.stream @@ -1529,9 +1532,9 @@ class ChangeStreamStateForm(forms.Form): f.label = state_type.label if self.stream.slug == 'ietf': if self.can_set_sub_pub: - f.help_text = u"Only select 'Submitted to IESG for Publication' to correct errors. Use the document's main page to request publication." + f.help_text = "Only select 'Submitted to IESG for Publication' to correct errors. Use the document's main page to request publication." else: - f.help_text = u"You may not set the 'Submitted to IESG for Publication' using this form - Use the document's main page to request publication." + f.help_text = "You may not set the 'Submitted to IESG for Publication' using this form - Use the document's main page to request publication." f = self.fields['tags'] f.queryset = f.queryset.filter(slug__in=get_tags_for_stream_id(doc.stream_id)) @@ -1620,9 +1623,9 @@ def change_stream_state(request, name, state_type): removed_tags = existing_tags - new_tags l = [] if added_tags: - l.append(u"Tag%s %s set." % (pluralize(added_tags), ", ".join(t.name for t in added_tags))) + l.append("Tag%s %s set." % (pluralize(added_tags), ", ".join(t.name for t in added_tags))) if removed_tags: - l.append(u"Tag%s %s cleared." % (pluralize(removed_tags), ", ".join(t.name for t in removed_tags))) + l.append("Tag%s %s cleared." % (pluralize(removed_tags), ", ".join(t.name for t in removed_tags))) e.desc = " ".join(l) e.save() events.append(e) diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py index 5cc5609cb..29c6174b9 100644 --- a/ietf/doc/views_material.py +++ b/ietf/doc/views_material.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + # views for managing group materials (slides, ...) +import io import os import re @@ -51,7 +55,7 @@ class UploadMaterialForm(forms.Form): self.fields["state"].widget = forms.HiddenInput() self.fields["state"].queryset = self.fields["state"].queryset.filter(slug="active") self.fields["state"].initial = self.fields["state"].queryset[0].pk - self.fields["name"].initial = u"%s-%s-" % (doc_type.slug, group.acronym) + self.fields["name"].initial = "%s-%s-" % (doc_type.slug, group.acronym) else: del self.fields["name"] @@ -140,7 +144,7 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None): f = form.cleaned_data["material"] file_ext = os.path.splitext(f.name)[1] - with open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + file_ext), 'wb+') as dest: + with io.open(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + file_ext), 'wb+') as dest: for chunk in f.chunks(): dest.write(chunk) @@ -157,17 +161,17 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None): if prev_title != doc.title: e = DocEvent(doc=doc, rev=doc.rev, by=request.user.person, type='changed_document') - e.desc = u"Changed title to %s" % doc.title + e.desc = "Changed title to %s" % doc.title if prev_title: - e.desc += u" from %s" % prev_title + e.desc += " from %s" % prev_title e.save() events.append(e) if prev_abstract != doc.abstract: e = DocEvent(doc=doc, rev=doc.rev, by=request.user.person, type='changed_document') - e.desc = u"Changed abstract to %s" % doc.abstract + e.desc = "Changed abstract to %s" % doc.abstract if prev_abstract: - e.desc += u" from %s" % prev_abstract + e.desc += " from %s" % prev_abstract e.save() events.append(e) diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index cec4fbfbc..d4ac64db6 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -1,7 +1,10 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import datetime import requests @@ -620,13 +623,13 @@ def complete_review(request, name, assignment_id): # save file on disk if review_submission == "upload": - encoded_content = form.cleaned_data['review_file'] + content = form.cleaned_data['review_file'] else: - encoded_content = form.cleaned_data['review_content'].encode("utf-8") + content = form.cleaned_data['review_content'] filename = os.path.join(review.get_file_path(), '{}.txt'.format(review.name, review.rev)) - with open(filename, 'wb') as destination: - destination.write(encoded_content) + with io.open(filename, 'w', encoding='utf-8') as destination: + destination.write(content) completion_datetime = datetime.datetime.now() if "completion_date" in form.cleaned_data: @@ -699,7 +702,7 @@ def complete_review(request, name, assignment_id): cc=form.cleaned_data["cc"], body = render_to_string("review/completed_review.txt", { "assignment": assignment, - "content": encoded_content.decode("utf-8"), + "content": content, }), ) msg.related_groups.add(*related_groups) @@ -769,9 +772,9 @@ def search_mail_archive(request, name, assignment_id): try: res["messages"] = mailarch.retrieve_messages(res["query_data_url"])[:MAX_RESULTS] except KeyError as e: - res["error"] = "No results found" + res["error"] = "No results found (%s)" % str(e) except Exception as e: - res["error"] = "Retrieval from mail archive failed: %s" % unicode(e) + res["error"] = "Retrieval from mail archive failed: %s" % str(e) # raise # useful when debugging return JsonResponse(res) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 24ec9f51e..09b582aa4 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -1,6 +1,6 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- - +# # Some parts Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -33,6 +33,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import re import datetime @@ -265,7 +268,7 @@ def search_for_name(request, name): return HttpResponseRedirect(url) # chop away extension - extension_split = re.search("^(.+)\.(txt|ps|pdf)$", n) + extension_split = re.search(r"^(.+)\.(txt|ps|pdf)$", n) if extension_split: n = extension_split.group(1) @@ -494,7 +497,7 @@ def index_all_drafts(request): if name.startswith("rfc"): name = name.upper() - sort_key = -int(name[3:]) + sort_key = '%09d' % (100000000-int(name[3:])) names.append((name, sort_key)) diff --git a/ietf/doc/views_status_change.py b/ietf/doc/views_status_change.py index d62319fad..b52e7d0cf 100644 --- a/ietf/doc/views_status_change.py +++ b/ietf/doc/views_status_change.py @@ -1,7 +1,13 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- -import datetime, os, re + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os +import re from django import forms from django.shortcuts import render, get_object_or_404, redirect @@ -9,6 +15,7 @@ from django.http import Http404, HttpResponseRedirect from django.urls import reverse from django.template.loader import render_to_string from django.conf import settings +from django.utils.encoding import force_text import debug # pyflakes:ignore @@ -124,7 +131,7 @@ class UploadForm(forms.Form): def save(self, doc): filename = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) - with open(filename, 'wb') as destination: + with io.open(filename, 'w', encoding='utf-8') as destination: if self.cleaned_data['txt']: destination.write(self.cleaned_data['txt']) else: @@ -188,7 +195,7 @@ def submit(request, name): else: filename = os.path.join(settings.STATUS_CHANGE_PATH, '%s-%s.txt' % (doc.canonical_name(), doc.rev)) try: - with open(filename, 'r') as f: + with io.open(filename, 'r') as f: init["content"] = f.read() except IOError: pass @@ -392,10 +399,10 @@ def clean_helper(form, formtype): new_relations = {} rfc_fields = {} status_fields={} - for k in sorted(form.data.iterkeys()): + for k in sorted(form.data.keys()): v = form.data[k] if k.startswith('new_relation_row'): - if re.match('\d{1,4}',v): + if re.match(r'\d{1,4}',v): v = 'rfc'+v rfc_fields[k[17:]]=v elif k.startswith('statchg_relation_row'): @@ -412,7 +419,7 @@ def clean_helper(form, formtype): errors=[] for key in new_relations: - if not re.match('(?i)rfc\d{1,4}',key): + if not re.match(r'(?i)rfc\d{1,4}',key): errors.append(key+" is not a valid RFC - please use the form RFCn\n") elif not DocAlias.objects.filter(name=key): errors.append(key+" does not exist\n") @@ -634,7 +641,7 @@ def generate_last_call_text(request, doc): e.doc = doc e.rev = doc.rev e.desc = 'Last call announcement was generated' - e.text = unicode(new_text) + e.text = force_text(new_text) e.save() return e diff --git a/ietf/group/admin.py b/ietf/group/admin.py index ad3056ce1..b44babe1b 100644 --- a/ietf/group/admin.py +++ b/ietf/group/admin.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2010-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + from functools import update_wrapper from django.contrib import admin @@ -9,7 +12,7 @@ from django.core.exceptions import PermissionDenied from django.core.management import load_command_class from django.http import Http404 from django.shortcuts import render -from django.utils.encoding import force_unicode +from django.utils.encoding import force_text from django.utils.html import escape from django.utils.translation import ugettext as _ @@ -38,7 +41,7 @@ class GroupAdmin(admin.ModelAdmin): roles = Role.objects.filter(group=obj).order_by("name", "person__name").select_related('person') res = [] for r in roles: - res.append(u'%s (%s)' % (r.person.pk, escape(r.person.plain_name()), r.pk, r.name.name)) + res.append('%s (%s)' % (r.person.pk, escape(r.person.plain_name()), r.pk, r.name.name)) return ", ".join(res) role_list.short_description = "Persons" role_list.allow_tags = True @@ -96,7 +99,7 @@ class GroupAdmin(admin.ModelAdmin): raise PermissionDenied if obj is None: - raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_unicode(opts.verbose_name), 'key': escape(object_id)}) + raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_text(opts.verbose_name), 'key': escape(object_id)}) return self.send_reminder(request, sdo=obj) @@ -144,7 +147,7 @@ class GroupHistoryAdmin(admin.ModelAdmin): admin.site.register(GroupHistory, GroupHistoryAdmin) class GroupURLAdmin(admin.ModelAdmin): - list_display = [u'id', 'group', 'name', 'url'] + list_display = ['id', 'group', 'name', 'url'] raw_id_fields = ['group'] search_fields = ['name'] admin.site.register(GroupURL, GroupURLAdmin) @@ -157,7 +160,7 @@ admin.site.register(GroupMilestone, GroupMilestoneAdmin) admin.site.register(GroupMilestoneHistory, GroupMilestoneAdmin) class GroupStateTransitionsAdmin(admin.ModelAdmin): - list_display = [u'id', 'group', 'state'] + list_display = ['id', 'group', 'state'] raw_id_fields = ['group', 'state'] admin.site.register(GroupStateTransitions, GroupStateTransitionsAdmin) @@ -183,7 +186,7 @@ class ChangeStateGroupEventAdmin(admin.ModelAdmin): admin.site.register(ChangeStateGroupEvent, ChangeStateGroupEventAdmin) class MilestoneGroupEventAdmin(admin.ModelAdmin): - list_display = [u'id', 'group', 'time', 'type', 'by', 'desc', 'milestone'] + list_display = ['id', 'group', 'time', 'type', 'by', 'desc', 'milestone'] list_filter = ['time'] raw_id_fields = ['group', 'by', 'milestone'] admin.site.register(MilestoneGroupEvent, MilestoneGroupEventAdmin) diff --git a/ietf/group/dot.py b/ietf/group/dot.py index abb5c4efe..0efc40545 100644 --- a/ietf/group/dot.py +++ b/ietf/group/dot.py @@ -1,6 +1,9 @@ -# Copyright The IETF Trust 2007-2019, All Rights Reserved +# Copyright The IETF Trust 2017-2019, All Rights Reserved +# -*- coding: utf-8 -*- # -*- check-flake8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals from django.db.models import Q from django.template.loader import render_to_string diff --git a/ietf/group/feeds.py b/ietf/group/feeds.py index c41b2ada2..8cc8a98ee 100644 --- a/ietf/group/feeds.py +++ b/ietf/group/feeds.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + from django.contrib.syndication.views import Feed, FeedDoesNotExist from django.utils.feedgenerator import Atom1Feed from django.urls import reverse as urlreverse @@ -18,15 +21,15 @@ class GroupChangesFeed(Feed): return Group.objects.get(acronym=acronym) def title(self, obj): - return u"Changes for %s %s" % (obj.acronym, obj.type) + return "Changes for %s %s" % (obj.acronym, obj.type) def link(self, obj): - if not obj: - raise FeedDoesNotExist + if not obj: + raise FeedDoesNotExist return obj.about_url() def description(self, obj): - return self.title(obj) + return self.title(obj) def items(self, obj): events = list(obj.groupevent_set.all().select_related("group")) @@ -44,11 +47,11 @@ class GroupChangesFeed(Feed): return obj.group.about_url() def item_pubdate(self, obj): - return obj.time + return obj.time def item_title(self, obj): - title = u"%s - %s" % (truncatewords(strip_tags(obj.desc), 10), obj.by) + title = "%s - %s" % (truncatewords(strip_tags(obj.desc), 10), obj.by) if isinstance(obj, DocEvent): - title = u"Chartering: %s" % title + title = "Chartering: %s" % title return title diff --git a/ietf/group/forms.py b/ietf/group/forms.py index 81dab5aea..50c56baa2 100644 --- a/ietf/group/forms.py +++ b/ietf/group/forms.py @@ -1,6 +1,8 @@ -# Copyright The IETF Trust 2007-2019, All Rights Reserved +# Copyright The IETF Trust 2017-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals # Stdlib imports import re @@ -19,7 +21,7 @@ from ietf.review.models import ReviewerSettings, UnavailablePeriod, ReviewSecret from ietf.review.utils import close_review_request_states, setup_reviewer_field from ietf.utils.textupload import get_cleaned_text_file_content from ietf.utils.text import strip_suffix -from ietf.utils.ordereddict import insert_after_in_ordered_dict +#from ietf.utils.ordereddict import insert_after_in_ordered_dict from ietf.utils.fields import DatepickerDateField, MultiEmailField # --- Constants -------------------------------------------------------- @@ -48,11 +50,11 @@ class StatusUpdateForm(forms.Form): def clean(self): if (self.cleaned_data['content'] and self.cleaned_data['content'].strip() and self.cleaned_data['txt']): - raise forms.ValidationError("Cannot enter both text box and TXT file") - elif (self.cleaned_data['content'] and not self.cleaned_data['content'].strip() and not self.cleaned_data['txt']): + raise forms.ValidationError("Cannot enter both text box and TXT file") + elif (self.cleaned_data['content'] and not self.cleaned_data['content'].strip() and not self.cleaned_data['txt']): raise forms.ValidationError("NULL input is not a valid option") elif (self.cleaned_data['txt'] and not self.cleaned_data['txt'].strip()) : - raise forms.ValidationError("NULL TXT file input is not a valid option") + raise forms.ValidationError("NULL TXT file input is not a valid option") class ConcludeGroupForm(forms.Form): instructions = forms.CharField(widget=forms.Textarea(attrs={'rows': 30}), required=True, strip=False) @@ -116,11 +118,13 @@ class GroupForm(forms.Form): for r in role_fields_to_remove: del self.fields[r + "_roles"] if field: - for f in self.fields: + keys = list(self.fields.keys()) + for f in keys: if f != field: del self.fields[f] def clean_acronym(self): + try: # Changing the acronym of an already existing group will cause 404s all # over the place, loose history, and generally muck up a lot of # things, so we don't permit it @@ -139,42 +143,45 @@ class GroupForm(forms.Form): confirmed = self.data.get("confirm_acronym", False) - def insert_confirm_field(label, initial): - # set required to false, we don't need it since we do the - # validation of the field in here, and otherwise the - # browser and Django may barf - insert_after_in_ordered_dict(self.fields, "confirm_acronym", forms.BooleanField(label=label, required=False), after="acronym") - # we can't set initial, it's ignored since the form is bound, instead mutate the data - self.data = self.data.copy() - self.data["confirm_acronym"] = initial +# def insert_confirm_field(label, initial): +# # set required to false, we don't need it since we do the +# # validation of the field in here, and otherwise the +# # browser and Django may barf +# insert_after_in_ordered_dict(self.fields, "confirm_acronym", forms.BooleanField(label=label, required=False), after="acronym") +# # we can't set initial, it's ignored since the form is bound, instead mutate the data +# self.data = self.data.copy() +# self.data["confirm_acronym"] = initial if existing and existing.type_id == self.group_type: if existing.state_id == "bof": - insert_confirm_field(label="Turn BoF %s into proposed %s and start chartering it" % (existing.acronym, existing.type.name), initial=True) + #insert_confirm_field(label="Turn BoF %s into proposed %s and start chartering it" % (existing.acronym, existing.type.name), initial=True) if confirmed: return acronym else: - raise forms.ValidationError("Warning: Acronym used for an existing BoF (%s)." % existing.name) + raise forms.ValidationError("Warning: Acronym used for an existing BoF (%s)." % existing.acronym) else: - insert_confirm_field(label="Set state of %s %s to proposed and start chartering it" % (existing.acronym, existing.type.name), initial=False) + #insert_confirm_field(label="Set state of %s %s to proposed and start chartering it" % (existing.acronym, existing.type.name), initial=False) if confirmed: return acronym else: - raise forms.ValidationError("Warning: Acronym used for an existing %s (%s, %s)." % (existing.type.name, existing.name, existing.state.name if existing.state else "unknown state")) + raise forms.ValidationError("Warning: Acronym used for an existing %s (%s, %s)." % (existing.type.name, existing.acronym, existing.state.name if existing.state else "unknown state")) if existing: - raise forms.ValidationError("Acronym used for an existing group (%s)." % existing.name) + raise forms.ValidationError("Acronym used for an existing group (%s)." % existing.acronym) - # TODO: Why is this limited to types wg and rg? We would want to be warned about _any_ old collision I think? - old = GroupHistory.objects.filter(acronym__iexact=acronym, type__in=("wg", "rg")) + old = GroupHistory.objects.filter(acronym__iexact=acronym) if old: - insert_confirm_field(label="Confirm reusing acronym %s" % old[0].acronym, initial=False) + #insert_confirm_field(label="Confirm reusing acronym %s" % old[0].acronym, initial=False) if confirmed: return acronym else: raise forms.ValidationError("Warning: Acronym used for a historic group.") - return acronym + except forms.ValidationError: + pass + except Exception: + import traceback + traceback.print_exc() def clean_urls(self): return [x.strip() for x in self.cleaned_data["urls"].splitlines() if x.strip()] @@ -198,7 +205,7 @@ class GroupForm(forms.Form): else: raise forms.ValidationError("A group cannot be its own ancestor. " "Found that the group '%s' would end up being the ancestor of (%s)" % (p.acronym, ', '.join([g.acronym for g in seen]))) - + def clean(self): cleaned_data = super(GroupForm, self).clean() state = cleaned_data.get('state', None) diff --git a/ietf/group/mails.py b/ietf/group/mails.py index 3db25f464..9adcbc4d1 100644 --- a/ietf/group/mails.py +++ b/ietf/group/mails.py @@ -1,5 +1,10 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- # generation of mails + +from __future__ import absolute_import, print_function, unicode_literals + import re @@ -13,7 +18,7 @@ from ietf.mailtrigger.utils import gather_address_lists def email_admin_re_charter(request, group, subject, text, mailtrigger): (to,cc) = gather_address_lists(mailtrigger,group=group) - full_subject = u"Regarding %s %s: %s" % (group.type.name, group.acronym, subject) + full_subject = "Regarding %s %s: %s" % (group.type.name, group.acronym, subject) text = strip_tags(text) send_mail(request, to, None, full_subject, @@ -28,32 +33,32 @@ def email_admin_re_charter(request, group, subject, text, mailtrigger): def email_personnel_change(request, group, text, changed_personnel): (to, cc) = gather_address_lists('group_personnel_change',group=group,changed_personnel=changed_personnel) - full_subject = u"Personnel change for %s %s" % (group.acronym,group.type.name) + full_subject = "Personnel change for %s %s" % (group.acronym,group.type.name) send_mail_text(request, to, None, full_subject, text, cc=cc) def email_milestones_changed(request, group, changes, states): def wrap_up_email(addrs, text): - subject = u"Milestones changed for %s %s" % (group.acronym, group.type.name) + subject = "Milestones changed for %s %s" % (group.acronym, group.type.name) if re.search("Added .* for review, due",text): - subject = u"Review Required - " + subject + subject = "Review Required - " + subject text = wordwrap(strip_tags(text), 78) text += "\n\n" - text += u"URL: %s" % (settings.IDTRACKER_BASE_URL + group.about_url()) + text += "URL: %s" % (settings.IDTRACKER_BASE_URL + group.about_url()) send_mail_text(request, addrs.to, None, subject, text, cc=addrs.cc) # first send to those who should see any edits (such as management and chairs) addrs = gather_address_lists('group_milestones_edited',group=group) if addrs.to or addrs.cc: - wrap_up_email(addrs, u"\n\n".join(c + "." for c in changes)) + wrap_up_email(addrs, "\n\n".join(c + "." for c in changes)) # then send only the approved milestones to those who shouldn't be # bothered with milestones pending approval addrs = gather_address_lists('group_approved_milestones_edited',group=group) - msg = u"\n\n".join(c + "." for c,s in zip(changes,states) if not s == "review") + msg = "\n\n".join(c + "." for c,s in zip(changes,states) if not s == "review") if (addrs.to or addrs.cc) and msg: wrap_up_email(addrs, msg) diff --git a/ietf/group/management/commands/show_group_features.py b/ietf/group/management/commands/show_group_features.py index cdbf5e2d4..75789d23f 100644 --- a/ietf/group/management/commands/show_group_features.py +++ b/ietf/group/management/commands/show_group_features.py @@ -1,6 +1,8 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals import collections diff --git a/ietf/group/migrations/0001_initial.py b/ietf/group/migrations/0001_initial.py index 2f45322c1..331aeb166 100644 --- a/ietf/group/migrations/0001_initial.py +++ b/ietf/group/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import datetime from django.db import migrations, models diff --git a/ietf/group/migrations/0002_groupfeatures_historicalgroupfeatures.py b/ietf/group/migrations/0002_groupfeatures_historicalgroupfeatures.py index d3efd2a42..54f7f784e 100644 --- a/ietf/group/migrations/0002_groupfeatures_historicalgroupfeatures.py +++ b/ietf/group/migrations/0002_groupfeatures_historicalgroupfeatures.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.13 on 2018-07-10 15:58 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import django.core.validators import django.db.models.deletion diff --git a/ietf/group/migrations/0003_groupfeatures_data.py b/ietf/group/migrations/0003_groupfeatures_data.py index 55b700840..6d78632ed 100644 --- a/ietf/group/migrations/0003_groupfeatures_data.py +++ b/ietf/group/migrations/0003_groupfeatures_data.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.13 on 2018-07-10 15:58 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.conf import settings from django.db import migrations @@ -10,7 +13,7 @@ import debug # pyflakes:ignore from ietf.review.utils import active_review_teams group_type_features = { - u'ag': { + 'ag': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': 'ietf', @@ -25,7 +28,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'area': { + 'area': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': 'ietf', @@ -40,7 +43,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'dir': { + 'dir': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair,secr', 'agenda_type': None, @@ -55,7 +58,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'review': { + 'review': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair,secr', 'agenda_type': None, @@ -70,7 +73,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': True, 'material_types': 'slides'}, - u'iab': { + 'iab': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': 'ietf', @@ -85,7 +88,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'ietf': { + 'ietf': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': 'ietf', @@ -100,7 +103,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'individ': { + 'individ': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': None, @@ -115,7 +118,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'irtf': { + 'irtf': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': 'ietf', @@ -130,7 +133,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'isoc': { + 'isoc': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': None, @@ -145,7 +148,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'nomcom': { + 'nomcom': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': 'side', @@ -160,7 +163,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'program': { + 'program': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'lead', 'agenda_type': None, @@ -175,7 +178,7 @@ group_type_features = { 'has_milestones': True, 'has_reviews': False, 'material_types': 'slides'}, - u'rfcedtyp': { + 'rfcedtyp': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': 'side', @@ -190,7 +193,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'rg': { + 'rg': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': 'ietf', @@ -205,7 +208,7 @@ group_type_features = { 'has_milestones': True, 'has_reviews': False, 'material_types': 'slides'}, - u'sdo': { + 'sdo': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': None, @@ -220,7 +223,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'team': { + 'team': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': 'ietf', @@ -235,7 +238,7 @@ group_type_features = { 'has_milestones': False, 'has_reviews': False, 'material_types': 'slides'}, - u'wg': { + 'wg': { 'about_page': 'ietf.group.views.group_about', 'admin_roles': 'chair', 'agenda_type': 'ietf', diff --git a/ietf/group/migrations/0004_add_group_feature_fields.py b/ietf/group/migrations/0004_add_group_feature_fields.py index 6547e6405..625273c74 100644 --- a/ietf/group/migrations/0004_add_group_feature_fields.py +++ b/ietf/group/migrations/0004_add_group_feature_fields.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2019-01-10 07:51 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import django.core.validators from django.db import migrations, models diff --git a/ietf/group/migrations/0005_group_features_list_data_to_json.py b/ietf/group/migrations/0005_group_features_list_data_to_json.py index 67e6f9512..1a1597833 100644 --- a/ietf/group/migrations/0005_group_features_list_data_to_json.py +++ b/ietf/group/migrations/0005_group_features_list_data_to_json.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2019-01-09 09:02 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import json import re diff --git a/ietf/group/migrations/0006_group_features_lists_to_jsonfield.py b/ietf/group/migrations/0006_group_features_lists_to_jsonfield.py index 0cc82763e..bd045f6f5 100644 --- a/ietf/group/migrations/0006_group_features_lists_to_jsonfield.py +++ b/ietf/group/migrations/0006_group_features_lists_to_jsonfield.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2019-01-16 05:53 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import jsonfield.fields diff --git a/ietf/group/migrations/0007_new_group_features_data.py b/ietf/group/migrations/0007_new_group_features_data.py index b93e60041..697f1a04a 100644 --- a/ietf/group/migrations/0007_new_group_features_data.py +++ b/ietf/group/migrations/0007_new_group_features_data.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2019-01-09 09:02 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations @@ -9,7 +11,7 @@ from django.db import migrations import debug # pyflakes:ignore group_type_features = { - u'ag': { + 'ag': { 'custom_group_roles': True, 'has_session_materials': True, 'acts_like_wg': True, @@ -22,7 +24,7 @@ group_type_features = { 'matman_roles': ['ad', 'chair', 'delegate', 'secr'], 'role_order': ['chair', 'secr'], }, - u'area': { + 'area': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -35,7 +37,7 @@ group_type_features = { 'matman_roles': ['ad', 'chair', 'delegate', 'secr'], 'role_order': ['chair', 'secr'], }, - u'dir': { + 'dir': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -48,7 +50,7 @@ group_type_features = { 'matman_roles': ['ad', 'chair', 'delegate', 'secr'], 'role_order': ['chair', 'secr'], }, - u'review': { + 'review': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -61,7 +63,7 @@ group_type_features = { 'matman_roles': ['ad', 'secr'], 'role_order': ['chair', 'secr'], }, - u'iab': { + 'iab': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -74,7 +76,7 @@ group_type_features = { 'matman_roles': ['chair', 'delegate'], 'role_order': ['chair', 'secr'], }, - u'ietf': { + 'ietf': { 'custom_group_roles': True, 'has_session_materials': True, 'acts_like_wg': False, @@ -87,7 +89,7 @@ group_type_features = { 'matman_roles': ['chair', 'delegate'], 'role_order': ['chair', 'secr'], }, - u'individ': { + 'individ': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -100,7 +102,7 @@ group_type_features = { 'matman_roles': [], 'role_order': ['chair', 'secr'], }, - u'irtf': { + 'irtf': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -113,7 +115,7 @@ group_type_features = { 'matman_roles': ['chair', 'delegate', 'secr'], 'role_order': ['chair', 'secr'], }, - u'isoc': { + 'isoc': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -126,7 +128,7 @@ group_type_features = { 'matman_roles': ['chair', 'secr'], 'role_order': ['chair', 'secr'], }, - u'nomcom': { + 'nomcom': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -139,7 +141,7 @@ group_type_features = { 'matman_roles': ['chair'], 'role_order': ['chair', 'member', 'advisor'], }, - u'program': { + 'program': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -152,7 +154,7 @@ group_type_features = { 'matman_roles': ['chair', 'secr'], 'role_order': ['chair', 'secr'], }, - u'rfcedtyp': { + 'rfcedtyp': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -165,7 +167,7 @@ group_type_features = { 'matman_roles': [], 'role_order': ['chair', 'secr'], }, - u'rg': { + 'rg': { 'custom_group_roles': False, 'has_session_materials': True, 'acts_like_wg': True, @@ -178,7 +180,7 @@ group_type_features = { 'matman_roles': ['chair', 'secr'], 'role_order': ['chair', 'secr'], }, - u'sdo': { + 'sdo': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -191,7 +193,7 @@ group_type_features = { 'matman_roles': [], 'role_order': ['liaiman'], }, - u'team': { + 'team': { 'custom_group_roles': True, 'has_session_materials': False, 'acts_like_wg': False, @@ -204,7 +206,7 @@ group_type_features = { 'matman_roles': [], 'role_order': ['chair', 'member', 'matman'], }, - u'wg': { + 'wg': { 'custom_group_roles': False, 'has_session_materials': True, 'acts_like_wg': True, diff --git a/ietf/group/migrations/0008_group_features_onetoone.py b/ietf/group/migrations/0008_group_features_onetoone.py index f010e77d4..27883d18f 100644 --- a/ietf/group/migrations/0008_group_features_onetoone.py +++ b/ietf/group/migrations/0008_group_features_onetoone.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2019-01-19 10:08 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/group/migrations/0009_auto_20190122_1012.py b/ietf/group/migrations/0009_auto_20190122_1012.py index 44df99f2b..254ec9e69 100644 --- a/ietf/group/migrations/0009_auto_20190122_1012.py +++ b/ietf/group/migrations/0009_auto_20190122_1012.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.18 on 2019-01-22 10:12 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/group/migrations/0010_add_group_types.py b/ietf/group/migrations/0010_add_group_types.py index bcc480b5d..3f16d377f 100644 --- a/ietf/group/migrations/0010_add_group_types.py +++ b/ietf/group/migrations/0010_add_group_types.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2019-01-09 09:02 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/group/migrations/0011_auto_20190225_1302.py b/ietf/group/migrations/0011_auto_20190225_1302.py index f0349ad9d..0dfa13f25 100644 --- a/ietf/group/migrations/0011_auto_20190225_1302.py +++ b/ietf/group/migrations/0011_auto_20190225_1302.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-02-25 13:02 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/group/migrations/0012_add_old_nomcom_announcements.py b/ietf/group/migrations/0012_add_old_nomcom_announcements.py index c13ff9570..50308af2c 100644 --- a/ietf/group/migrations/0012_add_old_nomcom_announcements.py +++ b/ietf/group/migrations/0012_add_old_nomcom_announcements.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-03-23 17:50 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations @@ -11,14 +14,14 @@ from datetime import datetime def forward(apps, schema_editor): - p=Person.objects.filter(name="Jim Fenton")[0] + p=Person.objects.filter(name="Jim Fenton")[0] # Unfortunately, get_nomcom_by_year only works for 2013 and later, so we need to do things the hard way. - n = Group.objects.filter(acronym="nomcom2002")[0] + n = Group.objects.filter(acronym="nomcom2002")[0] - n.message_set.create(by=p, subject="nomcom call for volunteers", time=datetime(2002,7,30), - frm="Phil Roberts ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="nomcom call for volunteers", time=datetime(2002,7,30), + frm="Phil Roberts ", to="IETF Announcement list ", body=""" This is the call for volunteers to participate in the 2002 IETF Nominations Committee, the committee that will select this year's nominees for the IAB and the IESG. Details about the Nominations @@ -115,8 +118,8 @@ Nordstrom (JWN) """) - n.message_set.create(by=p, subject="Selection of the Nominations Committee", time=datetime(2002,9,17), - frm="Phil Roberts ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Selection of the Nominations Committee", time=datetime(2002,9,17), + frm="Phil Roberts ", to="IETF Announcement list ", body=""" STOCKS USED IN THE NOMCOM SELECTION PROCESS, published result (in 100s), number used as input (using 000s, rounded): @@ -224,10 +227,10 @@ Nominations committee: 140. Eva Gustaffson """ - ) + ) - n.message_set.create(by=p, subject="Announcement of the Nominations Committee", time=datetime(2002,9,18), - frm="Phil Roberts ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Announcement of the Nominations Committee", time=datetime(2002,9,18), + frm="Phil Roberts ", to="IETF Announcement list ", body=""" We have completed selection of the 2002 Nominations committee. @@ -255,10 +258,10 @@ Theodore Ts'o (previous nomcom chair) Information on this year's selection process is available at: """ - ) + ) - n.message_set.create(by=p, subject="Announcement of IESG and IAB Nominations Requests", time=datetime(2002,10,21), - frm="Phil Roberts ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Announcement of IESG and IAB Nominations Requests", time=datetime(2002,10,21), + frm="Phil Roberts ", to="IETF Announcement list ", body=""" The 2002-2003 Nominations Committee is now soliciting nominations for the open slots on the IESG and IAB. @@ -342,8 +345,8 @@ also serves as a non-voting liaison. """) - n.message_set.create(by=p, subject="Announcement of IESG and IAB Nominations Requests", time=datetime(2002,11,05), - frm="Phil Roberts ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Announcement of IESG and IAB Nominations Requests", time=datetime(2002,11,0o5), + frm="Phil Roberts ", to="IETF Announcement list ", body=""" The 2002-2003 Nominations Committee is now soliciting nominations for the open slots on the IESG and IAB. @@ -429,8 +432,8 @@ open position. The Chair of the prior year's nominating committee also serves as a non-voting liaison. """) - n.message_set.create(by=p, subject="Announcement of IESG and IAB Nominations Requests", time=datetime(2002,11,12), - frm="Phil Roberts ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Announcement of IESG and IAB Nominations Requests", time=datetime(2002,11,12), + frm="Phil Roberts ", to="IETF Announcement list ", body=""" The 2002-2003 Nominations Committee is now soliciting nominations for the open slots on the IESG and IAB. @@ -515,10 +518,10 @@ committee from their current membership who are not sitting in an open position. The Chair of the prior year's nominating committee also serves as a non-voting liaison. """ - ) + ) - n.message_set.create(by=p, subject="IETF Nomcom Announcement", time=datetime(2003,2,27), - frm="Phil Roberts ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="IETF Nomcom Announcement", time=datetime(2003,2,27), + frm="Phil Roberts ", to="IETF Announcement list ", body=""" The nomcom is pleased to announce the results of the 2002-2003 selection process. The IAB has approved the IESG candidates and the ISOC has approved the IAB candidates. Please welcome them @@ -580,8 +583,8 @@ Eric Rescorla - IAB liaison Ted Ts'o - past chair """) - n.message_set.create(by=p, subject="Announcement of IESG and IAB Nominations Request", time=datetime(2003,6,11), - frm="Phil Roberts ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Announcement of IESG and IAB Nominations Request", time=datetime(2003,6,11), + frm="Phil Roberts ", to="IETF Announcement list ", body=""" The nominations committee has received notice to fill the vacancy in the Internet Area created with Erik Nordmark's departure. The nominations committee is seeking nominees to fill the open position. The chosen @@ -640,8 +643,8 @@ Ted Ts'o - past chair also serves as a non-voting liaison. """) - n.message_set.create(by=p, subject="Nomcom result announcement", time=datetime(2003,7,15), - frm="Phil Roberts ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Nomcom result announcement", time=datetime(2003,7,15), + frm="Phil Roberts ", to="IETF Announcement list ", body=""" The nomcom is pleased to announce that it has selected Margaret Wasserman to fill the mid-term vacancy left by Erik Nordmark's resignation. @@ -680,10 +683,10 @@ Ted Ts'o - past chair Ted Ts'o - past chair """) - n = Group.objects.filter(acronym="nomcom2003")[0] + n = Group.objects.filter(acronym="nomcom2003")[0] - n.message_set.create(by=p, subject="IETF Nominations Committee Chair Announcement", time=datetime(2003,8,25), - frm="Lynn St. Amour", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="IETF Nominations Committee Chair Announcement", time=datetime(2003,8,25), + frm="Lynn St. Amour", to="IETF Announcement list ", body=""" One of the roles of the ISOC President is to appoint the IETF Nominations Committee chair. This is done through consultation with the IETF and IAB Chairs as well as the ISOC Executive Committee and it gives us great pleasure to announce @@ -705,8 +708,8 @@ President & CEO The Internet Society """) - n.message_set.create(by=p, subject="NomCom call for volunteers", time=datetime(2003,9,22), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="NomCom call for volunteers", time=datetime(2003,9,22), + frm="Richard Draves ", to="IETF Announcement list ", body=""" This is the call for volunteers to participate in the 2003 IETF Nominations Committee, the committee that will select this year's nominees for the IAB and the IESG. @@ -796,8 +799,8 @@ Corning (GLW) Tyco International (TYC) """) - n.message_set.create(by=p, subject="NomCom volunteer list", time=datetime(2003,10,6), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="NomCom volunteer list", time=datetime(2003,10,6), + frm="Richard Draves ", to="IETF Announcement list ", body=""" The final list of volunteers for the 2003 NomCom can be found at http://www.ietf.org/nomcom. I would like to thank everyone who volunteered. @@ -844,8 +847,8 @@ Corning (GLW) Tyco International (TYC) """) - n.message_set.create(by=p, subject="Selection of the Nominations Committee", time=datetime(2003,10,10), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Selection of the Nominations Committee", time=datetime(2003,10,10), + frm="Richard Draves ", to="IETF Announcement list ", body=""" STOCKS USED IN THE NOMCOM SELECTION PROCESS, published result (in 100s), number used as input (using 000s, rounded): @@ -971,8 +974,8 @@ Nominations committee: 108. Andrew Thiessen """) - n.message_set.create(by=p, subject="NomCom Selection", time=datetime(2003,10,10), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="NomCom Selection", time=datetime(2003,10,10), + frm="Richard Draves ", to="IETF Announcement list ", body=""" Please welcome the voting members of this year's nomcom: Kireeti Kompella @@ -1009,8 +1012,8 @@ Thanks, Rich """) - n.message_set.create(by=p, subject="Call for Nominees", time=datetime(2003,10,17), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Call for Nominees", time=datetime(2003,10,17), + frm="Richard Draves ", to="IETF Announcement list ", body=""" Please send the NomCom nominations for the open IESG and IAB positions: @@ -1041,8 +1044,8 @@ Thanks, Rich """) - n.message_set.create(by=p, subject="NomCom members", time=datetime(2003,10,24), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="NomCom members", time=datetime(2003,10,24), + frm="Richard Draves ", to="IETF Announcement list ", body=""" AB has appointed Geoff Huston as its non-voting liaison to the nomcom. @@ -1074,8 +1077,8 @@ Thanks, Rich """) - n.message_set.create(by=p, subject="NomCom at IETF", time=datetime(2003,11,7), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="NomCom at IETF", time=datetime(2003,11,7), + frm="Richard Draves ", to="IETF Announcement list ", body=""" Members of the nomcom at IETF will be happy to chat with you in person about what the nomcom should look for in candidates for the IESG and IAB, specific feedback about the incumbents or potential nominees, @@ -1094,8 +1097,8 @@ Thanks, Rich """) - n.message_set.create(by=p, subject="NomCom News", time=datetime(2003,11,14), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="NomCom News", time=datetime(2003,11,14), + frm="Richard Draves ", to="IETF Announcement list ", body=""" Randy Bush has resigned from his position as O&M Area Director. Hence, the nomcom will be filling Randy's position in addition to the previously announced positions. @@ -1119,8 +1122,8 @@ Thanks, Rich """) - n.message_set.create(by=p, subject="reminder - nominations to replace Randy Bush", time=datetime(2003,11,26), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="reminder - nominations to replace Randy Bush", time=datetime(2003,11,26), + frm="Richard Draves ", to="IETF Announcement list ", body=""" At IETF Randy Bush resigned from his position as Operations & Management Area Director for the IETF. This creates a mid-term vacancy that the IETF NomCom needs to fill. Randy's replacement will have a one-year @@ -1143,8 +1146,8 @@ Thanks, Rich """) - n.message_set.create(by=p, subject="Randy Bush replacement schedule", time=datetime(2003,12,1), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Randy Bush replacement schedule", time=datetime(2003,12,1), + frm="Richard Draves ", to="IETF Announcement list ", body=""" For mid-term vacancies, RFC 2727 says "the selection and confirmation process is expected to be completed within 1 month." However, there are several factors that make a one-month schedule impractical in this @@ -1170,8 +1173,8 @@ Thanks, Rich """) - n.message_set.create(by=p, subject="Randy Bush replacement", time=datetime(2004,1,14), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Randy Bush replacement", time=datetime(2004,1,14), + frm="Richard Draves ", to="IETF Announcement list ", body=""" I am pleased to announce that the IAB has confirmed the NomCom's selection of David Kessens for a one-year term as O&M Area Director, filling the mid-term vacancy left by Randy Bush's resignation. @@ -1180,8 +1183,8 @@ Thanks, Rich """) - n.message_set.create(by=p, subject="NomCom results", time=datetime(2004,2,13), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="NomCom results", time=datetime(2004,2,13), + frm="Richard Draves ", to="IETF Announcement list ", body=""" I am pleased to announce the results of the 2003-2004 NomCom selection process. The IAB has approved the IESG candidates and the ISOC board has approved the IAB candidates. Please welcome them in their roles: @@ -1240,8 +1243,8 @@ Thanks, Rich """) - n.message_set.create(by=p, subject="call for Security AD nominations", time=datetime(2004,9,28), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="call for Security AD nominations", time=datetime(2004,9,28), + frm="Richard Draves ", to="IETF Announcement list ", body=""" Steve Bellovin has resigned from his position as Security Area Director, effective the end of the November IETF meeting. The IESG has asked the 2003-2004 NomCom to fill the mid-term vacancy. Steve's replacement will @@ -1265,8 +1268,8 @@ Thanks, Rich """) - n.message_set.create(by=p, subject="Steve Bellovin replacement", time=datetime(2004,11,7), - frm="Richard Draves ", to="IETF Announcement list ", body=""" + n.message_set.create(by=p, subject="Steve Bellovin replacement", time=datetime(2004,11,7), + frm="Richard Draves ", to="IETF Announcement list ", body=""" I am pleased to announce that the IAB has confirmed the NomCom's selection of Sam Hartman for a one-year term as Security Area Director, filling the mid-term vacancy left by Steve Bellovin's resignation. @@ -1280,20 +1283,20 @@ Thanks, Rich """) - return - + return + def reverse(apps, schema_editor): - n = Group.objects.filter(acronym="nomcom2002")[0] + n = Group.objects.filter(acronym="nomcom2002")[0] - announcements = Message.objects.filter(related_groups=n) - announcements.delete() + announcements = Message.objects.filter(related_groups=n) + announcements.delete() - n = Group.objects.filter(acronym="nomcom2003")[0] - announcements = Message.objects.filter(related_groups=n) - announcements.delete() + n = Group.objects.filter(acronym="nomcom2003")[0] + announcements = Message.objects.filter(related_groups=n) + announcements.delete() - return + return class Migration(migrations.Migration): @@ -1303,5 +1306,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(forward, reverse) + migrations.RunPython(forward, reverse) ] diff --git a/ietf/group/migrations/0013_add_groupmilestone_docs2_m2m.py b/ietf/group/migrations/0013_add_groupmilestone_docs2_m2m.py index 1ccca9b13..823ac3ee5 100644 --- a/ietf/group/migrations/0013_add_groupmilestone_docs2_m2m.py +++ b/ietf/group/migrations/0013_add_groupmilestone_docs2_m2m.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-02-25 13:02 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/group/migrations/0014_set_document_m2m_keys.py b/ietf/group/migrations/0014_set_document_m2m_keys.py index e8324f625..46311d284 100644 --- a/ietf/group/migrations/0014_set_document_m2m_keys.py +++ b/ietf/group/migrations/0014_set_document_m2m_keys.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-10 06:48 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys @@ -21,7 +23,7 @@ def forward(apps, schema_editor): # Document id fixup ------------------------------------------------------------ objs = Document.objects.in_bulk() - nameid = { o.name: o.id for id, o in objs.iteritems() } + nameid = { o.name: o.id for id, o in objs.items() } sys.stderr.write('\n') diff --git a/ietf/group/migrations/0015_1_del_docs_m2m_table.py b/ietf/group/migrations/0015_1_del_docs_m2m_table.py index 5b2462ce4..1f9868880 100644 --- a/ietf/group/migrations/0015_1_del_docs_m2m_table.py +++ b/ietf/group/migrations/0015_1_del_docs_m2m_table.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-22 08:00 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/group/migrations/0015_2_add_docs_m2m_table.py b/ietf/group/migrations/0015_2_add_docs_m2m_table.py index cf01621d8..02d16942c 100644 --- a/ietf/group/migrations/0015_2_add_docs_m2m_table.py +++ b/ietf/group/migrations/0015_2_add_docs_m2m_table.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-22 08:00 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/group/migrations/0016_copy_docs_m2m_table.py b/ietf/group/migrations/0016_copy_docs_m2m_table.py index 4eef2b618..0b34ff7ff 100644 --- a/ietf/group/migrations/0016_copy_docs_m2m_table.py +++ b/ietf/group/migrations/0016_copy_docs_m2m_table.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-27 05:57 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys, time diff --git a/ietf/group/migrations/0017_remove_docs2_m2m.py b/ietf/group/migrations/0017_remove_docs2_m2m.py index 26b5784fe..0839ace4f 100644 --- a/ietf/group/migrations/0017_remove_docs2_m2m.py +++ b/ietf/group/migrations/0017_remove_docs2_m2m.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-30 03:23 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/group/migrations/0018_remove_old_document_field.py b/ietf/group/migrations/0018_remove_old_document_field.py index 02f85b6d3..46b165e29 100644 --- a/ietf/group/migrations/0018_remove_old_document_field.py +++ b/ietf/group/migrations/0018_remove_old_document_field.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-25 06:51 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/group/migrations/0019_rename_field_document2.py b/ietf/group/migrations/0019_rename_field_document2.py index c4d25d3e5..ee6747514 100644 --- a/ietf/group/migrations/0019_rename_field_document2.py +++ b/ietf/group/migrations/0019_rename_field_document2.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-25 06:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/group/models.py b/ietf/group/models.py index 6b58e36bf..aecbf3287 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -1,19 +1,22 @@ -# Copyright The IETF Trust 2007-2019, All Rights Reserved +# Copyright The IETF Trust 2010-2019, All Rights Reserved # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals import datetime import email.utils import jsonfield import os import re +import six -from urlparse import urljoin +from six.moves.urllib.parse import urljoin from django.conf import settings from django.core.validators import RegexValidator from django.db import models from django.db.models.deletion import CASCADE from django.dispatch import receiver +from django.utils.encoding import python_2_unicode_compatible from simple_history.models import HistoricalRecords @@ -27,6 +30,7 @@ from ietf.utils import log from ietf.utils.models import ForeignKey, OneToOneField +@python_2_unicode_compatible class GroupInfo(models.Model): time = models.DateTimeField(default=datetime.datetime.now) name = models.CharField(max_length=80) @@ -42,7 +46,7 @@ class GroupInfo(models.Model): unused_states = models.ManyToManyField('doc.State', help_text="Document states that have been disabled for the group.", blank=True) unused_tags = models.ManyToManyField(DocTagName, help_text="Document tags that have been disabled for the group.", blank=True) - def __unicode__(self): + def __str__(self): return self.name def ad_role(self): @@ -91,7 +95,7 @@ class Group(GroupInfo): return e[0] if e else None def has_role(self, user, role_names): - if isinstance(role_names, str) or isinstance(role_names, unicode): + if isinstance(role_names, six.string_types): role_names = [role_names] return user.is_authenticated and self.role_set.filter(name__in=role_names, person__user=user).exists() @@ -192,7 +196,7 @@ class Group(GroupInfo): text = self.charter.text() # split into paragraphs and grab the first non-empty one if text: - desc = [ p for p in re.split('\r?\n\s*\r?\n\s*', text) if p.strip() ][0] + desc = [ p for p in re.split(r'\r?\n\s*\r?\n\s*', text) if p.strip() ][0] return desc @@ -247,14 +251,16 @@ class GroupHistory(GroupInfo): class Meta: verbose_name_plural="group histories" +@python_2_unicode_compatible class GroupURL(models.Model): group = ForeignKey(Group) name = models.CharField(max_length=255) url = models.URLField() - def __unicode__(self): + def __str__(self): return u"%s (%s)" % (self.url, self.name) +@python_2_unicode_compatible class GroupMilestoneInfo(models.Model): group = ForeignKey(Group) # a group has two sets of milestones, current milestones @@ -268,7 +274,7 @@ class GroupMilestoneInfo(models.Model): docs = models.ManyToManyField('doc.Document', blank=True) - def __unicode__(self): + def __str__(self): return self.desc[:20] + "..." class Meta: abstract = True @@ -281,6 +287,7 @@ class GroupMilestoneHistory(GroupMilestoneInfo): time = models.DateTimeField() milestone = ForeignKey(GroupMilestone, related_name="history_set") +@python_2_unicode_compatible class GroupStateTransitions(models.Model): """Captures that a group has overriden the default available document state transitions for a certain state.""" @@ -288,7 +295,7 @@ class GroupStateTransitions(models.Model): state = ForeignKey('doc.State', help_text="State for which the next states should be overridden") next_states = models.ManyToManyField('doc.State', related_name='previous_groupstatetransitions_states') - def __unicode__(self): + def __str__(self): return u'%s "%s" -> %s' % (self.group.acronym, self.state.name, [s.name for s in self.next_states.all()]) GROUP_EVENT_CHOICES = [ @@ -301,6 +308,7 @@ GROUP_EVENT_CHOICES = [ ("status_update", "Status update"), ] +@python_2_unicode_compatible class GroupEvent(models.Model): """An occurrence for a group, used for tracking who, when and what.""" group = ForeignKey(Group) @@ -309,7 +317,7 @@ class GroupEvent(models.Model): by = ForeignKey(Person) desc = models.TextField() - def __unicode__(self): + def __str__(self): return u"%s %s at %s" % (self.by.plain_name(), self.get_type_display().lower(), self.time) class Meta: @@ -321,12 +329,13 @@ class ChangeStateGroupEvent(GroupEvent): class MilestoneGroupEvent(GroupEvent): milestone = ForeignKey(GroupMilestone) +@python_2_unicode_compatible class Role(models.Model): name = ForeignKey(RoleName) group = ForeignKey(Group) person = ForeignKey(Person) email = ForeignKey(Email, help_text="Email address used by person for this role.") - def __unicode__(self): + def __str__(self): return u"%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym or self.group.name) def formatted_ascii_email(self): @@ -338,6 +347,7 @@ class Role(models.Model): class Meta: ordering = ['name_id', ] +@python_2_unicode_compatible class RoleHistory(models.Model): # RoleHistory doesn't have a time field as it's not supposed to be # used on its own - there should always be a GroupHistory @@ -347,7 +357,7 @@ class RoleHistory(models.Model): group = ForeignKey(GroupHistory) person = ForeignKey(Person) email = ForeignKey(Email, help_text="Email address used by person for this role.") - def __unicode__(self): + def __str__(self): return u"%s is %s in %s" % (self.person.plain_name(), self.name.name, self.group.acronym) class Meta: diff --git a/ietf/group/resources.py b/ietf/group/resources.py index d8f86504b..d32a6da29 100644 --- a/ietf/group/resources.py +++ b/ietf/group/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2014-11-13 23:15 + + from ietf.api import ModelResource from ietf.api import ToOneField from tastypie.fields import ToManyField, CharField diff --git a/ietf/group/tests.py b/ietf/group/tests.py index eb0acba7f..1fbe40cb6 100644 --- a/ietf/group/tests.py +++ b/ietf/group/tests.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import os from unittest import skipIf @@ -19,7 +22,7 @@ from ietf.group.factories import GroupFactory, RoleFactory from ietf.utils.test_runner import set_coverage_checking from ietf.person.factories import EmailFactory from ietf.person.models import Person -from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent +from ietf.utils.test_utils import login_testing_unauthorized, TestCase if getattr(settings,'SKIP_DOT_TO_PDF', False): skip_dot_to_pdf = True @@ -39,7 +42,7 @@ class StreamTests(TestCase): def test_streams(self): r = self.client.get(urlreverse("ietf.group.views.streams")) self.assertEqual(r.status_code, 200) - self.assertTrue("Independent Submission Editor" in unicontent(r)) + self.assertContains(r, "Independent Submission Editor") def test_stream_documents(self): draft = DocumentFactory(type_id='draft',group__acronym='iab',states=[('draft','active')]) @@ -48,7 +51,7 @@ class StreamTests(TestCase): r = self.client.get(urlreverse("ietf.group.views.stream_documents", kwargs=dict(acronym="iab"))) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.name in unicontent(r)) + self.assertContains(r, draft.name) def test_stream_edit(self): EmailFactory(address="ad2@ietf.org") diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 04d2cc21f..1829399b2 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -1,12 +1,14 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import os import shutil import calendar import datetime -import json -import StringIO +import io import bleach import six @@ -66,10 +68,10 @@ class GroupPagesTests(TestCase): url = urlreverse('ietf.group.views.active_groups', kwargs=dict(group_type="wg")) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(group.parent.name in unicontent(r)) - self.assertTrue(group.acronym in unicontent(r)) - self.assertTrue(group.name in unicontent(r)) - self.assertTrue(group.ad_role().person.plain_name() in unicontent(r)) + self.assertContains(r, group.parent.name) + self.assertContains(r, group.acronym) + self.assertContains(r, group.name) + self.assertContains(r, group.ad_role().person.plain_name()) for t in ('rg','area','ag','dir','review','team','program'): g = GroupFactory.create(type_id=t,state_id='active') @@ -79,13 +81,13 @@ class GroupPagesTests(TestCase): url = urlreverse('ietf.group.views.active_groups', kwargs=dict(group_type=t)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(g.acronym in unicontent(r)) + self.assertContains(r, g.acronym) url = urlreverse('ietf.group.views.active_groups', kwargs=dict()) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue("Directorate" in unicontent(r)) - self.assertTrue("AG" in unicontent(r)) + self.assertContains(r, "Directorate") + self.assertContains(r, "AG") for slug in GroupTypeName.objects.exclude(slug__in=['wg','rg','ag','area','dir','review','team', 'program']).values_list('slug',flat=True): with self.assertRaises(NoReverseMatch): @@ -103,12 +105,12 @@ class GroupPagesTests(TestCase): r = self.client.get(url) self.assertRedirects(r, next) r = self.client.get(next) - self.assertTrue(group.acronym in unicontent(r)) - self.assertTrue(group.name in unicontent(r)) + self.assertContains(r, group.acronym) + self.assertContains(r, group.name) for word in ['Documents', 'Date', 'Status', 'IPR', 'AD', 'Shepherd']: - self.assertTrue(word in unicontent(r)) - self.assertTrue(draft.name in unicontent(r)) - self.assertTrue(draft.title in unicontent(r)) + self.assertContains(r, word) + self.assertContains(r, draft.name) + self.assertContains(r, draft.title) def test_wg_summaries(self): group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area')).group @@ -117,41 +119,41 @@ class GroupPagesTests(TestCase): chair = Email.objects.filter(role__group=group, role__name="chair")[0] - with open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: + with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: f.write("This is a charter.") url = urlreverse('ietf.group.views.wg_summary_area', kwargs=dict(group_type="wg")) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(group.parent.name in unicontent(r)) - self.assertTrue(group.acronym in unicontent(r)) - self.assertTrue(group.name in unicontent(r)) - self.assertTrue(chair.address in unicontent(r)) + self.assertContains(r, group.parent.name) + self.assertContains(r, group.acronym) + self.assertContains(r, group.name) + self.assertContains(r, chair.address) url = urlreverse('ietf.group.views.wg_summary_acronym', kwargs=dict(group_type="wg")) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(group.acronym in unicontent(r)) - self.assertTrue(group.name in unicontent(r)) - self.assertTrue(chair.address in unicontent(r)) + self.assertContains(r, group.acronym) + self.assertContains(r, group.name) + self.assertContains(r, chair.address) url = urlreverse('ietf.group.views.wg_charters', kwargs=dict(group_type="wg")) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(group.acronym in unicontent(r)) - self.assertTrue(group.name in unicontent(r)) - self.assertTrue(group.ad_role().person.plain_name() in unicontent(r)) - self.assertTrue(chair.address in unicontent(r)) - self.assertTrue("This is a charter." in unicontent(r)) + self.assertContains(r, group.acronym) + self.assertContains(r, group.name) + self.assertContains(r, group.ad_role().person.plain_name()) + self.assertContains(r, chair.address) + self.assertContains(r, "This is a charter.") url = urlreverse('ietf.group.views.wg_charters_by_acronym', kwargs=dict(group_type="wg")) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(group.acronym in unicontent(r)) - self.assertTrue(group.name in unicontent(r)) - self.assertTrue(group.ad_role().person.plain_name() in unicontent(r)) - self.assertTrue(chair.address in unicontent(r)) - self.assertTrue("This is a charter." in unicontent(r)) + self.assertContains(r, group.acronym) + self.assertContains(r, group.name) + self.assertContains(r, group.ad_role().person.plain_name()) + self.assertContains(r, chair.address) + self.assertContains(r, "This is a charter.") def test_chartering_groups(self): group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area'),states=[('charter','intrev')]).group @@ -209,10 +211,10 @@ class GroupPagesTests(TestCase): for url in group_urlreverse_list(group, 'ietf.group.views.group_documents'): r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.name in unicontent(r)) - self.assertTrue(group.name in unicontent(r)) - self.assertTrue(group.acronym in unicontent(r)) - self.assertTrue(draft2.name in unicontent(r)) + self.assertContains(r, draft.name) + self.assertContains(r, group.name) + self.assertContains(r, group.acronym) + self.assertContains(r, draft2.name) # Make sure that a logged in user is presented with an opportunity to add results to their community list self.client.login(username="secretary", password="secretary+password") @@ -224,14 +226,14 @@ class GroupPagesTests(TestCase): for url in group_urlreverse_list(group, 'ietf.group.views.group_documents_txt'): r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.name in unicontent(r)) - self.assertTrue(draft2.name in unicontent(r)) + self.assertContains(r, draft.name) + self.assertContains(r, draft2.name) def test_group_charter(self): group = CharterFactory().group draft = WgDraftFactory(group=group) - with open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: + with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: f.write("This is a charter.") milestone = GroupMilestone.objects.create( @@ -244,11 +246,11 @@ class GroupPagesTests(TestCase): for url in [group.about_url(),] + group_urlreverse_list(group, 'ietf.group.views.group_about'): r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(group.name in unicontent(r)) - self.assertTrue(group.acronym in unicontent(r)) - self.assertTrue("This is a charter." in unicontent(r)) - self.assertTrue(milestone.desc in unicontent(r)) - self.assertTrue(milestone.docs.all()[0].name in unicontent(r)) + self.assertContains(r, group.name) + self.assertContains(r, group.acronym) + self.assertContains(r, "This is a charter.") + self.assertContains(r, milestone.desc) + self.assertContains(r, milestone.docs.all()[0].name) def test_group_about(self): @@ -306,9 +308,9 @@ class GroupPagesTests(TestCase): url = group.about_url() r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(group.name in unicontent(r)) - self.assertTrue(group.acronym in unicontent(r)) - self.assertTrue(group.description in unicontent(r)) + self.assertContains(r, group.name) + self.assertContains(r, group.acronym) + self.assertContains(r, group.description) for url in group_urlreverse_list(group, 'ietf.group.views.edit'): @@ -334,8 +336,8 @@ class GroupPagesTests(TestCase): for url in group_urlreverse_list(group, 'ietf.group.views.materials'): r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(doc.title in unicontent(r)) - self.assertTrue(doc.name in unicontent(r)) + self.assertContains(r, doc.title) + self.assertContains(r, doc.name) url = urlreverse("ietf.group.views.materials", kwargs={ 'acronym': group.acronym }) @@ -344,7 +346,7 @@ class GroupPagesTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(doc.title not in unicontent(r)) + self.assertNotContains(r, doc.title) def test_history(self): group = GroupFactory() @@ -358,7 +360,7 @@ class GroupPagesTests(TestCase): for url in group_urlreverse_list(group, 'ietf.group.views.history'): r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(e.desc in unicontent(r)) + self.assertContains(r, e.desc) def test_feed(self): group = CharterFactory().group @@ -378,8 +380,8 @@ class GroupPagesTests(TestCase): r = self.client.get("/feed/group-changes/%s/" % group.acronym) self.assertEqual(r.status_code, 200) - self.assertTrue(ge.desc in unicontent(r)) - self.assertTrue(de.desc in unicontent(r)) + self.assertContains(r, ge.desc) + self.assertContains(r, de.desc) def test_chair_photos(self): @@ -528,7 +530,6 @@ class GroupEditTests(TestCase): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(len(q('form .has-error')) > 0) - self.assertEqual(len(q('form input[name="confirm_acronym"]')), 0) # can't confirm us out of this # try elevating BoF to WG group.state_id = "bof" @@ -538,16 +539,15 @@ class GroupEditTests(TestCase): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(len(q('form .has-error')) > 0) - self.assertEqual(len(q('form input[name="confirm_acronym"]')), 1) self.assertEqual(Group.objects.get(acronym=group.acronym).state_id, "bof") - # confirm elevation - state = GroupStateName.objects.get(slug="proposed") - r = self.client.post(url, dict(name="Test", acronym=group.acronym, confirm_acronym="1", state=state.pk)) - self.assertEqual(r.status_code, 302) - self.assertEqual(Group.objects.get(acronym=group.acronym).state_id, "proposed") - self.assertEqual(Group.objects.get(acronym=group.acronym).name, "Test") +# # confirm elevation +# state = GroupStateName.objects.get(slug="proposed") +# r = self.client.post(url, dict(name="Test", acronym=group.acronym, confirm_acronym="1", state=state.pk)) +# self.assertEqual(r.status_code, 302) +# self.assertEqual(Group.objects.get(acronym=group.acronym).state_id, "proposed") +# self.assertEqual(Group.objects.get(acronym=group.acronym).name, "Test") def test_edit_info(self): group = GroupFactory(acronym='mars',parent=GroupFactory(type_id='area')) @@ -586,7 +586,7 @@ class GroupEditTests(TestCase): self.assertTrue(len(q('form .has-error')) > 0) # edit info - with open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: + with io.open(os.path.join(self.charter_dir, "%s-%s.txt" % (group.charter.canonical_name(), group.charter.rev)), "w") as f: f.write("This is a charter.") area = group.parent ad = Person.objects.get(name="Areað Irector") @@ -790,8 +790,8 @@ class MilestoneTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(m1.desc in unicontent(r)) - self.assertTrue(m2.desc not in unicontent(r)) + self.assertContains(r, m1.desc) + self.assertNotContains(r, m2.desc) self.client.logout() login_testing_unauthorized(self, "secretary", url) @@ -799,8 +799,8 @@ class MilestoneTests(TestCase): for url in group_urlreverse_list(group, 'ietf.group.milestones.edit_milestones;charter'): r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(m1.desc not in unicontent(r)) - self.assertTrue(m2.desc in unicontent(r)) + self.assertNotContains(r, m1.desc) + self.assertContains(r, m2.desc) def test_add_milestone(self): m1, m2, group = self.create_test_milestones() @@ -1109,7 +1109,7 @@ class EmailAliasesTests(TestCase): GroupFactory(acronym='mars',parent=GroupFactory(type_id='area')) GroupFactory(acronym='ames',parent=GroupFactory(type_id='area')) self.group_alias_file = NamedTemporaryFile(delete=False) - self.group_alias_file.write("""# Generated by hand at 2015-02-12_16:30:52 + self.group_alias_file.write(b"""# Generated by hand at 2015-02-12_16:30:52 virtual.ietf.org anything mars-ads@ietf.org xfilter-mars-ads expand-mars-ads@virtual.ietf.org aread@example.org @@ -1148,20 +1148,20 @@ expand-ames-chairs@virtual.ietf.org mars_chair@ietf url = urlreverse('ietf.group.views.email_aliases', kwargs=dict(group_type="wg")) r = self.client.get(url) self.assertEqual(r.status_code,200) - self.assertTrue('mars-ads@' in unicontent(r)) + self.assertContains(r, 'mars-ads@') url = urlreverse('ietf.group.views.email_aliases', kwargs=dict(group_type="rg")) r = self.client.get(url) self.assertEqual(r.status_code,200) - self.assertFalse('mars-ads@' in unicontent(r)) + self.assertNotContains(r, 'mars-ads@') def testExpansions(self): url = urlreverse('ietf.group.views.email', kwargs=dict(acronym="mars")) r = self.client.get(url) self.assertEqual(r.status_code,200) - self.assertTrue('Email aliases' in unicontent(r)) - self.assertTrue('mars-ads@ietf.org' in unicontent(r)) - self.assertTrue('group_personnel_change' in unicontent(r)) + self.assertContains(r, 'Email aliases') + self.assertContains(r, 'mars-ads@ietf.org') + self.assertContains(r, 'group_personnel_change') @@ -1173,7 +1173,7 @@ class AjaxTests(TestCase): r = self.client.get(urlreverse('ietf.group.views.group_menu_data')) self.assertEqual(r.status_code, 200) - parents = json.loads(r.content) + parents = r.json() area = Group.objects.get(type="area", acronym="farfut") self.assertTrue(str(area.id) in parents) @@ -1293,7 +1293,7 @@ class StatusUpdateTests(TestCase): self.assertEqual(response.status_code, 302) self.assertEqual(chair.group.latest_event(type='status_update').desc,'Direct content typed into form') - test_file = StringIO.StringIO("This came from a file.") + test_file = io.StringIO("This came from a file.") test_file.name = "unnamed" response = self.client.post(url,dict(txt=test_file,submit_response="1")) self.assertEqual(response.status_code, 302) diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py index 36243d598..66ee9a13e 100644 --- a/ietf/group/tests_review.py +++ b/ietf/group/tests_review.py @@ -1,14 +1,18 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import debug # pyflakes:ignore +import six from pyquery import PyQuery from django.urls import reverse as urlreverse -from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent, reload_db_objects +from ietf.utils.test_utils import login_testing_unauthorized, TestCase, reload_db_objects from ietf.doc.models import TelechatDocEvent from ietf.group.models import Role from ietf.iesg.models import TelechatDate @@ -39,8 +43,8 @@ class ReviewTests(TestCase): urlreverse(ietf.group.views.review_requests, kwargs={ 'acronym': group.acronym , 'group_type': group.type_id})]: r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertIn(review_req.doc.name, unicontent(r)) - self.assertIn(assignment.reviewer.person.__unicode__(), unicontent(r)) + self.assertContains(r, review_req.doc.name) + self.assertContains(r, str(assignment.reviewer.person)) url = urlreverse(ietf.group.views.review_requests, kwargs={ 'acronym': group.acronym }) @@ -51,7 +55,7 @@ class ReviewTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(review_req.doc.name in unicontent(r)) + self.assertContains(r, review_req.doc.name) def test_suggested_review_requests(self): review_req = ReviewRequestFactory(state_id='assigned') @@ -151,34 +155,34 @@ class ReviewTests(TestCase): urlreverse(ietf.group.views.reviewer_overview, kwargs={ 'acronym': group.acronym, 'group_type': group.type_id })]: r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertIn(unicode(reviewer), unicontent(r)) - self.assertIn(review_req1.doc.name, unicontent(r)) + self.assertContains(r, str(reviewer)) + self.assertContains(r, review_req1.doc.name) # without a login, reason for being unavailable should not be seen - self.assertNotIn("Availability", unicontent(r)) + self.assertNotContains(r, "Availability") url = urlreverse(ietf.group.views.reviewer_overview, kwargs={ 'acronym': group.acronym }) self.client.login(username="plain", password="plain+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) # not on review team, should not see reason for being unavailable - self.assertNotIn("Availability", unicontent(r)) + self.assertNotContains(r, "Availability") self.client.login(username="reviewer", password="reviewer+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) # review team members can see reason for being unavailable - self.assertIn("Availability", unicontent(r)) + self.assertContains(r, "Availability") self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) # secretariat can see reason for being unavailable - self.assertIn("Availability", unicontent(r)) + self.assertContains(r, "Availability") def test_manage_review_requests(self): group = ReviewTeamFactory() RoleFactory(name_id='reviewer',group=group,person__user__username='reviewer').person - marsperson = RoleFactory(name_id='reviewer',group=group,person=PersonFactory(name=u"Mars Anders Chairman",user__username='marschairman')).person + marsperson = RoleFactory(name_id='reviewer',group=group,person=PersonFactory(name="Mars Anders Chairman",user__username='marschairman')).person review_req1 = ReviewRequestFactory(doc__pages=2,doc__shepherd=marsperson.email(),team=group) review_req2 = ReviewRequestFactory(team=group) review_req3 = ReviewRequestFactory(team=group) @@ -188,17 +192,17 @@ class ReviewTests(TestCase): login_testing_unauthorized(self, "secretary", unassigned_url) # Need one more person in review team one so we can test incrementing skip_count without immediately decrementing it - another_reviewer = PersonFactory.create(name = u"Extra TestReviewer") # needs to be lexically greater than the exsting one + another_reviewer = PersonFactory.create(name = "Extra TestReviewer") # needs to be lexically greater than the exsting one another_reviewer.role_set.create(name_id='reviewer', email=another_reviewer.email(), group=review_req1.team) ReviewerSettingsFactory(team=review_req3.team, person = another_reviewer) - yet_another_reviewer = PersonFactory.create(name = u"YetAnotherExtra TestReviewer") # needs to be lexically greater than the exsting one + yet_another_reviewer = PersonFactory.create(name = "YetAnotherExtra TestReviewer") # needs to be lexically greater than the exsting one yet_another_reviewer.role_set.create(name_id='reviewer', email=yet_another_reviewer.email(), group=review_req1.team) ReviewerSettingsFactory(team=review_req3.team, person = yet_another_reviewer) # get r = self.client.get(unassigned_url) self.assertEqual(r.status_code, 200) - self.assertTrue(review_req1.doc.name in unicontent(r)) + self.assertContains(r, review_req1.doc.name) # Test that conflicts are detected r = self.client.post(unassigned_url, { @@ -211,9 +215,7 @@ class ReviewTests(TestCase): "action": "save", }) - self.assertEqual(r.status_code, 200) - content = unicontent(r).lower() - self.assertTrue("2 requests opened" in content) + self.assertContains(r, "2 requests opened") r = self.client.post(unassigned_url, { "reviewrequest": [str(review_req1.pk),str(review_req2.pk),str(review_req3.pk)], @@ -262,7 +264,7 @@ class ReviewTests(TestCase): q = PyQuery(r.content) generated_text = q("[name=body]").text() self.assertTrue(review_req1.doc.name in generated_text) - self.assertTrue(unicode(Person.objects.get(user__username="marschairman")) in generated_text) + self.assertTrue(six.text_type(Person.objects.get(user__username="marschairman")) in generated_text) empty_outbox() r = self.client.post(url, { @@ -349,7 +351,7 @@ class ReviewTests(TestCase): 'start_date': start_date.isoformat(), 'end_date': "", 'availability': "unavailable", - 'reason': "Whimsy", + 'reason': "Whimsy", }) self.assertEqual(r.status_code, 302) period = UnavailablePeriod.objects.get(person=reviewer, team=review_req.team, start_date=start_date) @@ -359,7 +361,7 @@ class ReviewTests(TestCase): msg_content = outbox[0].get_payload(decode=True).decode("utf-8").lower() self.assertTrue(start_date.isoformat(), msg_content) self.assertTrue("indefinite", msg_content) - self.assertEqual(period.reason, "Whimsy") + self.assertEqual(period.reason, "Whimsy") # end unavailable period empty_outbox() diff --git a/ietf/group/utils.py b/ietf/group/utils.py index 5c501435d..e6660cccf 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -1,6 +1,10 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os from django.db.models import Q @@ -20,6 +24,7 @@ from ietf.person.models import Email from ietf.review.utils import can_manage_review_requests_for_team from ietf.utils import log from ietf.utils.history import get_history_object_for, copy_many_to_many_for_history +from functools import reduce def save_group_in_history(group): """This should be called before saving changes to a Group instance, @@ -52,7 +57,7 @@ def get_charter_text(group): filename = os.path.join(c.get_file_path(), "%s-%s.txt" % (c.canonical_name(), c.rev)) try: - with open(filename) as f: + with io.open(filename) as f: return f.read() except IOError: return 'Error Loading Group Charter' @@ -62,7 +67,7 @@ def get_group_role_emails(group, roles): if not group or not group.acronym or group.acronym == 'none': return set() emails = Email.objects.filter(role__group=group, role__name__in=roles) - return set(filter(None, [e.email_address() for e in emails])) + return set([_f for _f in [e.email_address() for e in emails] if _f]) def get_child_group_role_emails(parent, roles, group_type='wg'): """Get a list of email addresses for a given set of @@ -207,35 +212,35 @@ def construct_group_menu_context(request, group, selected, group_type, others): if group.features.has_milestones: if group.state_id != "proposed" and can_manage: - actions.append((u"Edit milestones", urlreverse('ietf.group.milestones.edit_milestones;current', kwargs=kwargs))) + actions.append(("Edit milestones", urlreverse('ietf.group.milestones.edit_milestones;current', kwargs=kwargs))) if group.features.has_documents: clist = CommunityList.objects.filter(group=group).first() if clist and can_manage_community_list(request.user, clist): import ietf.community.views - actions.append((u'Manage document list', urlreverse(ietf.community.views.manage_list, kwargs=kwargs))) + actions.append(('Manage document list', urlreverse(ietf.community.views.manage_list, kwargs=kwargs))) if group.features.has_nonsession_materials and can_manage_materials(request.user, group): - actions.append((u"Upload material", urlreverse("ietf.doc.views_material.choose_material_type", kwargs=kwargs))) + actions.append(("Upload material", urlreverse("ietf.doc.views_material.choose_material_type", kwargs=kwargs))) if group.features.has_reviews and can_manage_review_requests_for_team(request.user, group): import ietf.group.views - actions.append((u"Manage unassigned reviews", urlreverse(ietf.group.views.manage_review_requests, kwargs=dict(assignment_status="unassigned", **kwargs)))) + actions.append(("Manage unassigned reviews", urlreverse(ietf.group.views.manage_review_requests, kwargs=dict(assignment_status="unassigned", **kwargs)))) #actions.append((u"Manage assigned reviews", urlreverse(ietf.group.views.manage_review_requests, kwargs=dict(assignment_status="assigned", **kwargs)))) if Role.objects.filter(name="secr", group=group, person__user=request.user).exists(): - actions.append((u"Secretary settings", urlreverse(ietf.group.views.change_review_secretary_settings, kwargs=kwargs))) - actions.append((u"Email open assignments summary", urlreverse(ietf.group.views.email_open_review_assignments, kwargs=dict(acronym=group.acronym, group_type=group.type_id)))) + actions.append(("Secretary settings", urlreverse(ietf.group.views.change_review_secretary_settings, kwargs=kwargs))) + actions.append(("Email open assignments summary", urlreverse(ietf.group.views.email_open_review_assignments, kwargs=dict(acronym=group.acronym, group_type=group.type_id)))) if group.state_id != "conclude" and can_manage: can_edit_group = True - actions.append((u"Edit group", urlreverse("ietf.group.views.edit", kwargs=dict(kwargs, action="edit")))) + actions.append(("Edit group", urlreverse("ietf.group.views.edit", kwargs=dict(kwargs, action="edit")))) if group.features.customize_workflow and can_manage: - actions.append((u"Customize workflow", urlreverse("ietf.group.views.customize_workflow", kwargs=kwargs))) + actions.append(("Customize workflow", urlreverse("ietf.group.views.customize_workflow", kwargs=kwargs))) if group.state_id in ("active", "dormant") and not group.type_id in ["sdo", "rfcedtyp", "isoc", ] and can_manage_group_type(request.user, group): - actions.append((u"Request closing group", urlreverse("ietf.group.views.conclude", kwargs=kwargs))) + actions.append(("Request closing group", urlreverse("ietf.group.views.conclude", kwargs=kwargs))) d = { "group": group, diff --git a/ietf/group/views.py b/ietf/group/views.py index 9dd5c523b..7844c74d2 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright The IETF Trust 2007-2019, All Rights Reserved -from __future__ import unicode_literals, print_function - +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# # Portion Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -34,12 +33,18 @@ from __future__ import unicode_literals, print_function # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os -import re + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import itertools +import io import json import math -import itertools -import datetime +import os +import re +import six + from tempfile import mkstemp from collections import OrderedDict, defaultdict from simple_history.utils import update_change_reason @@ -109,7 +114,7 @@ from ietf.doc.models import LastCallDocEvent from ietf.name.models import ReviewAssignmentStateName -from ietf.utils.mail import send_mail_text, parse_preformatted +from ietf.utils.mail import send_mail_text, parse_preformatted, get_payload from ietf.ietfauth.utils import user_is_person from ietf.dbtemplate.models import DBTemplate @@ -150,7 +155,7 @@ def fill_in_charter_info(group, include_drafts=False): personnel["ad"] = ad_roles group.personnel = [] - for role_name_slug, roles in personnel.iteritems(): + for role_name_slug, roles in personnel.items(): label = roles[0].name.name if len(roles) > 1: if label.endswith("y"): @@ -168,7 +173,7 @@ def fill_in_charter_info(group, include_drafts=False): if group.charter: group.charter_text = get_charter_text(group) else: - group.charter_text = u"Not chartered yet." + group.charter_text = "Not chartered yet." def extract_last_name(role): return role.person.name_parts()[3] @@ -201,10 +206,10 @@ def fill_in_wg_drafts(group): def check_group_email_aliases(): - pattern = re.compile('expand-(.*?)(-\w+)@.*? +(.*)$') + pattern = re.compile(r'expand-(.*?)(-\w+)@.*? +(.*)$') tot_count = 0 good_count = 0 - with open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file: + with io.open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file: for line in virtual_file.readlines(): m = pattern.match(line) tot_count += 1 @@ -268,7 +273,7 @@ def wg_charters_by_acronym(request, group_type): raise Http404 areas = dict((a.id, a) for a in Group.objects.filter(type="area", state="active").order_by("name")) - for area in areas.itervalues(): + for area in areas.values(): area.ads = sorted(roles(area, "ad"), key=extract_last_name) groups = Group.objects.filter(type="wg", state="active").exclude(parent=None).order_by("acronym") @@ -337,8 +342,8 @@ def active_programs(request): return render(request, 'group/active_programs.html', {'programs' : programs }) def active_areas(request): - areas = Group.objects.filter(type="area", state="active").order_by("name") - return render(request, 'group/active_areas.html', {'areas': areas }) + areas = Group.objects.filter(type="area", state="active").order_by("name") + return render(request, 'group/active_areas.html', {'areas': areas }) def active_wgs(request): areas = Group.objects.filter(type="area", state="active").order_by("name") @@ -496,7 +501,7 @@ def group_documents_txt(request, acronym, group_type=None): d.prefix = d.get_state().name for d in docs_related: - d.prefix = u"Related %s" % d.get_state().name + d.prefix = "Related %s" % d.get_state().name rows = [] for d in itertools.chain(docs, docs_related): @@ -506,9 +511,9 @@ def group_documents_txt(request, acronym, group_type=None): else: name = "%s-%s" % (d.name, d.rev) - rows.append(u"\t".join((d.prefix, name, clean_whitespace(d.title)))) + rows.append("\t".join((d.prefix, name, clean_whitespace(d.title)))) - return HttpResponse(u"\n".join(rows), content_type='text/plain; charset=UTF-8') + return HttpResponse("\n".join(rows), content_type='text/plain; charset=UTF-8') def group_about(request, acronym, group_type=None): group = get_group_or_404(acronym, group_type) @@ -625,12 +630,12 @@ def group_about_status_edit(request, acronym, group_type=None): def get_group_email_aliases(acronym, group_type): if acronym: - pattern = re.compile('expand-(%s)(-\w+)@.*? +(.*)$'%acronym) + pattern = re.compile(r'expand-(%s)(-\w+)@.*? +(.*)$'%acronym) else: - pattern = re.compile('expand-(.*?)(-\w+)@.*? +(.*)$') + pattern = re.compile(r'expand-(.*?)(-\w+)@.*? +(.*)$') aliases = [] - with open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file: + with io.open(settings.GROUP_VIRTUAL_PATH,"r") as virtual_file: for line in virtual_file.readlines(): m = pattern.match(line) if m: @@ -679,7 +684,7 @@ def materials(request, acronym, group_type=None): return render(request, 'group/materials.html', construct_group_menu_context(request, group, "materials", group_type, { - "doc_types": doc_types.items(), + "doc_types": list(doc_types.items()), "can_manage_materials": can_manage_materials(request.user, group) })) @@ -691,7 +696,7 @@ def dependencies(request, acronym, group_type=None, output_type="pdf"): dothandle, dotname = mkstemp() os.close(dothandle) - dotfile = open(dotname, "w") + dotfile = io.open(dotname, "w") dotfile.write(make_dot(group)) dotfile.close() @@ -708,7 +713,7 @@ def dependencies(request, acronym, group_type=None, output_type="pdf"): pipe("%s -f -l 10 -o %s %s" % (settings.UNFLATTEN_BINARY, unflatname, dotname)) pipe("%s -T%s -o %s %s" % (settings.DOT_BINARY, output_type, outname, unflatname)) - outhandle = open(outname, "r") + outhandle = io.open(outname, "rb") out = outhandle.read() outhandle.close() @@ -853,7 +858,7 @@ def edit(request, group_type=None, acronym=None, action="edit", field=None): res = [] for u in urls: if u.name: - res.append(u"%s (%s)" % (u.url, u.name)) + res.append("%s (%s)" % (u.url, u.name)) else: res.append(u.url) return fs.join(res) @@ -930,7 +935,7 @@ def edit(request, group_type=None, acronym=None, action="edit", field=None): personnel_change_text="" changed_personnel = set() # update roles - for attr, f in form.fields.iteritems(): + for attr, f in form.fields.items(): if not (attr.endswith("_roles") or attr == "ad"): continue @@ -976,7 +981,7 @@ def edit(request, group_type=None, acronym=None, action="edit", field=None): group.groupurl_set.all().delete() # Add new ones for u in new_urls: - m = re.search('(?P[\w\d:#@%/;$()~_?\+-=\\\.&]+)( \((?P.+)\))?', u) + m = re.search(r'(?P[\w\d:#@%/;$()~_?\+-=\\\.&]+)( \((?P.+)\))?', u) if m: if m.group('name'): url = GroupURL(url=m.group('url'), name=m.group('name'), group=group) @@ -999,7 +1004,7 @@ def edit(request, group_type=None, acronym=None, action="edit", field=None): return redirect('ietf.doc.views_charter.submit', name=charter_name_for_group(group), option="initcharter") return HttpResponseRedirect(group.about_url()) - else: # form.is_valid() + else: # Not POST: if not new_group: ad_role = group.ad_role() init = dict(name=group.name, @@ -1443,9 +1448,9 @@ def manage_review_requests(request, acronym, group_type=None, assignment_status= saving = form_action.startswith("save") # check for conflicts - review_requests_dict = { unicode(r.pk): r for r in review_requests } + review_requests_dict = { six.text_type(r.pk): r for r in review_requests } posted_reqs = set(request.POST.getlist("reviewrequest", [])) - current_reqs = set(review_requests_dict.iterkeys()) + current_reqs = set(review_requests_dict.keys()) closed_reqs = posted_reqs - current_reqs newly_closed = len(closed_reqs) @@ -1599,7 +1604,7 @@ def email_open_review_assignments(request, acronym, group_type=None): (msg,_,_) = parse_preformatted(partial_msg) - body = msg.get_payload() + body = get_payload(msg) subject = msg['Subject'] form = EmailOpenAssignmentsForm(initial={ @@ -1692,7 +1697,7 @@ def change_reviewer_settings(request, acronym, reviewer_email, group_type=None): period.start_date.isoformat() if period.start_date else "indefinite", period.end_date.isoformat() if period.end_date else "indefinite", period.get_availability_display(), - period.reason, + period.reason, ) if period.availability == "unavailable": diff --git a/ietf/idindex/generate_all_id2_txt.py b/ietf/idindex/generate_all_id2_txt.py index 10e66b171..67fef35d2 100755 --- a/ietf/idindex/generate_all_id2_txt.py +++ b/ietf/idindex/generate_all_id2_txt.py @@ -1,4 +1,7 @@ #!/usr/bin/env python +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- +# # Portions Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -31,11 +34,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import os +import six os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings") import django django.setup() from ietf.idindex.index import all_id2_txt -print all_id2_txt().encode('utf-8'), +six.print_(all_id2_txt().encode('utf-8'), end=' ') diff --git a/ietf/idindex/generate_all_id_txt.py b/ietf/idindex/generate_all_id_txt.py index 8bef4a9f7..f7d805be9 100755 --- a/ietf/idindex/generate_all_id_txt.py +++ b/ietf/idindex/generate_all_id_txt.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Portions Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -31,11 +33,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import os +import six os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings") import django django.setup() from ietf.idindex.index import all_id_txt -print all_id_txt().encode("utf-8"), +six.print_(all_id_txt().encode("utf-8"), end=' ') diff --git a/ietf/idindex/generate_id_abstracts_txt.py b/ietf/idindex/generate_id_abstracts_txt.py index c295991e3..d93b4b8a7 100755 --- a/ietf/idindex/generate_id_abstracts_txt.py +++ b/ietf/idindex/generate_id_abstracts_txt.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Portions Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -31,11 +33,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import os +import six os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings") import django django.setup() from ietf.idindex.index import id_index_txt -print id_index_txt(with_abstracts=True).encode('utf-8'), +six.print_(id_index_txt(with_abstracts=True).encode('utf-8'), end=' ') diff --git a/ietf/idindex/generate_id_index_txt.py b/ietf/idindex/generate_id_index_txt.py index f213614fb..c201a9cb7 100755 --- a/ietf/idindex/generate_id_index_txt.py +++ b/ietf/idindex/generate_id_index_txt.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Portions Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -31,11 +33,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import os +import six os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ietf.settings") import django django.setup() from ietf.idindex.index import id_index_txt -print id_index_txt().encode('utf-8'), +six.print_(id_index_txt().encode('utf-8'), end=' ') diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py index 547dd6787..03d5f2b2d 100644 --- a/ietf/idindex/index.py +++ b/ietf/idindex/index.py @@ -1,22 +1,26 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + # code to generate plain-text index files that are placed on # www.ietf.org in the same directory as the I-Ds -import datetime, os - -import debug # pyflakes:ignore - +import datetime +import os import pytz +import six from django.conf import settings from django.template.loader import render_to_string -from ietf.doc.templatetags.ietf_filters import clean_whitespace +import debug # pyflakes:ignore + from ietf.doc.models import Document, DocEvent, DocumentAuthor, RelatedDocument, DocAlias, State from ietf.doc.models import LastCallDocEvent, NewRevisionDocEvent from ietf.doc.models import IESG_SUBSTATE_TAGS +from ietf.doc.templatetags.ietf_filters import clean_whitespace from ietf.group.models import Group from ietf.person.models import Person, Email from ietf.utils import log @@ -89,7 +93,7 @@ def all_id_txt(): last_field, ) - return u"\n".join(res) + "\n" + return "\n".join(res) + "\n" def file_types_for_drafts(): """Look in the draft directory and return file types found as dict (name + rev -> [t1, t2, ...]).""" @@ -129,7 +133,7 @@ def all_id2_txt(): else: l = authors[a.document.name] if a.email: - l.append(u'%s <%s>' % (a.person.plain_name().replace("@", ""), a.email.address.replace(",", ""))) + l.append('%s <%s>' % (a.person.plain_name().replace("@", ""), a.email.address.replace(",", ""))) else: l.append(a.person.plain_name()) @@ -191,7 +195,7 @@ def all_id2_txt(): area = d.group.parent.acronym fields.append(area) # 9 responsible AD name - fields.append(unicode(d.ad) if d.ad else "") + fields.append(six.text_type(d.ad) if d.ad else "") # 10 fields.append(d.intended_std_level.name if d.intended_std_level else "") # 11 @@ -208,16 +212,16 @@ def all_id2_txt(): # 13 fields.append(clean_whitespace(d.title)) # FIXME: we should make sure this is okay in the database and in submit # 14 - fields.append(u", ".join(authors.get(d.name, []))) + fields.append(", ".join(authors.get(d.name, []))) # 15 fields.append(shepherds.get(d.shepherd_id, "")) # 16 Responsible AD name and email fields.append(ads.get(d.ad_id, "")) # - res.append(u"\t".join(fields)) + res.append("\t".join(fields)) - return render_to_string("idindex/all_id2.txt", {'data': u"\n".join(res) }) + return render_to_string("idindex/all_id2.txt", {'data': "\n".join(res) }) def active_drafts_index_by_group(extra_values=()): """Return active drafts grouped into their corresponding @@ -258,7 +262,7 @@ def active_drafts_index_by_group(extra_values=()): d["authors"].append(a.person.plain_ascii()) # This should probably change to .plain_name() when non-ascii names are permitted # put docs into groups - for d in docs_dict.itervalues(): + for d in docs_dict.values(): group = groups_dict.get(d["group_id"]) if not group: continue @@ -268,7 +272,7 @@ def active_drafts_index_by_group(extra_values=()): group.active_drafts.append(d) - groups = [g for g in groups_dict.itervalues() if hasattr(g, "active_drafts")] + groups = [g for g in groups_dict.values() if hasattr(g, "active_drafts")] groups.sort(key=lambda g: g.acronym) fallback_time = datetime.datetime(1950, 1, 1) diff --git a/ietf/idindex/tests.py b/ietf/idindex/tests.py index b4817d24f..503a84a93 100644 --- a/ietf/idindex/tests.py +++ b/ietf/idindex/tests.py @@ -1,14 +1,19 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- -import os -import datetime -import shutil -import debug # pyflakes:ignore +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os +import shutil +import six from django.conf import settings +import debug # pyflakes:ignore + from ietf.doc.factories import WgDraftFactory from ietf.doc.models import Document, DocAlias, RelatedDocument, State, LastCallDocEvent, NewRevisionDocEvent from ietf.group.factories import GroupFactory @@ -28,7 +33,7 @@ class IndexTests(TestCase): shutil.rmtree(self.id_dir) def write_draft_file(self, name, size): - with open(os.path.join(self.id_dir, name), 'w') as f: + with io.open(os.path.join(self.id_dir, name), 'w') as f: f.write("a" * size) def test_all_id_txt(self): @@ -70,7 +75,7 @@ class IndexTests(TestCase): draft = WgDraftFactory( states=[('draft','active'),('draft-iesg','review-e')], ad=PersonFactory(), - shepherd=EmailFactory(address='shepherd@example.com',person__name=u'Draft δÏαφτυ Shepherd'), + shepherd=EmailFactory(address='shepherd@example.com',person__name='Draft δÏαφτυ Shepherd'), group__parent=GroupFactory(type_id='area'), intended_std_level_id = 'ps', authors=[EmailFactory().person] @@ -97,15 +102,15 @@ class IndexTests(TestCase): self.assertEqual(t[6], draft.latest_event(type="new_revision").time.strftime("%Y-%m-%d")) self.assertEqual(t[7], draft.group.acronym) self.assertEqual(t[8], draft.group.parent.acronym) - self.assertEqual(t[9], unicode(draft.ad)) + self.assertEqual(t[9], six.text_type(draft.ad)) self.assertEqual(t[10], draft.intended_std_level.name) self.assertEqual(t[11], "") self.assertEqual(t[12], ".pdf,.txt") self.assertEqual(t[13], draft.title) author = draft.documentauthor_set.order_by("order").get() - self.assertEqual(t[14], u"%s <%s>" % (author.person.name, author.email.address)) - self.assertEqual(t[15], u"%s <%s>" % (draft.shepherd.person.plain_ascii(), draft.shepherd.address)) - self.assertEqual(t[16], u"%s <%s>" % (draft.ad.plain_ascii(), draft.ad.email_address())) + self.assertEqual(t[14], "%s <%s>" % (author.person.name, author.email.address)) + self.assertEqual(t[15], "%s <%s>" % (draft.shepherd.person.plain_ascii(), draft.shepherd.address)) + self.assertEqual(t[16], "%s <%s>" % (draft.ad.plain_ascii(), draft.ad.email_address())) # test RFC diff --git a/ietf/iesg/__init__.py b/ietf/iesg/__init__.py index 0dc251ae6..78e8b3ff8 100644 --- a/ietf/iesg/__init__.py +++ b/ietf/iesg/__init__.py @@ -1,5 +1,8 @@ -# Copyright The IETF Trust 2007, All Rights Reserved -# coding: latin-1 +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from types import ModuleType @@ -9,7 +12,7 @@ DEBUG_EMAILS = [ ('Ole Laursen', 'olau@iola.dk'), ] -for k in locals().keys(): +for k in list(locals().keys()): m = locals()[k] if isinstance(m, ModuleType): if hasattr(m, "DEBUG_EMAILS"): diff --git a/ietf/iesg/agenda.py b/ietf/iesg/agenda.py index 0b43ef374..381d3fa65 100644 --- a/ietf/iesg/agenda.py +++ b/ietf/iesg/agenda.py @@ -1,9 +1,12 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + # utilities for constructing agendas for IESG telechats -import codecs +import io import datetime from collections import OrderedDict @@ -146,10 +149,10 @@ def fill_in_agenda_administrivia(date, sections): for s, key, filename in extra_info_files: try: - with codecs.open(filename, 'r', 'utf-8', 'replace') as f: + with io.open(filename, 'r', encoding='utf-8', errors='replace') as f: t = f.read().strip() except IOError: - t = u"(Error reading %s)" % filename + t = "(Error reading %s)" % filename sections[s]["text"] = t @@ -196,13 +199,13 @@ def fill_in_agenda_docs(date, sections, docs=None): sections[number]["docs"].append(doc) # prune empty "For action" sections - empty_for_action = [n for n, section in sections.iteritems() + empty_for_action = [n for n, section in sections.items() if section["title"] == "For action" and not section["docs"]] for num in empty_for_action: del sections[num] # Be careful to keep this the same as what's used in agenda_documents - for s in sections.itervalues(): + for s in sections.values(): if "docs" in s: s["docs"].sort(key=lambda d: d.balloting_started) diff --git a/ietf/iesg/feeds.py b/ietf/iesg/feeds.py index 01ae1e4cb..0cd17f5fe 100644 --- a/ietf/iesg/feeds.py +++ b/ietf/iesg/feeds.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime from django.contrib.syndication.views import Feed @@ -26,7 +32,7 @@ class IESGAgendaFeed(Feed): return doc.latest_telechat_event.time def item_author_name(self, doc): - return doc.ad.plain_name() if doc.ad else "None" + return doc.ad.plain_name() if doc.ad else "None" def item_author_email(self, doc): if not doc.ad: diff --git a/ietf/iesg/models.py b/ietf/iesg/models.py index 3b43f1cb5..bf716a9d9 100644 --- a/ietf/iesg/models.py +++ b/ietf/iesg/models.py @@ -1,5 +1,6 @@ -# Copyright The IETF Trust 2007, All Rights Reserved - +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- +# # Portion Copyright (C) 2008 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -32,11 +33,15 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import datetime from django.db import models +from django.utils.encoding import python_2_unicode_compatible - +@python_2_unicode_compatible class TelechatAgendaItem(models.Model): TYPE_CHOICES = ( (1, "Any Other Business (WG News, New Proposals, etc.)"), @@ -49,9 +54,9 @@ class TelechatAgendaItem(models.Model): type = models.IntegerField(db_column='template_type', choices=TYPE_CHOICES, default=3) title = models.CharField(max_length=255, db_column='template_title') - def __unicode__(self): + def __str__(self): type_name = self.TYPE_CHOICES_DICT.get(self.type, str(self.type)) - return u'%s: %s' % (type_name, self.title or "") + return "%s: %s" % (type_name, self.title or "") class Telechat(models.Model): telechat_id = models.IntegerField(primary_key=True) @@ -64,7 +69,7 @@ class Telechat(models.Model): mi_frozen = models.IntegerField(null=True, blank=True) class Meta: - db_table = u'telechat' + db_table = 'telechat' def next_telechat_date(): @@ -77,12 +82,13 @@ class TelechatDateManager(models.Manager): def active(self): return self.get_queryset().filter(date__gte=datetime.date.today()) +@python_2_unicode_compatible class TelechatDate(models.Model): objects = TelechatDateManager() date = models.DateField(default=next_telechat_date) - def __unicode__(self): + def __str__(self): return self.date.isoformat() class Meta: diff --git a/ietf/iesg/resources.py b/ietf/iesg/resources.py index 96293bc45..a67dbc3a4 100644 --- a/ietf/iesg/resources.py +++ b/ietf/iesg/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2014-11-13 23:53 + + from ietf.api import ModelResource from tastypie.constants import ALL from tastypie.cache import SimpleCache diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py index cf1f0e3b9..3735f3005 100644 --- a/ietf/iesg/tests.py +++ b/ietf/iesg/tests.py @@ -1,14 +1,20 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io import os import shutil -import json -import datetime +import tarfile + from pyquery import PyQuery from django.conf import settings from django.urls import reverse as urlreverse +from django.utils.encoding import force_bytes import debug # pyflakes:ignore @@ -42,8 +48,8 @@ class IESGTests(TestCase): r = self.client.get(urlreverse("ietf.iesg.views.discusses")) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.name in unicontent(r)) - self.assertTrue(pos.ad.plain_name() in unicontent(r)) + self.assertContains(r, draft.name) + self.assertContains(r, pos.ad.plain_name()) def test_milestones_needing_review(self): draft = WgDraftFactory() @@ -58,12 +64,12 @@ class IESGTests(TestCase): login_testing_unauthorized(self, "ad", url) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(m.desc in unicontent(r)) + self.assertContains(r, m.desc) draft.group.state_id = 'conclude' draft.group.save() r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertFalse(m.desc in unicontent(r)) + self.assertNotContains(r, m.desc) def test_review_decisions(self): @@ -79,7 +85,7 @@ class IESGTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(draft.name in unicontent(r)) + self.assertContains(r, draft.name) def test_photos(self): url = urlreverse("ietf.iesg.views.photos") @@ -119,7 +125,7 @@ class IESGAgendaTests(TestCase): self.saved_internet_draft_path = settings.INTERNET_DRAFT_PATH settings.INTERNET_DRAFT_PATH = self.draft_dir - for d in self.telechat_docs.values(): + for d in list(self.telechat_docs.values()): TelechatDocEvent.objects.create(type="scheduled_for_telechat", doc=d, rev=d.rev, @@ -304,58 +310,58 @@ class IESGAgendaTests(TestCase): r = self.client.get("/feed/iesg-agenda/") self.assertEqual(r.status_code, 200) - for d in self.telechat_docs.values(): - self.assertTrue(d.name in unicontent(r)) - self.assertTrue(d.title in unicontent(r)) + for d in list(self.telechat_docs.values()): + self.assertContains(r, d.name) + self.assertContains(r, d.title) def test_agenda_json(self): r = self.client.get(urlreverse("ietf.iesg.views.agenda_json")) self.assertEqual(r.status_code, 200) - for k, d in self.telechat_docs.iteritems(): + for k, d in self.telechat_docs.items(): if d.type_id == "charter": - self.assertTrue(d.group.name in unicontent(r), "%s '%s' not in response" % (k, d.group.name)) - self.assertTrue(d.group.acronym in unicontent(r), "%s '%s' acronym not in response" % (k, d.group.acronym)) + self.assertContains(r, d.group.name, msg_prefix="%s '%s' not in response" % (k, d.group.name)) + self.assertContains(r, d.group.acronym, msg_prefix="%s '%s' acronym not in response" % (k, d.group.acronym)) else: - self.assertTrue(d.name in unicontent(r), "%s '%s' not in response" % (k, d.name)) - self.assertTrue(d.title in unicontent(r), "%s '%s' title not in response" % (k, d.title)) + self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name)) + self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title)) - self.assertTrue(json.loads(r.content)) + self.assertTrue(r.json()) def test_agenda(self): r = self.client.get(urlreverse("ietf.iesg.views.agenda")) self.assertEqual(r.status_code, 200) - for k, d in self.telechat_docs.iteritems(): + for k, d in self.telechat_docs.items(): if d.type_id == "charter": - self.assertTrue(d.group.name in unicontent(r), "%s '%s' not in response" % (k, d.group.name)) - self.assertTrue(d.group.acronym in unicontent(r), "%s '%s' acronym not in response" % (k, d.group.acronym)) + self.assertContains(r, d.group.name, msg_prefix="%s '%s' not in response" % (k, d.group.name)) + self.assertContains(r, d.group.acronym, msg_prefix="%s '%s' acronym not in response" % (k, d.group.acronym)) else: - self.assertTrue(d.name in unicontent(r), "%s '%s' not in response" % (k, d.name)) - self.assertTrue(d.title in unicontent(r), "%s '%s' title not in response" % (k, d.title)) + self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name)) + self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title)) def test_agenda_txt(self): r = self.client.get(urlreverse("ietf.iesg.views.agenda_txt")) self.assertEqual(r.status_code, 200) - for k, d in self.telechat_docs.iteritems(): + for k, d in self.telechat_docs.items(): if d.type_id == "charter": - self.assertTrue(d.group.name in unicontent(r), "%s '%s' not in response" % (k, d.group.name)) - self.assertTrue(d.group.acronym in unicontent(r), "%s '%s' acronym not in response" % (k, d.group.acronym)) + self.assertContains(r, d.group.name, msg_prefix="%s '%s' not in response" % (k, d.group.name)) + self.assertContains(r, d.group.acronym, msg_prefix="%s '%s' acronym not in response" % (k, d.group.acronym)) else: - self.assertTrue(d.name in unicontent(r), "%s '%s' not in response" % (k, d.name)) - self.assertTrue(d.title in unicontent(r), "%s '%s' title not in response" % (k, d.title)) + self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name)) + self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title)) def test_agenda_scribe_template(self): r = self.client.get(urlreverse("ietf.iesg.views.agenda_scribe_template")) self.assertEqual(r.status_code, 200) - for k, d in self.telechat_docs.iteritems(): + for k, d in self.telechat_docs.items(): if d.type_id == "charter": continue # scribe template doesn't contain chartering info - self.assertTrue(d.name in unicontent(r), "%s '%s' not in response" % (k, d.name)) - self.assertTrue(d.title in unicontent(r), "%s '%s' title not in response" % (k, d.title)) + self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name)) + self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title)) def test_agenda_moderator_package(self): url = urlreverse("ietf.iesg.views.agenda_moderator_package") @@ -363,19 +369,19 @@ class IESGAgendaTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) - for k, d in self.telechat_docs.iteritems(): + for k, d in self.telechat_docs.items(): if d.type_id == "charter": - self.assertTrue(d.group.name in unicontent(r), "%s '%s' not in response" % (k, d.group.name)) - self.assertTrue(d.group.acronym in unicontent(r), "%s '%s' acronym not in response" % (k, d.group.acronym)) + self.assertContains(r, d.group.name, msg_prefix="%s '%s' not in response" % (k, d.group.name)) + self.assertContains(r, d.group.acronym, msg_prefix="%s '%s' acronym not in response" % (k, d.group.acronym)) else: if d.type_id == "draft" and d.name == "draft-ietf-mars-test": - self.assertTrue(d.name in unicontent(r), "%s '%s' not in response" % (k, d.name)) - self.assertTrue(d.title in unicontent(r), "%s '%s' title not in response" % (k, d.title)) - self.assertTrue("Has downref: Yes" in unicontent(r), "%s downref not in response" % (k, )) - self.assertTrue("Add rfc6666" in unicontent(r), "%s downref not in response" % (k, )) + self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name)) + self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title)) + self.assertContains(r, "Has downref: Yes", msg_prefix="%s downref not in response" % (k, )) + self.assertContains(r, "Add rfc6666", msg_prefix="%s downref not in response" % (k, )) else: - self.assertTrue(d.name in unicontent(r), "%s '%s' not in response" % (k, d.name)) - self.assertTrue(d.title in unicontent(r), "%s '%s' title not in response" % (k, d.title)) + self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name)) + self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title)) def test_agenda_package(self): url = urlreverse("ietf.iesg.views.agenda_package") @@ -383,37 +389,37 @@ class IESGAgendaTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) - for k, d in self.telechat_docs.iteritems(): + for k, d in self.telechat_docs.items(): if d.type_id == "charter": - self.assertTrue(d.group.name in unicontent(r), "%s '%s' not in response" % (k, d.group.name, )) - self.assertTrue(d.group.acronym in unicontent(r), "%s '%s' acronym not in response" % (k, d.group.acronym, )) + self.assertContains(r, d.group.name, msg_prefix="%s '%s' not in response" % (k, d.group.name, )) + self.assertContains(r, d.group.acronym, msg_prefix="%s '%s' acronym not in response" % (k, d.group.acronym, )) else: - self.assertTrue(d.name in unicontent(r), "%s '%s' not in response" % (k, d.name, )) - self.assertTrue(d.title in unicontent(r), "%s '%s' title not in response" % (k, d.title, )) + self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name, )) + self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title, )) def test_agenda_documents_txt(self): url = urlreverse("ietf.iesg.views.agenda_documents_txt") r = self.client.get(url) self.assertEqual(r.status_code, 200) - for k, d in self.telechat_docs.iteritems(): - self.assertTrue(d.name in unicontent(r), "%s '%s' not in response" % (k, d.name, )) + for k, d in self.telechat_docs.items(): + self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name, )) def test_agenda_documents(self): url = urlreverse("ietf.iesg.views.agenda_documents") r = self.client.get(url) self.assertEqual(r.status_code, 200) - for k, d in self.telechat_docs.iteritems(): - self.assertTrue(d.name in unicontent(r), "%s '%s' not in response" % (k, d.name, )) - self.assertTrue(d.title in unicontent(r), "%s '%s' title not in response" % (k, d.title, )) + for k, d in self.telechat_docs.items(): + self.assertContains(r, d.name, msg_prefix="%s '%s' not in response" % (k, d.name, )) + self.assertContains(r, d.title, msg_prefix="%s '%s' title not in response" % (k, d.title, )) def test_past_documents(self): url = urlreverse("ietf.iesg.views.past_documents") # We haven't put any documents on past telechats, so this should be empty r = self.client.get(url) self.assertEqual(r.status_code, 200) - for k, d in self.telechat_docs.iteritems(): + for k, d in self.telechat_docs.items(): self.assertNotIn(d.name, unicontent(r)) self.assertNotIn(d.title, unicontent(r)) # Add the documents to a past telechat @@ -421,7 +427,7 @@ class IESGAgendaTests(TestCase): date = datetime.date.today() - datetime.timedelta(days=14) approved = State.objects.get(type='draft-iesg', slug='approved') iesg_eval = State.objects.get(type='draft-iesg', slug='iesg-eva') - for d in self.telechat_docs.values(): + for d in list(self.telechat_docs.values()): if d.type_id in ['draft', 'charter']: create_ballot_if_not_open(None, d, by, 'approve') TelechatDocEvent.objects.create(type="scheduled_for_telechat", @@ -435,7 +441,7 @@ class IESGAgendaTests(TestCase): # Now check that they are present on the past documents page r = self.client.get(url) self.assertEqual(r.status_code, 200) - for k, d in self.telechat_docs.iteritems(): + for k, d in self.telechat_docs.items(): if d.states.get(type='draft-iesg').slug in ['approved', 'iesg-eva', ]: self.assertIn(d.name, unicontent(r)) else: @@ -448,28 +454,28 @@ class IESGAgendaTests(TestCase): d1_filename = "%s-%s.txt" % (d1.name, d1.rev) d2_filename = "%s-%s.txt" % (d2.name, d2.rev) - with open(os.path.join(self.draft_dir, d1_filename), "w") as f: + with io.open(os.path.join(self.draft_dir, d1_filename), "w") as f: f.write("test content") url = urlreverse("ietf.iesg.views.telechat_docs_tarfile", kwargs=dict(date=get_agenda_date().isoformat())) r = self.client.get(url) self.assertEqual(r.status_code, 200) - import tarfile, StringIO - - tar = tarfile.open(None, fileobj=StringIO.StringIO(r.content)) + tar = tarfile.open(None, fileobj=io.BytesIO(r.content)) names = tar.getnames() self.assertIn(d1_filename, names) self.assertNotIn(d2_filename, names) self.assertIn("manifest.txt", names) f = tar.extractfile(d1_filename) - self.assertEqual(f.read(), "test content") + self.assertEqual(f.read(), b"test content") f = tar.extractfile("manifest.txt") lines = list(f.readlines()) - self.assertTrue("Included" in [l for l in lines if d1_filename in l][0]) - self.assertTrue("Not found" in [l for l in lines if d2_filename in l][0]) + d1fn = force_bytes(d1_filename) + d2fn = force_bytes(d2_filename) + self.assertTrue(b"Included" in [l for l in lines if d1fn in l][0]) + self.assertTrue(b"Not found" in [l for l in lines if d2fn in l][0]) def test_admin_change(self): draft = Document.objects.get(name="draft-ietf-mars-test") @@ -522,8 +528,9 @@ class RescheduleOnAgendaTests(TestCase): # agenda docs page r = self.client.get(url) self.assertEqual(r.status_code, 200) - d_header_pos = r.content.find("IESG telechat %s" % d.isoformat()) - draft_pos = unicontent(r)[d_header_pos:].find(draft.name) + content = unicontent(r) + d_header_pos = content.find("IESG telechat %s" % d.isoformat()) + draft_pos = content[d_header_pos:].find(draft.name) self.assertTrue(draft_pos>0) self.assertTrue(draft.latest_event(TelechatDocEvent, "scheduled_for_telechat")) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index 415f71925..41c25ca6d 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -1,5 +1,6 @@ -# Copyright The IETF Trust 2007, All Rights Reserved - +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- +# # Portion Copyright (C) 2008-2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -32,16 +33,17 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os + +from __future__ import absolute_import, print_function, unicode_literals + import datetime -import tarfile -import StringIO -import time +import io import itertools import json - -import debug # pyflakes:ignore - +import os +import six +import tarfile +import time from django import forms from django.conf import settings @@ -49,9 +51,12 @@ from django.db import models from django.http import HttpResponse from django.shortcuts import render, redirect from django.contrib.sites.models import Site +from django.utils.encoding import force_bytes #from django.views.decorators.cache import cache_page #from django.views.decorators.vary import vary_on_cookie +import debug # pyflakes:ignore + from ietf.doc.models import Document, State, LastCallDocEvent, ConsensusDocEvent, DocEvent, IESG_BALLOT_ACTIVE_STATES from ietf.doc.utils import update_telechat, augment_events_with_revision from ietf.group.models import GroupMilestone, Role @@ -80,7 +85,7 @@ def review_decisions(request, year=None): #proto_levels = ["bcp", "ds", "ps", "std"] #doc_levels = ["exp", "inf"] - timeframe = u"%s" % year if year else u"the past 6 months" + timeframe = "%s" % year if year else "the past 6 months" return render(request, 'iesg/review_decisions.html', dict(events=events, @@ -99,7 +104,7 @@ def agenda_json(request, date=None): "sections": {}, } - for num, section in data["sections"].iteritems(): + for num, section in data["sections"].items(): s = res["sections"][num] = { "title": section["title"], } @@ -178,7 +183,7 @@ def agenda_json(request, date=None): s["docs"].append(docinfo) - return HttpResponse(json.dumps(res, indent=2), content_type='text/plain') + return HttpResponse(json.dumps(res, indent=2), content_type='application/json') # def past_agendas(request): # # This is not particularly useful with the current way of constructing @@ -198,7 +203,7 @@ def agenda(request, date=None): request.session['ballot_edit_return_point'] = request.path_info return render(request, "iesg/agenda.html", { "date": data["date"], - "sections": sorted(data["sections"].iteritems()), + "sections": sorted(data["sections"].items()), "settings": settings, } ) @@ -206,13 +211,13 @@ def agenda_txt(request, date=None): data = agenda_data(date) return render(request, "iesg/agenda.txt", { "date": data["date"], - "sections": sorted(data["sections"].iteritems()), + "sections": sorted(data["sections"].items()), "domain": Site.objects.get_current().domain, }, content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET) def agenda_scribe_template(request, date=None): data = agenda_data(date) - sections = sorted((num, section) for num, section in data["sections"].iteritems() if "2" <= num < "4") + sections = sorted((num, section) for num, section in data["sections"].items() if "2" <= num < "4") appendix_docs = [] for num, section in sections: if "docs" in section: @@ -237,7 +242,7 @@ def agenda_moderator_package(request, date=None): or (num == "6" and "6.1" not in data["sections"])) # sort and prune non-leaf headlines - sections = sorted((num, section) for num, section in data["sections"].iteritems() + sections = sorted((num, section) for num, section in data["sections"].items() if leaf_section(num, section)) # add parents field to each section @@ -245,7 +250,7 @@ def agenda_moderator_package(request, date=None): s["parents"] = [] split = num.split(".") - for i in xrange(num.count(".")): + for i in range(num.count(".")): parent_num = ".".join(split[:i + 1]) parent = data["sections"].get(parent_num) if parent: @@ -281,12 +286,12 @@ def agenda_package(request, date=None): data = agenda_data(date) return render(request, "iesg/agenda_package.txt", { "date": data["date"], - "sections": sorted(data["sections"].iteritems()), + "sections": sorted(data["sections"].items()), "roll_call": data["sections"]["1.1"]["text"], "roll_call_url": settings.IESG_ROLL_CALL_URL, "minutes": data["sections"]["1.3"]["text"], "minutes_url": settings.IESG_MINUTES_URL, - "management_items": [(num, section) for num, section in data["sections"].iteritems() if "6" < num < "7"], + "management_items": [(num, section) for num, section in data["sections"].items() if "6" < num < "7"], }, content_type='text/plain') @@ -311,14 +316,14 @@ def agenda_documents_txt(request): row = ( d.computed_telechat_date.isoformat(), d.name, - unicode(d.intended_std_level), + six.text_type(d.intended_std_level), "1" if d.stream_id in ("ise", "irtf") else "0", - unicode(d.area_acronym()).lower(), + six.text_type(d.area_acronym()).lower(), d.ad.plain_name() if d.ad else "None Assigned", d.rev, ) rows.append("\t".join(row)) - return HttpResponse(u"\n".join(rows), content_type='text/plain') + return HttpResponse("\n".join(rows), content_type='text/plain') class RescheduleForm(forms.Form): telechat_date = forms.TypedChoiceField(coerce=lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').date(), empty_value=None, required=False) @@ -378,7 +383,7 @@ def agenda_documents(request): reschedule_status = { "changed": False } - for i in itertools.chain(*docs_by_date.values()): + for i in itertools.chain(*list(docs_by_date.values())): i.reschedule_form = handle_reschedule_form(request, i, dates, reschedule_status) if reschedule_status["changed"]: @@ -397,7 +402,7 @@ def agenda_documents(request): telechats.append({ "date": date, "pages": pages, - "sections": sorted((num, section) for num, section in sections.iteritems() + "sections": sorted((num, section) for num, section in sections.items() if "2" <= num < "5") }) request.session['ballot_edit_return_point'] = request.path_info @@ -454,22 +459,22 @@ def telechat_docs_tarfile(request, date): tarstream = tarfile.open('', 'w:gz', response) - manifest = StringIO.StringIO() + manifest = io.BytesIO() for doc in docs: - doc_path = os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + ".txt") + doc_path = force_bytes(os.path.join(doc.get_file_path(), doc.name + "-" + doc.rev + ".txt")) if os.path.exists(doc_path): try: tarstream.add(doc_path, str(doc.name + "-" + doc.rev + ".txt")) - manifest.write("Included: %s\n" % doc_path) + manifest.write(b"Included: %s\n" % doc_path) except Exception as e: - manifest.write("Failed (%s): %s\n" % (e, doc_path)) + manifest.write(b"Failed (%s): %s\n" % (force_bytes(e), doc_path)) else: - manifest.write("Not found: %s\n" % doc_path) + manifest.write(b"Not found: %s\n" % doc_path) manifest.seek(0) t = tarfile.TarInfo(name="manifest.txt") - t.size = len(manifest.buf) + t.size = len(manifest.getvalue()) t.mtime = time.time() tarstream.addfile(t, manifest) @@ -524,10 +529,10 @@ def milestones_needing_review(request): milestones.append(m) ad_list = [] - for ad, groups in ads.iteritems(): + for ad, groups in ads.items(): ad_list.append(ad) ad.groups_needing_review = sorted(groups, key=lambda g: g.acronym) - for g, milestones in groups.iteritems(): + for g, milestones in groups.items(): g.milestones_needing_review = sorted(milestones, key=lambda m: m.due) return render(request, 'iesg/milestones_needing_review.html', diff --git a/ietf/ietfauth/forms.py b/ietf/ietfauth/forms.py index 65bad7dfe..db4874412 100644 --- a/ietf/ietfauth/forms.py +++ b/ietf/ietfauth/forms.py @@ -1,6 +1,8 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals import re from unidecode import unidecode @@ -126,7 +128,7 @@ def get_person_form(*args, **kwargs): self.unidecoded_ascii = name != reconstructed_name def clean_name(self): - name = self.cleaned_data.get("name") or u"" + name = self.cleaned_data.get("name") or "" prevent_at_symbol(name) prevent_system_name(name) return name @@ -135,13 +137,13 @@ def get_person_form(*args, **kwargs): if self.unidecoded_ascii: raise forms.ValidationError("Name contained non-ASCII characters, and was automatically reconstructed using only Latin characters. Check the result - if you are happy, just hit Submit again.") - name = self.cleaned_data.get("ascii") or u"" + name = self.cleaned_data.get("ascii") or "" prevent_at_symbol(name) prevent_system_name(name) return ascii_cleaner(name) def clean_ascii_short(self): - name = self.cleaned_data.get("ascii_short") or u"" + name = self.cleaned_data.get("ascii_short") or "" prevent_at_symbol(name) prevent_system_name(name) return ascii_cleaner(name) @@ -184,11 +186,11 @@ class RoleEmailForm(forms.Form): super(RoleEmailForm, self).__init__(*args, **kwargs) f = self.fields["email"] - f.label = u"%s in %s" % (role.name, role.group.acronym.upper()) - f.help_text = u"Email to use for %s role in %s" % (role.name, role.group.name) + f.label = "%s in %s" % (role.name, role.group.acronym.upper()) + f.help_text = "Email to use for %s role in %s" % (role.name, role.group.name) f.queryset = f.queryset.filter(models.Q(person=role.person_id) | models.Q(role=role)).distinct() f.initial = role.email_id - f.choices = [(e.pk, e.address if e.active else u"({})".format(e.address)) for e in f.queryset] + f.choices = [(e.pk, e.address if e.active else "({})".format(e.address)) for e in f.queryset] class ResetPasswordForm(forms.Form): diff --git a/ietf/ietfauth/htpasswd.py b/ietf/ietfauth/htpasswd.py index 145af84c7..cc42a5d8a 100644 --- a/ietf/ietfauth/htpasswd.py +++ b/ietf/ietfauth/htpasswd.py @@ -1,4 +1,12 @@ +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import subprocess, hashlib +from django.utils.encoding import force_bytes from django.conf import settings @@ -6,9 +14,9 @@ def update_htpasswd_file(username, password): if getattr(settings, 'USE_PYTHON_HTDIGEST', None): pass_file = settings.HTPASSWD_FILE realm = settings.HTDIGEST_REALM - prefix = '%s:%s:' % (username, realm) - key = hashlib.md5(prefix + password).hexdigest() - f = open(pass_file, 'r+') + prefix = force_bytes('%s:%s:' % (username, realm)) + key = force_bytes(hashlib.md5(prefix + force_bytes(password)).hexdigest()) + f = io.open(pass_file, 'r+b') pos = f.tell() line = f.readline() while line: @@ -17,7 +25,7 @@ def update_htpasswd_file(username, password): pos=f.tell() line = f.readline() f.seek(pos) - f.write('%s%s\n' % (prefix, key)) + f.write(b'%s%s\n' % (prefix, key)) f.close() else: p = subprocess.Popen([settings.HTPASSWD_COMMAND, "-b", settings.HTPASSWD_FILE, username, password], stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/ietf/ietfauth/management/commands/send_apikey_usage_emails.py b/ietf/ietfauth/management/commands/send_apikey_usage_emails.py index 2718ef02a..ad671c218 100644 --- a/ietf/ietfauth/management/commands/send_apikey_usage_emails.py +++ b/ietf/ietfauth/management/commands/send_apikey_usage_emails.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- -# Copyright The IETF Trust 2017, All Rights Reserved -from __future__ import print_function, unicode_literals +# Copyright The IETF Trust 2017-2019, All Rights Reserved + + +from __future__ import absolute_import, print_function, unicode_literals import datetime diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py index e01952734..ea7eea7e8 100644 --- a/ietf/ietfauth/tests.py +++ b/ietf/ietfauth/tests.py @@ -1,9 +1,12 @@ -# Copyright The IETF Trust 2017-2019, All Rights Reserved +# Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os, shutil, time, datetime -from urlparse import urlsplit +from six.moves.urllib.parse import urlsplit from pyquery import PyQuery from unittest import skipIf @@ -14,7 +17,7 @@ from django.conf import settings import debug # pyflakes:ignore -from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent +from ietf.utils.test_utils import TestCase, login_testing_unauthorized from ietf.utils.mail import outbox, empty_outbox from ietf.group.models import Group, Role, RoleName from ietf.group.factories import GroupFactory, RoleFactory @@ -46,7 +49,7 @@ class IetfAuthTests(TestCase): self.saved_htpasswd_file = settings.HTPASSWD_FILE self.htpasswd_dir = self.tempdir('htpasswd') settings.HTPASSWD_FILE = os.path.join(self.htpasswd_dir, "htpasswd") - open(settings.HTPASSWD_FILE, 'a').close() # create empty file + io.open(settings.HTPASSWD_FILE, 'a').close() # create empty file self.saved_htdigest_realm = getattr(settings, "HTDIGEST_REALM", None) settings.HTDIGEST_REALM = "test-realm" @@ -98,7 +101,7 @@ class IetfAuthTests(TestCase): def extract_confirm_url(self, confirm_email): # dig out confirm_email link - msg = confirm_email.get_payload(decode=True) + msg = confirm_email.get_payload(decode=True).decode(confirm_email.get_content_charset()) line_start = "http" confirm_url = None for line in msg.split("\n"): @@ -109,11 +112,11 @@ class IetfAuthTests(TestCase): return confirm_url def username_in_htpasswd_file(self, username): - with open(settings.HTPASSWD_FILE) as f: + with io.open(settings.HTPASSWD_FILE) as f: for l in f: if l.startswith(username + ":"): return True - with open(settings.HTPASSWD_FILE) as f: + with io.open(settings.HTPASSWD_FILE) as f: print(f.read()) return False @@ -131,7 +134,7 @@ class IetfAuthTests(TestCase): empty_outbox() r = self.client.post(url, { 'email': email }) self.assertEqual(r.status_code, 200) - self.assertIn("Account creation failed", unicontent(r)) + self.assertContains(r, "Account creation failed") def register_and_verify(self, email): url = urlreverse(ietf.ietfauth.views.create_account) @@ -140,7 +143,7 @@ class IetfAuthTests(TestCase): empty_outbox() r = self.client.post(url, { 'email': email }) self.assertEqual(r.status_code, 200) - self.assertIn("Account request received", unicontent(r)) + self.assertContains(r, "Account request received") self.assertEqual(len(outbox), 1) # go to confirm page @@ -172,11 +175,11 @@ class IetfAuthTests(TestCase): r = self.client.get(urlreverse(ietf.ietfauth.views.add_account_whitelist)) self.assertEqual(r.status_code, 200) - self.assertIn("Add a whitelist entry", unicontent(r)) + self.assertContains(r, "Add a whitelist entry") r = self.client.post(urlreverse(ietf.ietfauth.views.add_account_whitelist), {"email": email}) self.assertEqual(r.status_code, 200) - self.assertIn("Whitelist entry creation successful", unicontent(r)) + self.assertContains(r, "Whitelist entry creation successful") # log out r = self.client.get(urlreverse(django.contrib.auth.views.logout)) @@ -216,9 +219,9 @@ class IetfAuthTests(TestCase): self.assertEqual(len(q('[name="active_emails"][value="%s"][checked]' % email_address)), 1) base_data = { - "name": u"Test Nãme", - "ascii": u"Test Name", - "ascii_short": u"T. Name", + "name": "Test Nãme", + "ascii": "Test Name", + "ascii_short": "T. Name", "affiliation": "Test Org", "active_emails": email_address, "consent": True, @@ -226,7 +229,7 @@ class IetfAuthTests(TestCase): # edit details - faulty ASCII faulty_ascii = base_data.copy() - faulty_ascii["ascii"] = u"Test Nãme" + faulty_ascii["ascii"] = "Test Nãme" r = self.client.post(url, faulty_ascii) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -234,7 +237,7 @@ class IetfAuthTests(TestCase): # edit details - blank ASCII blank_ascii = base_data.copy() - blank_ascii["ascii"] = u"" + blank_ascii["ascii"] = "" r = self.client.post(url, blank_ascii) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -245,14 +248,14 @@ class IetfAuthTests(TestCase): r = self.client.post(url, base_data) self.assertEqual(r.status_code, 200) person = Person.objects.get(user__username=username) - self.assertEqual(person.name, u"Test Nãme") - self.assertEqual(person.ascii, u"Test Name") - self.assertEqual(Person.objects.filter(alias__name=u"Test Name", user__username=username).count(), 1) - self.assertEqual(Person.objects.filter(alias__name=u"Test Nãme", user__username=username).count(), 1) + self.assertEqual(person.name, "Test Nãme") + self.assertEqual(person.ascii, "Test Name") + self.assertEqual(Person.objects.filter(alias__name="Test Name", user__username=username).count(), 1) + self.assertEqual(Person.objects.filter(alias__name="Test Nãme", user__username=username).count(), 1) self.assertEqual(Email.objects.filter(address=email_address, person__user__username=username, active=True).count(), 1) # deactivate address - without_email_address = { k: v for k, v in base_data.iteritems() if k != "active_emails" } + without_email_address = { k: v for k, v in base_data.items() if k != "active_emails" } r = self.client.post(url, without_email_address) self.assertEqual(r.status_code, 200) @@ -379,7 +382,7 @@ class IetfAuthTests(TestCase): # get r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(review_req.doc.name in unicontent(r)) + self.assertContains(r, review_req.doc.name) # wish to review r = self.client.post(url, { @@ -460,7 +463,7 @@ class IetfAuthTests(TestCase): self.assertRedirects(r, prof_url) # refresh user object user = User.objects.get(username="someone@example.com") - self.assertTrue(user.check_password(u'foobar')) + self.assertTrue(user.check_password('foobar')) def test_change_username(self): @@ -508,7 +511,7 @@ class IetfAuthTests(TestCase): prev = user user = User.objects.get(username="othername@example.org") self.assertEqual(prev, user) - self.assertTrue(user.check_password(u'password')) + self.assertTrue(user.check_password('password')) def test_apikey_management(self): person = PersonFactory() @@ -581,36 +584,31 @@ class IetfAuthTests(TestCase): self.assertRedirects(r, urlreverse('ietf.ietfauth.views.apikey_index')) for key in person.apikeys.all()[:3]: - url = key.endpoint # bad method - r = self.client.put(url, {'apikey':key.hash()}) + r = self.client.put(key.endpoint, {'apikey':key.hash()}) self.assertEqual(r.status_code, 405) # missing apikey - r = self.client.post(url, {'dummy':'dummy',}) - self.assertEqual(r.status_code, 400) - self.assertIn('Missing apikey parameter', unicontent(r)) + r = self.client.post(key.endpoint, {'dummy':'dummy',}) + self.assertContains(r, 'Missing apikey parameter', status_code=400) # invalid apikey - r = self.client.post(url, {'apikey':BAD_KEY, 'dummy':'dummy',}) - self.assertEqual(r.status_code, 400) - self.assertIn('Invalid apikey', unicontent(r)) + r = self.client.post(key.endpoint, {'apikey':BAD_KEY, 'dummy':'dummy',}) + self.assertContains(r, 'Invalid apikey', status_code=400) # too long since regular login person.user.last_login = datetime.datetime.now() - datetime.timedelta(days=settings.UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS+1) person.user.save() - r = self.client.post(url, {'apikey':key.hash(), 'dummy':'dummy',}) - self.assertEqual(r.status_code, 400) - self.assertIn('Too long since last regular login', unicontent(r)) + r = self.client.post(key.endpoint, {'apikey':key.hash(), 'dummy':'dummy',}) + self.assertContains(r, 'Too long since last regular login', status_code=400) person.user.last_login = datetime.datetime.now() person.user.save() # endpoint mismatch key2 = PersonalApiKey.objects.create(person=person, endpoint='/') - r = self.client.post(url, {'apikey':key2.hash(), 'dummy':'dummy',}) - self.assertEqual(r.status_code, 400) - self.assertIn('Apikey endpoint mismatch', unicontent(r)) + r = self.client.post(key.endpoint, {'apikey':key2.hash(), 'dummy':'dummy',}) + self.assertContains(r, 'Apikey endpoint mismatch', status_code=400) key2.delete() def test_send_apikey_report(self): @@ -636,8 +634,7 @@ class IetfAuthTests(TestCase): time.sleep(2) for i in range(count): for key in person.apikeys.all(): - url = key.endpoint - self.client.post(url, {'apikey':key.hash(), 'dummy': 'dummy', }) + self.client.post(key.endpoint, {'apikey':key.hash(), 'dummy': 'dummy', }) date = str(datetime.date.today()) empty_outbox() diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index 010f3fbd2..59eaa99a5 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -1,4 +1,8 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals # various authentication and authorization utilities @@ -30,7 +34,7 @@ def has_role(user, role_names, *args, **kwargs): """Determines whether user has any of the given standard roles given. Role names must be a list or, in case of a single value, a string.""" - if isinstance(role_names, str) or isinstance(role_names, unicode): + if not isinstance(role_names, (list, tuple)): role_names = [ role_names ] if not user or not user.is_authenticated: @@ -48,24 +52,24 @@ def has_role(user, role_names, *args, **kwargs): return False role_qs = { - "Area Director": Q(person=person, name__in=("pre-ad", "ad"), group__type="area", group__state="active"), - "Secretariat": Q(person=person, name="secr", group__acronym="secretariat"), + "Area Director": Q(person=person, name__in=("pre-ad", "ad"), group__type="area", group__state="active"), + "Secretariat": Q(person=person, name="secr", group__acronym="secretariat"), "IAB" : Q(person=person, name="member", group__acronym="iab"), - "IANA": Q(person=person, name="auth", group__acronym="iana"), + "IANA": Q(person=person, name="auth", group__acronym="iana"), "RFC Editor": Q(person=person, name="auth", group__acronym="rfceditor"), "ISE" : Q(person=person, name="chair", group__acronym="ise"), - "IAD": Q(person=person, name="admdir", group__acronym="ietf"), - "IETF Chair": Q(person=person, name="chair", group__acronym="ietf"), - "IETF Trust Chair": Q(person=person, name="chair", group__acronym="ietf-trust"), - "IRTF Chair": Q(person=person, name="chair", group__acronym="irtf"), - "IAB Chair": Q(person=person, name="chair", group__acronym="iab"), - "IAB Executive Director": Q(person=person, name="execdir", group__acronym="iab"), + "IAD": Q(person=person, name="admdir", group__acronym="ietf"), + "IETF Chair": Q(person=person, name="chair", group__acronym="ietf"), + "IETF Trust Chair": Q(person=person, name="chair", group__acronym="ietf-trust"), + "IRTF Chair": Q(person=person, name="chair", group__acronym="irtf"), + "IAB Chair": Q(person=person, name="chair", group__acronym="iab"), + "IAB Executive Director": Q(person=person, name="execdir", group__acronym="iab"), "IAB Group Chair": Q(person=person, name="chair", group__type="iab", group__state="active"), "IAOC Chair": Q(person=person, name="chair", group__acronym="iaoc"), - "WG Chair": Q(person=person,name="chair", group__type="wg", group__state__in=["active","bof", "proposed"]), - "WG Secretary": Q(person=person,name="secr", group__type="wg", group__state__in=["active","bof", "proposed"]), - "RG Chair": Q(person=person,name="chair", group__type="rg", group__state__in=["active","proposed"]), - "RG Secretary": Q(person=person,name="secr", group__type="rg", group__state__in=["active","proposed"]), + "WG Chair": Q(person=person,name="chair", group__type="wg", group__state__in=["active","bof", "proposed"]), + "WG Secretary": Q(person=person,name="secr", group__type="wg", group__state__in=["active","bof", "proposed"]), + "RG Chair": Q(person=person,name="chair", group__type="rg", group__state__in=["active","proposed"]), + "RG Secretary": Q(person=person,name="secr", group__type="rg", group__state__in=["active","proposed"]), "AG Secretary": Q(person=person,name="secr", group__type="ag", group__state__in=["active"]), "Team Chair": Q(person=person,name="chair", group__type="team", group__state="active"), "Nomcom Chair": Q(person=person, name="chair", group__type="nomcom", group__acronym__icontains=kwargs.get('year', '0000')), diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py index 4fe2e1349..04359f88a 100644 --- a/ietf/ietfauth/views.py +++ b/ietf/ietfauth/views.py @@ -1,3 +1,6 @@ +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- +# # Portions Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -30,7 +33,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Copyright The IETF Trust 2007, All Rights Reserved + +from __future__ import absolute_import, print_function, unicode_literals import importlib @@ -54,6 +58,7 @@ from django.urls import reverse as urlreverse from django.utils.safestring import mark_safe from django.http import Http404, HttpResponseRedirect #, HttpResponse, from django.shortcuts import render, redirect, get_object_or_404 +from django.utils.encoding import force_bytes import debug # pyflakes:ignore @@ -228,7 +233,7 @@ def profile(request): auth = django.core.signing.dumps([person.user.username, to_email], salt="add_email") domain = Site.objects.get_current().domain - subject = u'Confirm email address for %s' % person.name + subject = 'Confirm email address for %s' % person.name from_email = settings.DEFAULT_FROM_EMAIL send_mail(request, to_email, from_email, subject, 'registration/add_email_email.txt', { @@ -674,7 +679,7 @@ def apikey_disable(request): class KeyDeleteForm(forms.Form): hash = forms.ChoiceField(label='Key', choices=choices) def clean_key(self): - hash = self.cleaned_data['hash'] + hash = force_bytes(self.cleaned_data['hash']) key = PersonalApiKey.validate_key(hash) if key and key.person == request.user.person: return hash @@ -684,7 +689,7 @@ def apikey_disable(request): if request.method == 'POST': form = KeyDeleteForm(request.POST) if form.is_valid(): - hash = form.data['hash'] + hash = force_bytes(form.data['hash']) key = PersonalApiKey.validate_key(hash) key.valid = False key.save() diff --git a/ietf/ipr/admin.py b/ietf/ipr/admin.py index e47f20679..bcc0c33aa 100644 --- a/ietf/ipr/admin.py +++ b/ietf/ipr/admin.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2010-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + from django import forms from django.contrib import admin from ietf.name.models import DocRelationshipName @@ -43,7 +46,7 @@ class IprDisclosureBaseAdmin(admin.ModelAdmin): inlines = [IprDocRelInline,RelatedIprInline] def related_docs(self, obj): - return u", ".join(a.formatted_name() for a in IprDocRel.objects.filter(disclosure=obj).order_by("id").select_related("document")) + return ", ".join(a.formatted_name() for a in IprDocRel.objects.filter(disclosure=obj).order_by("id").select_related("document")) admin.site.register(IprDisclosureBase, IprDisclosureBaseAdmin) @@ -53,7 +56,7 @@ class HolderIprDisclosureAdmin(admin.ModelAdmin): inlines = [IprDocRelInline,RelatedIprInline] def related_docs(self, obj): - return u", ".join(a.formatted_name() for a in IprDocRel.objects.filter(disclosure=obj).order_by("id").select_related("document")) + return ", ".join(a.formatted_name() for a in IprDocRel.objects.filter(disclosure=obj).order_by("id").select_related("document")) admin.site.register(HolderIprDisclosure, HolderIprDisclosureAdmin) @@ -63,7 +66,7 @@ class ThirdPartyIprDisclosureAdmin(admin.ModelAdmin): inlines = [IprDocRelInline,RelatedIprInline] def related_docs(self, obj): - return u", ".join(a.formatted_name() for a in IprDocRel.objects.filter(disclosure=obj).order_by("id").select_related("document")) + return ", ".join(a.formatted_name() for a in IprDocRel.objects.filter(disclosure=obj).order_by("id").select_related("document")) admin.site.register(ThirdPartyIprDisclosure, ThirdPartyIprDisclosureAdmin) @@ -73,7 +76,7 @@ class GenericIprDisclosureAdmin(admin.ModelAdmin): inlines = [RelatedIprInline] def related_docs(self, obj): - return u", ".join(a.formatted_name() for a in IprDocRel.objects.filter(disclosure=obj).order_by("id").select_related("document")) + return ", ".join(a.formatted_name() for a in IprDocRel.objects.filter(disclosure=obj).order_by("id").select_related("document")) admin.site.register(GenericIprDisclosure, GenericIprDisclosureAdmin) @@ -83,7 +86,7 @@ class NonDocSpecificIprDisclosureAdmin(admin.ModelAdmin): inlines = [RelatedIprInline] def related_docs(self, obj): - return u", ".join(a.formatted_name() for a in IprDocRel.objects.filter(disclosure=obj).order_by("id").select_related("document")) + return ", ".join(a.formatted_name() for a in IprDocRel.objects.filter(disclosure=obj).order_by("id").select_related("document")) admin.site.register(NonDocSpecificIprDisclosure, NonDocSpecificIprDisclosureAdmin) @@ -105,7 +108,7 @@ class IprEventAdmin(admin.ModelAdmin): admin.site.register(IprEvent, IprEventAdmin) class LegacyMigrationIprEventAdmin(admin.ModelAdmin): - list_display = [u'id', 'time', 'type', 'by', 'disclosure', 'desc', 'message', 'in_reply_to', 'response_due'] + list_display = ['id', 'time', 'type', 'by', 'disclosure', 'desc', 'message', 'in_reply_to', 'response_due'] list_filter = ['time', 'type', 'response_due'] raw_id_fields = ['by', 'disclosure', 'message', 'in_reply_to'] admin.site.register(LegacyMigrationIprEvent, LegacyMigrationIprEventAdmin) diff --git a/ietf/ipr/factories.py b/ietf/ipr/factories.py index 62a96f191..038117f75 100644 --- a/ietf/ipr/factories.py +++ b/ietf/ipr/factories.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import factory diff --git a/ietf/ipr/feeds.py b/ietf/ipr/feeds.py index 0da26c542..7b8f1938f 100644 --- a/ietf/ipr/feeds.py +++ b/ietf/ipr/feeds.py @@ -1,9 +1,14 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from django.contrib.syndication.views import Feed from django.utils.feedgenerator import Atom1Feed from django.urls import reverse_lazy from django.utils.safestring import mark_safe +from django.utils.encoding import force_text from ietf.ipr.models import IprDisclosureBase @@ -22,7 +27,7 @@ class LatestIprDisclosuresFeed(Feed): return mark_safe(item.title) def item_description(self, item): - return unicode(item.title) + return force_text(item.title) def item_pubdate(self, item): return item.time diff --git a/ietf/ipr/fields.py b/ietf/ipr/fields.py index 3dbb8e902..4b9987967 100644 --- a/ietf/ipr/fields.py +++ b/ietf/ipr/fields.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import json +import six from django.utils.html import escape from django import forms @@ -12,7 +16,7 @@ import debug # pyflakes:ignore from ietf.ipr.models import IprDisclosureBase def select2_id_ipr_title_json(value): - return json.dumps([{ "id": o.pk, "text": escape(u"%s <%s>" % (o.title, o.time.date().isoformat())) } for o in value]) + return json.dumps([{ "id": o.pk, "text": escape("%s <%s>" % (o.title, o.time.date().isoformat())) } for o in value]) class SearchableIprDisclosuresField(forms.CharField): """Server-based multi-select field for choosing documents using @@ -50,7 +54,7 @@ class SearchableIprDisclosuresField(forms.CharField): def prepare_value(self, value): if not value: value = "" - if isinstance(value, basestring): + if isinstance(value, six.string_types): pks = self.parse_select2_value(value) # if the user posted a non integer value we need to remove it for key in pks: @@ -66,23 +70,23 @@ class SearchableIprDisclosuresField(forms.CharField): # patterns may not have been fully constructed there yet self.widget.attrs["data-ajax-url"] = urlreverse('ietf.ipr.views.ajax_search') - return u",".join(unicode(e.pk) for e in value) + return ",".join(six.text_type(e.pk) for e in value) def clean(self, value): value = super(SearchableIprDisclosuresField, self).clean(value) pks = self.check_pks(self.parse_select2_value(value)) if not all([ key.isdigit() for key in pks ]): - raise forms.ValidationError(u'You must enter IPR ID(s) as integers') + raise forms.ValidationError('You must enter IPR ID(s) as integers') objs = self.model.objects.filter(pk__in=pks) found_pks = [str(o.pk) for o in objs] failed_pks = [x for x in pks if x not in found_pks] if failed_pks: - raise forms.ValidationError(u"Could not recognize the following {model_name}s: {pks}. You can only input {model_name}s already registered in the Datatracker.".format(pks=", ".join(failed_pks), model_name=self.model.__name__.lower())) + raise forms.ValidationError("Could not recognize the following {model_name}s: {pks}. You can only input {model_name}s already registered in the Datatracker.".format(pks=", ".join(failed_pks), model_name=self.model.__name__.lower())) if self.max_entries != None and len(objs) > self.max_entries: - raise forms.ValidationError(u"You can select at most %s entries only." % self.max_entries) + raise forms.ValidationError("You can select at most %s entries only." % self.max_entries) return objs diff --git a/ietf/ipr/forms.py b/ietf/ipr/forms.py index 31b30fba1..410caa91c 100644 --- a/ietf/ipr/forms.py +++ b/ietf/ipr/forms.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import email @@ -8,6 +11,7 @@ import email from django import forms from django.core.validators import RegexValidator from django.utils.safestring import mark_safe +from django.utils.encoding import force_str import debug # pyflakes:ignore @@ -73,7 +77,7 @@ class AddEmailForm(forms.Form): def clean_message(self): '''Returns a ietf.message.models.Message object''' text = self.cleaned_data['message'] - message = email.message_from_string(text) + message = email.message_from_string(force_str(text)) for field in ('to','from','subject','date'): if not message[field]: raise forms.ValidationError('Error parsing email: {} field not found.'.format(field)) @@ -110,12 +114,12 @@ class DraftForm(forms.ModelForm): help_texts = { 'sections': 'Sections' } validate_patent_number = RegexValidator( - regex=("^(" - "([A-Z][A-Z]\d\d/\d{6}" - "|[A-Z][A-Z]\d{6,12}([A-Z]\d?)?" - "|[A-Z][A-Z]\d{4}(\w{1,2}\d{5,7})?" - "|[A-Z][A-Z]\d{15}" - ")[, ]*)+$"), + regex=(r"^(" + r"([A-Z][A-Z]\d\d/\d{6}" + r"|[A-Z][A-Z]\d{6,12}([A-Z]\d?)?" + r"|[A-Z][A-Z]\d{4}(\w{1,2}\d{5,7})?" + r"|[A-Z][A-Z]\d{15}" + r")[, ]*)+$"), message="Please enter one or more patent publication or application numbers as country code and serial number, e.g.: US62/123456 or WO2017123456." ) def validate_string(s, letter_min, digit_min, space_min, message): diff --git a/ietf/ipr/mail.py b/ietf/ipr/mail.py index f37552f51..1f4497e7e 100644 --- a/ietf/ipr/mail.py +++ b/ietf/ipr/mail.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import base64 import email import datetime @@ -5,7 +11,11 @@ from dateutil.tz import tzoffset import os import pytz import re + from django.template.loader import render_to_string +from django.utils.encoding import force_text, force_str + +import debug # pyflakes:ignore from ietf.ipr.models import IprEvent from ietf.message.models import Message @@ -91,7 +101,7 @@ def get_reply_to(): address with "plus addressing" using a random string. Guaranteed to be unique""" local,domain = get_base_ipr_request_address().split('@') while True: - rand = base64.urlsafe_b64encode(os.urandom(12)) + rand = force_text(base64.urlsafe_b64encode(os.urandom(12))) address = "{}+{}@{}".format(local,rand,domain) q = Message.objects.filter(reply_to=address) if not q: @@ -165,7 +175,7 @@ def process_response_email(msg): a matching value in the reply_to field, associated to an IPR disclosure through IprEvent. Create a Message object for the incoming message and associate it to the original message via new IprEvent""" - message = email.message_from_string(msg) + message = email.message_from_string(force_str(msg)) to = message.get('To') # exit if this isn't a response we're interested in (with plus addressing) @@ -194,5 +204,5 @@ def process_response_email(msg): in_reply_to = to_message ) - log(u"Received IPR email from %s" % ietf_message.frm) + log("Received IPR email from %s" % ietf_message.frm) return ietf_message diff --git a/ietf/ipr/management/commands/process_email.py b/ietf/ipr/management/commands/process_email.py index 2ca42ddfb..0bb200ff6 100644 --- a/ietf/ipr/management/commands/process_email.py +++ b/ietf/ipr/management/commands/process_email.py @@ -1,3 +1,10 @@ +# Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import sys from django.core.management.base import BaseCommand, CommandError @@ -7,7 +14,7 @@ from ietf.ipr.mail import process_response_email import debug # pyflakes:ignore class Command(BaseCommand): - help = (u"Process incoming email responses to ipr mail") + help = ("Process incoming email responses to ipr mail") def add_arguments(self, parser): parser.add_argument('--email-file', dest='email', help='File containing email (default: stdin)') @@ -19,7 +26,7 @@ class Command(BaseCommand): if not email: msg = sys.stdin.read() else: - msg = open(email, "r").read() + msg = io.open(email, "r").read() try: process_response_email(msg) diff --git a/ietf/ipr/migrations/0001_initial.py b/ietf/ipr/migrations/0001_initial.py index 369cd2deb..ac077dd85 100644 --- a/ietf/ipr/migrations/0001_initial.py +++ b/ietf/ipr/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/ipr/migrations/0002_auto_20180225_1207.py b/ietf/ipr/migrations/0002_auto_20180225_1207.py index 42983867e..01c1fa9b1 100644 --- a/ietf/ipr/migrations/0002_auto_20180225_1207.py +++ b/ietf/ipr/migrations/0002_auto_20180225_1207.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-25 12:07 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/ipr/migrations/0003_add_ipdocrel_document2_fk.py b/ietf/ipr/migrations/0003_add_ipdocrel_document2_fk.py index a29cede7d..35576ffb3 100644 --- a/ietf/ipr/migrations/0003_add_ipdocrel_document2_fk.py +++ b/ietf/ipr/migrations/0003_add_ipdocrel_document2_fk.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-08 11:58 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations import django.db.models.deletion diff --git a/ietf/ipr/migrations/0004_remove_iprdocrel_document.py b/ietf/ipr/migrations/0004_remove_iprdocrel_document.py index 313973632..2439193a6 100644 --- a/ietf/ipr/migrations/0004_remove_iprdocrel_document.py +++ b/ietf/ipr/migrations/0004_remove_iprdocrel_document.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-20 09:53 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/ipr/migrations/0005_rename_field_document2.py b/ietf/ipr/migrations/0005_rename_field_document2.py index 374e6806e..6db908888 100644 --- a/ietf/ipr/migrations/0005_rename_field_document2.py +++ b/ietf/ipr/migrations/0005_rename_field_document2.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-21 05:31 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/ipr/migrations/0006_document_primary_key_cleanup.py b/ietf/ipr/migrations/0006_document_primary_key_cleanup.py index fb31a1b78..32dca70c4 100644 --- a/ietf/ipr/migrations/0006_document_primary_key_cleanup.py +++ b/ietf/ipr/migrations/0006_document_primary_key_cleanup.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-06-10 03:42 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations import django.db.models.deletion diff --git a/ietf/ipr/models.py b/ietf/ipr/models.py index 70fa386d1..4b70dbe3c 100644 --- a/ietf/ipr/models.py +++ b/ietf/ipr/models.py @@ -1,10 +1,15 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import datetime from django.conf import settings -from django.urls import reverse from django.db import models +from django.urls import reverse +from django.utils.encoding import python_2_unicode_compatible from ietf.doc.models import DocAlias from ietf.name.models import DocRelationshipName,IprDisclosureStateName,IprLicenseTypeName,IprEventTypeName @@ -12,6 +17,7 @@ from ietf.person.models import Person from ietf.message.models import Message from ietf.utils.models import ForeignKey +@python_2_unicode_compatible class IprDisclosureBase(models.Model): by = ForeignKey(Person) # who was logged in, or System if nobody was logged in compliant = models.BooleanField("Complies to RFC3979", default=True) @@ -29,7 +35,7 @@ class IprDisclosureBase(models.Model): class Meta: ordering = ['-time', '-id'] - def __unicode__(self): + def __str__(self): return self.title def get_absolute_url(self): @@ -148,6 +154,7 @@ class GenericIprDisclosure(IprDisclosureBase): holder_contact_info = models.TextField(blank=True, help_text="Address, phone, etc.") statement = models.TextField() # includes licensing info +@python_2_unicode_compatible class IprDocRel(models.Model): disclosure = ForeignKey(IprDisclosureBase) document = ForeignKey(DocAlias) @@ -172,20 +179,22 @@ class IprDocRel(models.Model): else: return name - def __unicode__(self): + def __str__(self): if self.revisions: - return u"%s which applies to %s-%s" % (self.disclosure, self.document.name, self.revisions) + return "%s which applies to %s-%s" % (self.disclosure, self.document.name, self.revisions) else: - return u"%s which applies to %s" % (self.disclosure, self.document.name) + return "%s which applies to %s" % (self.disclosure, self.document.name) +@python_2_unicode_compatible class RelatedIpr(models.Model): source = ForeignKey(IprDisclosureBase,related_name='relatedipr_source_set') target = ForeignKey(IprDisclosureBase,related_name='relatedipr_target_set') relationship = ForeignKey(DocRelationshipName) # Re-use; change to a dedicated RelName if needed - def __unicode__(self): - return u"%s %s %s" % (self.source.title, self.relationship.name.lower(), self.target.title) + def __str__(self): + return "%s %s %s" % (self.source.title, self.relationship.name.lower(), self.target.title) +@python_2_unicode_compatible class IprEvent(models.Model): time = models.DateTimeField(auto_now_add=True) type = ForeignKey(IprEventTypeName) @@ -196,8 +205,8 @@ class IprEvent(models.Model): in_reply_to = ForeignKey(Message, null=True, blank=True,related_name='irtoevents') response_due= models.DateTimeField(blank=True,null=True) - def __unicode__(self): - return u"%s %s by %s at %s" % (self.disclosure.title, self.type.name.lower(), self.by.plain_name(), self.time) + def __str__(self): + return "%s %s by %s at %s" % (self.disclosure.title, self.type.name.lower(), self.by.plain_name(), self.time) def response_past_due(self): """Returns true if it's beyond the response_due date and no response has been diff --git a/ietf/ipr/resources.py b/ietf/ipr/resources.py index 65cb551f5..665b0ab02 100644 --- a/ietf/ipr/resources.py +++ b/ietf/ipr/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2015-03-21 14:05 PDT + + from ietf.api import ModelResource from ietf.api import ToOneField from tastypie.fields import ToManyField # pyflakes:ignore diff --git a/ietf/ipr/templatetags/ipr_filters.py b/ietf/ipr/templatetags/ipr_filters.py index 146fbff7e..43bf90234 100644 --- a/ietf/ipr/templatetags/ipr_filters.py +++ b/ietf/ipr/templatetags/ipr_filters.py @@ -1,4 +1,8 @@ -# Copyright The IETF Trust 2014, All Rights Reserved +# Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from django import template from django.utils.html import format_html @@ -11,10 +15,10 @@ def render_message_for_history(msg): """Format message for display in history. Suppress the 'To' line for incoming responses """ if msg.to.startswith('ietf-ipr+'): - return format_html(u'Date: {}
From: {}
Subject: {}
Cc: {}

{}', + return format_html('Date: {}
From: {}
Subject: {}
Cc: {}

{}', msg.time,msg.frm,msg.subject,msg.cc,msg.body) else: - return format_html(u'Date: {}
From: {}
To: {}
Subject: {}
Cc: {}

{}', + return format_html('Date: {}
From: {}
To: {}
Subject: {}
Cc: {}

{}', msg.time,msg.frm,msg.to,msg.subject,msg.cc,msg.body) diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py index 9598d7855..38186752c 100644 --- a/ietf/ipr/tests.py +++ b/ietf/ipr/tests.py @@ -1,10 +1,14 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime -import urllib + from pyquery import PyQuery +from six.moves.urllib.parse import quote from django.urls import reverse as urlreverse @@ -21,7 +25,7 @@ from ietf.ipr.utils import get_genitive, get_ipr_summary from ietf.mailtrigger.utils import gather_address_lists from ietf.message.models import Message from ietf.utils.mail import outbox, empty_outbox -from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent +from ietf.utils.test_utils import TestCase, login_testing_unauthorized from ietf.utils.text import text_to_dict @@ -72,14 +76,12 @@ class IprTests(TestCase): def test_showlist(self): ipr = HolderIprDisclosureFactory() r = self.client.get(urlreverse("ietf.ipr.views.showlist")) - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + self.assertContains(r, ipr.title) def test_show_posted(self): ipr = HolderIprDisclosureFactory() r = self.client.get(urlreverse("ietf.ipr.views.show", kwargs=dict(id=ipr.pk))) - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + self.assertContains(r, ipr.title) def test_show_parked(self): ipr = HolderIprDisclosureFactory(state_id='parked') @@ -99,38 +101,33 @@ class IprTests(TestCase): def test_show_removed(self): ipr = HolderIprDisclosureFactory(state_id='removed') r = self.client.get(urlreverse("ietf.ipr.views.show", kwargs=dict(id=ipr.pk))) - self.assertEqual(r.status_code, 200) - self.assertTrue('This IPR disclosure was removed' in unicontent(r)) + self.assertContains(r, 'This IPR disclosure was removed') def test_ipr_history(self): ipr = HolderIprDisclosureFactory() r = self.client.get(urlreverse("ietf.ipr.views.history", kwargs=dict(id=ipr.pk))) - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + self.assertContains(r, ipr.title) def test_iprs_for_drafts(self): draft=WgDraftFactory() ipr = HolderIprDisclosureFactory(docs=[draft,]) r = self.client.get(urlreverse("ietf.ipr.views.by_draft_txt")) - self.assertEqual(r.status_code, 200) - self.assertTrue(draft.name in unicontent(r)) - self.assertTrue(str(ipr.pk) in unicontent(r)) + self.assertContains(r, draft.name) + self.assertContains(r, str(ipr.pk)) def test_iprs_for_drafts_recursive(self): draft = WgDraftFactory(relations=[('replaces', IndividualDraftFactory())]) ipr = HolderIprDisclosureFactory(docs=[draft,]) replaced = draft.all_related_that_doc('replaces') r = self.client.get(urlreverse("ietf.ipr.views.by_draft_recursive_txt")) - self.assertEqual(r.status_code, 200) - self.assertTrue(draft.name in unicontent(r)) + self.assertContains(r, draft.name) for alias in replaced: - self.assertTrue(alias.name in unicontent(r)) - self.assertTrue(str(ipr.pk) in unicontent(r)) + self.assertContains(r, alias.name) + self.assertContains(r, str(ipr.pk)) def test_about(self): r = self.client.get(urlreverse("ietf.ipr.views.about")) - self.assertEqual(r.status_code, 200) - self.assertTrue("File a disclosure" in unicontent(r)) + self.assertContains(r, "File a disclosure") def test_search(self): WgDraftFactory() # The test matching the prefix "draft" needs more than one thing to find @@ -146,67 +143,55 @@ class IprTests(TestCase): # find by id r = self.client.get(url + "?submit=draft&id=%s" % draft.name) - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + self.assertContains(r, ipr.title) # find draft r = self.client.get(url + "?submit=draft&draft=%s" % draft.name) - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + self.assertContains(r, ipr.title) # search + select document r = self.client.get(url + "?submit=draft&draft=draft") - self.assertEqual(r.status_code, 200) - self.assertTrue(draft.name in unicontent(r)) - self.assertTrue(ipr.title not in unicontent(r)) + self.assertContains(r, draft.name) + self.assertNotContains(r, ipr.title) DocAlias.objects.create(name="rfc321").docs.add(draft) # find RFC r = self.client.get(url + "?submit=rfc&rfc=321") - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + self.assertContains(r, ipr.title) # find by patent owner r = self.client.get(url + "?submit=holder&holder=%s" % ipr.holder_legal_name) - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + self.assertContains(r, ipr.title) # find by patent info r = self.client.get(url + "?submit=patent&patent=%s" % ipr.patent_info) - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + self.assertContains(r, ipr.title) r = self.client.get(url + "?submit=patent&patent=US12345") - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + self.assertContains(r, ipr.title) # find by group acronym r = self.client.get(url + "?submit=group&group=%s" % draft.group.pk) - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + self.assertContains(r, ipr.title) # find by doc title - r = self.client.get(url + "?submit=doctitle&doctitle=%s" % urllib.quote(draft.title)) - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + r = self.client.get(url + "?submit=doctitle&doctitle=%s" % quote(draft.title)) + self.assertContains(r, ipr.title) # find by ipr title - r = self.client.get(url + "?submit=iprtitle&iprtitle=%s" % urllib.quote(ipr.title)) - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + r = self.client.get(url + "?submit=iprtitle&iprtitle=%s" % quote(ipr.title)) + self.assertContains(r, ipr.title) def test_feed(self): ipr = HolderIprDisclosureFactory() r = self.client.get("/feed/ipr/") - self.assertEqual(r.status_code, 200) - self.assertTrue(ipr.title in unicontent(r)) + self.assertContains(r, ipr.title) def test_sitemap(self): ipr = HolderIprDisclosureFactory() r = self.client.get("/sitemap-ipr.xml") - self.assertEqual(r.status_code, 200) - self.assertTrue("/ipr/%s/" % ipr.pk in unicontent(r)) + self.assertContains(r, "/ipr/%s/" % ipr.pk) def test_new_generic(self): """Add a new generic disclosure. Note: submitter does not need to be logged in. @@ -232,8 +217,7 @@ class IprTests(TestCase): "submitter_email": "test@holder.com", "notes": "some notes" }) - self.assertEqual(r.status_code, 200) - self.assertTrue("Your IPR disclosure has been submitted" in unicontent(r)) + self.assertContains(r, "Your IPR disclosure has been submitted") self.assertEqual(len(outbox),1) self.assertTrue('New IPR Submission' in outbox[0]['Subject']) self.assertTrue('ietf-ipr@' in outbox[0]['To']) @@ -275,15 +259,14 @@ class IprTests(TestCase): "submitter_name": "Test Holder", "submitter_email": "test@holder.com", }) - self.assertEqual(r.status_code, 200) - self.assertTrue("Your IPR disclosure has been submitted" in unicontent(r)) + self.assertContains(r, "Your IPR disclosure has been submitted") iprs = IprDisclosureBase.objects.filter(title__icontains=draft.name) self.assertEqual(len(iprs), 1) ipr = iprs[0] self.assertEqual(ipr.holder_legal_name, "Test Legal") self.assertEqual(ipr.state.slug, 'pending') - for item in [u'SE12345678901','A method of transfering bits','2000-01-01']: + for item in ['SE12345678901','A method of transfering bits','2000-01-01']: self.assertIn(item, ipr.get_child().patent_info) self.assertTrue(isinstance(ipr.get_child(),HolderIprDisclosure)) self.assertEqual(len(outbox),1) @@ -318,15 +301,14 @@ class IprTests(TestCase): "submitter_name": "Test Holder", "submitter_email": "test@holder.com", }) - self.assertEqual(r.status_code, 200) - self.assertTrue("Your IPR disclosure has been submitted" in unicontent(r)) + self.assertContains(r, "Your IPR disclosure has been submitted") iprs = IprDisclosureBase.objects.filter(title__icontains="belonging to Test Legal") self.assertEqual(len(iprs), 1) ipr = iprs[0] self.assertEqual(ipr.holder_legal_name, "Test Legal") self.assertEqual(ipr.state.slug, "pending") - for item in [u'SE12345678901','A method of transfering bits','2000-01-01' ]: + for item in ['SE12345678901','A method of transfering bits','2000-01-01' ]: self.assertIn(item, ipr.get_child().patent_info) self.assertTrue(isinstance(ipr.get_child(),ThirdPartyIprDisclosure)) self.assertEqual(len(outbox),1) @@ -374,7 +356,7 @@ class IprTests(TestCase): self.assertEqual(len(iprs), 1) ipr = iprs[0].get_child() self.assertEqual(ipr.holder_legal_name, "Test Legal") - patent_info_dict = dict( (k.replace('patent_','').capitalize(), v) for k, v in post_data.items() if k.startswith('patent_') ) + patent_info_dict = dict( (k.replace('patent_','').capitalize(), v) for k, v in list(post_data.items()) if k.startswith('patent_') ) self.assertEqual(text_to_dict(ipr.patent_info), patent_info_dict) self.assertEqual(ipr.state.slug, 'posted') @@ -415,8 +397,7 @@ class IprTests(TestCase): "submitter_name": "Test Holder", "submitter_email": "test@holder.com", }) - self.assertEqual(r.status_code, 200) - self.assertTrue("Your IPR disclosure has been submitted" in unicontent(r)) + self.assertContains(r, "Your IPR disclosure has been submitted") iprs = IprDisclosureBase.objects.filter(title__icontains=draft.name) self.assertEqual(len(iprs), 1) @@ -472,11 +453,10 @@ class IprTests(TestCase): # private comment r = self.client.post(url, dict(comment='Private comment',private=True),follow=True) - self.assertEqual(r.status_code,200) - self.assertTrue('Private comment' in unicontent(r)) + self.assertContains(r, 'Private comment') self.client.logout() - r = self.client.get(url) - self.assertFalse('Private comment' in unicontent(r)) + r = self.client.get(url, follow=True) + self.assertNotContains(r, 'Private comment') def test_addemail(self): ipr = HolderIprDisclosureFactory() @@ -610,6 +590,7 @@ Date: {} Subject: test """.format(data['reply_to'],datetime.datetime.now().ctime()) result = process_response_email(message_string) + self.assertIsInstance(result,Message) self.assertFalse(event.response_past_due()) diff --git a/ietf/ipr/utils.py b/ietf/ipr/utils.py index 2cb05aa29..b31d5195a 100644 --- a/ietf/ipr/utils.py +++ b/ietf/ipr/utils.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import debug # pyflakes:ignore def get_genitive(name): diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index 2a2a8d139..81df9ab95 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -1,8 +1,12 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import itertools +import six from django.conf import settings from django.contrib import messages @@ -342,11 +346,11 @@ def edit(request, id, updates=None): else: initial = model_to_dict(ipr) - patent_info = text_to_dict(initial.get('patent_info', u'')) - if patent_info.keys(): - patent_dict = dict([ ('patent_'+k.lower(), v) for k,v in patent_info.items() ]) + patent_info = text_to_dict(initial.get('patent_info', '')) + if list(patent_info.keys()): + patent_dict = dict([ ('patent_'+k.lower(), v) for k,v in list(patent_info.items()) ]) else: - patent_dict = {'patent_notes': initial.get('patent_info', u'')} + patent_dict = {'patent_notes': initial.get('patent_info', '')} initial.update(patent_dict) if ipr.updates: initial.update({'updates':[ x.target for x in ipr.updates ]}) @@ -408,7 +412,7 @@ def email(request, id): 'to': addrs.to, 'cc': addrs.cc, 'frm': settings.IPR_EMAIL_FROM, - 'subject': u'Regarding {}'.format(ipr.title), + 'subject': 'Regarding {}'.format(ipr.title), 'reply_to': reply_to, } form = MessageModelForm(initial=initial) @@ -450,9 +454,9 @@ def by_draft_txt(request): docipr[name].append(o.disclosure_id) - lines = [ u"# Machine-readable list of IPR disclosures by draft name" ] - for name, iprs in docipr.iteritems(): - lines.append(name + "\t" + "\t".join(unicode(ipr_id) for ipr_id in sorted(iprs))) + lines = [ "# Machine-readable list of IPR disclosures by draft name" ] + for name, iprs in docipr.items(): + lines.append(name + "\t" + "\t".join(six.text_type(ipr_id) for ipr_id in sorted(iprs))) return HttpResponse("\n".join(lines), content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET) @@ -472,9 +476,9 @@ def by_draft_recursive_txt(request): docipr[name] = [] docipr[name].append(o.disclosure_id) - lines = [ u"# Machine-readable list of IPR disclosures by draft name" ] - for name, iprs in docipr.iteritems(): - lines.append(name + "\t" + "\t".join(unicode(ipr_id) for ipr_id in sorted(iprs))) + lines = [ "# Machine-readable list of IPR disclosures by draft name" ] + for name, iprs in docipr.items(): + lines.append(name + "\t" + "\t".join(six.text_type(ipr_id) for ipr_id in sorted(iprs))) return HttpResponse("\n".join(lines), content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET) @@ -546,11 +550,11 @@ def new(request, type, updates=None): original = IprDisclosureBase(id=updates).get_child() initial = model_to_dict(original) initial.update({'updates':str(updates), }) - patent_info = text_to_dict(initial.get('patent_info', u'')) - if patent_info.keys(): - patent_dict = dict([ ('patent_'+k.lower(), v) for k,v in patent_info.items() ]) + patent_info = text_to_dict(initial.get('patent_info', '')) + if list(patent_info.keys()): + patent_dict = dict([ ('patent_'+k.lower(), v) for k,v in list(patent_info.items()) ]) else: - patent_dict = {'patent_notes': initial.get('patent_info', u'')} + patent_dict = {'patent_notes': initial.get('patent_info', '')} initial.update(patent_dict) form = ipr_form_mapping[type](initial=initial) else: diff --git a/ietf/liaisons/__init__.py b/ietf/liaisons/__init__.py index a285c3e43..fd066bea4 100644 --- a/ietf/liaisons/__init__.py +++ b/ietf/liaisons/__init__.py @@ -1,15 +1,18 @@ -# Copyright The IETF Trust 2007, All Rights Reserved -# coding: latin-1 +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from types import ModuleType # These people will be sent a stack trace if there's an uncaught exception in # code any of the modules imported above: DEBUG_EMAILS = [ - ('Emilio A. Sánchez', 'esanchez@yaco.es'), + ('Emilio A. Sánchez', 'esanchez@yaco.es'), ] -for k in locals().keys(): +for k in list(locals().keys()): m = locals()[k] if isinstance(m, ModuleType): if hasattr(m, "DEBUG_EMAILS"): diff --git a/ietf/liaisons/admin.py b/ietf/liaisons/admin.py index 4fdf7e406..7bf4ad8aa 100644 --- a/ietf/liaisons/admin.py +++ b/ietf/liaisons/admin.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.contrib import admin from django.urls import reverse @@ -29,13 +35,13 @@ class LiaisonStatementAdmin(admin.ModelAdmin): related_to.allow_tags = True class LiaisonStatementAttachmentAdmin(admin.ModelAdmin): - list_display = [u'id', 'statement', 'document', 'removed'] + list_display = ['id', 'statement', 'document', 'removed'] list_filter = ['removed'] raw_id_fields = ['statement', 'document'] admin.site.register(LiaisonStatementAttachment, LiaisonStatementAttachmentAdmin) class RelatedLiaisonStatementAdmin(admin.ModelAdmin): - list_display = [u'id', 'source', 'target', 'relationship'] + list_display = ['id', 'source', 'target', 'relationship'] list_filter = ['relationship'] raw_id_fields = ['source', 'target'] admin.site.register(RelatedLiaisonStatement, RelatedLiaisonStatementAdmin) diff --git a/ietf/liaisons/feeds.py b/ietf/liaisons/feeds.py index adbde6c40..21920935a 100644 --- a/ietf/liaisons/feeds.py +++ b/ietf/liaisons/feeds.py @@ -1,4 +1,8 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import re @@ -33,7 +37,7 @@ class LiaisonStatementsFeed(Feed): try: group = Group.objects.get(acronym=search) obj['filter'] = { 'from_groups': group } - obj['title'] = u'Liaison Statements from %s' % group.name + obj['title'] = 'Liaison Statements from %s' % group.name return obj except Group.DoesNotExist: # turn all-nonword characters into one-character @@ -46,7 +50,7 @@ class LiaisonStatementsFeed(Feed): name = statement.from_groups.first().name obj['filter'] = { 'from_name': name } - obj['title'] = u'Liaison Statements from %s' % name + obj['title'] = 'Liaison Statements from %s' % name return obj if kind == 'to': @@ -55,7 +59,7 @@ class LiaisonStatementsFeed(Feed): group = Group.objects.get(acronym=search) obj['filter'] = { 'to_groups': group } - obj['title'] = u'Liaison Statements to %s' % group.name + obj['title'] = 'Liaison Statements to %s' % group.name return obj if kind == 'subject': @@ -70,16 +74,16 @@ class LiaisonStatementsFeed(Feed): def items(self, obj): qs = LiaisonStatement.objects.all().order_by("-id") - if obj.has_key('q'): + if 'q' in obj: qs = qs.filter(*obj['q']) - if obj.has_key('filter'): + if 'filter' in obj: qs = qs.filter(**obj['filter']) - if obj.has_key('limit'): + if 'limit' in obj: qs = qs[:obj['limit']] qs = qs.prefetch_related("attachments") - return qs + return qs def title(self, obj): return obj['title'] diff --git a/ietf/liaisons/fields.py b/ietf/liaisons/fields.py index 382afdfaa..6f21b7c74 100644 --- a/ietf/liaisons/fields.py +++ b/ietf/liaisons/fields.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import json +import six from django.utils.html import escape from django import forms @@ -10,7 +14,7 @@ from django.urls import reverse as urlreverse from ietf.liaisons.models import LiaisonStatement def select2_id_liaison_json(objs): - return json.dumps([{ "id": o.pk, "text":u"[{}] {}".format(o.pk, escape(o.title)) } for o in objs]) + return json.dumps([{ "id": o.pk, "text":"[{}] {}".format(o.pk, escape(o.title)) } for o in objs]) def select2_id_group_json(objs): return json.dumps([{ "id": o.pk, "text": escape(o.acronym) } for o in objs]) @@ -47,9 +51,9 @@ class SearchableLiaisonStatementsField(forms.CharField): def prepare_value(self, value): if not value: value = "" - if isinstance(value, (int, long)): + if isinstance(value, six.integer_types): value = str(value) - if isinstance(value, basestring): + if isinstance(value, six.string_types): pks = self.parse_select2_value(value) value = self.model.objects.filter(pk__in=pks) if isinstance(value, LiaisonStatement): @@ -61,7 +65,7 @@ class SearchableLiaisonStatementsField(forms.CharField): # patterns may not have been fully constructed there yet self.widget.attrs["data-ajax-url"] = urlreverse("ietf.liaisons.views.ajax_select2_search_liaison_statements") - return u",".join(unicode(o.pk) for o in value) + return ",".join(six.text_type(o.pk) for o in value) def clean(self, value): value = super(SearchableLiaisonStatementsField, self).clean(value) @@ -72,9 +76,9 @@ class SearchableLiaisonStatementsField(forms.CharField): found_pks = [str(o.pk) for o in objs] failed_pks = [x for x in pks if x not in found_pks] if failed_pks: - raise forms.ValidationError(u"Could not recognize the following groups: {pks}.".format(pks=", ".join(failed_pks))) + raise forms.ValidationError("Could not recognize the following groups: {pks}.".format(pks=", ".join(failed_pks))) if self.max_entries != None and len(objs) > self.max_entries: - raise forms.ValidationError(u"You can select at most %s entries only." % self.max_entries) + raise forms.ValidationError("You can select at most %s entries only." % self.max_entries) return objs diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index c465c1280..d9bb4da6d 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -1,9 +1,14 @@ # Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import datetime, os import operator import six + from email.utils import parseaddr from form_utils.forms import BetterModelForm @@ -30,6 +35,7 @@ from ietf.person.models import Email from ietf.person.fields import SearchableEmailField from ietf.doc.models import Document, DocAlias from ietf.utils.fields import DatepickerDateField +from functools import reduce ''' NOTES: @@ -209,12 +215,12 @@ class LiaisonModelForm(BetterModelForm): NOTE: from_groups and to_groups are marked as not required because select2 has a problem with validating ''' - from_groups = forms.ModelMultipleChoiceField(queryset=Group.objects.all(),label=u'Groups',required=False) + from_groups = forms.ModelMultipleChoiceField(queryset=Group.objects.all(),label='Groups',required=False) from_contact = forms.EmailField() to_contacts = forms.CharField(label="Contacts", widget=forms.Textarea(attrs={'rows':'3', }), strip=False) - to_groups = forms.ModelMultipleChoiceField(queryset=Group.objects,label=u'Groups',required=False) + to_groups = forms.ModelMultipleChoiceField(queryset=Group.objects,label='Groups',required=False) deadline = DatepickerDateField(date_format="yyyy-mm-dd", picker_settings={"autoclose": "1" }, label='Deadline', required=True) - related_to = SearchableLiaisonStatementsField(label=u'Related Liaison Statement', required=False) + related_to = SearchableLiaisonStatementsField(label='Related Liaison Statement', required=False) submitted_date = DatepickerDateField(date_format="yyyy-mm-dd", picker_settings={"autoclose": "1" }, label='Submission date', required=True, initial=datetime.date.today()) attachments = CustomModelMultipleChoiceField(queryset=Document.objects,label='Attachments', widget=ShowAttachmentsWidget, required=False) attach_title = forms.CharField(label='Title', required=False) @@ -298,13 +304,13 @@ class LiaisonModelForm(BetterModelForm): def clean(self): if not self.cleaned_data.get('body', None) and not self.has_attachments(): - self._errors['body'] = ErrorList([u'You must provide a body or attachment files']) - self._errors['attachments'] = ErrorList([u'You must provide a body or attachment files']) + self._errors['body'] = ErrorList(['You must provide a body or attachment files']) + self._errors['attachments'] = ErrorList(['You must provide a body or attachment files']) # if purpose=response there must be a related statement purpose = LiaisonStatementPurposeName.objects.get(slug='response') if self.cleaned_data.get('purpose') == purpose and not self.cleaned_data.get('related_to'): - self._errors['related_to'] = ErrorList([u'You must provide a related statement when purpose is In Response']) + self._errors['related_to'] = ErrorList(['You must provide a related statement when purpose is In Response']) return self.cleaned_data def full_clean(self): @@ -313,8 +319,8 @@ class LiaisonModelForm(BetterModelForm): self.reset_required_fields() def has_attachments(self): - for key in self.files.keys(): - if key.startswith('attach_file_') and key.replace('file', 'title') in self.data.keys(): + for key in list(self.files.keys()): + if key.startswith('attach_file_') and key.replace('file', 'title') in list(self.data.keys()): return True return False @@ -351,10 +357,10 @@ class LiaisonModelForm(BetterModelForm): request.POST[attach_title_N] ''' written = self.instance.attachments.all().count() - for key in self.files.keys(): + for key in list(self.files.keys()): title_key = key.replace('file', 'title') attachment_title = self.data.get(title_key) - if not key.startswith('attach_file_') or not title_key in self.data.keys(): + if not key.startswith('attach_file_') or not title_key in list(self.data.keys()): continue attached_file = self.files.get(key) extension=attached_file.name.rsplit('.', 1) @@ -375,7 +381,7 @@ class LiaisonModelForm(BetterModelForm): if created: DocAlias.objects.create(name=attach.name).docs.add(attach) LiaisonStatementAttachment.objects.create(statement=self.instance,document=attach) - attach_file = open(os.path.join(settings.LIAISON_ATTACH_PATH, attach.name + extension), 'w') + attach_file = io.open(os.path.join(settings.LIAISON_ATTACH_PATH, attach.name + extension), 'wb') attach_file.write(attached_file.read()) attach_file.close() @@ -422,7 +428,7 @@ class LiaisonModelForm(BetterModelForm): class IncomingLiaisonForm(LiaisonModelForm): def clean(self): - if 'send' in self.data.keys() and self.get_post_only(): + if 'send' in list(self.data.keys()) and self.get_post_only(): raise forms.ValidationError('As an IETF Liaison Manager you can not send incoming liaison statements, you only can post them') return super(IncomingLiaisonForm, self).clean() @@ -446,7 +452,7 @@ class IncomingLiaisonForm(LiaisonModelForm): self.fields['from_contact'].initial = self.person.role_set.filter(group=queryset[0]).first().email.address self.fields['from_contact'].widget.attrs['readonly'] = True self.fields['from_groups'].queryset = queryset - self.fields['from_groups'].widget.submitter = unicode(self.person) + self.fields['from_groups'].widget.submitter = six.text_type(self.person) # if there's only one possibility make it the default if len(queryset) == 1: diff --git a/ietf/liaisons/mails.py b/ietf/liaisons/mails.py index 413a429d4..caba5d366 100644 --- a/ietf/liaisons/mails.py +++ b/ietf/liaisons/mails.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2011-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime from django.conf import settings @@ -8,7 +14,7 @@ from ietf.group.models import Role from ietf.mailtrigger.utils import gather_address_lists def send_liaison_by_email(request, liaison): - subject = u'New Liaison Statement, "%s"' % (liaison.title) + subject = 'New Liaison Statement, "%s"' % (liaison.title) from_email = settings.LIAISON_UNIVERSAL_FROM (to_email, cc) = gather_address_lists('liaison_statement_posted',liaison=liaison) bcc = ['statements@ietf.org'] @@ -20,7 +26,7 @@ def notify_pending_by_email(request, liaison): '''Send mail requesting approval of pending liaison statement. Send mail to the intersection of approvers for all from_groups ''' - subject = u'New Liaison Statement, "%s" needs your approval' % (liaison.title) + subject = 'New Liaison Statement, "%s" needs your approval' % (liaison.title) from_email = settings.LIAISON_UNIVERSAL_FROM (to, cc) = gather_address_lists('liaison_approval_requested',liaison=liaison) body = render_to_string('liaisons/pending_liaison_mail.txt', dict(liaison=liaison)) @@ -58,7 +64,7 @@ def possibly_send_deadline_reminder(liaison): } days_to_go = (liaison.deadline - datetime.date.today()).days - if not (days_to_go < 0 or days_to_go in PREVIOUS_DAYS.keys()): + if not (days_to_go < 0 or days_to_go in list(PREVIOUS_DAYS.keys())): return None # no reminder if days_to_go < 0: diff --git a/ietf/liaisons/management/commands/check_liaison_deadlines.py b/ietf/liaisons/management/commands/check_liaison_deadlines.py index aebadb46f..77032fe4b 100644 --- a/ietf/liaisons/management/commands/check_liaison_deadlines.py +++ b/ietf/liaisons/management/commands/check_liaison_deadlines.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime from django.core.management.base import BaseCommand @@ -7,7 +13,7 @@ from ietf.liaisons.mails import possibly_send_deadline_reminder class Command(BaseCommand): - help = (u"Check liaison deadlines and send a reminder if we are close to a deadline") + help = ("Check liaison deadlines and send a reminder if we are close to a deadline") def handle(self, *args, **options): today = datetime.date.today() diff --git a/ietf/liaisons/management/commands/remind_update_sdo_list.py b/ietf/liaisons/management/commands/remind_update_sdo_list.py index a33b069b4..264b11b21 100644 --- a/ietf/liaisons/management/commands/remind_update_sdo_list.py +++ b/ietf/liaisons/management/commands/remind_update_sdo_list.py @@ -1,3 +1,8 @@ +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from django.core.management.base import BaseCommand @@ -6,7 +11,7 @@ from ietf.liaisons.mails import send_sdo_reminder class Command(BaseCommand): - help = (u"Send a remind to each SDO Liaison Manager to update the list of persons authorized to send liaison statements on behalf of his SDO") + help = ("Send a remind to each SDO Liaison Manager to update the list of persons authorized to send liaison statements on behalf of his SDO") def add_arguments(self, parser): parser.add_argument('-s', '--sdo-pk', dest='sdo_pk', @@ -25,16 +30,16 @@ def send_reminders_to_sdos(sdo_pk=None): sdos = sdos.filter(pk=sdo_pk) if not sdos: - print "No SDOs found!" + print("No SDOs found!") msgs = [] for sdo in sdos: body = send_sdo_reminder(sdo) if not body: - msg = u'%05s#: %s has no liaison manager' % (sdo.pk, sdo.name) + msg = '%05s#: %s has no liaison manager' % (sdo.pk, sdo.name) else: - msg = u'%05s#: %s mail sent!' % (sdo.pk, sdo.name) + msg = '%05s#: %s mail sent!' % (sdo.pk, sdo.name) msgs.append(msg) return msgs \ No newline at end of file diff --git a/ietf/liaisons/migrations/0001_initial.py b/ietf/liaisons/migrations/0001_initial.py index cfba2a61f..df733a3cd 100644 --- a/ietf/liaisons/migrations/0001_initial.py +++ b/ietf/liaisons/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/liaisons/migrations/0002_auto_20180225_1207.py b/ietf/liaisons/migrations/0002_auto_20180225_1207.py index 94f2fd2b1..f848a0391 100644 --- a/ietf/liaisons/migrations/0002_auto_20180225_1207.py +++ b/ietf/liaisons/migrations/0002_auto_20180225_1207.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-25 12:07 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/liaisons/migrations/0003_liaison_document2_fk.py b/ietf/liaisons/migrations/0003_liaison_document2_fk.py index 659899d4f..b46ced747 100644 --- a/ietf/liaisons/migrations/0003_liaison_document2_fk.py +++ b/ietf/liaisons/migrations/0003_liaison_document2_fk.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-08 11:58 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations import django.db.models.deletion diff --git a/ietf/liaisons/migrations/0004_remove_liaisonstatementattachment_document.py b/ietf/liaisons/migrations/0004_remove_liaisonstatementattachment_document.py index b5438e6b7..30a697d58 100644 --- a/ietf/liaisons/migrations/0004_remove_liaisonstatementattachment_document.py +++ b/ietf/liaisons/migrations/0004_remove_liaisonstatementattachment_document.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-20 09:53 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/liaisons/migrations/0005_rename_field_document2.py b/ietf/liaisons/migrations/0005_rename_field_document2.py index e5df89916..c557b3360 100644 --- a/ietf/liaisons/migrations/0005_rename_field_document2.py +++ b/ietf/liaisons/migrations/0005_rename_field_document2.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-21 05:31 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/liaisons/migrations/0006_document_primary_key_cleanup.py b/ietf/liaisons/migrations/0006_document_primary_key_cleanup.py index c61220811..bc2918667 100644 --- a/ietf/liaisons/migrations/0006_document_primary_key_cleanup.py +++ b/ietf/liaisons/migrations/0006_document_primary_key_cleanup.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-06-10 03:47 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations import django.db.models.deletion diff --git a/ietf/liaisons/models.py b/ietf/liaisons/models.py index b42d72044..77a29e9a0 100644 --- a/ietf/liaisons/models.py +++ b/ietf/liaisons/models.py @@ -1,8 +1,13 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from django.conf import settings from django.urls import reverse as urlreverse from django.db import models +from django.utils.encoding import python_2_unicode_compatible from django.utils.text import slugify from ietf.person.models import Email, Person @@ -15,15 +20,16 @@ from ietf.utils.models import ForeignKey # maps (previous state id, new state id) to event type id STATE_EVENT_MAPPING = { - (u'pending','approved'):'approved', - (u'pending','dead'):'killed', - (u'pending','posted'):'posted', - (u'approved','posted'):'posted', - (u'dead','pending'):'resurrected', - (u'pending','pending'):'submitted' + ('pending','approved'):'approved', + ('pending','dead'):'killed', + ('pending','posted'):'posted', + ('approved','posted'):'posted', + ('dead','pending'):'resurrected', + ('pending','pending'):'submitted' } +@python_2_unicode_compatible class LiaisonStatement(models.Model): title = models.CharField(max_length=255) from_groups = models.ManyToManyField(Group, blank=True, related_name='liaisonstatement_from_set') @@ -47,10 +53,9 @@ class LiaisonStatement(models.Model): class Meta: ordering = ['id'] - - def __unicode__(self): - return self.title or u"" + def __str__(self): + return self.title or "" def change_state(self,state_id=None,person=None): '''Helper function to change state of liaison statement and create appropriate @@ -198,33 +203,37 @@ class LiaisonStatement(models.Model): approval_set.intersection_update(group.liaison_approvers()) return list(set([ r.email.address for r in approval_set ])) +@python_2_unicode_compatible class LiaisonStatementAttachment(models.Model): statement = ForeignKey(LiaisonStatement) document = ForeignKey(Document) removed = models.BooleanField(default=False) - def __unicode__(self): + def __str__(self): return self.document.name +@python_2_unicode_compatible class RelatedLiaisonStatement(models.Model): source = ForeignKey(LiaisonStatement, related_name='source_of_set') target = ForeignKey(LiaisonStatement, related_name='target_of_set') relationship = ForeignKey(DocRelationshipName) - def __unicode__(self): - return u"%s %s %s" % (self.source.title, self.relationship.name.lower(), self.target.title) + def __str__(self): + return "%s %s %s" % (self.source.title, self.relationship.name.lower(), self.target.title) +@python_2_unicode_compatible class LiaisonStatementGroupContacts(models.Model): group = ForeignKey(Group, unique=True, null=True) contacts = models.CharField(max_length=255,blank=True) cc_contacts = models.CharField(max_length=255,blank=True) - def __unicode__(self): - return u"%s" % self.group.name + def __str__(self): + return "%s" % self.group.name +@python_2_unicode_compatible class LiaisonStatementEvent(models.Model): time = models.DateTimeField(auto_now_add=True) type = ForeignKey(LiaisonStatementEventTypeName) @@ -232,8 +241,8 @@ class LiaisonStatementEvent(models.Model): statement = ForeignKey(LiaisonStatement) desc = models.TextField() - def __unicode__(self): - return u"%s %s by %s at %s" % (self.statement.title, self.type.slug, self.by.plain_name(), self.time) + def __str__(self): + return "%s %s by %s at %s" % (self.statement.title, self.type.slug, self.by.plain_name(), self.time) class Meta: ordering = ['-time', '-id'] diff --git a/ietf/liaisons/resources.py b/ietf/liaisons/resources.py index 3bfd6321f..6ca4188ba 100644 --- a/ietf/liaisons/resources.py +++ b/ietf/liaisons/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2015-10-11 13:15 PDT + + from ietf.api import ModelResource from ietf.api import ToOneField from tastypie.fields import ToManyField # pyflakes:ignore diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index fa2a1bfb2..c5cd80b27 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -1,5 +1,14 @@ -import datetime, os, shutil -import json +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os +import shutil +import six import debug # pyflakes:ignore @@ -7,10 +16,10 @@ from django.conf import settings from django.contrib.auth.models import User from django.urls import reverse as urlreverse from django.db.models import Q -from StringIO import StringIO +from io import StringIO from pyquery import PyQuery -from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent +from ietf.utils.test_utils import TestCase, login_testing_unauthorized from ietf.utils.mail import outbox from ietf.group.factories import GroupFactory, RoleFactory @@ -57,40 +66,40 @@ class LiaisonTests(TestCase): r = self.client.get(urlreverse('ietf.liaisons.views.liaison_list')) self.assertEqual(r.status_code, 200) - self.assertTrue(liaison.title in unicontent(r)) + self.assertContains(r, liaison.title) def test_details(self): liaison = LiaisonStatementFactory() r = self.client.get(urlreverse("ietf.liaisons.views.liaison_detail", kwargs={ 'object_id': liaison.pk })) self.assertEqual(r.status_code, 200) - self.assertTrue(liaison.title in unicontent(r)) + self.assertContains(r, liaison.title) def test_feeds(self): liaison = LiaisonStatementFactory(title="Comment from United League of Marsmen") r = self.client.get('/feed/liaison/recent/') self.assertEqual(r.status_code, 200) - self.assertTrue(liaison.title in unicontent(r)) + self.assertContains(r, liaison.title) r = self.client.get('/feed/liaison/from/%s/' % liaison.from_groups.first().acronym) self.assertEqual(r.status_code, 200) - self.assertTrue(liaison.title in unicontent(r)) + self.assertContains(r, liaison.title) r = self.client.get('/feed/liaison/to/%s/' % liaison.to_groups.first().acronym) self.assertEqual(r.status_code, 200) - self.assertTrue(liaison.title in unicontent(r)) + self.assertContains(r, liaison.title) r = self.client.get('/feed/liaison/subject/marsmen/') self.assertEqual(r.status_code, 200) - self.assertTrue(liaison.title in unicontent(r)) + self.assertContains(r, liaison.title) def test_sitemap(self): liaison = LiaisonStatementFactory() r = self.client.get('/sitemap-liaison.xml') self.assertEqual(r.status_code, 200) - self.assertTrue(urlreverse("ietf.liaisons.views.liaison_detail", kwargs={ 'object_id': liaison.pk }) in unicontent(r)) + self.assertContains(r, urlreverse("ietf.liaisons.views.liaison_detail", kwargs={ 'object_id': liaison.pk })) def test_help_pages(self): self.assertEqual(self.client.get('/liaison/help/').status_code, 200) @@ -186,7 +195,7 @@ class AjaxTests(TestCase): self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) + data = r.json() self.assertEqual(data["error"], False) self.assertEqual(data["post_only"], False) self.assertTrue('cc' in data) @@ -204,8 +213,8 @@ class AjaxTests(TestCase): self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) - self.assertEqual(data["to_contacts"],[u'test@example.com']) + data = r.json() + self.assertEqual(data["to_contacts"],['test@example.com']) def test_ajax_select2_search_liaison_statements(self): liaison = LiaisonStatementFactory() @@ -215,14 +224,14 @@ class AjaxTests(TestCase): self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) + data = r.json() self.assertTrue(liaison.pk in [ x['id'] for x in data ]) # test id search url = urlreverse('ietf.liaisons.views.ajax_select2_search_liaison_statements') + "?q=%s" % liaison.pk r = self.client.get(url) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) + data = r.json() self.assertTrue(liaison.pk in [ x['id'] for x in data ]) @@ -232,7 +241,7 @@ class ManagementCommandTests(TestCase): LiaisonStatementFactory(deadline=datetime.date.today()+datetime.timedelta(days=1)) - out = StringIO() + out = six.StringIO() mailbox_before = len(outbox) call_command('check_liaison_deadlines',stdout=out) self.assertEqual(len(outbox), mailbox_before + 1) @@ -242,7 +251,7 @@ class ManagementCommandTests(TestCase): RoleFactory(name_id='liaiman',group__type_id='sdo') - out = StringIO() + out = six.StringIO() mailbox_before = len(outbox) call_command('remind_update_sdo_list',stdout=out) self.assertTrue(len(outbox) > mailbox_before) @@ -299,10 +308,10 @@ class LiaisonManagementTests(TestCase): # private comment r = self.client.post(addurl, dict(comment='Private comment',private=True),follow=True) self.assertEqual(r.status_code,200) - self.assertTrue('Private comment' in r.content) + self.assertContains(r, 'Private comment') self.client.logout() r = self.client.get(url) - self.assertFalse('Private comment' in r.content) + self.assertNotContains(r, 'Private comment') def test_taken_care_of(self): liaison = LiaisonStatementFactory(deadline=datetime.date.today()+datetime.timedelta(days=1)) @@ -343,14 +352,14 @@ class LiaisonManagementTests(TestCase): login_testing_unauthorized(self, "ad", url) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(liaison.title in unicontent(r)) + self.assertContains(r, liaison.title) # check the detail page / unauthorized url = urlreverse('ietf.liaisons.views.liaison_detail', kwargs=dict(object_id=liaison.pk)) self.client.logout() r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(liaison.title in unicontent(r)) + self.assertContains(r, liaison.title) q = PyQuery(r.content) self.assertEqual(len(q('form input[name=approved]')), 0) @@ -358,7 +367,7 @@ class LiaisonManagementTests(TestCase): self.client.login(username="ulm-liaiman", password="ulm-liaiman+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue(liaison.title in unicontent(r)) + self.assertContains(r, liaison.title) q = PyQuery(r.content) from ietf.liaisons.utils import can_edit_liaison user = User.objects.get(username='ulm-liaiman') @@ -432,7 +441,7 @@ class LiaisonManagementTests(TestCase): self.assertEqual(new_liaison.attachments.count(), attachments_before + 1) attachment = new_liaison.attachments.order_by("-name")[0] self.assertEqual(attachment.title, "attachment") - with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: + with io.open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: written_content = f.read() test_file.seek(0) @@ -736,7 +745,7 @@ class LiaisonManagementTests(TestCase): self.assertEqual(l.attachments.count(), 1) attachment = l.attachments.all()[0] self.assertEqual(attachment.title, "attachment") - with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: + with io.open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: written_content = f.read() test_file.seek(0) @@ -815,7 +824,7 @@ class LiaisonManagementTests(TestCase): self.assertEqual(l.attachments.count(), 1) attachment = l.attachments.all()[0] self.assertEqual(attachment.title, "attachment") - with open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: + with io.open(os.path.join(self.liaison_dir, attachment.uploaded_filename)) as f: written_content = f.read() test_file.seek(0) @@ -1090,7 +1099,7 @@ class LiaisonManagementTests(TestCase): r = self.client.post(url,get_liaison_post_data(),follow=True) self.assertEqual(r.status_code, 200) - self.assertTrue('As an IETF Liaison Manager you can not send incoming liaison statements' in unicontent(r)) + self.assertContains(r, 'As an IETF Liaison Manager you can not send incoming liaison statements') def test_deadline_field(self): '''Required for action, comment, not info, response''' @@ -1130,7 +1139,7 @@ class LiaisonManagementTests(TestCase): r = self.client.post(url,post_data,follow=True) self.assertEqual(r.status_code, 200) - self.assertTrue('You must provide a body or attachment files' in unicontent(r)) + self.assertContains(r, 'You must provide a body or attachment files') def test_send_sdo_reminder(self): RoleFactory(name_id='liaiman',person__user__username='ulm-liaiman',person__user__email='ulm-liaiman@somewhere.example',group__type_id='sdo',group__acronym='ulm') diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py index d8931d4f1..0b68cf8b6 100644 --- a/ietf/liaisons/views.py +++ b/ietf/liaisons/views.py @@ -1,4 +1,9 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import json from email.utils import parseaddr @@ -86,7 +91,7 @@ def _find_person_in_emails(liaison, person): def contacts_from_roles(roles): '''Returns contact string for given roles''' - emails = [ u'{} <{}>'.format(r.person.plain_name(),r.email.address) for r in roles ] + emails = [ '{} <{}>'.format(r.person.plain_name(),r.email.address) for r in roles ] return ','.join(emails) def get_cc(group): @@ -106,17 +111,17 @@ def get_cc(group): elif group.type_id == 'area': emails.append(EMAIL_ALIASES['IETFCHAIR']) ad_roles = group.role_set.filter(name='ad') - emails.extend([ u'{} <{}>'.format(r.person.plain_name(),r.email.address) for r in ad_roles ]) + emails.extend([ '{} <{}>'.format(r.person.plain_name(),r.email.address) for r in ad_roles ]) elif group.type_id == 'wg': ad_roles = group.parent.role_set.filter(name='ad') - emails.extend([ u'{} <{}>'.format(r.person.plain_name(),r.email.address) for r in ad_roles ]) + emails.extend([ '{} <{}>'.format(r.person.plain_name(),r.email.address) for r in ad_roles ]) chair_roles = group.role_set.filter(name='chair') - emails.extend([ u'{} <{}>'.format(r.person.plain_name(),r.email.address) for r in chair_roles ]) + emails.extend([ '{} <{}>'.format(r.person.plain_name(),r.email.address) for r in chair_roles ]) if group.list_email: - emails.append(u'{} Discussion List <{}>'.format(group.name,group.list_email)) + emails.append('{} Discussion List <{}>'.format(group.name,group.list_email)) elif group.type_id == 'sdo': liaiman_roles = group.role_set.filter(name='liaiman') - emails.extend([ u'{} <{}>'.format(r.person.plain_name(),r.email.address) for r in liaiman_roles ]) + emails.extend([ '{} <{}>'.format(r.person.plain_name(),r.email.address) for r in liaiman_roles ]) # explicit CCs if group.liaisonstatementgroupcontacts_set.exists() and group.liaisonstatementgroupcontacts_set.first().cc_contacts: @@ -274,7 +279,7 @@ def ajax_select2_search_liaison_statements(request): def redirect_add(request): """Redirects old add urls""" - if 'incoming' in request.GET.keys(): + if 'incoming' in list(request.GET.keys()): return redirect('ietf.liaisons.views.liaison_add', type='incoming') else: return redirect('ietf.liaisons.views.liaison_add', type='outgoing') diff --git a/ietf/liaisons/widgets.py b/ietf/liaisons/widgets.py index ea281cc40..c29700653 100644 --- a/ietf/liaisons/widgets.py +++ b/ietf/liaisons/widgets.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.urls import reverse as urlreverse from django.db.models.query import QuerySet from django.forms.widgets import Widget @@ -14,29 +20,29 @@ class ButtonWidget(Widget): super(ButtonWidget, self).__init__(*args, **kwargs) def render(self, name, value, attrs=None): - html = u'' % conditional_escape(self.show_on) - html += u'' % conditional_escape(self.label) + html = '' % conditional_escape(self.show_on) + html += '' % conditional_escape(self.label) if self.require: for i in self.require: - html += u'' % conditional_escape(i) - required_str = u'Please fill in %s to attach a new file' % conditional_escape(self.required_label) - html += u'' % conditional_escape(required_str) - html += u'' % conditional_escape(self.label) + html += '' % conditional_escape(i) + required_str = 'Please fill in %s to attach a new file' % conditional_escape(self.required_label) + html += '' % conditional_escape(required_str) + html += '' % conditional_escape(self.label) return mark_safe(html) class ShowAttachmentsWidget(Widget): def render(self, name, value, attrs=None): - html = u'
' % name - html += u'' - html += u'
' + html = '
' % name + html += '' + html += '
' if value and isinstance(value, QuerySet): for attachment in value: - html += u'%s ' % (conditional_escape(attachment.document.href()), conditional_escape(attachment.document.title)) - html += u'Edit '.format(urlreverse("ietf.liaisons.views.liaison_edit_attachment", kwargs={'object_id':attachment.statement.pk,'doc_id':attachment.document.pk})) - html += u'Delete '.format(urlreverse("ietf.liaisons.views.liaison_delete_attachment", kwargs={'object_id':attachment.statement.pk,'attach_id':attachment.pk})) - html += u'
' + html += '%s ' % (conditional_escape(attachment.document.href()), conditional_escape(attachment.document.title)) + html += 'Edit '.format(urlreverse("ietf.liaisons.views.liaison_edit_attachment", kwargs={'object_id':attachment.statement.pk,'doc_id':attachment.document.pk})) + html += 'Delete '.format(urlreverse("ietf.liaisons.views.liaison_delete_attachment", kwargs={'object_id':attachment.statement.pk,'attach_id':attachment.pk})) + html += '
' else: - html += u'No files attached' - html += u'
' + html += 'No files attached' + html += '
' return mark_safe(html) diff --git a/ietf/mailinglists/factories.py b/ietf/mailinglists/factories.py index c2942fdbd..d713cbbde 100644 --- a/ietf/mailinglists/factories.py +++ b/ietf/mailinglists/factories.py @@ -1,6 +1,8 @@ -# Copyright The IETF Trust 2018, All Rights Reserved +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals import factory import random diff --git a/ietf/mailinglists/migrations/0001_initial.py b/ietf/mailinglists/migrations/0001_initial.py index 9efd709e9..c127e7621 100644 --- a/ietf/mailinglists/migrations/0001_initial.py +++ b/ietf/mailinglists/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import django.core.validators from django.db import migrations, models diff --git a/ietf/mailinglists/models.py b/ietf/mailinglists/models.py index f515eff2c..239583fae 100644 --- a/ietf/mailinglists/models.py +++ b/ietf/mailinglists/models.py @@ -1,36 +1,44 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + from django.conf import settings from django.core.validators import validate_email from django.db import models +from django.utils.encoding import python_2_unicode_compatible from ietf.person.models import Person from ietf.utils.models import ForeignKey +@python_2_unicode_compatible class List(models.Model): name = models.CharField(max_length=32) description = models.CharField(max_length=256) advertised = models.BooleanField(default=True) - def __unicode__(self): + + def __str__(self): return "" % self.name def info_url(self): return settings.MAILING_LIST_INFO_URL % {'list_addr': self.name } +@python_2_unicode_compatible class Subscribed(models.Model): time = models.DateTimeField(auto_now_add=True) email = models.CharField(max_length=128, validators=[validate_email]) lists = models.ManyToManyField(List) - def __unicode__(self): + def __str__(self): return "" % (self.email, self.time) class Meta: verbose_name_plural = "Subscribed" +@python_2_unicode_compatible class Whitelisted(models.Model): time = models.DateTimeField(auto_now_add=True) email = models.CharField("Email address", max_length=64, validators=[validate_email]) by = ForeignKey(Person) - def __unicode__(self): + def __str__(self): return "" % (self.email, self.time) class Meta: verbose_name_plural = "Whitelisted" diff --git a/ietf/mailinglists/resources.py b/ietf/mailinglists/resources.py index 7f48ac35e..b2fce0900 100644 --- a/ietf/mailinglists/resources.py +++ b/ietf/mailinglists/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2016-06-12 12:29 PDT + + from ietf.api import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore diff --git a/ietf/mailinglists/tests.py b/ietf/mailinglists/tests.py index 82629d1b2..34b555d91 100644 --- a/ietf/mailinglists/tests.py +++ b/ietf/mailinglists/tests.py @@ -1,6 +1,8 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals from pyquery import PyQuery diff --git a/ietf/mailtrigger/migrations/0001_initial.py b/ietf/mailtrigger/migrations/0001_initial.py index a563fa2e3..03eb02262 100644 --- a/ietf/mailtrigger/migrations/0001_initial.py +++ b/ietf/mailtrigger/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/mailtrigger/migrations/0002_conflrev_changes.py b/ietf/mailtrigger/migrations/0002_conflrev_changes.py index 8fc3090b9..3330e196d 100644 --- a/ietf/mailtrigger/migrations/0002_conflrev_changes.py +++ b/ietf/mailtrigger/migrations/0002_conflrev_changes.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.13 on 2018-05-21 12:07 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/mailtrigger/migrations/0003_add_review_notify_ad.py b/ietf/mailtrigger/migrations/0003_add_review_notify_ad.py index fbcf0ca1f..b8eebea5e 100644 --- a/ietf/mailtrigger/migrations/0003_add_review_notify_ad.py +++ b/ietf/mailtrigger/migrations/0003_add_review_notify_ad.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-02 11:34 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/mailtrigger/migrations/0004_ballot_rfceditornote_changed_postapproval.py b/ietf/mailtrigger/migrations/0004_ballot_rfceditornote_changed_postapproval.py index 592afe92b..411fc68b1 100644 --- a/ietf/mailtrigger/migrations/0004_ballot_rfceditornote_changed_postapproval.py +++ b/ietf/mailtrigger/migrations/0004_ballot_rfceditornote_changed_postapproval.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-03 00:24 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/mailtrigger/migrations/0005_slides_proposed.py b/ietf/mailtrigger/migrations/0005_slides_proposed.py index f701b81d2..8efa3397d 100644 --- a/ietf/mailtrigger/migrations/0005_slides_proposed.py +++ b/ietf/mailtrigger/migrations/0005_slides_proposed.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-03-25 06:11 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/mailtrigger/migrations/0006_sub_new_wg_00.py b/ietf/mailtrigger/migrations/0006_sub_new_wg_00.py index 0169c456b..a380a20a1 100644 --- a/ietf/mailtrigger/migrations/0006_sub_new_wg_00.py +++ b/ietf/mailtrigger/migrations/0006_sub_new_wg_00.py @@ -1,6 +1,8 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py index 3e3a6e373..7de884552 100644 --- a/ietf/mailtrigger/models.py +++ b/ietf/mailtrigger/models.py @@ -1,7 +1,12 @@ # Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import models from django.template import Template, Context +from django.utils.encoding import python_2_unicode_compatible from email.utils import parseaddr from ietf.utils.mail import formataddr, get_email_addresses_from_text @@ -28,6 +33,7 @@ def clean_duplicates(addrlist): addresses.append(addr) return addresses +@python_2_unicode_compatible class MailTrigger(models.Model): slug = models.CharField(max_length=32, primary_key=True) desc = models.TextField(blank=True) @@ -37,9 +43,10 @@ class MailTrigger(models.Model): class Meta: ordering = ["slug"] - def __unicode__(self): + def __str__(self): return self.slug +@python_2_unicode_compatible class Recipient(models.Model): slug = models.CharField(max_length=32, primary_key=True) desc = models.TextField(blank=True) @@ -48,7 +55,7 @@ class Recipient(models.Model): class Meta: ordering = ["slug"] - def __unicode__(self): + def __str__(self): return self.slug def gather(self, **kwargs): @@ -265,7 +272,7 @@ class Recipient(models.Model): doc = kwargs['doc'] active_ballot = doc.active_ballot() if active_ballot: - for ad, pos in active_ballot.active_ad_positions().iteritems(): + for ad, pos in active_ballot.active_ad_positions().items(): if pos and pos.pos_id == "discuss": addrs.append(ad.role_email("ad").address) return addrs diff --git a/ietf/mailtrigger/tests.py b/ietf/mailtrigger/tests.py index 058f9c037..802c2a6bb 100644 --- a/ietf/mailtrigger/tests.py +++ b/ietf/mailtrigger/tests.py @@ -1,6 +1,12 @@ +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.urls import reverse as urlreverse -from ietf.utils.test_utils import TestCase, unicontent +from ietf.utils.test_utils import TestCase class EventMailTests(TestCase): @@ -9,22 +15,22 @@ class EventMailTests(TestCase): url = urlreverse('ietf.mailtrigger.views.show_triggers') r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue('ballot_saved' in unicontent(r)) + self.assertContains(r, 'ballot_saved') url = urlreverse('ietf.mailtrigger.views.show_triggers',kwargs=dict(mailtrigger_slug='ballot_saved')) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue('ballot_saved' in unicontent(r)) + self.assertContains(r, 'ballot_saved') def test_show_recipients(self): url = urlreverse('ietf.mailtrigger.views.show_recipients') r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue('doc_group_mail_list' in unicontent(r)) + self.assertContains(r, 'doc_group_mail_list') url = urlreverse('ietf.mailtrigger.views.show_recipients',kwargs=dict(recipient_slug='doc_group_mail_list')) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue('doc_group_mail_list' in unicontent(r)) + self.assertContains(r, 'doc_group_mail_list') diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py index 1713d0215..63bb68067 100644 --- a/ietf/meeting/admin.py +++ b/ietf/meeting/admin.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.contrib import admin from ietf.meeting.models import (Meeting, Room, Session, TimeSlot, Constraint, Schedule, @@ -37,7 +43,7 @@ class MeetingAdmin(admin.ModelAdmin): if instance.country: loc.append(instance.get_country_display()) - return u", ".join(loc) + return ", ".join(loc) admin.site.register(Meeting, MeetingAdmin) @@ -53,7 +59,7 @@ class TimeSlotAdmin(admin.ModelAdmin): if instance.session.name: return instance.session.name elif instance.session.group: - return u"%s (%s)" % (instance.session.group.name, instance.session.group.acronym) + return "%s (%s)" % (instance.session.group.name, instance.session.group.acronym) return "" session_desc.short_description = "session" diff --git a/ietf/meeting/factories.py b/ietf/meeting/factories.py index 44f8e4145..df54dd646 100644 --- a/ietf/meeting/factories.py +++ b/ietf/meeting/factories.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import factory import random import datetime @@ -146,8 +152,8 @@ class FloorPlanFactory(factory.DjangoModelFactory): class Meta: model = FloorPlan - name = factory.Sequence(lambda n: u'Venue Floor %d' % n) - short = factory.Sequence(lambda n: u'%d' % n) + name = factory.Sequence(lambda n: 'Venue Floor %d' % n) + short = factory.Sequence(lambda n: '%d' % n) meeting = factory.SubFactory(MeetingFactory) order = factory.Sequence(lambda n: n) image = factory.LazyAttribute( diff --git a/ietf/meeting/feeds.py b/ietf/meeting/feeds.py index a2780065d..0a1e0a42a 100644 --- a/ietf/meeting/feeds.py +++ b/ietf/meeting/feeds.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import os from django.contrib.syndication.views import Feed @@ -33,7 +39,7 @@ class LatestMeetingMaterialFeed(Feed): return "Meeting Materials Activity" def item_title(self, item): - return u"%s: %s" % (item["group_acronym"], escape(item["title"])) + return "%s: %s" % (item["group_acronym"], escape(item["title"])) def item_description(self, item): return "" diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index a50965bec..8cea55790 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -1,8 +1,11 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os -import codecs import datetime from django import forms @@ -287,7 +290,7 @@ class InterimSessionModelForm(forms.ModelForm): directory = os.path.dirname(path) if not os.path.exists(directory): os.makedirs(directory) - with codecs.open(path, "w", encoding='utf-8') as file: + with io.open(path, "w", encoding='utf-8') as file: file.write(self.cleaned_data['agenda']) diff --git a/ietf/meeting/helpers.py b/ietf/meeting/helpers.py index ee6b5cff7..50cc4f041 100644 --- a/ietf/meeting/helpers.py +++ b/ietf/meeting/helpers.py @@ -1,6 +1,11 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import datetime +import io import os import re from tempfile import mkstemp @@ -211,7 +216,7 @@ def read_session_file(type, num, doc): # FIXME: uploaded_filename should be replaced with a function call that computes names that are fixed path = os.path.join(settings.AGENDA_PATH, "%s/%s/%s" % (num, type, doc.uploaded_filename)) if os.path.exists(path): - with open(path) as f: + with io.open(path) as f: return f.read(), path else: return None, path @@ -224,13 +229,13 @@ def convert_draft_to_pdf(doc_name): outpath = os.path.join(settings.INTERNET_DRAFT_PDF_PATH, doc_name + ".pdf") try: - infile = open(inpath, "r") + infile = io.open(inpath, "r") except IOError: return t,tempname = mkstemp() os.close(t) - tempfile = open(tempname, "w") + tempfile = io.open(tempname, "w") pageend = 0; newpage = 0; @@ -238,7 +243,7 @@ def convert_draft_to_pdf(doc_name): for line in infile: line = re.sub("\r","",line) line = re.sub("[ \t]+$","",line) - if re.search("\[?[Pp]age [0-9ivx]+\]?[ \t]*$",line): + if re.search(r"\[?[Pp]age [0-9ivx]+\]?[ \t]*$",line): pageend=1 tempfile.write(line) continue @@ -288,13 +293,13 @@ def agenda_permissions(meeting, schedule, user): return cansee, canedit, secretariat def session_constraint_expire(request,session): - from ajax import session_constraints + from .ajax import session_constraints path = reverse(session_constraints, args=[session.meeting.number, session.pk]) temp_request = HttpRequest() temp_request.path = path temp_request.META['HTTP_HOST'] = request.META['HTTP_HOST'] key = get_cache_key(temp_request) - if key is not None and cache.has_key(key): + if key is not None and key in cache: cache.delete(key) # ------------------------------------------------- diff --git a/ietf/meeting/management/commands/update_important_dates.py b/ietf/meeting/management/commands/update_important_dates.py index d5668c1b4..8689ba7b3 100644 --- a/ietf/meeting/management/commands/update_important_dates.py +++ b/ietf/meeting/management/commands/update_important_dates.py @@ -1,5 +1,8 @@ -# Copyright The IETF Trust 2018, All Rights Reserved -from __future__ import unicode_literals +# Copyright The IETF Trust 2018-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import datetime diff --git a/ietf/meeting/migrations/0001_initial.py b/ietf/meeting/migrations/0001_initial.py index ce77e14d6..5d8910e68 100644 --- a/ietf/meeting/migrations/0001_initial.py +++ b/ietf/meeting/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import datetime import django.core.validators diff --git a/ietf/meeting/migrations/0002_auto_20180225_1207.py b/ietf/meeting/migrations/0002_auto_20180225_1207.py index d6082a214..981dcf831 100644 --- a/ietf/meeting/migrations/0002_auto_20180225_1207.py +++ b/ietf/meeting/migrations/0002_auto_20180225_1207.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-25 12:07 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/meeting/migrations/0003_rename_modified_fields.py b/ietf/meeting/migrations/0003_rename_modified_fields.py index 98d3f66e3..c63aa49a8 100644 --- a/ietf/meeting/migrations/0003_rename_modified_fields.py +++ b/ietf/meeting/migrations/0003_rename_modified_fields.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-03-02 14:33 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/meeting/migrations/0004_meeting_attendees.py b/ietf/meeting/migrations/0004_meeting_attendees.py index 7eca3c2e3..502b0b126 100644 --- a/ietf/meeting/migrations/0004_meeting_attendees.py +++ b/ietf/meeting/migrations/0004_meeting_attendees.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.11 on 2018-03-20 09:17 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/meeting/migrations/0005_backfill_old_meetings.py b/ietf/meeting/migrations/0005_backfill_old_meetings.py index 0335adc4e..af245617c 100644 --- a/ietf/meeting/migrations/0005_backfill_old_meetings.py +++ b/ietf/meeting/migrations/0005_backfill_old_meetings.py @@ -1,5 +1,8 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations @@ -9,64 +12,64 @@ def backfill_old_meetings(apps, schema_editor): for id, number, type_id, date, city, country, time_zone, continent, attendees in [ ( 59,'59','ietf','2004-03-29','Seoul','KR','Asia/Seoul','Asia','1390' ), - ( 58,'58','ietf','2003-11-09','Minneapolis','US','America/Menominee','America','1233' ), - ( 57,'57','ietf','2003-07-13','Vienna','AT','Europe/Vienna','Europe','1304' ), - ( 56,'56','ietf','2003-03-16','San Francisco','US','America/Los_Angeles','America','1679' ), - ( 55,'55','ietf','2002-11-17','Atlanta','US','America/New_York','America','1570' ), - ( 54,'54','ietf','2002-07-14','Yokohama','JP','Asia/Tokyo','Asia','1885' ), - ( 53,'53','ietf','2002-03-17','Minneapolis','US','America/Menominee','America','1656' ), - ( 52,'52','ietf','2001-12-09','Salt Lake City','US','America/Denver','America','1691' ), - ( 51,'51','ietf','2001-08-05','London','GB','Europe/London','Europe','2226' ), - ( 50,'50','ietf','2001-03-18','Minneapolis','US','America/Menominee','America','1822' ), - ( 49,'49','ietf','2000-12-10','San Diego','US','America/Los_Angeles','America','2810' ), - ( 48,'48','ietf','2000-07-31','Pittsburgh','US','America/New_York','America','2344' ), - ( 47,'47','ietf','2000-03-26','Adelaide','AU','Australia/Adelaide','Australia','1431' ), - ( 46,'46','ietf','1999-11-07','Washington','US','America/New_York','America','2379' ), - ( 45,'45','ietf','1999-07-11','Oslo','NO','Europe/Oslo','Europe','1710' ), - ( 44,'44','ietf','1999-03-14','Minneapolis','US','America/Menominee','America','1705' ), - ( 43,'43','ietf','1998-12-07','Orlando','US','America/New_York','America','2124' ), - ( 42,'42','ietf','1998-08-24','Chicago','US','America/Chicago','America','2106' ), - ( 41,'41','ietf','1998-03-30','Los Angeles','US','America/Los_Angeles','America','1775' ), - ( 40,'40','ietf','1997-12-08','Washington','US','America/New_York','America','1897' ), - ( 39,'39','ietf','1997-08-11','Munich','DE','Europe/Berlin','Europe','1308' ), - ( 38,'38','ietf','1997-04-07','Memphis','US','America/Chicago','America','1321' ), - ( 37,'37','ietf','1996-12-09','San Jose','US','America/Los_Angeles','America','1993' ), - ( 36,'36','ietf','1996-06-24','Montreal','CA','America/New_York','America','1283' ), - ( 35,'35','ietf','1996-03-04','Los Angeles','US','America/Los_Angeles','America','1038' ), - ( 34,'34','ietf','1995-12-04','Dallas','US','America/Chicago','America','1007' ), - ( 33,'33','ietf','1995-07-17','Stockholm','SE','Europe/Stockholm','Europe','617' ), - ( 32,'32','ietf','1995-04-03','Danvers','US','America/New_York','America','983' ), - ( 31,'31','ietf','1994-12-05','San Jose','US','America/Los_Angeles','America','1079' ), - ( 30,'30','ietf','1994-07-25','Toronto','CA','America/New_York','America','710' ), - ( 29,'29','ietf','1994-03-28','Seattle','US','America/Los_Angeles','America','785' ), - ( 28,'28','ietf','1993-11-01','Houston','US','America/Chicago','America','636' ), - ( 27,'27','ietf','1993-07-12','Amsterdam','NL','Europe/Amsterdam','Europe','493' ), - ( 26,'26','ietf','1993-03-29','Columbus','US','America/New_York','America','638' ), - ( 25,'25','ietf','1992-11-16','Washington','US','America/New_York','America','633' ), - ( 24,'24','ietf','1992-07-13','Cambridge','US','America/New_York','America','677' ), - ( 23,'23','ietf','1992-03-16','San Diego','US','America/Los_Angeles','America','530' ), - ( 22,'22','ietf','1991-11-18','Santa Fe','US','America/Denver','America','372' ), - ( 21,'21','ietf','1991-07-29','Atlanta','US','America/New_York','America','387' ), - ( 20,'20','ietf','1991-03-11','St. Louis','US','America/Chicago','America','348' ), - ( 19,'19','ietf','1990-12-03','Boulder','US','America/Denver','America','292' ), - ( 18,'18','ietf','1990-07-30','Vancouver','CA','America/Los_Angeles','America','293' ), - ( 17,'17','ietf','1990-05-01','Pittsburgh','US','America/New_York','America','244' ), - ( 16,'16','ietf','1990-02-06','Tallahassee','US','America/New_York','America','196' ), - ( 15,'15','ietf','1989-10-31','Honolulu','US','Pacific/Honolulu','America','138' ), - ( 14,'14','ietf','1989-07-25','Stanford','US','America/Los_Angeles','America','217' ), - ( 13,'13','ietf','1989-04-11','Cocoa Beach','US','America/New_York','America','114' ), - ( 12,'12','ietf','1989-01-18','Austin','US','America/Chicago','America','120' ), - ( 11,'11','ietf','1988-10-17','Ann Arbor','US','America/New_York','America','114' ), - ( 10,'10','ietf','1988-06-15','Annapolis','US','America/New_York','America','112' ), - ( 9,'9','ietf','1988-03-01','San Diego','US','America/Los_Angeles','America','82' ), - ( 8,'8','ietf','1987-11-02','Boulder','US','America/Denver','America','56' ), - ( 7,'7','ietf','1987-07-27','McLean','US','America/New_York','America','101' ), - ( 6,'6','ietf','1987-04-22','Boston','US','America/New_York','America','88' ), - ( 5,'5','ietf','1987-02-04','Moffett Field','US','America/Los_Angeles','America','35' ), - ( 4,'4','ietf','1986-10-15','Menlo Park','US','America/Los_Angeles','America','35' ), - ( 3,'3','ietf','1986-07-23','Ann Arbor','US','America/New_York','America','18' ), - ( 2,'2','ietf','1986-04-08','Aberdeen','US','America/New_York','America','21' ), - ( 1,'1','ietf','1986-01-16','San Diego','US','America/Los_Angeles','America','21' ), + ( 58,'58','ietf','2003-11-09','Minneapolis','US','America/Menominee','America','1233' ), + ( 57,'57','ietf','2003-07-13','Vienna','AT','Europe/Vienna','Europe','1304' ), + ( 56,'56','ietf','2003-03-16','San Francisco','US','America/Los_Angeles','America','1679' ), + ( 55,'55','ietf','2002-11-17','Atlanta','US','America/New_York','America','1570' ), + ( 54,'54','ietf','2002-07-14','Yokohama','JP','Asia/Tokyo','Asia','1885' ), + ( 53,'53','ietf','2002-03-17','Minneapolis','US','America/Menominee','America','1656' ), + ( 52,'52','ietf','2001-12-09','Salt Lake City','US','America/Denver','America','1691' ), + ( 51,'51','ietf','2001-08-05','London','GB','Europe/London','Europe','2226' ), + ( 50,'50','ietf','2001-03-18','Minneapolis','US','America/Menominee','America','1822' ), + ( 49,'49','ietf','2000-12-10','San Diego','US','America/Los_Angeles','America','2810' ), + ( 48,'48','ietf','2000-07-31','Pittsburgh','US','America/New_York','America','2344' ), + ( 47,'47','ietf','2000-03-26','Adelaide','AU','Australia/Adelaide','Australia','1431' ), + ( 46,'46','ietf','1999-11-07','Washington','US','America/New_York','America','2379' ), + ( 45,'45','ietf','1999-07-11','Oslo','NO','Europe/Oslo','Europe','1710' ), + ( 44,'44','ietf','1999-03-14','Minneapolis','US','America/Menominee','America','1705' ), + ( 43,'43','ietf','1998-12-07','Orlando','US','America/New_York','America','2124' ), + ( 42,'42','ietf','1998-08-24','Chicago','US','America/Chicago','America','2106' ), + ( 41,'41','ietf','1998-03-30','Los Angeles','US','America/Los_Angeles','America','1775' ), + ( 40,'40','ietf','1997-12-08','Washington','US','America/New_York','America','1897' ), + ( 39,'39','ietf','1997-08-11','Munich','DE','Europe/Berlin','Europe','1308' ), + ( 38,'38','ietf','1997-04-07','Memphis','US','America/Chicago','America','1321' ), + ( 37,'37','ietf','1996-12-09','San Jose','US','America/Los_Angeles','America','1993' ), + ( 36,'36','ietf','1996-06-24','Montreal','CA','America/New_York','America','1283' ), + ( 35,'35','ietf','1996-03-04','Los Angeles','US','America/Los_Angeles','America','1038' ), + ( 34,'34','ietf','1995-12-04','Dallas','US','America/Chicago','America','1007' ), + ( 33,'33','ietf','1995-07-17','Stockholm','SE','Europe/Stockholm','Europe','617' ), + ( 32,'32','ietf','1995-04-03','Danvers','US','America/New_York','America','983' ), + ( 31,'31','ietf','1994-12-05','San Jose','US','America/Los_Angeles','America','1079' ), + ( 30,'30','ietf','1994-07-25','Toronto','CA','America/New_York','America','710' ), + ( 29,'29','ietf','1994-03-28','Seattle','US','America/Los_Angeles','America','785' ), + ( 28,'28','ietf','1993-11-01','Houston','US','America/Chicago','America','636' ), + ( 27,'27','ietf','1993-07-12','Amsterdam','NL','Europe/Amsterdam','Europe','493' ), + ( 26,'26','ietf','1993-03-29','Columbus','US','America/New_York','America','638' ), + ( 25,'25','ietf','1992-11-16','Washington','US','America/New_York','America','633' ), + ( 24,'24','ietf','1992-07-13','Cambridge','US','America/New_York','America','677' ), + ( 23,'23','ietf','1992-03-16','San Diego','US','America/Los_Angeles','America','530' ), + ( 22,'22','ietf','1991-11-18','Santa Fe','US','America/Denver','America','372' ), + ( 21,'21','ietf','1991-07-29','Atlanta','US','America/New_York','America','387' ), + ( 20,'20','ietf','1991-03-11','St. Louis','US','America/Chicago','America','348' ), + ( 19,'19','ietf','1990-12-03','Boulder','US','America/Denver','America','292' ), + ( 18,'18','ietf','1990-07-30','Vancouver','CA','America/Los_Angeles','America','293' ), + ( 17,'17','ietf','1990-05-01','Pittsburgh','US','America/New_York','America','244' ), + ( 16,'16','ietf','1990-02-06','Tallahassee','US','America/New_York','America','196' ), + ( 15,'15','ietf','1989-10-31','Honolulu','US','Pacific/Honolulu','America','138' ), + ( 14,'14','ietf','1989-07-25','Stanford','US','America/Los_Angeles','America','217' ), + ( 13,'13','ietf','1989-04-11','Cocoa Beach','US','America/New_York','America','114' ), + ( 12,'12','ietf','1989-01-18','Austin','US','America/Chicago','America','120' ), + ( 11,'11','ietf','1988-10-17','Ann Arbor','US','America/New_York','America','114' ), + ( 10,'10','ietf','1988-06-15','Annapolis','US','America/New_York','America','112' ), + ( 9,'9','ietf','1988-03-01','San Diego','US','America/Los_Angeles','America','82' ), + ( 8,'8','ietf','1987-11-02','Boulder','US','America/Denver','America','56' ), + ( 7,'7','ietf','1987-07-27','McLean','US','America/New_York','America','101' ), + ( 6,'6','ietf','1987-04-22','Boston','US','America/New_York','America','88' ), + ( 5,'5','ietf','1987-02-04','Moffett Field','US','America/Los_Angeles','America','35' ), + ( 4,'4','ietf','1986-10-15','Menlo Park','US','America/Los_Angeles','America','35' ), + ( 3,'3','ietf','1986-07-23','Ann Arbor','US','America/New_York','America','18' ), + ( 2,'2','ietf','1986-04-08','Aberdeen','US','America/New_York','America','21' ), + ( 1,'1','ietf','1986-01-16','San Diego','US','America/Los_Angeles','America','21' ), ]: Meeting.objects.get_or_create(id=id, number=number, type_id=type_id, date=date, city=city, country=country, diff --git a/ietf/meeting/migrations/0006_backfill_attendees.py b/ietf/meeting/migrations/0006_backfill_attendees.py index 05dfe4457..2e4103c16 100644 --- a/ietf/meeting/migrations/0006_backfill_attendees.py +++ b/ietf/meeting/migrations/0006_backfill_attendees.py @@ -1,5 +1,8 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/meeting/migrations/0007_auto_20180716_1337.py b/ietf/meeting/migrations/0007_auto_20180716_1337.py index 0521ca185..b348d07cb 100644 --- a/ietf/meeting/migrations/0007_auto_20180716_1337.py +++ b/ietf/meeting/migrations/0007_auto_20180716_1337.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.14 on 2018-07-16 13:37 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/meeting/migrations/0008_rename_meeting_agenda_note.py b/ietf/meeting/migrations/0008_rename_meeting_agenda_note.py index cda62e1ac..528bb2afd 100644 --- a/ietf/meeting/migrations/0008_rename_meeting_agenda_note.py +++ b/ietf/meeting/migrations/0008_rename_meeting_agenda_note.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.15 on 2018-10-09 13:09 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/meeting/migrations/0009_add_agenda_info_note.py b/ietf/meeting/migrations/0009_add_agenda_info_note.py index ae1f32676..109fcd910 100644 --- a/ietf/meeting/migrations/0009_add_agenda_info_note.py +++ b/ietf/meeting/migrations/0009_add_agenda_info_note.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-10-09 14:07 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/meeting/migrations/0010_set_ietf_103_agenda_info_note.py b/ietf/meeting/migrations/0010_set_ietf_103_agenda_info_note.py index f1ae68251..92bbd6a24 100644 --- a/ietf/meeting/migrations/0010_set_ietf_103_agenda_info_note.py +++ b/ietf/meeting/migrations/0010_set_ietf_103_agenda_info_note.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-10-09 14:23 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/meeting/migrations/0011_auto_20190114_0550.py b/ietf/meeting/migrations/0011_auto_20190114_0550.py index 066ece556..0a12b5fc0 100644 --- a/ietf/meeting/migrations/0011_auto_20190114_0550.py +++ b/ietf/meeting/migrations/0011_auto_20190114_0550.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.18 on 2019-01-14 05:50 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/meeting/migrations/0012_add_slide_submissions.py b/ietf/meeting/migrations/0012_add_slide_submissions.py index 9ae818c08..617809de0 100644 --- a/ietf/meeting/migrations/0012_add_slide_submissions.py +++ b/ietf/meeting/migrations/0012_add_slide_submissions.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-03-23 07:41 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/meeting/migrations/0013_make_separate_break_sessobj.py b/ietf/meeting/migrations/0013_make_separate_break_sessobj.py index 3d2e77c64..359f75ce5 100644 --- a/ietf/meeting/migrations/0013_make_separate_break_sessobj.py +++ b/ietf/meeting/migrations/0013_make_separate_break_sessobj.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-03-23 06:11 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import datetime from django.db import migrations diff --git a/ietf/meeting/migrations/0014_auto_20190426_0305.py b/ietf/meeting/migrations/0014_auto_20190426_0305.py index c83544689..4f7d47a62 100644 --- a/ietf/meeting/migrations/0014_auto_20190426_0305.py +++ b/ietf/meeting/migrations/0014_auto_20190426_0305.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-04-26 03:05 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/meeting/migrations/0015_sessionpresentation_document2_fk.py b/ietf/meeting/migrations/0015_sessionpresentation_document2_fk.py index bd13536ea..8354d546e 100644 --- a/ietf/meeting/migrations/0015_sessionpresentation_document2_fk.py +++ b/ietf/meeting/migrations/0015_sessionpresentation_document2_fk.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-08 11:58 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations import django.db.models.deletion diff --git a/ietf/meeting/migrations/0016_remove_sessionpresentation_document.py b/ietf/meeting/migrations/0016_remove_sessionpresentation_document.py index 228c63ed2..724bfcfc0 100644 --- a/ietf/meeting/migrations/0016_remove_sessionpresentation_document.py +++ b/ietf/meeting/migrations/0016_remove_sessionpresentation_document.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-21 03:57 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/meeting/migrations/0017_rename_field_document2.py b/ietf/meeting/migrations/0017_rename_field_document2.py index a857a074f..5095a33ff 100644 --- a/ietf/meeting/migrations/0017_rename_field_document2.py +++ b/ietf/meeting/migrations/0017_rename_field_document2.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-21 05:31 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/meeting/migrations/0018_document_primary_key_cleanup.py b/ietf/meeting/migrations/0018_document_primary_key_cleanup.py index 491a34f93..786530908 100644 --- a/ietf/meeting/migrations/0018_document_primary_key_cleanup.py +++ b/ietf/meeting/migrations/0018_document_primary_key_cleanup.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-06-10 03:47 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations import django.db.models.deletion diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 5b3478cae..464b2ae15 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -1,11 +1,13 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals # old meeting models can be found in ../proceedings/models.py import pytz import datetime -from urlparse import urljoin +import io +from six.moves.urllib.parse import urljoin import os import re import string @@ -19,6 +21,7 @@ from django.conf import settings # mostly used by json_dict() #from django.template.defaultfilters import slugify, date as date_format, time as time_format from django.template.defaultfilters import date as date_format +from django.utils.encoding import python_2_unicode_compatible from django.utils.text import slugify from ietf.dbtemplate.models import DBTemplate @@ -33,8 +36,8 @@ from ietf.utils.text import xslugify from ietf.utils.timezone import date2datetime from ietf.utils.models import ForeignKey -countries = pytz.country_names.items() -countries.sort(lambda x,y: cmp(x[1], y[1])) +countries = list(pytz.country_names.items()) +countries.sort(key=lambda x: x[1]) timezones = [] for name in pytz.common_timezones: @@ -54,6 +57,7 @@ def fmt_date(o): d = datetime_safe.new_date(o) return d.strftime(DATE_FORMAT) +@python_2_unicode_compatible class Meeting(models.Model): # number is either the number for IETF meetings, or some other # identifier for interim meetings/IESG retreats/liaison summits/... @@ -100,16 +104,16 @@ class Meeting(models.Model): agenda_warning_note = models.TextField(blank=True, help_text="Text in this field will be placed more prominently at the top of the html agenda page for the meeting. HTML can be used, but will not be validated.") agenda = ForeignKey('Schedule',null=True,blank=True, related_name='+') session_request_lock_message = models.CharField(blank=True,max_length=255) # locked if not empty - proceedings_final = models.BooleanField(default=False, help_text=u"Are the proceedings for this meeting complete?") + proceedings_final = models.BooleanField(default=False, help_text="Are the proceedings for this meeting complete?") acknowledgements = models.TextField(blank=True, help_text="Acknowledgements for use in meeting proceedings. Use ReStructuredText markup.") overview = ForeignKey(DBTemplate, related_name='overview', null=True, editable=False) show_important_dates = models.BooleanField(default=False) attendees = models.IntegerField(blank=True, null=True, default=None, - help_text=u"Number of Attendees for backfilled meetings, leave it blank for new meetings, and then it is calculated from the registrations") + help_text="Number of Attendees for backfilled meetings, leave it blank for new meetings, and then it is calculated from the registrations") - def __unicode__(self): + def __str__(self): if self.type_id == "ietf": - return "IETF-%s" % (self.number) + return u"IETF-%s" % (self.number) else: return self.number @@ -255,7 +259,7 @@ class Meeting(models.Model): days.sort() for ymd in time_slices: time_slices[ymd].sort() - slots[ymd].sort(lambda x,y: cmp(x.time, y.time)) + slots[ymd].sort(key=lambda x: x.time) return days,time_slices,slots # this functions makes a list of timeslices and rooms, and @@ -276,7 +280,7 @@ class Meeting(models.Model): try: tzfn = os.path.join(settings.TZDATA_ICS_PATH, self.time_zone + ".ics") if os.path.exists(tzfn): - with open(tzfn) as tzf: + with io.open(tzfn) as tzf: icstext = tzf.read() vtimezone = re.search("(?sm)(\nBEGIN:VTIMEZONE.*\nEND:VTIMEZONE\n)", icstext).group(1).strip() if vtimezone: @@ -310,12 +314,13 @@ class Meeting(models.Model): # === Rooms, Resources, Floorplans ============================================= +@python_2_unicode_compatible class ResourceAssociation(models.Model): name = ForeignKey(RoomResourceName) icon = models.CharField(max_length=64) # icon to be found in /static/img desc = models.CharField(max_length=256) - def __unicode__(self): + def __str__(self): return self.desc def json_dict(self, host_scheme): @@ -326,6 +331,7 @@ class ResourceAssociation(models.Model): res1['resource_id'] = self.pk return res1 +@python_2_unicode_compatible class Room(models.Model): meeting = ForeignKey(Meeting) modified = models.DateTimeField(auto_now=True) @@ -344,8 +350,8 @@ class Room(models.Model): y2 = models.SmallIntegerField(null=True, blank=True, default=None) # end floorplan-related stuff - def __unicode__(self): - return "%s size: %s" % (self.name, self.capacity) + def __str__(self): + return u"%s size: %s" % (self.name, self.capacity) def delete_timeslots(self): for ts in self.timeslot_set.all(): @@ -413,8 +419,9 @@ class UrlResource(models.Model): def floorplan_path(instance, filename): root, ext = os.path.splitext(filename) - return u"%s/floorplan-%s-%s%s" % (settings.FLOORPLAN_MEDIA_DIR, instance.meeting.number, xslugify(instance.name), ext) + return "%s/floorplan-%s-%s%s" % (settings.FLOORPLAN_MEDIA_DIR, instance.meeting.number, xslugify(instance.name), ext) +@python_2_unicode_compatible class FloorPlan(models.Model): name = models.CharField(max_length=255) short = models.CharField(max_length=3, default='') @@ -426,11 +433,12 @@ class FloorPlan(models.Model): class Meta: ordering = ['-id',] # - def __unicode__(self): - return 'floorplan-%s-%s' % (self.meeting.number, xslugify(self.name)) + def __str__(self): + return u'floorplan-%s-%s' % (self.meeting.number, xslugify(self.name)) # === Schedules, Sessions, Timeslots and Assignments =========================== +@python_2_unicode_compatible class TimeSlot(models.Model): """ Everything that would appear on the meeting agenda of a meeting is @@ -444,7 +452,7 @@ class TimeSlot(models.Model): duration = models.DurationField(default=datetime.timedelta(0)) location = ForeignKey(Room, blank=True, null=True) show_location = models.BooleanField(default=True, help_text="Show location in agenda.") - sessions = models.ManyToManyField('Session', related_name='slots', through='SchedTimeSessAssignment', blank=True, help_text=u"Scheduled session, if any.") + sessions = models.ManyToManyField('Session', related_name='slots', through='SchedTimeSessAssignment', blank=True, help_text="Scheduled session, if any.") modified = models.DateTimeField(auto_now=True) # @@ -456,7 +464,7 @@ class TimeSlot(models.Model): @property def time_desc(self): - return u"%s-%s" % (self.time.strftime("%H%M"), (self.time + self.duration).strftime("%H%M")) + return "%s-%s" % (self.time.strftime("%H%M"), (self.time + self.duration).strftime("%H%M")) def meeting_date(self): return self.time.date() @@ -472,10 +480,10 @@ class TimeSlot(models.Model): self._reg_info = None return self._reg_info - def __unicode__(self): + def __str__(self): location = self.get_location() if not location: - location = "(no location)" + location = u"(no location)" return u"%s: %s-%s %s, %s" % (self.meeting.number, self.time.strftime("%m-%d %H:%M"), (self.time + self.duration).strftime("%H:%M"), self.name, location) @@ -598,6 +606,7 @@ class TimeSlot(models.Model): # end of TimeSlot +@python_2_unicode_compatible class Schedule(models.Model): """ Each person may have multiple agendas saved. @@ -611,12 +620,12 @@ class Schedule(models.Model): meeting = ForeignKey(Meeting, null=True) name = models.CharField(max_length=16, blank=False) owner = ForeignKey(Person) - visible = models.BooleanField(default=True, help_text=u"Make this agenda available to those who know about it.") - public = models.BooleanField(default=True, help_text=u"Make this agenda publically available.") + visible = models.BooleanField(default=True, help_text="Make this agenda available to those who know about it.") + public = models.BooleanField(default=True, help_text="Make this agenda publically available.") badness = models.IntegerField(null=True, blank=True) # considering copiedFrom = ForeignKey('Schedule', blank=True, null=True) - def __unicode__(self): + def __str__(self): return u"%s:%s(%s)" % (self.meeting, self.name, self.owner) def base_url(self): @@ -707,6 +716,7 @@ class Schedule(models.Model): self.delete() # to be renamed SchedTimeSessAssignments (stsa) +@python_2_unicode_compatible class SchedTimeSessAssignment(models.Model): """ This model provides an N:M relationship between Session and TimeSlot. @@ -714,9 +724,9 @@ class SchedTimeSessAssignment(models.Model): a specific person/user. """ timeslot = ForeignKey('TimeSlot', null=False, blank=False, related_name='sessionassignments') - session = ForeignKey('Session', null=True, default=None, related_name='timeslotassignments', help_text=u"Scheduled session.") + session = ForeignKey('Session', null=True, default=None, related_name='timeslotassignments', help_text="Scheduled session.") schedule = ForeignKey('Schedule', null=False, blank=False, related_name='assignments') - extendedfrom = ForeignKey('self', null=True, default=None, help_text=u"Timeslot this session is an extension of.") + extendedfrom = ForeignKey('self', null=True, default=None, help_text="Timeslot this session is an extension of.") modified = models.DateTimeField(auto_now=True) notes = models.TextField(blank=True) badness = models.IntegerField(default=0, blank=True, null=True) @@ -725,7 +735,7 @@ class SchedTimeSessAssignment(models.Model): class Meta: ordering = ["timeslot__time", "timeslot__type__slug", "session__group__parent__name", "session__group__acronym", "session__name", ] - def __unicode__(self): + def __str__(self): return u"%s [%s<->%s]" % (self.schedule, self.session, self.timeslot) @property @@ -807,8 +817,9 @@ class SchedTimeSessAssignment(models.Model): components.append(g.acronym) - return u"-".join(components).lower() + return "-".join(components).lower() +@python_2_unicode_compatible class Constraint(models.Model): """ Specifies a constraint on the scheduling. @@ -828,16 +839,16 @@ class Constraint(models.Model): active_status = None - def __unicode__(self): + def __str__(self): return u"%s %s target=%s person=%s" % (self.source, self.name.name.lower(), self.target, self.person) def brief_display(self): if self.target and self.person: - return u"%s ; %s" % (self.target.acronym, self.person) + return "%s ; %s" % (self.target.acronym, self.person) elif self.target and not self.person: - return u"%s " % (self.target.acronym) + return "%s " % (self.target.acronym) elif not self.target and self.person: - return u"%s " % (self.person) + return "%s " % (self.person) def json_url(self): return "/meeting/%s/constraint/%s.json" % (self.meeting.number, self.id) @@ -857,6 +868,7 @@ class Constraint(models.Model): return ct1 +@python_2_unicode_compatible class SessionPresentation(models.Model): session = ForeignKey('Session') document = ForeignKey(Document) @@ -868,12 +880,13 @@ class SessionPresentation(models.Model): ordering = ('order',) unique_together = (('session', 'document'),) - def __unicode__(self): + def __str__(self): return u"%s -> %s-%s" % (self.session, self.document.name, self.rev) constraint_cache_uses = 0 constraint_cache_initials = 0 +@python_2_unicode_compatible class Session(models.Model): """Session records that a group should have a session on the meeting (time and location is stored in a TimeSlot) - if multiple @@ -1007,7 +1020,7 @@ class Session(models.Model): def is_material_submission_cutoff(self): return datetime.date.today() > self.meeting.get_submission_correction_date() - def __unicode__(self): + def __str__(self): if self.meeting.type_id == "interim": return self.meeting.number @@ -1018,7 +1031,7 @@ class Session(models.Model): ss = self.timeslotassignments.filter(schedule=self.meeting.agenda).order_by('timeslot__time') if ss: ss0name = ','.join([x.timeslot.time.strftime("%a-%H%M") for x in ss]) - return u"%s: %s %s %s" % (self.meeting, self.group.acronym, self.name, ss0name) + return "%s: %s %s %s" % (self.meeting, self.group.acronym, self.name, ss0name) @property def short_name(self): @@ -1028,7 +1041,7 @@ class Session(models.Model): return self.short if self.group: return self.group.acronym - return u"req#%u" % (id) + return "req#%u" % (id) @property def special_request_token(self): @@ -1080,22 +1093,22 @@ class Session(models.Model): sess1['group_href'] = urljoin(host_scheme, self.group.json_url()) if self.group.parent is not None: sess1['area'] = self.group.parent.acronym.upper() - sess1['description'] = self.group.name.encode('utf-8') + sess1['description'] = self.group.name sess1['group_id'] = str(self.group.pk) reslist = [] for r in self.resources.all(): reslist.append(r.json_dict(host_scheme)) sess1['resources'] = reslist sess1['session_id'] = str(self.pk) - sess1['name'] = self.name.encode('utf-8') - sess1['title'] = self.short_name.encode('utf-8') - sess1['short_name'] = self.short_name.encode('utf-8') + sess1['name'] = self.name + sess1['title'] = self.short_name + sess1['short_name'] = self.short_name sess1['bof'] = str(self.group.is_bof()) - sess1['agenda_note'] = self.agenda_note.encode('utf-8') + sess1['agenda_note'] = self.agenda_note sess1['attendees'] = str(self.attendees) - sess1['status'] = self.status.name.encode('utf-8') + sess1['status'] = self.status.name if self.comments is not None: - sess1['comments'] = self.comments.encode('utf-8') + sess1['comments'] = self.comments sess1['requested_time'] = self.requested.strftime("%Y-%m-%d") # the related person object sometimes does not exist in the dataset. try: @@ -1113,7 +1126,7 @@ class Session(models.Model): if doc: path = os.path.join(settings.AGENDA_PATH, self.meeting.number, "agenda", doc.uploaded_filename) if os.path.exists(path): - with open(path) as f: + with io.open(path) as f: return f.read() else: return "No agenda file found" @@ -1147,6 +1160,7 @@ class Session(models.Model): else: return self.group.acronym +@python_2_unicode_compatible class ImportantDate(models.Model): meeting = ForeignKey(Meeting) date = models.DateField() @@ -1154,7 +1168,7 @@ class ImportantDate(models.Model): class Meta: ordering = ["-meeting_id","date", ] - def __unicode__(self): + def __str__(self): return u'%s : %s : %s' % ( self.meeting, self.name, self.date ) class SlideSubmission(models.Model): diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py index 62fea679c..a8783c295 100644 --- a/ietf/meeting/resources.py +++ b/ietf/meeting/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2014-11-13 23:15 + + from ietf.api import ModelResource from ietf.api import ToOneField from tastypie.fields import ToManyField, DateTimeField diff --git a/ietf/meeting/templatetags/agenda_custom_tags.py b/ietf/meeting/templatetags/agenda_custom_tags.py index b6e17aa1a..a1501ab64 100644 --- a/ietf/meeting/templatetags/agenda_custom_tags.py +++ b/ietf/meeting/templatetags/agenda_custom_tags.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django import template register = template.Library() @@ -42,7 +48,7 @@ def durationFormat(inp): def callMethod(obj, methodName): method = getattr(obj, methodName) - if obj.__dict__.has_key("__callArg"): + if "__callArg" in obj.__dict__: ret = method(*obj.__callArg) del obj.__callArg return ret @@ -50,7 +56,7 @@ def callMethod(obj, methodName): @register.filter(name="args") def args(obj, arg): - if not obj.__dict__.has_key("__callArg"): + if "__callArg" not in obj.__dict__: obj.__callArg = [] obj.__callArg += [arg] diff --git a/ietf/meeting/test_data.py b/ietf/meeting/test_data.py index 349f1fbe7..6cb2c8f39 100644 --- a/ietf/meeting/test_data.py +++ b/ietf/meeting/test_data.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime from django.utils.text import slugify @@ -70,7 +76,7 @@ def make_meeting_test_data(meeting=None): #secretary = Person.objects.get(user__username="secretary") ## not used if not meeting: - meeting = Meeting.objects.get(number="42", type="ietf") + meeting = Meeting.objects.get(number="72", type="ietf") schedule = Schedule.objects.create(meeting=meeting, owner=plainman, name="test-agenda", visible=True, public=True) unofficial_schedule = Schedule.objects.create(meeting=meeting, owner=plainman, name="test-unofficial-agenda", visible=True, public=True) @@ -157,24 +163,24 @@ def make_meeting_test_data(meeting=None): meeting.unofficial_schedule = unofficial_schedule - doc = DocumentFactory.create(name='agenda-42-mars', type_id='agenda', title="Agenda", - uploaded_filename="agenda-42-mars.txt", group=mars, rev='00', states=[('draft','active')]) + doc = DocumentFactory.create(name='agenda-72-mars', type_id='agenda', title="Agenda", + uploaded_filename="agenda-72-mars.txt", group=mars, rev='00', states=[('draft','active')]) pres = SessionPresentation.objects.create(session=mars_session,document=doc,rev=doc.rev) mars_session.sessionpresentation_set.add(pres) # - doc = DocumentFactory.create(name='minutes-42-mars', type_id='minutes', title="Minutes", - uploaded_filename="minutes-42-mars.txt", group=mars, rev='00', states=[('minutes','active')]) + doc = DocumentFactory.create(name='minutes-72-mars', type_id='minutes', title="Minutes", + uploaded_filename="minutes-72-mars.txt", group=mars, rev='00', states=[('minutes','active')]) pres = SessionPresentation.objects.create(session=mars_session,document=doc,rev=doc.rev) mars_session.sessionpresentation_set.add(pres) - doc = DocumentFactory.create(name='slides-42-mars-1-active', type_id='slides', title="Slideshow", - uploaded_filename="slides-42-mars.txt", group=mars, rev='00', + doc = DocumentFactory.create(name='slides-72-mars-1-active', type_id='slides', title="Slideshow", + uploaded_filename="slides-72-mars.txt", group=mars, rev='00', states=[('slides','active'), ('reuse_policy', 'single')]) pres = SessionPresentation.objects.create(session=mars_session,document=doc,rev=doc.rev) mars_session.sessionpresentation_set.add(pres) - doc = DocumentFactory.create(name='slides-42-mars-2-deleted', type_id='slides', - title="Bad Slideshow", uploaded_filename="slides-42-mars-2-deleted.txt", group=mars, rev='00', + doc = DocumentFactory.create(name='slides-72-mars-2-deleted', type_id='slides', + title="Bad Slideshow", uploaded_filename="slides-72-mars-2-deleted.txt", group=mars, rev='00', states=[('slides','deleted'), ('reuse_policy', 'single')]) pres = SessionPresentation.objects.create(session=mars_session,document=doc,rev=doc.rev) mars_session.sessionpresentation_set.add(pres) diff --git a/ietf/meeting/tests_api.py b/ietf/meeting/tests_api.py index 0bf6da2e7..dc9b788bf 100644 --- a/ietf/meeting/tests_api.py +++ b/ietf/meeting/tests_api.py @@ -1,6 +1,12 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime -import json -from urlparse import urlsplit + +from six.moves.urllib.parse import urlsplit from django.urls import reverse as urlreverse @@ -17,7 +23,7 @@ from ietf.utils.mail import outbox class ApiTests(TestCase): def test_update_agenda(self): meeting = make_meeting_test_data() - schedule = Schedule.objects.get(meeting__number=42,name="test-agenda") + schedule = Schedule.objects.get(meeting__number=72,name="test-agenda") mars_session = Session.objects.filter(meeting=meeting, group__acronym="mars").first() ames_session = Session.objects.filter(meeting=meeting, group__acronym="ames").first() @@ -68,7 +74,7 @@ class ApiTests(TestCase): self.client.login(username="ad", password="ad+password") r = do_unschedule(mars_scheduled) self.assertEqual(r.status_code, 403) - self.assertTrue("error" in json.loads(r.content)) + self.assertTrue("error" in r.json()) # faulty post r = do_schedule(schedule,ames_session,mars_slot) self.assertEqual(r.status_code, 403) @@ -77,7 +83,7 @@ class ApiTests(TestCase): self.client.login(username="plain", password='plain+password') r = do_unschedule(ames_scheduled) self.assertEqual(r.status_code, 200) - self.assertTrue("error" not in json.loads(r.content)) + self.assertNotIn("error", r.json()) r = do_schedule(schedule,ames_session,mars_slot) self.assertEqual(r.status_code, 201) @@ -89,13 +95,13 @@ class ApiTests(TestCase): # Extend the mars session r = do_extend(schedule,mars_scheduled) self.assertEqual(r.status_code, 201) - self.assertTrue("error" not in json.loads(r.content)) + self.assertTrue("error" not in r.json()) self.assertEqual(mars_session.timeslotassignments.filter(schedule__name='test-agenda').count(),2) # Unschedule mars r = do_unschedule(mars_scheduled) self.assertEqual(r.status_code, 200) - self.assertTrue("error" not in json.loads(r.content)) + self.assertNotIn("error", r.json()) # Make sure it got both the original and extended session self.assertEqual(mars_session.timeslotassignments.filter(schedule__name='test-agenda').count(),0) @@ -115,7 +121,7 @@ class ApiTests(TestCase): r = self.client.get(urlreverse("ietf.meeting.ajax.session_constraints", kwargs=dict(num=meeting.number, sessionid=session.pk))) self.assertEqual(r.status_code, 200) - constraints = json.loads(r.content) + constraints = r.json() self.assertEqual(set([c_ames.pk, c_person.pk]), set(c["constraint_id"] for c in constraints)) def test_meeting_json(self): @@ -123,7 +129,7 @@ class ApiTests(TestCase): r = self.client.get(urlreverse("ietf.meeting.ajax.meeting_json", kwargs=dict(num=meeting.number))) self.assertEqual(r.status_code, 200) - info = json.loads(r.content) + info = r.json() self.assertEqual(info["name"], meeting.number) def test_get_room_json(self): @@ -132,7 +138,7 @@ class ApiTests(TestCase): r = self.client.get(urlreverse("ietf.meeting.ajax.timeslot_roomurl", kwargs=dict(num=meeting.number, roomid=room.pk))) self.assertEqual(r.status_code, 200) - info = json.loads(r.content) + info = r.json() self.assertEqual(info["name"], room.name) def test_create_new_room(self): @@ -185,7 +191,7 @@ class ApiTests(TestCase): url = urlreverse("ietf.group.views.group_json", kwargs=dict(acronym=group.acronym)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - info = json.loads(r.content) + info = r.json() self.assertEqual(info["name"], group.name) # This really belongs in person tests @@ -196,7 +202,7 @@ class ApiTests(TestCase): url = urlreverse("ietf.person.ajax.person_json", kwargs=dict(personid=person.pk)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - info = json.loads(r.content) + info = r.json() self.assertEqual(info["name"], person.name) def test_sessions_json(self): @@ -205,7 +211,7 @@ class ApiTests(TestCase): url = urlreverse("ietf.meeting.ajax.sessions_json",kwargs=dict(num=meeting.number)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - info = json.loads(r.content) + info = r.json() self.assertEqual(set([x['short_name'] for x in info]),set([s.session.short_name for s in meeting.agenda.assignments.filter(session__type_id='session')])) schedule = meeting.agenda @@ -213,7 +219,7 @@ class ApiTests(TestCase): kwargs=dict(num=meeting.number,owner=schedule.owner_email(),name=schedule.name)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - info = json.loads(r.content) + info = r.json() self.assertEqual(len(info),schedule.assignments.count()) @@ -225,7 +231,7 @@ class ApiTests(TestCase): kwargs=dict(num=meeting.number, slotid=slot.pk)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - info = json.loads(r.content) + info = r.json() self.assertEqual(info["timeslot_id"], slot.pk) def test_create_new_slot(self): @@ -281,7 +287,7 @@ class ApiTests(TestCase): name=meeting.agenda.name)) r = self.client.get(url) - info = json.loads(r.content) + info = r.json() self.assertEqual(info["schedule_id"], meeting.agenda.pk) def test_create_new_schedule(self): @@ -405,7 +411,7 @@ class ApiTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) - info = json.loads(r.content) + info = r.json() self.assertEqual(info['secretariat'], True) self.assertEqual(urlsplit(info['owner_href'])[2], "/person/%s.json" % meeting.agenda.owner_id) self.assertEqual(info['read_only'], True) @@ -418,7 +424,7 @@ class ApiTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) - info = json.loads(r.content) + info = r.json() self.assertEqual(info['secretariat'], False) self.assertEqual(info['read_only'], False) self.assertEqual(info['save_perm'], False) diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index 230c960f9..b306814aa 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -1,4 +1,8 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import sys import time @@ -80,10 +84,10 @@ class ScheduleEditTests(StaticLiveServerTestCase): def testUnschedule(self): - self.assertEqual(SchedTimeSessAssignment.objects.filter(session__meeting__number=42,session__group__acronym='mars',schedule__name='test-agenda').count(),1) + self.assertEqual(SchedTimeSessAssignment.objects.filter(session__meeting__number=72,session__group__acronym='mars',schedule__name='test-agenda').count(),1) self.login() - url = self.absreverse('ietf.meeting.views.edit_agenda',kwargs=dict(num='42',name='test-agenda',owner='plain@example.com')) + url = self.absreverse('ietf.meeting.views.edit_agenda',kwargs=dict(num='72',name='test-agenda',owner='plain@example.com')) self.driver.get(url) q = PyQuery(self.driver.page_source) @@ -97,7 +101,7 @@ class ScheduleEditTests(StaticLiveServerTestCase): self.assertTrue(len(q('#sortable-list #session_1'))>0) time.sleep(0.1) # The API that modifies the database runs async - self.assertEqual(SchedTimeSessAssignment.objects.filter(session__meeting__number=42,session__group__acronym='mars',schedule__name='test-agenda').count(),0) + self.assertEqual(SchedTimeSessAssignment.objects.filter(session__meeting__number=72,session__group__acronym='mars',schedule__name='test-agenda').count(),0) @skipIf(skip_selenium, skip_message) class SlideReorderTests(StaticLiveServerTestCase): @@ -148,7 +152,7 @@ class SlideReorderTests(StaticLiveServerTestCase): time.sleep(0.1) # The API that modifies the database runs async names=self.session.sessionpresentation_set.values_list('document__name',flat=True) - self.assertEqual(list(names),[u'one',u'three',u'two']) + self.assertEqual(list(names),['one','three','two']) # The following are useful debugging tools @@ -169,5 +173,5 @@ class SlideReorderTests(StaticLiveServerTestCase): # condition_data() # # def testOpenSchedule(self): -# url = urlreverse('ietf.meeting.views.edit_agenda', kwargs=dict(num='42',name='test-agenda')) +# url = urlreverse('ietf.meeting.views.edit_agenda', kwargs=dict(num='72',name='test-agenda')) # r = self.client.get(url) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index 7e091f6fe..de3fc4286 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -1,24 +1,28 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- -import json -import os -import shutil -import datetime -import urlparse -import random -from unittest import skipIf -import debug # pyflakes:ignore +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import io +import os +import random +import shutil +import six + +from unittest import skipIf +from mock import patch +from pyquery import PyQuery +from io import StringIO, BytesIO +from bs4 import BeautifulSoup +from six.moves.urllib.parse import urlparse from django.urls import reverse as urlreverse from django.conf import settings from django.contrib.auth.models import User -from mock import patch -from pyquery import PyQuery -from StringIO import StringIO -from bs4 import BeautifulSoup +import debug # pyflakes:ignore from ietf.doc.models import Document from ietf.group.models import Group, Role @@ -70,7 +74,7 @@ class MeetingTests(TestCase): if not os.path.exists(dirname): os.makedirs(dirname) - with open(path, "w") as f: + with io.open(path, "w") as f: f.write(content) def write_materials_files(self, meeting, session): @@ -106,11 +110,11 @@ class MeetingTests(TestCase): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) agenda_content = q("#content").html() - self.assertTrue(session.group.acronym in agenda_content) - self.assertTrue(session.group.name in agenda_content) - self.assertTrue(session.group.parent.acronym.upper() in agenda_content) - self.assertTrue(slot.location.name in agenda_content) - self.assertTrue(time_interval in agenda_content) + self.assertIn(session.group.acronym, agenda_content) + self.assertIn(session.group.name, agenda_content) + self.assertIn(session.group.parent.acronym.upper(), agenda_content) + self.assertIn(slot.location.name, agenda_content) + self.assertIn(time_interval, agenda_content) # plain time_interval = "%s-%s" % (slot.time.strftime("%H:%M").lstrip("0"), (slot.time + slot.duration).strftime("%H:%M").lstrip("0")) @@ -119,11 +123,11 @@ class MeetingTests(TestCase): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) agenda_content = q("#content").html() - self.assertTrue(session.group.acronym in agenda_content) - self.assertTrue(session.group.name in agenda_content) - self.assertTrue(session.group.parent.acronym.upper() in agenda_content) - self.assertTrue(slot.location.name in agenda_content) - self.assertTrue(time_interval in agenda_content) + self.assertIn(session.group.acronym, agenda_content) + self.assertIn(session.group.name, agenda_content) + self.assertIn(session.group.parent.acronym.upper(), agenda_content) + self.assertIn(slot.location.name, agenda_content) + self.assertIn(time_interval, agenda_content) # Make sure there's a frame for the agenda and it points to the right place self.assertTrue(any([session.materials.get(type='agenda').href() in x.attrib["data-src"] for x in q('tr div.modal-body div.frame')])) @@ -134,8 +138,7 @@ class MeetingTests(TestCase): # future meeting, no agenda r = self.client.get(urlreverse("ietf.meeting.views.agenda", kwargs=dict(num=future_meeting.number))) - self.assertEqual(r.status_code, 200) - self.assertContains(r, u"There is no agenda available yet.") + self.assertContains(r, "There is no agenda available yet.") self.assertTemplateUsed(r, 'meeting/no-agenda.html') # text @@ -143,71 +146,59 @@ class MeetingTests(TestCase): time_interval = time_interval.replace(":", "") r = self.client.get(urlreverse("ietf.meeting.views.agenda", kwargs=dict(num=meeting.number, ext=".txt"))) - self.assertEqual(r.status_code, 200) - agenda_content = r.content - self.assertTrue(session.group.acronym in agenda_content) - self.assertTrue(session.group.name in agenda_content) - self.assertTrue(session.group.parent.acronym.upper() in agenda_content) - self.assertTrue(slot.location.name in agenda_content) + self.assertContains(r, session.group.acronym) + self.assertContains(r, session.group.name) + self.assertContains(r, session.group.parent.acronym.upper()) + self.assertContains(r, slot.location.name) - self.assertTrue(time_interval in agenda_content) + self.assertContains(r, time_interval) r = self.client.get(urlreverse("ietf.meeting.views.agenda", kwargs=dict(num=meeting.number,name=meeting.unofficial_schedule.name,owner=meeting.unofficial_schedule.owner.email()))) - self.assertEqual(r.status_code, 200) - self.assertTrue('not the official schedule' in unicontent(r)) + self.assertContains(r, 'not the official schedule') # future meeting, no agenda r = self.client.get(urlreverse("ietf.meeting.views.agenda", kwargs=dict(num=future_meeting.number, ext=".txt"))) - self.assertEqual(r.status_code, 200) self.assertContains(r, "There is no agenda available yet.") self.assertTemplateUsed(r, 'meeting/no-agenda.txt') # CSV r = self.client.get(urlreverse("ietf.meeting.views.agenda", kwargs=dict(num=meeting.number, ext=".csv"))) - self.assertEqual(r.status_code, 200) - agenda_content = r.content - self.assertTrue(session.group.acronym in agenda_content) - self.assertTrue(session.group.name in agenda_content) - self.assertTrue(session.group.parent.acronym.upper() in agenda_content) - self.assertTrue(slot.location.name in agenda_content) + self.assertContains(r, session.group.acronym) + self.assertContains(r, session.group.name) + self.assertContains(r, session.group.parent.acronym.upper()) + self.assertContains(r, slot.location.name) - self.assertTrue(session.materials.get(type='agenda').uploaded_filename in unicontent(r)) - self.assertTrue(session.materials.filter(type='slides').exclude(states__type__slug='slides',states__slug='deleted').first().uploaded_filename in unicontent(r)) - self.assertFalse(session.materials.filter(type='slides',states__type__slug='slides',states__slug='deleted').first().uploaded_filename in unicontent(r)) + self.assertContains(r, session.materials.get(type='agenda').uploaded_filename) + self.assertContains(r, session.materials.filter(type='slides').exclude(states__type__slug='slides',states__slug='deleted').first().uploaded_filename) + self.assertNotContains(r, session.materials.filter(type='slides',states__type__slug='slides',states__slug='deleted').first().uploaded_filename) # iCal r = self.client.get(urlreverse("ietf.meeting.views.ical_agenda", kwargs=dict(num=meeting.number)) + "?" + session.group.parent.acronym.upper()) - self.assertEqual(r.status_code, 200) - agenda_content = r.content - self.assertTrue(session.group.acronym in agenda_content) - self.assertTrue(session.group.name in agenda_content) - self.assertTrue(slot.location.name in agenda_content) - self.assertTrue("BEGIN:VTIMEZONE" in agenda_content) - self.assertTrue("END:VTIMEZONE" in agenda_content) + self.assertContains(r, session.group.acronym) + self.assertContains(r, session.group.name) + self.assertContains(r, slot.location.name) + self.assertContains(r, "BEGIN:VTIMEZONE") + self.assertContains(r, "END:VTIMEZONE") - self.assertTrue(session.agenda().href() in unicontent(r)) - self.assertTrue(session.materials.filter(type='slides').exclude(states__type__slug='slides',states__slug='deleted').first().href() in unicontent(r)) + self.assertContains(r, session.agenda().href()) + self.assertContains(r, session.materials.filter(type='slides').exclude(states__type__slug='slides',states__slug='deleted').first().href()) # TODO - the ics view uses .all on a queryset in a view so it's showing the deleted slides. - #self.assertFalse(session.materials.filter(type='slides',states__type__slug='slides',states__slug='deleted').first().get_absolute_url() in unicontent(r)) + #self.assertNotContains(r, session.materials.filter(type='slides',states__type__slug='slides',states__slug='deleted').first().get_absolute_url()) # week view r = self.client.get(urlreverse("ietf.meeting.views.week_view", kwargs=dict(num=meeting.number))) - self.assertEqual(r.status_code, 200) - agenda_content = unicontent(r) - self.assertNotIn('CANCELLED',agenda_content) - self.assertTrue(session.group.acronym in agenda_content) - self.assertTrue(slot.location.name in agenda_content) + self.assertNotContains(r, 'CANCELLED') + self.assertContains(r, session.group.acronym) + self.assertContains(r, slot.location.name) # week view with a cancelled session session.status_id='canceled' session.save() r = self.client.get(urlreverse("ietf.meeting.views.week_view", kwargs=dict(num=meeting.number))) - self.assertEqual(r.status_code, 200) - agenda_content = unicontent(r) - self.assertIn('CANCELLED',agenda_content) - self.assertTrue(session.group.acronym in agenda_content) - self.assertTrue(slot.location.name in agenda_content) + self.assertContains(r, 'CANCELLED') + self.assertContains(r, session.group.acronym) + self.assertContains(r, slot.location.name) def test_agenda_current_audio(self): date = datetime.date.today() @@ -215,7 +206,7 @@ class MeetingTests(TestCase): make_meeting_test_data(meeting=meeting) url = urlreverse("ietf.meeting.views.agenda", kwargs=dict(num=meeting.number)) r = self.client.get(url) - self.assertTrue("Audio stream" in unicontent(r)) + self.assertContains(r, "Audio stream") def test_agenda_by_room(self): meeting = make_meeting_test_data() @@ -227,7 +218,7 @@ class MeetingTests(TestCase): url = urlreverse("ietf.meeting.views.agenda_by_room",kwargs=dict(num=meeting.number,name=meeting.unofficial_schedule.name,owner=meeting.unofficial_schedule.owner.email())) r = self.client.get(url) self.assertTrue(all([x in unicontent(r) for x in ['mars','Test Room',]])) - self.assertFalse('IESG Breakfast' in unicontent(r)) + self.assertNotContains(r, 'IESG Breakfast') def test_agenda_by_type(self): meeting = make_meeting_test_data() @@ -240,7 +231,7 @@ class MeetingTests(TestCase): url = urlreverse("ietf.meeting.views.agenda_by_type",kwargs=dict(num=meeting.number,name=meeting.unofficial_schedule.name,owner=meeting.unofficial_schedule.owner.email())) r = self.client.get(url) self.assertTrue(all([x in unicontent(r) for x in ['mars','Test Room',]])) - self.assertFalse('IESG Breakfast' in unicontent(r)) + self.assertNotContains(r, 'IESG Breakfast') url = urlreverse("ietf.meeting.views.agenda_by_type",kwargs=dict(num=meeting.number,type='session')) r = self.client.get(url) @@ -265,9 +256,8 @@ class MeetingTests(TestCase): self.assertTrue(all([x in unicontent(r) for x in ['mars','IESG Breakfast','Test Room','Breakfast Room']])) url = urlreverse("ietf.meeting.views.room_view",kwargs=dict(num=meeting.number,name=meeting.unofficial_schedule.name,owner=meeting.unofficial_schedule.owner.email())) r = self.client.get(url) - self.assertEqual(r.status_code,200) self.assertTrue(all([x in unicontent(r) for x in ['mars','Test Room','Breakfast Room']])) - self.assertFalse('IESG Breakfast' in unicontent(r)) + self.assertNotContains(r, 'IESG Breakfast') def test_agenda_week_view(self): @@ -303,15 +293,13 @@ class MeetingTests(TestCase): if r.status_code != 200: q = PyQuery(r.content) debug.show('q(".alert").text()') - self.assertEqual(r.status_code, 200) - self.assertTrue("1. WG status" in unicontent(r)) + self.assertContains(r, "1. WG status") # session minutes url = urlreverse("ietf.meeting.views.materials_document", kwargs=dict(num=meeting.number, document=session.minutes())) r = self.client.get(url) - self.assertEqual(r.status_code, 200) - self.assertTrue("1. More work items underway" in unicontent(r)) + self.assertContains(r, "1. More work items underway") # test with explicit meeting number in url if meeting.number.isdigit(): @@ -362,23 +350,20 @@ class MeetingTests(TestCase): self.client.login(username="marschairman", password="marschairman+password") r = self.client.get(urlreverse("ietf.meeting.views.materials_editable_groups", kwargs={'num':meeting.number})) - self.assertEqual(r.status_code, 200) - self.assertTrue(meeting.number in unicontent(r)) - self.assertTrue("mars" in unicontent(r)) - self.assertFalse("No session requested" in unicontent(r)) + self.assertContains(r, meeting.number) + self.assertContains(r, "mars") + self.assertNotContains(r, "No session requested") self.client.login(username="ad", password="ad+password") r = self.client.get(urlreverse("ietf.meeting.views.materials_editable_groups", kwargs={'num':meeting.number})) - self.assertEqual(r.status_code, 200) - self.assertTrue(meeting.number in unicontent(r)) - self.assertTrue("frfarea" in unicontent(r)) - self.assertTrue("No session requested" in unicontent(r)) + self.assertContains(r, meeting.number) + self.assertContains(r, "frfarea") + self.assertContains(r, "No session requested") self.client.login(username="plain",password="plain+password") r = self.client.get(urlreverse("ietf.meeting.views.materials_editable_groups", kwargs={'num':meeting.number})) - self.assertEqual(r.status_code, 200) - self.assertTrue(meeting.number in unicontent(r)) - self.assertTrue("You cannot manage the meeting materials for any groups" in unicontent(r)) + self.assertContains(r, meeting.number) + self.assertContains(r, "You cannot manage the meeting materials for any groups") def test_proceedings(self): meeting = make_meeting_test_data() @@ -400,35 +385,32 @@ class MeetingTests(TestCase): meeting.save() url = urlreverse('ietf.meeting.views.proceedings_acknowledgements',kwargs={'num':meeting.number}) response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertTrue('test acknowledgements' in response.content) + self.assertContains(response, 'test acknowledgements') - @patch('urllib2.urlopen') + @patch('six.moves.urllib.request.urlopen') def test_proceedings_attendees(self, mock_urlopen): - mock_urlopen.return_value = StringIO('[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') + mock_urlopen.return_value = six.BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') make_meeting_test_data() meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96") finalize(meeting) url = urlreverse('ietf.meeting.views.proceedings_attendees',kwargs={'num':96}) response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertTrue('Attendee List' in response.content) + self.assertContains(response, 'Attendee List') q = PyQuery(response.content) self.assertEqual(1,len(q("#id_attendees tbody tr"))) - @patch('urllib2.urlopen') + @patch('six.moves.urllib.request.urlopen') def test_proceedings_overview(self, mock_urlopen): '''Test proceedings IETF Overview page. Note: old meetings aren't supported so need to add a new meeting then test. ''' - mock_urlopen.return_value = StringIO('[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') + mock_urlopen.return_value = six.BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') make_meeting_test_data() meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96") finalize(meeting) url = urlreverse('ietf.meeting.views.proceedings_overview',kwargs={'num':96}) response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertTrue('The Internet Engineering Task Force' in response.content) + self.assertContains(response, 'The Internet Engineering Task Force') def test_proceedings_progress_report(self): make_meeting_test_data() @@ -437,17 +419,15 @@ class MeetingTests(TestCase): url = urlreverse('ietf.meeting.views.proceedings_progress_report',kwargs={'num':96}) response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertTrue('Progress Report' in response.content) + self.assertContains(response, 'Progress Report') def test_feed(self): meeting = make_meeting_test_data() session = Session.objects.filter(meeting=meeting, group__acronym="mars").first() r = self.client.get("/feed/wg-proceedings/") - self.assertEqual(r.status_code, 200) - self.assertTrue("agenda" in unicontent(r)) - self.assertTrue(session.group.acronym in unicontent(r)) + self.assertContains(r, "agenda") + self.assertContains(r, session.group.acronym) def test_important_dates(self): meeting=MeetingFactory(type_id='ietf') @@ -456,8 +436,7 @@ class MeetingTests(TestCase): populate_important_dates(meeting) url = urlreverse('ietf.meeting.views.important_dates',kwargs={'num':meeting.number}) r = self.client.get(url) - self.assertEqual(r.status_code, 200) - self.assertIn(str(meeting.importantdate_set.first().date), unicontent(r)) + self.assertContains(r, str(meeting.importantdate_set.first().date)) idn = ImportantDateName.objects.filter(used=True).first() pre_date = meeting.importantdate_set.get(name=idn).date idn.default_offset_days -= 1 @@ -478,10 +457,9 @@ class MeetingTests(TestCase): # url = urlreverse('ietf.meeting.views.ical_agenda', kwargs={'num':meeting.number, 'acronym':s1.group.acronym, }) r = self.client.get(url) - self.assertEqual(r.status_code, 200) self.assertEqual(r.get('Content-Type'), "text/calendar") self.assertContains(r, 'BEGIN:VEVENT') - self.assertEqual(r.content.count('UID'), 2) + self.assertEqual(r.content.count(b'UID'), 2) self.assertContains(r, 'SUMMARY:mars - Martian Special Interest Group') self.assertContains(r, t1.time.strftime('%Y%m%dT%H%M%S')) self.assertContains(r, t2.time.strftime('%Y%m%dT%H%M%S')) @@ -489,10 +467,9 @@ class MeetingTests(TestCase): # url = urlreverse('ietf.meeting.views.ical_agenda', kwargs={'num':meeting.number, 'session_id':s1.id, }) r = self.client.get(url) - self.assertEqual(r.status_code, 200) self.assertEqual(r.get('Content-Type'), "text/calendar") self.assertContains(r, 'BEGIN:VEVENT') - self.assertEqual(r.content.count('UID'), 1) + self.assertEqual(r.content.count(b'UID'), 1) self.assertContains(r, 'SUMMARY:mars - Martian Special Interest Group') self.assertContains(r, t1.time.strftime('%Y%m%dT%H%M%S')) self.assertNotContains(r, t2.time.strftime('%Y%m%dT%H%M%S')) @@ -504,7 +481,7 @@ class MeetingTests(TestCase): session.sessionpresentation_set.create(document=doc) file,_ = submission_file(name=doc.name,format='txt',templatename='test_submission.txt',group=session.group,rev="00") filename = os.path.join(doc.get_file_path(),file.name) - with open(filename,'w') as draftbits: + with io.open(filename,'w') as draftbits: draftbits.write(file.getvalue()) url = urlreverse('ietf.meeting.views.session_draft_tarfile', kwargs={'num':session.meeting.number,'acronym':session.group.acronym}) @@ -521,7 +498,7 @@ class MeetingTests(TestCase): session.sessionpresentation_set.create(document=doc) file,_ = submission_file(name=doc.name,format='txt',templatename='test_submission.txt',group=session.group,rev="00") filename = os.path.join(doc.get_file_path(),file.name) - with open(filename,'w') as draftbits: + with io.open(filename,'w') as draftbits: draftbits.write(file.getvalue()) url = urlreverse('ietf.meeting.views.session_draft_pdf', kwargs={'num':session.meeting.number,'acronym':session.group.acronym}) @@ -596,8 +573,7 @@ class EditTests(TestCase): self.client.login(username="secretary", password="secretary+password") r = self.client.get(urlreverse("ietf.meeting.views.edit_agenda", kwargs=dict(num=meeting.number))) - self.assertEqual(r.status_code, 200) - self.assertTrue("load_assignments" in unicontent(r)) + self.assertContains(r, "load_assignments") def test_save_agenda_as_and_read_permissions(self): meeting = make_meeting_test_data() @@ -620,7 +596,7 @@ class EditTests(TestCase): }) self.assertEqual(r.status_code, 302) # Verify that we actually got redirected to a new place. - self.assertNotEqual(urlparse.urlparse(r.url).path, url) + self.assertNotEqual(urlparse(r.url).path, url) # get schedule = meeting.get_schedule_by_name("foo") @@ -666,7 +642,7 @@ class EditTests(TestCase): 'saveas': "saveas", }) self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r.url).path, url) + self.assertEqual(urlparse(r.url).path, url) # TODO: Verify that an error message was in fact returned. r = self.client.post(url, { @@ -675,16 +651,16 @@ class EditTests(TestCase): }) # TODO: Verify that an error message was in fact returned. self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r.url).path, url) + self.assertEqual(urlparse(r.url).path, url) # Non-ASCII alphanumeric characters r = self.client.post(url, { - 'savename': u"f\u00E9ling", + 'savename': "f\u00E9ling", 'saveas': "saveas", }) # TODO: Verify that an error message was in fact returned. self.assertEqual(r.status_code, 302) - self.assertEqual(urlparse.urlparse(r.url).path, url) + self.assertEqual(urlparse(r.url).path, url) def test_edit_timeslots(self): @@ -692,8 +668,7 @@ class EditTests(TestCase): self.client.login(username="secretary", password="secretary+password") r = self.client.get(urlreverse("ietf.meeting.views.edit_timeslots", kwargs=dict(num=meeting.number))) - self.assertEqual(r.status_code, 200) - self.assertTrue(meeting.room_set.all().first().name in unicontent(r)) + self.assertContains(r, meeting.room_set.all().first().name) def test_edit_timeslot_type(self): timeslot = TimeSlotFactory(meeting__type_id='ietf') @@ -738,7 +713,7 @@ class SessionDetailsTests(TestCase): url = urlreverse('ietf.meeting.views.session_details', kwargs=dict(num=session.meeting.number, acronym=group.acronym)) r = self.client.get(url) self.assertTrue(all([x in unicontent(r) for x in ('slides','agenda','minutes','draft')])) - self.assertFalse('deleted' in unicontent(r)) + self.assertNotContains(r, 'deleted') def test_add_session_drafts(self): group = GroupFactory.create(type_id='wg',state_id='active') @@ -760,13 +735,12 @@ class SessionDetailsTests(TestCase): self.client.login(username=group_chair.user.username, password='%s+password'%group_chair.user.username) r = self.client.get(url) - self.assertEqual(r.status_code, 200) - self.assertTrue(old_draft.name in unicontent(r)) + self.assertContains(r, old_draft.name) r = self.client.post(url,dict(drafts=[new_draft.pk, old_draft.pk])) self.assertTrue(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue("Already linked:" in q('form .alert-danger').text()) + self.assertIn("Already linked:", q('form .alert-danger').text()) self.assertEqual(1,session.sessionpresentation_set.count()) r = self.client.post(url,dict(drafts=[new_draft.pk,])) @@ -882,8 +856,7 @@ class InterimTests(TestCase): session.save() login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) - self.assertEqual(r.status_code, 200) - self.assertTrue(meeting.number in r.content) + self.assertContains(r, meeting.number) def test_interim_skip_announcement(self): make_meeting_test_data() @@ -915,7 +888,7 @@ class InterimTests(TestCase): r = self.client.post(url, initial) self.assertRedirects(r, urlreverse('ietf.meeting.views.interim_announce')) self.assertEqual(len(outbox), len_before + 1) - self.assertTrue('WG Virtual Meeting' in outbox[-1]['Subject']) + self.assertIn('WG Virtual Meeting', outbox[-1]['Subject']) def test_interim_approve_by_ad(self): make_meeting_test_data() @@ -928,7 +901,7 @@ class InterimTests(TestCase): for session in meeting.session_set.all(): self.assertEqual(session.status.slug, 'scheda') self.assertEqual(len(outbox), length_before + 1) - self.assertTrue('ready for announcement' in outbox[-1]['Subject']) + self.assertIn('ready for announcement', outbox[-1]['Subject']) def test_interim_approve_by_secretariat(self): make_meeting_test_data() @@ -947,26 +920,24 @@ class InterimTests(TestCase): interim = SessionFactory(meeting__type_id='interim',meeting__date=last_week,status_id='canceled',group__state_id='active',group__parent=GroupFactory(state_id='active')) url = urlreverse('ietf.meeting.views.past') r = self.client.get(url) - self.assertEqual(r.status_code, 200) - self.assertTrue('IETF - %02d'%int(ietf.meeting.number) in unicontent(r)) + self.assertContains(r, 'IETF - %02d'%int(ietf.meeting.number)) q = PyQuery(r.content) id="-%s" % interim.group.acronym - self.assertTrue('CANCELLED' in q('[id*="'+id+'"]').text()) + self.assertIn('CANCELLED', q('[id*="'+id+'"]').text()) def test_upcoming(self): make_meeting_test_data() url = urlreverse("ietf.meeting.views.upcoming") - r = self.client.get(url) - self.assertEqual(r.status_code, 200) today = datetime.date.today() mars_interim = Meeting.objects.filter(date__gt=today, type='interim', session__group__acronym='mars', session__status='sched').first() ames_interim = Meeting.objects.filter(date__gt=today, type='interim', session__group__acronym='ames', session__status='canceled').first() - self.assertTrue(mars_interim.number in r.content) - self.assertTrue(ames_interim.number in r.content) - self.assertTrue('IETF - 42' in r.content) + r = self.client.get(url) + self.assertContains(r, mars_interim.number) + self.assertContains(r, ames_interim.number) + self.assertContains(r, 'IETF - 72') # cancelled session q = PyQuery(r.content) - self.assertTrue('CANCELLED' in q('[id*="-ames"]').text()) + self.assertIn('CANCELLED', q('[id*="-ames"]').text()) self.check_interim_tabs(url) def test_upcoming_ical(self): @@ -975,14 +946,14 @@ class InterimTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertEqual(r.get('Content-Type'), "text/calendar") - self.assertEqual(r.content.count('UID'), 7) + self.assertEqual(r.content.count(b'UID'), 7) # check filtered output url = url + '?filters=mars' r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertEqual(r.get('Content-Type'), "text/calendar") # print r.content - self.assertEqual(r.content.count('UID'), 2) + self.assertEqual(r.content.count(b'UID'), 2) def test_interim_request_permissions(self): @@ -1096,8 +1067,8 @@ class InterimTests(TestCase): self.assertTrue(os.path.exists(path)) # check notice to secretariat self.assertEqual(len(outbox), length_before + 1) - self.assertTrue('interim meeting ready for announcement' in outbox[-1]['Subject']) - self.assertTrue('iesg-secretary@ietf.org' in outbox[-1]['To']) + self.assertIn('interim meeting ready for announcement', outbox[-1]['Subject']) + self.assertIn('iesg-secretary@ietf.org', outbox[-1]['To']) def test_interim_request_single_in_person(self): make_meeting_test_data() @@ -1243,8 +1214,7 @@ class InterimTests(TestCase): 'session_set-INITIAL_FORMS':0} r = self.client.post(urlreverse("ietf.meeting.views.interim_request"),data) - self.assertEqual(r.status_code, 200) - self.assertTrue('days must be consecutive' in r.content) + self.assertContains(r, 'days must be consecutive') def test_interim_request_series(self): make_meeting_test_data() @@ -1469,7 +1439,7 @@ class InterimTests(TestCase): self.assertEqual(session.status_id, 'canceled') self.assertEqual(session.agenda_note, comments) self.assertEqual(len(outbox), length_before + 1) - self.assertTrue('Interim Meeting Cancelled' in outbox[-1]['Subject']) + self.assertIn('Interim Meeting Cancelled', outbox[-1]['Subject']) def test_interim_request_edit_no_notice(self): '''Edit a request. No notice should go out if it hasn't been announced yet''' @@ -1544,7 +1514,7 @@ class InterimTests(TestCase): r = self.client.post(url, data) self.assertRedirects(r, urlreverse('ietf.meeting.views.interim_request_details', kwargs={'number': meeting.number})) self.assertEqual(len(outbox),length_before+1) - self.assertTrue('CHANGED' in outbox[-1]['Subject']) + self.assertIn('CHANGED', outbox[-1]['Subject']) session = meeting.session_set.first() timeslot = session.official_timeslotassignment().timeslot self.assertEqual(timeslot.time,new_time) @@ -1572,7 +1542,7 @@ class InterimTests(TestCase): length_before = len(outbox) send_interim_approval_request(meetings=[meeting]) self.assertEqual(len(outbox),length_before+1) - self.assertTrue('New Interim Meeting Request' in outbox[-1]['Subject']) + self.assertIn('New Interim Meeting Request', outbox[-1]['Subject']) def test_send_interim_cancellation_notice(self): make_meeting_test_data() @@ -1580,7 +1550,7 @@ class InterimTests(TestCase): length_before = len(outbox) send_interim_cancellation_notice(meeting=meeting) self.assertEqual(len(outbox),length_before+1) - self.assertTrue('Interim Meeting Cancelled' in outbox[-1]['Subject']) + self.assertIn('Interim Meeting Cancelled', outbox[-1]['Subject']) def test_send_interim_minutes_reminder(self): make_meeting_test_data() @@ -1590,7 +1560,7 @@ class InterimTests(TestCase): length_before = len(outbox) send_interim_minutes_reminder(meeting=meeting) self.assertEqual(len(outbox),length_before+1) - self.assertTrue('Action Required: Minutes' in outbox[-1]['Subject']) + self.assertIn('Action Required: Minutes', outbox[-1]['Subject']) def test_group_ical(self): @@ -1606,10 +1576,9 @@ class InterimTests(TestCase): # url = urlreverse('ietf.meeting.views.ical_agenda', kwargs={'num':meeting.number, 'acronym':s1.group.acronym, }) r = self.client.get(url) - self.assertEqual(r.status_code, 200) self.assertEqual(r.get('Content-Type'), "text/calendar") self.assertContains(r, 'BEGIN:VEVENT') - self.assertEqual(r.content.count('UID'), 2) + self.assertEqual(r.content.count(b'UID'), 2) self.assertContains(r, 'SUMMARY:mars - Martian Special Interest Group') self.assertContains(r, t1.time.strftime('%Y%m%dT%H%M%S')) self.assertContains(r, t2.time.strftime('%Y%m%dT%H%M%S')) @@ -1617,10 +1586,9 @@ class InterimTests(TestCase): # url = urlreverse('ietf.meeting.views.ical_agenda', kwargs={'num':meeting.number, 'session_id':s1.id, }) r = self.client.get(url) - self.assertEqual(r.status_code, 200) self.assertEqual(r.get('Content-Type'), "text/calendar") self.assertContains(r, 'BEGIN:VEVENT') - self.assertEqual(r.content.count('UID'), 1) + self.assertEqual(r.content.count(b'UID'), 1) self.assertContains(r, 'SUMMARY:mars - Martian Special Interest Group') self.assertContains(r, t1.time.strftime('%Y%m%dT%H%M%S')) self.assertNotContains(r, t2.time.strftime('%Y%m%dT%H%M%S')) @@ -1633,27 +1601,27 @@ class AjaxTests(TestCase): url = urlreverse('ietf.meeting.views.ajax_get_utc') + "?date=2016-1-1&time=badtime&timezone=UTC" r = self.client.get(url) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) + data = r.json() self.assertEqual(data["error"], True) url = urlreverse('ietf.meeting.views.ajax_get_utc') + "?date=2016-1-1&time=25:99&timezone=UTC" r = self.client.get(url) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) + data = r.json() self.assertEqual(data["error"], True) url = urlreverse('ietf.meeting.views.ajax_get_utc') + "?date=2016-1-1&time=10:00am&timezone=UTC" r = self.client.get(url) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) + data = r.json() self.assertEqual(data["error"], True) # test good query url = urlreverse('ietf.meeting.views.ajax_get_utc') + "?date=2016-1-1&time=12:00&timezone=America/Los_Angeles" r = self.client.get(url) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) - self.assertTrue('timezone' in data) - self.assertTrue('time' in data) - self.assertTrue('utc' in data) - self.assertTrue('error' not in data) + data = r.json() + self.assertIn('timezone', data) + self.assertIn('time', data) + self.assertIn('utc', data) + self.assertNotIn('error', data) self.assertEqual(data['utc'], '20:00') class FloorPlanTests(TestCase): @@ -1699,9 +1667,9 @@ class IphoneAppJsonTests(TestCase): self.assertEqual(r.status_code,200) class FinalizeProceedingsTests(TestCase): - @patch('urllib2.urlopen') + @patch('six.moves.urllib.request.urlopen') def test_finalize_proceedings(self, mock_urlopen): - mock_urlopen.return_value = StringIO('[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') + mock_urlopen.return_value = BytesIO(b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]') make_meeting_test_data() meeting = Meeting.objects.filter(type_id='ietf').order_by('id').last() meeting.session_set.filter(group__acronym='mars').first().sessionpresentation_set.create(document=Document.objects.filter(type='draft').first(),rev=None) @@ -1742,8 +1710,6 @@ class MaterialsTests(TestCase): def follow(url): seen.add(url) r = self.client.get(url) - if r.status_code != 200: - debug.show('url') self.assertEqual(r.status_code, 200) if not ('.' in url and url.rsplit('.', 1)[1] in ['tgz', 'pdf', ]): if r.content: @@ -1751,7 +1717,7 @@ class MaterialsTests(TestCase): soup = BeautifulSoup(page, 'html.parser') for a in soup('a'): href = a.get('href') - path = urlparse.urlparse(href).path + path = urlparse(href).path if (path and path not in seen and path.startswith(top)): follow(path) follow(url) @@ -1763,9 +1729,9 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Upload' in unicode(q("title"))) + self.assertIn('Upload', six.text_type(q("title"))) self.assertFalse(session.sessionpresentation_set.exists()) - test_file = StringIO(b'%PDF-1.4\n%âãÃÓ\nthis is some text for a test') + test_file = StringIO('%PDF-1.4\n%âãÃÓ\nthis is some text for a test') test_file.name = "not_really.pdf" r = self.client.post(url,dict(file=test_file)) self.assertEqual(r.status_code, 302) @@ -1774,7 +1740,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Revise' in unicode(q("title"))) + self.assertIn('Revise', six.text_type(q("title"))) test_file = StringIO('%PDF-1.4\n%âãÃÓ\nthis is some different text for a test') test_file.name = "also_not_really.pdf" r = self.client.post(url,dict(file=test_file)) @@ -1798,9 +1764,9 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Upload' in unicode(q("title"))) + self.assertIn('Upload', six.text_type(q("title"))) self.assertFalse(session.sessionpresentation_set.exists()) - test_file = StringIO(b'%PDF-1.4\n%âãÃÓ\nthis is some text for a test') + test_file = StringIO('%PDF-1.4\n%âãÃÓ\nthis is some text for a test') test_file.name = "not_really.pdf" r = self.client.post(url,dict(file=test_file)) self.assertEqual(r.status_code, 302) @@ -1816,7 +1782,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Upload' in unicode(q("title"))) + self.assertIn('Upload', six.text_type(q("title"))) def test_upload_minutes_agenda(self): @@ -1831,7 +1797,7 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Upload' in unicode(q("Title"))) + self.assertIn('Upload', six.text_type(q("Title"))) self.assertFalse(session.sessionpresentation_set.exists()) self.assertFalse(q('form input[type="checkbox"]')) @@ -1841,21 +1807,21 @@ class MaterialsTests(TestCase): q = PyQuery(r.content) self.assertTrue(q('form input[type="checkbox"]')) - test_file = StringIO('this is some text for a test') + test_file = BytesIO(b'this is some text for a test') test_file.name = "not_really.json" r = self.client.post(url,dict(file=test_file)) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('form .has-error')) - test_file = StringIO('this is some text for a test'*1510000) + test_file = BytesIO(b'this is some text for a test'*1510000) test_file.name = "not_really.pdf" r = self.client.post(url,dict(file=test_file)) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('form .has-error')) - test_file = StringIO('') + test_file = BytesIO(b'') test_file.name = "not_really.html" r = self.client.post(url,dict(file=test_file)) self.assertEqual(r.status_code, 200) @@ -1863,7 +1829,7 @@ class MaterialsTests(TestCase): self.assertTrue(q('form .has-error')) # Test html sanitization - test_file = StringIO('Title

Title

Some text
') + test_file = BytesIO(b'Title

Title

Some text
') test_file.name = "some.html" r = self.client.post(url,dict(file=test_file)) self.assertEqual(r.status_code, 302) @@ -1874,7 +1840,7 @@ class MaterialsTests(TestCase): self.assertNotIn('
', text) self.assertIn('charset="utf-8"', text) - test_file = StringIO(u'This is some text for a test, with the word\nvirtual at the beginning of a line.') + test_file = BytesIO(b'This is some text for a test, with the word\nvirtual at the beginning of a line.') test_file.name = "not_really.txt" r = self.client.post(url,dict(file=test_file,apply_to_all=False)) self.assertEqual(r.status_code, 302) @@ -1885,8 +1851,8 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Revise' in unicode(q("Title"))) - test_file = StringIO('this is some different text for a test') + self.assertIn('Revise', six.text_type(q("Title"))) + test_file = BytesIO(b'this is some different text for a test') test_file.name = "also_not_really.txt" r = self.client.post(url,dict(file=test_file,apply_to_all=True)) self.assertEqual(r.status_code, 302) @@ -1895,7 +1861,7 @@ class MaterialsTests(TestCase): self.assertTrue(session2.sessionpresentation_set.filter(document__type_id=doctype)) # Test bad encoding - test_file = StringIO(u'

Title

Some\x93text
'.encode('latin1')) + test_file = BytesIO('

Title

Some\x93text
'.encode('latin1')) test_file.name = "some.html" r = self.client.post(url,dict(file=test_file)) self.assertContains(r, 'Could not identify the file encoding') @@ -1919,11 +1885,11 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Upload' in unicode(q("Title"))) + self.assertIn('Upload', six.text_type(q("Title"))) self.assertFalse(session.sessionpresentation_set.exists()) self.assertFalse(q('form input[type="checkbox"]')) - test_file = StringIO('this is some text for a test') + test_file = BytesIO(b'this is some text for a test') test_file.name = "not_really.txt" r = self.client.post(url,dict(file=test_file,apply_to_all=False)) self.assertEqual(r.status_code, 410) @@ -1940,9 +1906,9 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Upload' in unicode(q("title"))) + self.assertIn('Upload', six.text_type(q("title"))) self.assertFalse(session.sessionpresentation_set.filter(document__type_id=doctype)) - test_file = StringIO('this is some text for a test') + test_file = BytesIO(b'this is some text for a test') test_file.name = "not_really.txt" r = self.client.post(url,dict(file=test_file)) self.assertEqual(r.status_code, 302) @@ -1963,9 +1929,9 @@ class MaterialsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Upload' in unicode(q("title"))) + self.assertIn('Upload', six.text_type(q("title"))) self.assertFalse(session1.sessionpresentation_set.filter(document__type_id='slides')) - test_file = StringIO('this is not really a slide') + test_file = BytesIO(b'this is not really a slide') test_file.name = 'not_really.txt' r = self.client.post(url,dict(file=test_file,title='a test slide file',apply_to_all=True)) self.assertEqual(r.status_code, 302) @@ -1976,7 +1942,7 @@ class MaterialsTests(TestCase): self.assertEqual(sp.order,1) url = urlreverse('ietf.meeting.views.upload_session_slides',kwargs={'num':session2.meeting.number,'session_id':session2.id}) - test_file = StringIO('some other thing still not slidelike') + test_file = BytesIO(b'some other thing still not slidelike') test_file.name = 'also_not_really.txt' r = self.client.post(url,dict(file=test_file,title='a different slide file',apply_to_all=False)) self.assertEqual(r.status_code, 302) @@ -1984,23 +1950,23 @@ class MaterialsTests(TestCase): self.assertEqual(session2.sessionpresentation_set.count(),2) sp = session2.sessionpresentation_set.get(document__name__endswith='-a-different-slide-file') self.assertEqual(sp.order,2) - self.assertEqual(sp.rev,u'00') - self.assertEqual(sp.document.rev,u'00') + self.assertEqual(sp.rev,'00') + self.assertEqual(sp.document.rev,'00') url = urlreverse('ietf.meeting.views.upload_session_slides',kwargs={'num':session2.meeting.number,'session_id':session2.id,'name':session2.sessionpresentation_set.get(order=2).document.name}) r = self.client.get(url) self.assertTrue(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Revise' in unicode(q("title"))) - test_file = StringIO('new content for the second slide deck') + self.assertIn('Revise', six.text_type(q("title"))) + test_file = BytesIO(b'new content for the second slide deck') test_file.name = 'doesnotmatter.txt' r = self.client.post(url,dict(file=test_file,title='rename the presentation',apply_to_all=False)) self.assertEqual(r.status_code, 302) self.assertEqual(session1.sessionpresentation_set.count(),1) self.assertEqual(session2.sessionpresentation_set.count(),2) sp = session2.sessionpresentation_set.get(order=2) - self.assertEqual(sp.rev,u'01') - self.assertEqual(sp.document.rev,u'01') + self.assertEqual(sp.rev,'01') + self.assertEqual(sp.document.rev,'01') def test_remove_sessionpresentation(self): session = SessionFactory(meeting__type_id='ietf') @@ -2052,7 +2018,7 @@ class MaterialsTests(TestCase): login_testing_unauthorized(self,newperson.user.username,propose_url) r = self.client.get(propose_url) self.assertEqual(r.status_code,200) - test_file = StringIO('this is not really a slide') + test_file = BytesIO(b'this is not really a slide') test_file.name = 'not_really.txt' empty_outbox() r = self.client.post(propose_url,dict(file=test_file,title='a test slide file',apply_to_all=True)) @@ -2145,9 +2111,8 @@ class SessionTests(TestCase): not_meeting = SessionFactory(meeting=meeting,group__parent=area,status_id='notmeet',add_to_schedule=False) url = urlreverse('ietf.meeting.views.meeting_requests',kwargs={'num':meeting.number}) r = self.client.get(url) - self.assertEqual(r.status_code,200) - self.assertTrue(requested_session.group.acronym in unicontent(r)) - self.assertTrue(not_meeting.group.acronym in unicontent(r)) + self.assertContains(r, requested_session.group.acronym) + self.assertContains(r, not_meeting.group.acronym) def test_request_minutes(self): meeting = MeetingFactory(type_id='ietf') @@ -2160,8 +2125,8 @@ class SessionTests(TestCase): url = urlreverse('ietf.meeting.views.request_minutes',kwargs={'num':meeting.number}) login_testing_unauthorized(self,"secretary",url) r = self.client.get(url) - self.assertNotIn(has_minutes.group.acronym, unicontent(r).lower()) - self.assertIn(has_no_minutes.group.acronym, unicontent(r).lower()) + self.assertNotContains(r, has_minutes.group.acronym.upper()) + self.assertContains(r, has_no_minutes.group.acronym.upper()) r = self.client.post(url,{'to':'wgchairs@ietf.org', 'cc': 'irsg@irtf.org', 'subject': 'I changed the subject', diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index f57d3b625..4a845ec33 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -1,7 +1,14 @@ +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import json -import urllib2 +import six.moves.urllib.request +from six.moves.urllib.error import HTTPError from django.conf import settings from django.template.loader import render_to_string @@ -87,8 +94,8 @@ def create_proceedings_templates(meeting): # Get meeting attendees from registration system url = settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number) try: - attendees = json.load(urllib2.urlopen(url)) - except (ValueError, urllib2.HTTPError): + attendees = json.load(six.moves.urllib.request.urlopen(url)) + except (ValueError, HTTPError): attendees = [] if attendees: diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 585dba5a5..083c961d0 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -1,18 +1,24 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import csv import datetime import glob +import io import json import os import pytz import re +import six import tarfile -import urllib + from calendar import timegm from collections import OrderedDict, Counter, deque +from six.moves.urllib.parse import unquote from tempfile import mkstemp from wsgiref.handlers import format_date_time @@ -171,11 +177,11 @@ def current_materials(request): def materials_document(request, document, num=None, ext=None): if num is None: num = get_meeting(num).number - if (re.search('^\w+-\d+-.+-\d\d$', document) or - re.search('^\w+-interim-\d+-.+-\d\d-\d\d$', document) or - re.search('^\w+-interim-\d+-.+-sess[a-z]-\d\d$', document) or - re.search('^minutes-interim-\d+-.+-\d\d$', document) or - re.search('^slides-interim-\d+-.+-\d\d$', document)): + if (re.search(r'^\w+-\d+-.+-\d\d$', document) or + re.search(r'^\w+-interim-\d+-.+-\d\d-\d\d$', document) or + re.search(r'^\w+-interim-\d+-.+-sess[a-z]-\d\d$', document) or + re.search(r'^minutes-interim-\d+-.+-\d\d$', document) or + re.search(r'^slides-interim-\d+-.+-\d\d$', document)): name, rev = document.rsplit('-', 1) else: name, rev = document, None @@ -199,7 +205,7 @@ def materials_document(request, document, num=None, ext=None): _, basename = os.path.split(filename) if not os.path.exists(filename): raise Http404("File not found: %s" % filename) - with open(filename, 'rb') as file: + with io.open(filename, 'rb') as file: bytes = file.read() mtype, chset = get_mime_type(bytes) @@ -518,12 +524,12 @@ def agenda(request, num=None, name=None, base=None, ext=None, owner=None, utc="" def agenda_csv(schedule, filtered_assignments): response = HttpResponse(content_type="text/csv; charset=%s"%settings.DEFAULT_CHARSET) - writer = csv.writer(response, delimiter=',', quoting=csv.QUOTE_ALL) + writer = csv.writer(response, delimiter=str(','), quoting=csv.QUOTE_ALL) headings = ["Date", "Start", "End", "Session", "Room", "Area", "Acronym", "Type", "Description", "Session ID", "Agenda", "Slides"] def write_row(row): - encoded_row = [v.encode('utf-8') if isinstance(v, unicode) else v for v in row] + encoded_row = [v.encode('utf-8') if isinstance(v, six.text_type) else v for v in row] while len(encoded_row) < len(headings): encoded_row.append(None) # produce empty entries at the end as necessary @@ -681,7 +687,7 @@ def session_draft_tarfile(request, num, acronym): tarstream = tarfile.open('','w:gz',response) mfh, mfn = mkstemp() os.close(mfh) - manifest = open(mfn, "w") + manifest = io.open(mfn, "w") for doc_name in drafts: pdf_path = os.path.join(settings.INTERNET_DRAFT_PDF_PATH, doc_name + ".pdf") @@ -693,7 +699,7 @@ def session_draft_tarfile(request, num, acronym): try: tarstream.add(pdf_path, str(doc_name + ".pdf")) manifest.write("Included: "+pdf_path+"\n") - except Exception, e: + except Exception as e: manifest.write(("Failed (%s): "%e)+pdf_path+"\n") else: manifest.write("Not found: "+pdf_path+"\n") @@ -709,7 +715,7 @@ def session_draft_pdf(request, num, acronym): curr_page = 1 pmh, pmn = mkstemp() os.close(pmh) - pdfmarks = open(pmn, "w") + pdfmarks = io.open(pmn, "w") pdf_list = "" for draft in drafts: @@ -730,7 +736,7 @@ def session_draft_pdf(request, num, acronym): code, out, err = pipe(gs + " -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -sOutputFile=" + pdfn + " " + pdf_list + " " + pmn) assertion('code == 0') - pdf = open(pdfn,"r") + pdf = io.open(pdfn,"rb") pdf_contents = pdf.read() pdf.close() @@ -873,7 +879,7 @@ def ical_agenda(request, num=None, name=None, acronym=None, session_id=None): raise Http404 q = request.META.get('QUERY_STRING','') or "" - filter = set(urllib.unquote(q).lower().split(',')) + filter = set(unquote(q).lower().split(',')) include = [ i for i in filter if not (i.startswith('-') or i.startswith('~')) ] include_types = set(["plenary","other"]) exclude = [] @@ -1583,7 +1589,7 @@ def propose_session_slides(request, session_id, num): name = 'slides-%s-%s' % (session.meeting.number, session.docname_token()) name = name + '-' + slugify(title).replace('_', '-')[:128] filename = '%s-00%s'% (name, ext) - destination = open(os.path.join(settings.SLIDE_STAGING_PATH, filename),'wb+') + destination = io.open(os.path.join(settings.SLIDE_STAGING_PATH, filename),'wb+') for chunk in file.chunks(): destination.write(chunk) destination.close() @@ -2148,7 +2154,7 @@ def proceedings(request, num=None): meeting = get_meeting(num) - if meeting.number <= 64 or not meeting.agenda or not meeting.agenda.assignments.exists(): + if (meeting.number.isdigit() and int(meeting.number) <= 64) or not meeting.agenda or not meeting.agenda.assignments.exists(): return HttpResponseRedirect( 'https://www.ietf.org/proceedings/%s' % num ) begin_date = meeting.get_submission_start_date() @@ -2179,7 +2185,7 @@ def finalize_proceedings(request, num=None): meeting = get_meeting(num) - if meeting.number <= 64 or not meeting.agenda or not meeting.agenda.assignments.exists() or meeting.proceedings_final: + if (meeting.number.isdigit() and int(meeting.number) <= 64) or not meeting.agenda or not meeting.agenda.assignments.exists() or meeting.proceedings_final: raise Http404 if request.method=='POST': diff --git a/ietf/message/migrations/0001_initial.py b/ietf/message/migrations/0001_initial.py index 6a004fd87..9fe4ff38e 100644 --- a/ietf/message/migrations/0001_initial.py +++ b/ietf/message/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import datetime from django.db import migrations, models diff --git a/ietf/message/migrations/0002_add_message_docs2_m2m.py b/ietf/message/migrations/0002_add_message_docs2_m2m.py index 6cb7f27a0..1402fa8d8 100644 --- a/ietf/message/migrations/0002_add_message_docs2_m2m.py +++ b/ietf/message/migrations/0002_add_message_docs2_m2m.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-21 14:23 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/message/migrations/0003_set_document_m2m_keys.py b/ietf/message/migrations/0003_set_document_m2m_keys.py index a6da567d4..493adc2ee 100644 --- a/ietf/message/migrations/0003_set_document_m2m_keys.py +++ b/ietf/message/migrations/0003_set_document_m2m_keys.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-21 14:27 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys @@ -20,7 +22,7 @@ def forward(apps, schema_editor): # Document id fixup ------------------------------------------------------------ objs = Document.objects.in_bulk() - nameid = { o.name: o.id for id, o in objs.iteritems() } + nameid = { o.name: o.id for id, o in objs.items() } sys.stderr.write('\n') diff --git a/ietf/message/migrations/0004_1_del_docs_m2m_table.py b/ietf/message/migrations/0004_1_del_docs_m2m_table.py index 328b6cdcf..670609fc7 100644 --- a/ietf/message/migrations/0004_1_del_docs_m2m_table.py +++ b/ietf/message/migrations/0004_1_del_docs_m2m_table.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-22 08:01 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/message/migrations/0004_2_add_docs_m2m_table.py b/ietf/message/migrations/0004_2_add_docs_m2m_table.py index 064b2e085..9da02c6e6 100644 --- a/ietf/message/migrations/0004_2_add_docs_m2m_table.py +++ b/ietf/message/migrations/0004_2_add_docs_m2m_table.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-22 08:01 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/message/migrations/0005_copy_docs_m2m_table.py b/ietf/message/migrations/0005_copy_docs_m2m_table.py index 7959d0365..f85def627 100644 --- a/ietf/message/migrations/0005_copy_docs_m2m_table.py +++ b/ietf/message/migrations/0005_copy_docs_m2m_table.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-27 05:56 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys, time diff --git a/ietf/message/migrations/0006_remove_docs2_m2m.py b/ietf/message/migrations/0006_remove_docs2_m2m.py index 527eb5595..58743da07 100644 --- a/ietf/message/migrations/0006_remove_docs2_m2m.py +++ b/ietf/message/migrations/0006_remove_docs2_m2m.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-30 03:32 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/message/models.py b/ietf/message/models.py index 6fc243dc0..9a03bde36 100644 --- a/ietf/message/models.py +++ b/ietf/message/models.py @@ -1,7 +1,14 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import email.utils from django.db import models +from django.utils.encoding import python_2_unicode_compatible import debug # pyflakes:ignore @@ -12,6 +19,7 @@ from ietf.name.models import RoleName from ietf.utils.models import ForeignKey from ietf.utils.mail import get_email_addresses_from_text +@python_2_unicode_compatible class Message(models.Model): time = models.DateTimeField(default=datetime.datetime.now) by = ForeignKey(Person) @@ -32,7 +40,7 @@ class Message(models.Model): class Meta: ordering = ['time'] - def __unicode__(self): + def __str__(self): return "'%s' %s -> %s" % (self.subject, self.frm, self.to) def get(self, field): @@ -40,6 +48,7 @@ class Message(models.Model): return r if isinstance(r, list) else get_email_addresses_from_text(r) +@python_2_unicode_compatible class MessageAttachment(models.Model): message = ForeignKey(Message) filename = models.CharField(max_length=255, db_index=True, blank=True) @@ -48,10 +57,11 @@ class MessageAttachment(models.Model): removed = models.BooleanField(default=False) body = models.TextField() - def __unicode__(self): + def __str__(self): return self.filename +@python_2_unicode_compatible class SendQueue(models.Model): time = models.DateTimeField(default=datetime.datetime.now) by = ForeignKey(Person) @@ -66,16 +76,17 @@ class SendQueue(models.Model): class Meta: ordering = ['time'] - def __unicode__(self): - return u"'%s' %s -> %s (sent at %s)" % (self.message.subject, self.message.frm, self.message.to, self.sent_at or "") + def __str__(self): + return "'%s' %s -> %s (sent at %s)" % (self.message.subject, self.message.frm, self.message.to, self.sent_at or "") +@python_2_unicode_compatible class AnnouncementFrom(models.Model): name = ForeignKey(RoleName) group = ForeignKey(Group) address = models.CharField(max_length=255) - def __unicode__(self): + def __str__(self): return self.address class Meta: diff --git a/ietf/message/resources.py b/ietf/message/resources.py index f1e77cb5b..30c7ac1e8 100644 --- a/ietf/message/resources.py +++ b/ietf/message/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2014-11-13 23:53 + + from ietf.api import ModelResource from ietf.api import ToOneField from tastypie.fields import ToManyField diff --git a/ietf/message/tests.py b/ietf/message/tests.py index 5a69b3cd8..5781a72d8 100644 --- a/ietf/message/tests.py +++ b/ietf/message/tests.py @@ -1,8 +1,14 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime from django.urls import reverse as urlreverse -from ietf.utils.test_utils import TestCase, unicontent +from ietf.utils.test_utils import TestCase from ietf.utils.mail import outbox from ietf.message.models import Message, SendQueue @@ -25,10 +31,10 @@ class MessageTests(TestCase): r = self.client.get(urlreverse("ietf.message.views.message", kwargs=dict(message_id=msg.id))) self.assertEqual(r.status_code, 200) - self.assertTrue(msg.subject in unicontent(r)) - self.assertTrue(msg.to in unicontent(r)) - self.assertTrue(msg.frm in unicontent(r)) - self.assertTrue("Hello World!" in unicontent(r)) + self.assertContains(r, msg.subject) + self.assertContains(r, msg.to) + self.assertContains(r, msg.frm) + self.assertContains(r, "Hello World!") class SendScheduledAnnouncementsTests(TestCase): @@ -40,7 +46,7 @@ class SendScheduledAnnouncementsTests(TestCase): frm="testmonkey@example.com", cc="cc.a@example.com, cc.b@example.com", bcc="bcc@example.com", - body=u"Hello World!", + body="Hello World!", content_type="", ) @@ -66,7 +72,7 @@ class SendScheduledAnnouncementsTests(TestCase): frm="testmonkey@example.com", cc="cc.a@example.com, cc.b@example.com", bcc="bcc@example.com", - body=u'--NextPart\r\n\r\nA New Internet-Draft is available from the on-line Internet-Drafts directories.\r\n--NextPart\r\nContent-Type: Message/External-body;\r\n\tname="draft-huang-behave-bih-01.txt";\r\n\tsite="ftp.ietf.org";\r\n\taccess-type="anon-ftp";\r\n\tdirectory="internet-drafts"\r\n\r\nContent-Type: text/plain\r\nContent-ID: <2010-07-30001541.I-D@ietf.org>\r\n\r\n--NextPart--', + body='--NextPart\r\n\r\nA New Internet-Draft is available from the on-line Internet-Drafts directories.\r\n--NextPart\r\nContent-Type: Message/External-body;\r\n\tname="draft-huang-behave-bih-01.txt";\r\n\tsite="ftp.ietf.org";\r\n\taccess-type="anon-ftp";\r\n\tdirectory="internet-drafts"\r\n\r\nContent-Type: text/plain\r\nContent-ID: <2010-07-30001541.I-D@ietf.org>\r\n\r\n--NextPart--', content_type='Multipart/Mixed; Boundary="NextPart"', ) diff --git a/ietf/message/utils.py b/ietf/message/utils.py index 5fae361fd..1460fed62 100644 --- a/ietf/message/utils.py +++ b/ietf/message/utils.py @@ -1,21 +1,29 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import re, datetime, email -from ietf.utils.mail import send_mail_text, send_mail_mime +from django.utils.encoding import force_str + +from ietf.utils.mail import send_mail_text, send_mail_mime, get_payload from ietf.message.models import Message first_dot_on_line_re = re.compile(r'^\.', re.MULTILINE) def infer_message(s): - parsed = email.message_from_string(s.encode("utf-8")) + parsed = email.message_from_string(force_str(s)) m = Message() - m.subject = parsed.get("Subject", "").decode("utf-8") - m.frm = parsed.get("From", "").decode("utf-8") - m.to = parsed.get("To", "").decode("utf-8") - m.cc = parsed.get("Cc", "").decode("utf-8") - m.bcc = parsed.get("Bcc", "").decode("utf-8") - m.reply_to = parsed.get("Reply-To", "").decode("utf-8") - m.body = parsed.get_payload().decode("utf-8") + m.subject = parsed.get("Subject", "") + m.frm = parsed.get("From", "") + m.to = parsed.get("To", "") + m.cc = parsed.get("Cc", "") + m.bcc = parsed.get("Bcc", "") + m.reply_to = parsed.get("Reply-To", "") + m.body = get_payload(parsed) return m @@ -40,7 +48,7 @@ def send_scheduled_message_from_send_queue(send_queue): # make body a real message so we can parse it body = ("MIME-Version: 1.0\r\nContent-Type: %s\r\n" % message.content_type) + body - msg = email.message_from_string(body.encode("utf-8")) + msg = email.message_from_string(force_str(body)) send_mail_mime(None, message.to, message.frm, message.subject, msg, cc=message.cc, bcc=message.bcc) diff --git a/ietf/middleware.py b/ietf/middleware.py index b8098d2df..b0759894e 100644 --- a/ietf/middleware.py +++ b/ietf/middleware.py @@ -1,4 +1,8 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import connection from django.db.utils import OperationalError @@ -14,9 +18,9 @@ import unicodedata def sql_log_middleware(get_response): def sql_log(request): response = get_response(request) - for q in connection.queries: - if re.match('(update|insert)', q['sql'], re.IGNORECASE): - log(q['sql']) + for q in connection.queries: + if re.match('(update|insert)', q['sql'], re.IGNORECASE): + log(q['sql']) return response return sql_log @@ -26,11 +30,11 @@ class SMTPExceptionMiddleware(object): def __call__(self, request): return self.get_response(request) def process_exception(self, request, exception): - if isinstance(exception, smtplib.SMTPException): + if isinstance(exception, smtplib.SMTPException): (extype, value, tb) = log_smtp_exception(exception) - return render(request, 'email_failed.html', + return render(request, 'email_failed.html', {'exception': extype, 'args': value, 'traceback': "".join(tb)} ) - return None + return None class Utf8ExceptionMiddleware(object): def __init__(self, get_response): @@ -38,27 +42,27 @@ class Utf8ExceptionMiddleware(object): def __call__(self, request): return self.get_response(request) def process_exception(self, request, exception): - if isinstance(exception, OperationalError): + if isinstance(exception, OperationalError): extype, value, tb = exc_parts() if value[0] == 1366: log("Database 4-byte utf8 exception: %s: %s" % (extype, value)) return render(request, 'utf8_4byte_failed.html', {'exception': extype, 'args': value, 'traceback': "".join(tb)} ) - return None + return None def redirect_trailing_period_middleware(get_response): def redirect_trailing_period(request): response = get_response(request) - if response.status_code == 404 and request.path.endswith("."): - return HttpResponsePermanentRedirect(request.path.rstrip(".")) - return response + if response.status_code == 404 and request.path.endswith("."): + return HttpResponsePermanentRedirect(request.path.rstrip(".")) + return response return redirect_trailing_period def unicode_nfkc_normalization_middleware(get_response): def unicode_nfkc_normalization(request): """Do Unicode NFKC normalization to turn ligatures into individual characters. This was prompted by somebody actually requesting an url for /wg/ipfix/charter - where the 'fi' was composed of an \ufb01 ligature... + where the 'fi' was composed of an \\ufb01 ligature... There are probably other elements of a request which may need this normalization too, but let's put that in as it comes up, rather than guess ahead. diff --git a/ietf/name/generate_fixtures.py b/ietf/name/generate_fixtures.py index 707c54aff..c453b6c53 100644 --- a/ietf/name/generate_fixtures.py +++ b/ietf/name/generate_fixtures.py @@ -1,8 +1,10 @@ +# Copyright The IETF Trust 2011-2019, All Rights Reserved #!/usr/bin/python # simple script for exporting name related base data for the tests # boiler plate +import io import os, sys import django @@ -17,7 +19,7 @@ from django.core.serializers import serialize def output(name, seq): try: - f = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/%s.json" % name), 'w') + f = io.open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures/%s.json" % name), 'w') f.write(serialize("json", seq, indent=1)) f.close() except: diff --git a/ietf/name/migrations/0001_initial.py b/ietf/name/migrations/0001_initial.py index fab75d062..72b30039f 100644 --- a/ietf/name/migrations/0001_initial.py +++ b/ietf/name/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/name/migrations/0002_agendatypename.py b/ietf/name/migrations/0002_agendatypename.py index 11fd824df..cf9f42950 100644 --- a/ietf/name/migrations/0002_agendatypename.py +++ b/ietf/name/migrations/0002_agendatypename.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.13 on 2018-07-10 13:47 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/name/migrations/0003_agendatypename_data.py b/ietf/name/migrations/0003_agendatypename_data.py index 175ffb66e..48d72129d 100644 --- a/ietf/name/migrations/0003_agendatypename_data.py +++ b/ietf/name/migrations/0003_agendatypename_data.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.13 on 2018-07-10 13:47 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/name/migrations/0004_add_prefix_to_doctypenames.py b/ietf/name/migrations/0004_add_prefix_to_doctypenames.py index 2749f561f..038b7c2a5 100644 --- a/ietf/name/migrations/0004_add_prefix_to_doctypenames.py +++ b/ietf/name/migrations/0004_add_prefix_to_doctypenames.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-10-19 11:34 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/name/migrations/0005_reviewassignmentstatename.py b/ietf/name/migrations/0005_reviewassignmentstatename.py index dbf253e51..c8660920d 100644 --- a/ietf/name/migrations/0005_reviewassignmentstatename.py +++ b/ietf/name/migrations/0005_reviewassignmentstatename.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.18 on 2019-01-04 13:59 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/name/migrations/0006_adjust_statenames.py b/ietf/name/migrations/0006_adjust_statenames.py index f3954906d..116e25f3c 100644 --- a/ietf/name/migrations/0006_adjust_statenames.py +++ b/ietf/name/migrations/0006_adjust_statenames.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.18 on 2019-01-04 14:02 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/name/models.py b/ietf/name/models.py index 5a257c22f..453b7042e 100644 --- a/ietf/name/models.py +++ b/ietf/name/models.py @@ -1,9 +1,15 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import models +from django.utils.encoding import python_2_unicode_compatible from ietf.utils.models import ForeignKey +@python_2_unicode_compatible class NameModel(models.Model): slug = models.CharField(max_length=32, primary_key=True) name = models.CharField(max_length=255) @@ -11,7 +17,7 @@ class NameModel(models.Model): used = models.BooleanField(default=True) order = models.IntegerField(default=0) - def __unicode__(self): + def __str__(self): return self.name class Meta: diff --git a/ietf/nomcom/admin.py b/ietf/nomcom/admin.py index d6e0170d9..9e8ba57b3 100644 --- a/ietf/nomcom/admin.py +++ b/ietf/nomcom/admin.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.contrib import admin from ietf.nomcom.models import ( ReminderDates, NomCom, Nomination, Nominee, NomineePosition, @@ -43,7 +49,7 @@ admin.site.register(Position, PositionAdmin) class FeedbackAdmin(admin.ModelAdmin): def nominee(self, obj): - return u", ".join(n.person.ascii for n in obj.nominees.all()) + return ", ".join(n.person.ascii for n in obj.nominees.all()) nominee.admin_order_field = 'nominees__person__ascii' list_display = ['id', 'nomcom', 'author', 'nominee', 'subject', 'type', 'user', 'time'] diff --git a/ietf/nomcom/decorators.py b/ietf/nomcom/decorators.py index 231661799..bbaebd0cd 100644 --- a/ietf/nomcom/decorators.py +++ b/ietf/nomcom/decorators.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import functools from django.urls import reverse @@ -9,7 +15,7 @@ def nomcom_private_key_required(view_func): def inner(request, *args, **kwargs): year = kwargs.get('year', None) if not year: - raise Exception, 'View decorated with nomcom_private_key_required must receive a year argument' + raise Exception('View decorated with nomcom_private_key_required must receive a year argument') if not 'NOMCOM_PRIVATE_KEY_%s' % year in request.session: return HttpResponseRedirect('%s?back_to=%s' % (reverse('ietf.nomcom.views.private_key', None, args=(year, )), urlquote(request.get_full_path()))) else: diff --git a/ietf/nomcom/factories.py b/ietf/nomcom/factories.py index 2509c1421..1f41466a0 100644 --- a/ietf/nomcom/factories.py +++ b/ietf/nomcom/factories.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import factory import random @@ -164,9 +170,13 @@ class FeedbackFactory(factory.DjangoModelFactory): nomcom = factory.SubFactory(NomComFactory) subject = factory.Faker('sentence') - comments = factory.Faker('paragraph') type_id = 'comment' + @factory.post_generation + def comments(obj, create, extracted, **kwargs): + comment_text = factory.Faker('paragraph').generate() + obj.comments = obj.nomcom.encrypt(comment_text) + class TopicFactory(factory.DjangoModelFactory): class Meta: model = Topic diff --git a/ietf/nomcom/fields.py b/ietf/nomcom/fields.py index eab7111da..be8c5ba6c 100644 --- a/ietf/nomcom/fields.py +++ b/ietf/nomcom/fields.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.conf import settings from django.db import models from django.utils.encoding import smart_str @@ -19,7 +25,7 @@ class EncryptedTextField(models.TextField): raise ValueError("Trying to read the NomCom public key: " + str(e)) command = "%s smime -encrypt -in /dev/stdin %s" % (settings.OPENSSL_COMMAND, cert_file) - code, out, error = pipe(command, comments) + code, out, error = pipe(command, comments.encode('utf-8')) if code != 0: log("openssl error: %s:\n Error %s: %s" %(command, code, error)) if not error: diff --git a/ietf/nomcom/forms.py b/ietf/nomcom/forms.py index e7791f12e..401bc6532 100644 --- a/ietf/nomcom/forms.py +++ b/ietf/nomcom/forms.py @@ -1,3 +1,11 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import six + from django.conf import settings from django import forms from django.urls import reverse @@ -32,12 +40,12 @@ class PositionNomineeField(forms.ChoiceField): results = [] for position in positions: accepted_nominees = [np.nominee for np in NomineePosition.objects.filter(position=position,state='accepted').exclude(nominee__duplicated__isnull=False)] - nominees = [('%s_%s' % (position.id, i.id), unicode(i)) for i in accepted_nominees] + nominees = [('%s_%s' % (position.id, i.id), six.text_type(i)) for i in accepted_nominees] if nominees: results.append((position.name+" (Accepted)", nominees)) for position in positions: other_nominees = [np.nominee for np in NomineePosition.objects.filter(position=position).exclude(state='accepted').exclude(nominee__duplicated__isnull=False)] - nominees = [('%s_%s' % (position.id, i.id), unicode(i)) for i in other_nominees] + nominees = [('%s_%s' % (position.id, i.id), six.text_type(i)) for i in other_nominees] if nominees: results.append((position.name+" (Declined or Pending)", nominees)) kwargs['choices'] = results @@ -292,7 +300,7 @@ class NominateForm(forms.ModelForm): # Complete nomination data feedback = Feedback.objects.create(nomcom=self.nomcom, - comments=qualifications, + comments=self.nomcom.encrypt(qualifications), type=FeedbackTypeName.objects.get(slug='nomina'), user=self.user) feedback.positions.add(position) @@ -407,7 +415,7 @@ class NominateNewPersonForm(forms.ModelForm): # Complete nomination data feedback = Feedback.objects.create(nomcom=self.nomcom, - comments=qualifications, + comments=self.nomcom.encrypt(qualifications), type=FeedbackTypeName.objects.get(slug='nomina'), user=self.user) feedback.positions.add(position) @@ -450,7 +458,7 @@ class NominateNewPersonForm(forms.ModelForm): class FeedbackForm(forms.ModelForm): nominator_email = forms.CharField(label='Commenter email',required=False) - comments = forms.CharField(label='Comments', widget=forms.Textarea(), strip=False) + comment_text = forms.CharField(label='Comments', widget=forms.Textarea(), strip=False) confirmation = forms.BooleanField(label='Email comments back to me as confirmation (if selected, your comments will be emailed to you in cleartext when you press Save).', required=False) @@ -483,13 +491,13 @@ class FeedbackForm(forms.ModelForm): if not NomineePosition.objects.accepted().filter(nominee=self.nominee, position=self.position): msg = "There isn't a accepted nomination for %s on the %s position" % (self.nominee, self.position) - self._errors["comments"] = self.error_class([msg]) + self._errors["comment_text"] = self.error_class([msg]) return self.cleaned_data def save(self, commit=True): feedback = super(FeedbackForm, self).save(commit=False) confirmation = self.cleaned_data['confirmation'] - comments = self.cleaned_data['comments'] + comment_text = self.cleaned_data['comment_text'] nomcom_template_path = '/nomcom/%s/' % self.nomcom.group.acronym author = None @@ -507,6 +515,7 @@ class FeedbackForm(forms.ModelForm): feedback.nomcom = self.nomcom feedback.user = self.user feedback.type = FeedbackTypeName.objects.get(slug='comment') + feedback.comments = self.nomcom.encrypt(comment_text) feedback.save() if self.nominee and self.position: feedback.positions.add(self.position) @@ -525,7 +534,7 @@ class FeedbackForm(forms.ModelForm): elif self.topic: about = self.topic.subject context = {'about': about, - 'comments': comments, + 'comments': comment_text, 'year': self.nomcom.year(), } path = nomcom_template_path + FEEDBACK_RECEIPT_TEMPLATE @@ -536,7 +545,6 @@ class FeedbackForm(forms.ModelForm): model = Feedback fields = ( 'nominator_email', - 'comments', 'confirmation', ) @@ -553,8 +561,9 @@ class FeedbackEmailForm(forms.Form): class QuestionnaireForm(forms.ModelForm): - comments = forms.CharField(label='Questionnaire response from this candidate', + comment_text = forms.CharField(label='Questionnaire response from this candidate', widget=forms.Textarea(), strip=False) + def __init__(self, *args, **kwargs): self.nomcom = kwargs.pop('nomcom', None) self.user = kwargs.pop('user', None) @@ -564,6 +573,7 @@ class QuestionnaireForm(forms.ModelForm): def save(self, commit=True): feedback = super(QuestionnaireForm, self).save(commit=False) + comment_text = self.cleaned_data['comment_text'] (position, nominee) = self.cleaned_data['nominee'] author = get_user_email(self.user) @@ -574,6 +584,7 @@ class QuestionnaireForm(forms.ModelForm): feedback.nomcom = self.nomcom feedback.user = self.user feedback.type = FeedbackTypeName.objects.get(slug='questio') + feedback.comments = self.nomcom.encrypt(comment_text) feedback.save() self.save_m2m() feedback.nominees.add(nominee) @@ -581,7 +592,7 @@ class QuestionnaireForm(forms.ModelForm): class Meta: model = Feedback - fields = ( 'comments', ) + fields = [] class NomComTemplateForm(DBTemplateForm): content = forms.CharField(label="Text", widget=forms.Textarea(attrs={'cols': '120', 'rows':'40', }), strip=False) diff --git a/ietf/nomcom/management/commands/feedback_email.py b/ietf/nomcom/management/commands/feedback_email.py index 99a003bd9..7774a6c08 100644 --- a/ietf/nomcom/management/commands/feedback_email.py +++ b/ietf/nomcom/management/commands/feedback_email.py @@ -1,3 +1,10 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import sys from django.core.management.base import BaseCommand, CommandError @@ -10,7 +17,7 @@ from ietf.nomcom.fields import EncryptedException import debug # pyflakes:ignore class Command(BaseCommand): - help = (u"Receive nomcom email, encrypt and save it.") + help = ("Receive nomcom email, encrypt and save it.") def add_arguments(self, parser): parser.add_argument('--nomcom-year', dest='year', help='NomCom year') @@ -30,7 +37,7 @@ class Command(BaseCommand): if not email: msg = sys.stdin.read() else: - msg = open(email, "r").read() + msg = io.open(email, "r").read() try: nomcom = NomCom.objects.get(group__acronym__icontains=year, @@ -40,6 +47,6 @@ class Command(BaseCommand): try: feedback = create_feedback_email(nomcom, msg) - log(u"Received nomcom email from %s" % feedback.author) + log("Received nomcom email from %s" % feedback.author) except (EncryptedException, ValueError) as e: raise CommandError(e) diff --git a/ietf/nomcom/management/commands/make_dummy_nomcom.py b/ietf/nomcom/management/commands/make_dummy_nomcom.py index a6724a326..92d71dcac 100644 --- a/ietf/nomcom/management/commands/make_dummy_nomcom.py +++ b/ietf/nomcom/management/commands/make_dummy_nomcom.py @@ -1,6 +1,8 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2017-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals import socket @@ -14,7 +16,7 @@ from ietf.group.models import Group from ietf.person.models import User class Command(BaseCommand): - help = (u"Create (or delete) a dummy nomcom for test and development purposes.") + help = ("Create (or delete) a dummy nomcom for test and development purposes.") def add_arguments(self, parser): parser.add_argument('--delete', dest='delete', action='store_true', help='Delete the test and development dummy nomcom') @@ -39,22 +41,22 @@ class Command(BaseCommand): populate_personnel=False, populate_positions=False)) - e = EmailFactory(person__name=u'Dummy Chair', address=u'dummychair@example.com', person__user__username=u'dummychair', person__default_emails=False, origin='dummychair') + e = EmailFactory(person__name='Dummy Chair', address='dummychair@example.com', person__user__username='dummychair', person__default_emails=False, origin='dummychair') e.person.user.set_password('password') e.person.user.save() - nc.group.role_set.create(name_id=u'chair',person=e.person,email=e) + nc.group.role_set.create(name_id='chair',person=e.person,email=e) - e = EmailFactory(person__name=u'Dummy Member', address=u'dummymember@example.com', person__user__username=u'dummymember', person__default_emails=False, origin='dummymember') + e = EmailFactory(person__name='Dummy Member', address='dummymember@example.com', person__user__username='dummymember', person__default_emails=False, origin='dummymember') e.person.user.set_password('password') e.person.user.save() - nc.group.role_set.create(name_id=u'member',person=e.person,email=e) + nc.group.role_set.create(name_id='member',person=e.person,email=e) - e = EmailFactory(person__name=u'Dummy Candidate', address=u'dummycandidate@example.com', person__user__username=u'dummycandidate', person__default_emails=False, origin='dummycandidate') + e = EmailFactory(person__name='Dummy Candidate', address='dummycandidate@example.com', person__user__username='dummycandidate', person__default_emails=False, origin='dummycandidate') e.person.user.set_password('password') e.person.user.save() NomineePositionFactory(nominee__nomcom=nc, nominee__person=e.person, - position__nomcom=nc, position__name=u'Dummy Area Director', + position__nomcom=nc, position__name='Dummy Area Director', ) self.stdout.write("%s\n" % key) diff --git a/ietf/nomcom/management/commands/send_reminders.py b/ietf/nomcom/management/commands/send_reminders.py index b8e83fd09..a4a442de8 100644 --- a/ietf/nomcom/management/commands/send_reminders.py +++ b/ietf/nomcom/management/commands/send_reminders.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import syslog @@ -17,7 +23,7 @@ def is_time_to_send(nomcom,send_date,nomination_date): return bool(nomcom.reminderdates_set.filter(date=send_date)) class Command(BaseCommand): - help = (u"Send acceptance and questionnaire reminders to nominees") + help = ("Send acceptance and questionnaire reminders to nominees") def handle(self, *args, **options): for nomcom in NomCom.objects.filter(group__state__slug='active'): diff --git a/ietf/nomcom/migrations/0001_initial.py b/ietf/nomcom/migrations/0001_initial.py index ca498d684..09bdebd6f 100644 --- a/ietf/nomcom/migrations/0001_initial.py +++ b/ietf/nomcom/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.conf import settings from django.db import migrations, models diff --git a/ietf/nomcom/migrations/0002_auto_20180918_0550.py b/ietf/nomcom/migrations/0002_auto_20180918_0550.py index ad4b76928..9d32121c9 100644 --- a/ietf/nomcom/migrations/0002_auto_20180918_0550.py +++ b/ietf/nomcom/migrations/0002_auto_20180918_0550.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.15 on 2018-09-18 05:50 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.conf import settings from django.db import migrations diff --git a/ietf/nomcom/migrations/0003_nomcom_show_accepted_nominees.py b/ietf/nomcom/migrations/0003_nomcom_show_accepted_nominees.py index a6691afd2..5076c70a8 100644 --- a/ietf/nomcom/migrations/0003_nomcom_show_accepted_nominees.py +++ b/ietf/nomcom/migrations/0003_nomcom_show_accepted_nominees.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.15 on 2018-09-26 11:10 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/nomcom/migrations/0004_set_show_accepted_nominees_false_on_existing_nomcoms.py b/ietf/nomcom/migrations/0004_set_show_accepted_nominees_false_on_existing_nomcoms.py index 49bf943d9..d3ab2f478 100644 --- a/ietf/nomcom/migrations/0004_set_show_accepted_nominees_false_on_existing_nomcoms.py +++ b/ietf/nomcom/migrations/0004_set_show_accepted_nominees_false_on_existing_nomcoms.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.15 on 2018-09-26 11:12 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/nomcom/migrations/0005_auto_20181008_0602.py b/ietf/nomcom/migrations/0005_auto_20181008_0602.py index e35b95660..9a5a37d4f 100644 --- a/ietf/nomcom/migrations/0005_auto_20181008_0602.py +++ b/ietf/nomcom/migrations/0005_auto_20181008_0602.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-10-08 06:02 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/nomcom/models.py b/ietf/nomcom/models.py index 9429d2541..d0977acea 100644 --- a/ietf/nomcom/models.py +++ b/ietf/nomcom/models.py @@ -1,4 +1,7 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + import os from django.db import models @@ -7,23 +10,27 @@ from django.conf import settings from django.contrib.auth.models import User from django.template.loader import render_to_string from django.template.defaultfilters import linebreaks +from django.utils.encoding import python_2_unicode_compatible import debug # pyflakes:ignore -from ietf.nomcom.fields import EncryptedTextField from ietf.person.models import Person,Email from ietf.group.models import Group from ietf.name.models import NomineePositionStateName, FeedbackTypeName, TopicAudienceName from ietf.dbtemplate.models import DBTemplate -from ietf.nomcom.managers import NomineePositionManager, NomineeManager, \ - PositionManager, FeedbackManager +from ietf.nomcom.managers import (NomineePositionManager, NomineeManager, + PositionManager, FeedbackManager, ) from ietf.nomcom.utils import (initialize_templates_for_group, initialize_questionnaire_for_position, initialize_requirements_for_position, initialize_description_for_topic, - delete_nomcom_templates) + delete_nomcom_templates, + EncryptedException, + ) +from ietf.utils.log import log from ietf.utils.models import ForeignKey +from ietf.utils.pipe import pipe from ietf.utils.storage import NoLocationMigrationFileSystemStorage @@ -36,6 +43,7 @@ class ReminderDates(models.Model): nomcom = ForeignKey('NomCom') +@python_2_unicode_compatible class NomCom(models.Model): public_key = models.FileField(storage=NoLocationMigrationFileSystemStorage(location=settings.NOMCOM_PUBLIC_KEYS_DIR), upload_to=upload_path_handler, blank=True, null=True) @@ -58,7 +66,7 @@ class NomCom(models.Model): verbose_name_plural = 'NomComs' verbose_name = 'NomCom' - def __unicode__(self): + def __str__(self): return self.group.acronym def save(self, *args, **kwargs): @@ -78,6 +86,21 @@ class NomCom(models.Model): def pending_email_count(self): return self.feedback_set.filter(type__isnull=True).count() + def encrypt(self, cleartext): + try: + cert_file = self.public_key.path + except ValueError as e: + raise ValueError("Trying to read the NomCom public key: " + str(e)) + + command = "%s smime -encrypt -in /dev/stdin %s" % (settings.OPENSSL_COMMAND, cert_file) + code, out, error = pipe(command, cleartext.encode('utf-8')) + if code != 0: + log("openssl error: %s:\n Error %s: %s" %(command, code, error)) + if not error: + return out + else: + raise EncryptedException(error) + def delete_nomcom(sender, **kwargs): nomcom = kwargs.get('instance', None) @@ -87,6 +110,7 @@ def delete_nomcom(sender, **kwargs): post_delete.connect(delete_nomcom, sender=NomCom) +@python_2_unicode_compatible class Nomination(models.Model): position = ForeignKey('Position') candidate_name = models.CharField(verbose_name='Candidate name', max_length=255) @@ -107,10 +131,11 @@ class Nomination(models.Model): class Meta: verbose_name_plural = 'Nominations' - def __unicode__(self): - return u"%s (%s)" % (self.candidate_name, self.candidate_email) + def __str__(self): + return "%s (%s)" % (self.candidate_name, self.candidate_email) +@python_2_unicode_compatible class Nominee(models.Model): email = ForeignKey(Email) @@ -126,18 +151,19 @@ class Nominee(models.Model): unique_together = ('email', 'nomcom') ordering = ['-nomcom__group__acronym', 'person__name', ] - def __unicode__(self): + def __str__(self): if self.email.person and self.email.person.name: - return u'%s <%s> %s' % (self.email.person.plain_name(), self.email.address, self.nomcom.year()) + return "%s <%s> %s" % (self.email.person.plain_name(), self.email.address, self.nomcom.year()) else: - return u'%s %s' % (self.email.address, self.nomcom.year()) + return "%s %s" % (self.email.address, self.nomcom.year()) def name(self): if self.email.person and self.email.person.name: - return u'%s' % (self.email.person.plain_name(),) + return '%s' % (self.email.person.plain_name(),) else: return self.email.address +@python_2_unicode_compatible class NomineePosition(models.Model): position = ForeignKey('Position') @@ -158,8 +184,8 @@ class NomineePosition(models.Model): self.state = NomineePositionStateName.objects.get(slug='pending') super(NomineePosition, self).save(**kwargs) - def __unicode__(self): - return u"%s - %s - %s" % (self.nominee, self.state, self.position) + def __str__(self): + return "%s - %s - %s" % (self.nominee, self.state, self.position) @property def questionnaires(self): @@ -167,6 +193,7 @@ class NomineePosition(models.Model): nominees__in=[self.nominee]) +@python_2_unicode_compatible class Position(models.Model): nomcom = ForeignKey('NomCom') name = models.CharField(verbose_name='Name', max_length=255, help_text='This short description will appear on the Nomination and Feedback pages. Be as descriptive as necessary. Past examples: "Transport AD", "IAB Member"') @@ -181,7 +208,7 @@ class Position(models.Model): class Meta: verbose_name_plural = 'Positions' - def __unicode__(self): + def __str__(self): return self.name def save(self, *args, **kwargs): @@ -213,6 +240,7 @@ class Position(models.Model): rendered = linebreaks(rendered) return rendered +@python_2_unicode_compatible class Topic(models.Model): nomcom = ForeignKey('NomCom') subject = models.CharField(verbose_name='Name', max_length=255, help_text='This short description will appear on the Feedback pages.') @@ -223,7 +251,7 @@ class Topic(models.Model): class Meta: verbose_name_plural = 'Topics' - def __unicode__(self): + def __str__(self): return self.subject def save(self, *args, **kwargs): @@ -242,6 +270,7 @@ class Topic(models.Model): rendered = linebreaks(rendered) return rendered +@python_2_unicode_compatible class Feedback(models.Model): nomcom = ForeignKey('NomCom') author = models.EmailField(verbose_name='Author', blank=True) @@ -249,15 +278,15 @@ class Feedback(models.Model): nominees = models.ManyToManyField('Nominee', blank=True) topics = models.ManyToManyField('Topic', blank=True) subject = models.TextField(verbose_name='Subject', blank=True) - comments = EncryptedTextField(verbose_name='Comments') + comments = models.BinaryField(verbose_name='Comments') type = ForeignKey(FeedbackTypeName, blank=True, null=True) user = ForeignKey(User, editable=False, blank=True, null=True, on_delete=models.SET_NULL) time = models.DateTimeField(auto_now_add=True) objects = FeedbackManager() - def __unicode__(self): - return u"from %s" % self.author + def __str__(self): + return "from %s" % self.author class Meta: ordering = ['time'] diff --git a/ietf/nomcom/resources.py b/ietf/nomcom/resources.py index 311485fb0..61d101052 100644 --- a/ietf/nomcom/resources.py +++ b/ietf/nomcom/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2014-11-13 23:53 + + from ietf.api import ModelResource from ietf.api import ToOneField from tastypie.fields import ToManyField @@ -129,7 +132,7 @@ class FeedbackResource(ModelResource): "id": ALL, "author": ALL, "subject": ALL, - "comments": ALL, + # "comments": ALL, "time": ALL, "nomcom": ALL_WITH_RELATIONS, "type": ALL_WITH_RELATIONS, diff --git a/ietf/nomcom/test_data.py b/ietf/nomcom/test_data.py index c04621c91..072f43e92 100644 --- a/ietf/nomcom/test_data.py +++ b/ietf/nomcom/test_data.py @@ -1,3 +1,10 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import tempfile import os @@ -38,7 +45,7 @@ POSITIONS = [ def generate_cert(): """Function to generate cert""" - config = """ + config = b""" [ req ] distinguished_name = req_distinguished_name string_mask = utf8only @@ -89,7 +96,7 @@ def check_comments(encryped, plain, privatekey_file): decrypted_file.close() encrypted_file.close() - decrypted_comments = open(decrypted_file.name, 'r').read().decode('utf8') + decrypted_comments = io.open(decrypted_file.name, 'rb').read().decode('utf-8') os.unlink(encrypted_file.name) os.unlink(decrypted_file.name) @@ -111,7 +118,7 @@ def nomcom_test_data(): nomcom_test_cert_file, privatekey_file = generate_cert() nomcom.public_key.storage = FileSystemStorage(location=settings.NOMCOM_PUBLIC_KEYS_DIR) - nomcom.public_key.save('cert', File(open(nomcom_test_cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(nomcom_test_cert_file.name, 'r'))) # chair and member create_person(group, "chair", username=CHAIR_USER, email_address='%s%s'%(CHAIR_USER,EMAIL_DOMAIN)) diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index a6d312a0d..7b13ee897 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1,17 +1,24 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- -#import tempfile + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime +import io import random import shutil -import urlparse + from pyquery import PyQuery +from six.moves.urllib.parse import urlparse from django.db import IntegrityError from django.db.models import Max from django.conf import settings -from django.urls import reverse from django.core.files import File from django.contrib.auth.models import User +from django.urls import reverse +from django.utils.encoding import force_text import debug # pyflakes:ignore @@ -34,7 +41,7 @@ from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, get_hash from ietf.person.factories import PersonFactory, EmailFactory from ietf.person.models import Email, Person from ietf.stats.models import MeetingRegistration -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.mail import outbox, empty_outbox, get_payload from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent client_test_cert_files = None @@ -175,10 +182,10 @@ class NomcomViewsTest(TestCase): def test_private_merge_view(self): """Verify private nominee merge view""" - nominees = [u'nominee0@example.com', - u'nominee1@example.com', - u'nominee2@example.com', - u'nominee3@example.com'] + nominees = ['nominee0@example.com', + 'nominee1@example.com', + 'nominee2@example.com', + 'nominee3@example.com'] # do nominations login_testing_unauthorized(self, COMMUNITY_USER, self.public_nominate_url) @@ -333,7 +340,7 @@ class NomcomViewsTest(TestCase): response = self.client.post(self.private_merge_nominee_url, test_data) self.assertEqual(response.status_code, 302) redirect_url = response["Location"] - redirect_path = urlparse.urlparse(redirect_url).path + redirect_path = urlparse(redirect_url).path self.assertEqual(redirect_path, reverse('ietf.nomcom.views.private_index', kwargs={"year": NOMCOM_YEAR})) response = self.client.get(redirect_url) @@ -369,16 +376,16 @@ class NomcomViewsTest(TestCase): # Check nominations state self.assertEqual(NomineePosition.objects.get(position__name='TSV', - nominee=nominee).state.slug, u'accepted') + nominee=nominee).state.slug, 'accepted') self.assertEqual(NomineePosition.objects.get(position__name='IAOC', - nominee=nominee).state.slug, u'accepted') + nominee=nominee).state.slug, 'accepted') self.assertEqual(NomineePosition.objects.get(position__name='IAB', - nominee=nominee).state.slug, u'declined') + nominee=nominee).state.slug, 'declined') self.client.logout() def change_members(self, members): - members_emails = u','.join(['%s%s' % (member, EMAIL_DOMAIN) for member in members]) + members_emails = ','.join(['%s%s' % (member, EMAIL_DOMAIN) for member in members]) test_data = {'members': members_emails,} self.client.post(self.edit_members_url, test_data) @@ -406,7 +413,7 @@ class NomcomViewsTest(TestCase): q = PyQuery(r.content) reminder_date = '%s-09-30' % self.year - f = open(self.cert_file.name) + f = io.open(self.cert_file.name) response = self.client.post(self.edit_nomcom_url, { 'public_key': f, 'reminderdates_set-TOTAL_FORMS': q('input[name="reminderdates_set-TOTAL_FORMS"]').val(), @@ -421,22 +428,22 @@ class NomcomViewsTest(TestCase): nominee = Nominee.objects.get(email__person__user__username=COMMUNITY_USER) position = Position.objects.get(name='OAM') - comments = u'Plain text. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' + comment_text = 'Plain text. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' nomcom = get_nomcom_by_year(self.year) feedback = Feedback.objects.create(nomcom=nomcom, - comments=comments, + comments=nomcom.encrypt(comment_text), type=FeedbackTypeName.objects.get(slug='nomina')) feedback.positions.add(position) feedback.nominees.add(nominee) # to check feedback comments are saved like enrypted data - self.assertNotEqual(feedback.comments, comments) + self.assertNotEqual(feedback.comments, comment_text) - self.assertEqual(check_comments(feedback.comments, comments, self.privatekey_file), True) + self.assertEqual(check_comments(feedback.comments, comment_text, self.privatekey_file), True) # Check that the set reminder date is present reminder_dates = dict([ (d.id,str(d.date)) for d in nomcom.reminderdates_set.all() ]) - self.assertIn(reminder_date, reminder_dates.values()) + self.assertIn(reminder_date, list(reminder_dates.values())) # Remove reminder date q = PyQuery(response.content) # from previous post @@ -444,14 +451,14 @@ class NomcomViewsTest(TestCase): 'reminderdates_set-TOTAL_FORMS': q('input[name="reminderdates_set-TOTAL_FORMS"]').val(), 'reminderdates_set-INITIAL_FORMS': q('input[name="reminderdates_set-INITIAL_FORMS"]').val(), 'reminderdates_set-MAX_NUM_FORMS': q('input[name="reminderdates_set-MAX_NUM_FORMS"]').val(), - 'reminderdates_set-0-id': str(reminder_dates.keys()[0]), + 'reminderdates_set-0-id': str(list(reminder_dates.keys())[0]), 'reminderdates_set-0-date': '', }) self.assertEqual(r.status_code, 200) # Check that reminder date has been removed reminder_dates = dict([ (d.id,str(d.date)) for d in ReminderDates.objects.filter(nomcom=nomcom) ]) - self.assertNotIn(reminder_date, reminder_dates.values()) + self.assertNotIn(reminder_date, list(reminder_dates.values())) self.client.logout() @@ -486,9 +493,9 @@ class NomcomViewsTest(TestCase): r = self.client.get(reverse('ietf.nomcom.views.announcements')) self.assertEqual(r.status_code, 200) - self.assertTrue(("Messages from %s" % nomcom.time.year) in unicontent(r)) - self.assertTrue(nomcom.role_set.filter(name="chair")[0].person.email_address() in unicontent(r)) - self.assertTrue(msg.subject in unicontent(r)) + self.assertContains(r, ("Messages from %s" % nomcom.time.year)) + self.assertContains(r, nomcom.role_set.filter(name="chair")[0].person.email_address()) + self.assertContains(r, msg.subject) def test_requirements_view(self): @@ -519,7 +526,7 @@ class NomcomViewsTest(TestCase): self.assertEqual('Nomination receipt', outbox[-1]['Subject']) self.assertEqual(self.email_from, outbox[-1]['From']) self.assertIn('plain', outbox[-1]['To']) - self.assertIn(u'Comments with accents äöå', unicode(outbox[-1].get_payload(decode=True),"utf-8","replace")) + self.assertIn('Comments with accents äöå', force_text(outbox[-1].get_payload(decode=True),"utf-8","replace")) # Nominate the same person for the same position again without asking for confirmation @@ -560,7 +567,7 @@ class NomcomViewsTest(TestCase): self.assertEqual('Nomination receipt', outbox[-1]['Subject']) self.assertEqual(self.email_from, outbox[-1]['From']) self.assertIn('plain', outbox[-1]['To']) - self.assertIn(u'Comments with accents äöå', unicode(outbox[-1].get_payload(decode=True),"utf-8","replace")) + self.assertIn('Comments with accents äöå', force_text(outbox[-1].get_payload(decode=True),"utf-8","replace")) # Nominate the same person for the same position again without asking for confirmation @@ -594,7 +601,7 @@ class NomcomViewsTest(TestCase): def nominate_view(self, *args, **kwargs): public = kwargs.pop('public', True) searched_email = kwargs.pop('searched_email', None) - nominee_email = kwargs.pop('nominee_email', u'nominee@example.com') + nominee_email = kwargs.pop('nominee_email', 'nominee@example.com') if not searched_email: searched_email = Email.objects.filter(address=nominee_email).first() if not searched_email: @@ -620,7 +627,7 @@ class NomcomViewsTest(TestCase): # save the cert file in tmp #nomcom.public_key.storage.location = tempfile.gettempdir() - nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) response = self.client.get(nominate_url) self.assertEqual(response.status_code, 200) @@ -628,13 +635,13 @@ class NomcomViewsTest(TestCase): self.assertEqual(len(q("#nominate-form")), 1) position = Position.objects.get(name=position_name) - comments = u'Test nominate view. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' - candidate_phone = u'123456' + comment_text = 'Test nominate view. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' + candidate_phone = '123456' test_data = {'searched_email': searched_email.pk, 'candidate_phone': candidate_phone, 'position': position.id, - 'qualifications': comments, + 'qualifications': comment_text, 'confirmation': confirmation} if not public: test_data['nominator_email'] = nominator_email @@ -654,9 +661,9 @@ class NomcomViewsTest(TestCase): self.assertEqual(feedback.author, nominator_email) # to check feedback comments are saved like enrypted data - self.assertNotEqual(feedback.comments, comments) + self.assertNotEqual(feedback.comments, comment_text) - self.assertEqual(check_comments(feedback.comments, comments, self.privatekey_file), True) + self.assertEqual(check_comments(feedback.comments, comment_text, self.privatekey_file), True) Nomination.objects.get(position=position, candidate_name=nominee.person.plain_name(), candidate_email=searched_email.address, @@ -667,7 +674,7 @@ class NomcomViewsTest(TestCase): def nominate_newperson_view(self, *args, **kwargs): public = kwargs.pop('public', True) - nominee_email = kwargs.pop('nominee_email', u'nominee@example.com') + nominee_email = kwargs.pop('nominee_email', 'nominee@example.com') nominator_email = kwargs.pop('nominator_email', "%s%s" % (COMMUNITY_USER, EMAIL_DOMAIN)) position_name = kwargs.pop('position', 'IAOC') confirmation = kwargs.pop('confirmation', False) @@ -686,7 +693,7 @@ class NomcomViewsTest(TestCase): # save the cert file in tmp #nomcom.public_key.storage.location = tempfile.gettempdir() - nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) response = self.client.get(nominate_url) self.assertEqual(response.status_code, 200) @@ -695,15 +702,15 @@ class NomcomViewsTest(TestCase): position = Position.objects.get(name=position_name) candidate_email = nominee_email - candidate_name = u'nominee' - comments = u'Test nominate view. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' - candidate_phone = u'123456' + candidate_name = 'nominee' + comment_text = 'Test nominate view. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' + candidate_phone = '123456' test_data = {'candidate_name': candidate_name, 'candidate_email': candidate_email, 'candidate_phone': candidate_phone, 'position': position.id, - 'qualifications': comments, + 'qualifications': comment_text, 'confirmation': confirmation} if not public: test_data['nominator_email'] = nominator_email @@ -726,9 +733,9 @@ class NomcomViewsTest(TestCase): self.assertEqual(feedback.author, nominator_email) # to check feedback comments are saved like enrypted data - self.assertNotEqual(feedback.comments, comments) + self.assertNotEqual(feedback.comments, comment_text) - self.assertEqual(check_comments(feedback.comments, comments, self.privatekey_file), True) + self.assertEqual(check_comments(feedback.comments, comment_text, self.privatekey_file), True) Nomination.objects.get(position=position, candidate_name=candidate_name, candidate_email=candidate_email, @@ -744,7 +751,7 @@ class NomcomViewsTest(TestCase): def add_questionnaire(self, *args, **kwargs): public = kwargs.pop('public', False) - nominee_email = kwargs.pop('nominee_email', u'nominee@example.com') + nominee_email = kwargs.pop('nominee_email', 'nominee@example.com') nominator_email = kwargs.pop('nominator_email', "%s%s" % (COMMUNITY_USER, EMAIL_DOMAIN)) position_name = kwargs.pop('position', 'IAOC') @@ -762,7 +769,7 @@ class NomcomViewsTest(TestCase): # save the cert file in tmp #nomcom.public_key.storage.location = tempfile.gettempdir() - nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) response = self.client.get(self.add_questionnaire_url) self.assertEqual(response.status_code, 200) @@ -771,14 +778,13 @@ class NomcomViewsTest(TestCase): position = Position.objects.get(name=position_name) nominee = Nominee.objects.get(email__address=nominee_email) - comments = u'Test add questionnaire view. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' + comment_text = 'Test add questionnaire view. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' - test_data = {'comments': comments, + test_data = {'comment_text': comment_text, 'nominee': '%s_%s' % (position.id, nominee.id)} response = self.client.post(self.add_questionnaire_url, test_data) - self.assertEqual(response.status_code, 200) self.assertContains(response, "alert-success") ## check objects @@ -787,9 +793,9 @@ class NomcomViewsTest(TestCase): type=FeedbackTypeName.objects.get(slug='questio')).latest('id') ## to check feedback comments are saved like enrypted data - self.assertNotEqual(feedback.comments, comments) + self.assertNotEqual(feedback.comments, comment_text) - self.assertEqual(check_comments(feedback.comments, comments, self.privatekey_file), True) + self.assertEqual(check_comments(feedback.comments, comment_text, self.privatekey_file), True) def test_public_feedback(self): login_testing_unauthorized(self, COMMUNITY_USER, self.public_feedback_url) @@ -801,12 +807,12 @@ class NomcomViewsTest(TestCase): # We're interested in the confirmation receipt here self.assertEqual(len(outbox),3) self.assertEqual('NomCom comment confirmation', outbox[2]['Subject']) - email_body = outbox[2].get_payload() + email_body = get_payload(outbox[2]) self.assertIn(position, email_body) self.assertNotIn('$', email_body) self.assertEqual(self.email_from, outbox[-2]['From']) self.assertIn('plain', outbox[2]['To']) - self.assertIn(u'Comments with accents äöå', unicode(outbox[2].get_payload(decode=True),"utf-8","replace")) + self.assertIn('Comments with accents äöå', force_text(outbox[2].get_payload(decode=True),"utf-8","replace")) empty_outbox() self.feedback_view(public=True) @@ -819,7 +825,7 @@ class NomcomViewsTest(TestCase): def feedback_view(self, *args, **kwargs): public = kwargs.pop('public', True) - nominee_email = kwargs.pop('nominee_email', u'nominee@example.com') + nominee_email = kwargs.pop('nominee_email', 'nominee@example.com') nominator_email = kwargs.pop('nominator_email', "%s%s" % (COMMUNITY_USER, EMAIL_DOMAIN)) position_name = kwargs.pop('position', 'IAOC') confirmation = kwargs.pop('confirmation', False) @@ -842,7 +848,7 @@ class NomcomViewsTest(TestCase): # save the cert file in tmp #nomcom.public_key.storage.location = tempfile.gettempdir() - nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) response = self.client.get(feedback_url) self.assertEqual(response.status_code, 200) @@ -856,9 +862,9 @@ class NomcomViewsTest(TestCase): self.assertEqual(response.status_code, 200) self.assertContains(response, "feedbackform") - comments = u'Test feedback view. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' + comments = 'Test feedback view. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' - test_data = {'comments': comments, + test_data = {'comment_text': comments, 'position_name': position.name, 'nominee_name': nominee.email.person.name, 'nominee_email': nominee.email.address, @@ -881,7 +887,6 @@ class NomcomViewsTest(TestCase): nominee_position.save() response = self.client.post(feedback_url, test_data) - self.assertEqual(response.status_code, 200) self.assertContains(response, "alert-success") self.assertNotContains(response, "feedbackform") @@ -958,9 +963,10 @@ class FeedbackTest(TestCase): # save the cert file in tmp #nomcom.public_key.storage.location = tempfile.gettempdir() - nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) - comments = u'Plain text. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' + comment_text = 'Plain text. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.' + comments = nomcom.encrypt(comment_text) feedback = Feedback.objects.create(nomcom=nomcom, comments=comments, type=FeedbackTypeName.objects.get(slug='nomina')) @@ -968,9 +974,8 @@ class FeedbackTest(TestCase): feedback.nominees.add(nominee) # to check feedback comments are saved like enrypted data - self.assertNotEqual(feedback.comments, comments) - - self.assertEqual(check_comments(feedback.comments, comments, self.privatekey_file), True) + self.assertNotEqual(feedback.comments, comment_text) + self.assertEqual(check_comments(feedback.comments, comment_text, self.privatekey_file), True) class ReminderTest(TestCase): @@ -980,7 +985,7 @@ class ReminderTest(TestCase): self.nomcom = get_nomcom_by_year(NOMCOM_YEAR) self.cert_file, self.privatekey_file = get_cert_files() #self.nomcom.public_key.storage.location = tempfile.gettempdir() - self.nomcom.public_key.save('cert', File(open(self.cert_file.name, 'r'))) + self.nomcom.public_key.save('cert', File(io.open(self.cert_file.name, 'r'))) gen = Position.objects.get(nomcom=self.nomcom,name='GEN') rai = Position.objects.get(nomcom=self.nomcom,name='RAI') @@ -989,8 +994,8 @@ class ReminderTest(TestCase): today = datetime.date.today() t_minus_3 = today - datetime.timedelta(days=3) t_minus_4 = today - datetime.timedelta(days=4) - e1 = EmailFactory(address="nominee1@example.org", person=PersonFactory(name=u"Nominee 1"), origin='test') - e2 = EmailFactory(address="nominee2@example.org", person=PersonFactory(name=u"Nominee 2"), origin='test') + e1 = EmailFactory(address="nominee1@example.org", person=PersonFactory(name="Nominee 1"), origin='test') + e2 = EmailFactory(address="nominee2@example.org", person=PersonFactory(name="Nominee 2"), origin='test') n = make_nomineeposition(self.nomcom,e1.person,gen,None) np = n.nomineeposition_set.get(position=gen) np.time = t_minus_3 @@ -1010,7 +1015,7 @@ class ReminderTest(TestCase): np.time = t_minus_4 np.save() feedback = Feedback.objects.create(nomcom=self.nomcom, - comments='some non-empty comments', + comments=self.nomcom.encrypt('some non-empty comments'), type=FeedbackTypeName.objects.get(slug='questio'), user=User.objects.get(username=CHAIR_USER)) feedback.positions.add(gen) @@ -1103,7 +1108,7 @@ class InactiveNomcomTests(TestCase): empty_outbox() fb_before = self.nc.feedback_set.count() - test_data = {'comments': u'Test feedback view. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.', + test_data = {'comment_text': 'Test feedback view. Comments with accents äöåÄÖÅ éáíóú âêîôû ü àèìòù.', 'nominator_email': self.plain_person.email_set.first().address, 'confirmation': True} response = self.client.post(url, test_data) @@ -1126,7 +1131,7 @@ class InactiveNomcomTests(TestCase): def test_acceptance_closed(self): today = datetime.date.today().strftime('%Y%m%d') - pid = self.nc.position_set.first().nomineeposition_set.order_by('pk').first().id + pid = self.nc.position_set.first().nomineeposition_set.order_by('pk').first().id url = reverse('ietf.nomcom.views.process_nomination_status', kwargs = { 'year' : self.nc.year(), 'nominee_position_id' : pid, @@ -1460,7 +1465,7 @@ class NewActiveNomComTests(TestCase): fb_count_before = Feedback.objects.count() response = self.client.post(url,{'email_text':"""To: rjsparks@nostrum.com From: Robert Sparks -Subject: Junk message for feedback testing +Subject: Junk message for feedback testing =?iso-8859-1?q?p=F6stal?= Message-ID: <566F2FE5.1050401@nostrum.com> Date: Mon, 14 Dec 2015 15:08:53 -0600 Content-Type: text/plain; charset=utf-8; format=flowed @@ -1763,7 +1768,7 @@ Junk body for testing 'duplicate_persons':[nominee2.person.pk]}) self.assertEqual(response.status_code, 302) self.assertEqual(len(outbox),1) - self.assertTrue(all([str(x.person.pk) in outbox[0].get_payload(decode=True) for x in [nominee1,nominee2]])) + self.assertTrue(all([str(x.person.pk) in outbox[0].get_payload() for x in [nominee1,nominee2]])) def test_extract_email(self): url = reverse('ietf.nomcom.views.extract_email_lists',kwargs={'year':self.nc.year()}) @@ -1904,7 +1909,7 @@ class AcceptingTests(TestCase): response = self.client.get(posurl) self.assertIn('not currently accepting feedback', unicontent(response)) - test_data = {'comments': 'junk', + test_data = {'comment_text': 'junk', 'position_name': pos.name, 'nominee_name': pos.nominee_set.first().email.person.name, 'nominee_email': pos.nominee_set.first().email.address, @@ -1919,7 +1924,7 @@ class AcceptingTests(TestCase): response = self.client.get(topicurl) self.assertIn('not currently accepting feedback', unicontent(response)) - test_data = {'comments': 'junk', + test_data = {'comment_text': 'junk', 'confirmation': False, } response = self.client.post(topicurl, test_data) @@ -2045,7 +2050,7 @@ class TopicTests(TestCase): url = reverse('ietf.nomcom.views.public_feedback',kwargs={'year':self.nc.year() }) url += '?topic=%d'%topic.pk login_testing_unauthorized(self, self.plain_person.user.username, url) - response=self.client.post(url, {'comments':'junk', 'confirmation':False}) + response=self.client.post(url, {'comment_text':'junk', 'confirmation':False}) self.assertEqual(response.status_code, 200) self.assertContains(response, "alert-success") self.assertNotContains(response, "feedbackform") @@ -2057,12 +2062,11 @@ class TopicTests(TestCase): feedback_url = reverse('ietf.nomcom.views.public_feedback',kwargs={'year':self.nc.year() }) login_testing_unauthorized(self, self.plain_person.user.username, feedback_url) r = self.client.get(feedback_url) - self.assertEqual(r.status_code,200) - self.assertNotIn(topic.subject, unicontent(r)) + self.assertNotContains(r, topic.subject) topic_url = feedback_url + '?topic=%d'%topic.pk r = self.client.get(topic_url) self.assertEqual(r.status_code,404) - r = self.client.post(topic_url, {'comments':'junk', 'confirmation':False}) + r = self.client.post(topic_url, {'comment_text':'junk', 'confirmation':False}) self.assertEqual(r.status_code,404) self.client.logout() @@ -2072,11 +2076,10 @@ class TopicTests(TestCase): valid_user = self.nc.nominee_set.first().person self.client.login(username=valid_user.user.username,password=valid_user.user.username+"+password") r = self.client.get(feedback_url) - self.assertEqual(r.status_code,200) - self.assertIn(topic.subject, unicontent(r)) + self.assertContains(r, topic.subject) r = self.client.get(topic_url) self.assertEqual(r.status_code,200) - r = self.client.post(topic_url, {'comments':'junk', 'confirmation':False}) + r = self.client.post(topic_url, {'comment_text':'junk', 'confirmation':False}) self.assertEqual(r.status_code,200) self.assertEqual(topic.feedback_set.count(),1) self.client.logout() diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index c95b5cd57..5f8da9b06 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -1,7 +1,14 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import hashlib import os import re +import six import tempfile from email import message_from_string @@ -15,7 +22,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.urls import reverse from django.template.loader import render_to_string from django.shortcuts import get_object_or_404 -from django.utils.encoding import smart_str +from django.utils.encoding import force_str, force_text from ietf.dbtemplate.models import DBTemplate from ietf.person.models import Email, Person @@ -60,7 +67,7 @@ def get_nomcom_by_year(year): def get_year_by_nomcom(nomcom): acronym = nomcom.group.acronym - m = re.search('(?P\d\d\d\d)', acronym) + m = re.search(r'(?P\d\d\d\d)', acronym) return m.group(0) @@ -86,7 +93,7 @@ def get_user_email(user): return user._email_cache def get_hash_nominee_position(date, nominee_position_id): - return hashlib.md5('%s%s%s' % (settings.SECRET_KEY, date, nominee_position_id)).hexdigest() + return hashlib.md5(('%s%s%s' % (settings.SECRET_KEY, date, nominee_position_id)).encode('utf-8')).hexdigest() def initialize_templates_for_group(group): @@ -159,7 +166,7 @@ def retrieve_nomcom_private_key(request, year): command = "%s bf -d -in /dev/stdin -k \"%s\" -a" code, out, error = pipe(command % (settings.OPENSSL_COMMAND, - settings.SECRET_KEY), private_key) + settings.SECRET_KEY), private_key.encode('utf-8')) if code != 0: log("openssl error: %s:\n Error %s: %s" %(command, code, error)) return out @@ -171,7 +178,7 @@ def store_nomcom_private_key(request, year, private_key): else: command = "%s bf -e -in /dev/stdin -k \"%s\" -a" code, out, error = pipe(command % (settings.OPENSSL_COMMAND, - settings.SECRET_KEY), private_key) + settings.SECRET_KEY), private_key.encode('utf-8')) if code != 0: log("openssl error: %s:\n Error %s: %s" %(command, code, error)) if error: @@ -181,7 +188,7 @@ def store_nomcom_private_key(request, year, private_key): def validate_private_key(key): key_file = tempfile.NamedTemporaryFile(delete=False) - key_file.write(key) + key_file.write(key.encode('utf-8')) key_file.close() command = "%s rsa -in %s -check -noout" @@ -398,10 +405,9 @@ def make_nomineeposition_for_newperson(nomcom, candidate_name, candidate_email, def getheader(header_text, default="ascii"): """Decode the specified header""" - headers = decode_header(header_text) - header_sections = [unicode(text, charset or default) - for text, charset in headers] - return u"".join(header_sections) + tuples = decode_header(header_text) + header_sections = [ text.decode(charset or default) if isinstance(text, six.binary_type) else text for text, charset in tuples] + return "".join(header_sections) def get_charset(message, default="ascii"): @@ -427,24 +433,22 @@ def get_body(message): body = [] for part in text_parts: charset = get_charset(part, get_charset(message)) - body.append(unicode(part.get_payload(decode=True), + body.append(force_text(part.get_payload(decode=True), charset, "replace")) - return u"\n".join(body).strip() + return "\n".join(body).strip() else: # if it is not multipart, the payload will be a string # representing the message body - body = unicode(message.get_payload(decode=True), + body = force_text(message.get_payload(decode=True), get_charset(message), "replace") return body.strip() def parse_email(text): - if isinstance(text, unicode): - text = smart_str(text) - msg = message_from_string(text) + msg = message_from_string(force_str(text)) body = get_body(msg) subject = getheader(msg['Subject']) @@ -460,6 +464,10 @@ def create_feedback_email(nomcom, msg): feedback = Feedback(nomcom=nomcom, author=by, subject=subject or '', - comments=body) + comments=nomcom.encrypt(body)) feedback.save() return feedback + +class EncryptedException(Exception): + pass + \ No newline at end of file diff --git a/ietf/nomcom/views.py b/ietf/nomcom/views.py index ba4a7f140..2fde29b64 100644 --- a/ietf/nomcom/views.py +++ b/ietf/nomcom/views.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import re from collections import OrderedDict, Counter @@ -179,7 +185,7 @@ def private_index(request, year): 'position__id':p.pk, 'position': p, } for p in positions] - states = list(NomineePositionStateName.objects.values('slug', 'name')) + [{'slug': questionnaire_state, 'name': u'Questionnaire'}] + states = list(NomineePositionStateName.objects.values('slug', 'name')) + [{'slug': questionnaire_state, 'name': 'Questionnaire'}] positions = set([ n.position for n in all_nominee_positions.order_by('position__name') ]) for s in stats: for state in states: @@ -639,7 +645,7 @@ def private_questionnaire(request, year): if form.is_valid(): form.save() messages.success(request, 'The questionnaire response has been registered.') - questionnaire_response = form.cleaned_data['comments'] + questionnaire_response = form.cleaned_data['comment_text'] form = QuestionnaireForm(nomcom=nomcom, user=request.user) else: form = QuestionnaireForm(nomcom=nomcom, user=request.user) @@ -687,7 +693,7 @@ def process_nomination_status(request, year, nominee_position_id, state, date, h f = Feedback.objects.create(nomcom = nomcom, author = nominee_position.nominee.email, subject = '%s nomination %s'%(nominee_position.nominee.name(),state), - comments = form.cleaned_data['comments'], + comments = nomcom.encrypt(form.cleaned_data['comments']), type_id = 'comment', user = who, ) diff --git a/ietf/person/factories.py b/ietf/person/factories.py index 627c9ed6a..e076b4ef7 100644 --- a/ietf/person/factories.py +++ b/ietf/person/factories.py @@ -1,18 +1,22 @@ -# Copyright The IETF Trust 2014-2018, All Rights Reserved +# Copyright The IETF Trust 2015-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function -import os + +from __future__ import absolute_import, print_function, unicode_literals + import factory import faker -import shutil -import random import faker.config +import os +import random +import shutil + from unidecode import unidecode from django.conf import settings from django.contrib.auth.models import User from django.utils.text import slugify +from django.utils.encoding import force_text import debug # pyflakes:ignore @@ -53,11 +57,11 @@ class PersonFactory(factory.DjangoModelFactory): model = Person user = factory.SubFactory(UserFactory) - name = factory.LazyAttribute(lambda p: normalize_name(u'%s %s'%(p.user.first_name, p.user.last_name))) - ascii = factory.LazyAttribute(lambda p: unicode(unidecode_name(p.name))) + name = factory.LazyAttribute(lambda p: normalize_name('%s %s'%(p.user.first_name, p.user.last_name))) + ascii = factory.LazyAttribute(lambda p: force_text(unidecode_name(p.name))) class Params: - with_bio = factory.Trait(biography = u"\n\n".join(fake.paragraphs())) + with_bio = factory.Trait(biography = "\n\n".join(fake.paragraphs())) @factory.post_generation def default_aliases(obj, create, extracted, **kwargs): # pylint: disable=no-self-argument @@ -82,7 +86,7 @@ class PersonFactory(factory.DjangoModelFactory): import atexit if obj.biography: photo_name = obj.photo_name() - media_name = u"%s/%s.jpg" % (settings.PHOTOS_DIRNAME, photo_name) + media_name = "%s/%s.jpg" % (settings.PHOTOS_DIRNAME, photo_name) obj.photo = media_name obj.photo_thumb = media_name photosrc = os.path.join(settings.TEST_DATA_DIR, "profile-default.jpg") diff --git a/ietf/person/fields.py b/ietf/person/fields.py index 672dcb9d2..077b6c4ca 100644 --- a/ietf/person/fields.py +++ b/ietf/person/fields.py @@ -1,11 +1,14 @@ # Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import json import six from collections import Counter -from urllib import urlencode +from six.moves.urllib.parse import urlencode from django import forms from django.core.validators import validate_email @@ -18,7 +21,7 @@ from ietf.person.models import Email, Person def select2_id_name_json(objs): def format_email(e): - return escape(u"%s <%s>" % (e.person.name, e.address)) + return escape("%s <%s>" % (e.person.name, e.address)) def format_person(p): if p.name_count > 1: return escape('%s (%s)' % (p.name,p.email().address if p.email() else 'no email address')) @@ -87,7 +90,7 @@ class SearchablePersonsField(forms.CharField): def prepare_value(self, value): if not value: value = "" - if isinstance(value, basestring): + if isinstance(value, six.string_types): pks = self.parse_select2_value(value) if self.model == Person: value = self.model.objects.filter(pk__in=pks) @@ -109,7 +112,7 @@ class SearchablePersonsField(forms.CharField): if query_args: self.widget.attrs["data-ajax-url"] += "?%s" % urlencode(query_args) - return u",".join(str(p.pk) for p in value) + return ",".join(str(p.pk) for p in value) def clean(self, value): value = super(SearchablePersonsField, self).clean(value) @@ -123,13 +126,13 @@ class SearchablePersonsField(forms.CharField): #if self.only_users: # objs = objs.exclude(person__user=None) - found_pks = [ six.text_type(o.pk) for o in objs] + found_pks = [ str(o.pk) for o in objs] failed_pks = [x for x in pks if x not in found_pks] if failed_pks: - raise forms.ValidationError(u"Could not recognize the following {model_name}s: {pks}. You can only input {model_name}s already registered in the Datatracker.".format(pks=", ".join(failed_pks), model_name=self.model.__name__.lower())) + raise forms.ValidationError("Could not recognize the following {model_name}s: {pks}. You can only input {model_name}s already registered in the Datatracker.".format(pks=", ".join(failed_pks), model_name=self.model.__name__.lower())) if self.max_entries != None and len(objs) > self.max_entries: - raise forms.ValidationError(u"You can select at most %s entries only." % self.max_entries) + raise forms.ValidationError("You can select at most %s entries only." % self.max_entries) return objs @@ -176,9 +179,9 @@ class PersonEmailChoiceField(forms.ModelChoiceField): def label_from_instance(self, email): if self.label_with == "person": - return unicode(email.person) + return six.text_type(email.person) elif self.label_with == "email": return email.address else: - return u"{} <{}>".format(email.person, email.address) + return "{} <{}>".format(email.person, email.address) diff --git a/ietf/person/forms.py b/ietf/person/forms.py index aa7015b5c..d146aa0b1 100644 --- a/ietf/person/forms.py +++ b/ietf/person/forms.py @@ -1,6 +1,9 @@ -# Copyright The IETF Trust 2017, All Rights Reserved +# Copyright The IETF Trust 2018-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals -from __future__ import unicode_literals from django import forms from ietf.person.models import Person diff --git a/ietf/person/management/commands/deactivate_email_addresses.py b/ietf/person/management/commands/deactivate_email_addresses.py index 0218630ed..04357dc9f 100644 --- a/ietf/person/management/commands/deactivate_email_addresses.py +++ b/ietf/person/management/commands/deactivate_email_addresses.py @@ -1,8 +1,11 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals import flufl.bounce +import io import mailbox import sys @@ -18,7 +21,7 @@ from ietf.person.models import Email, PersonEvent class Command(BaseCommand): - help = (u""" + help = (""" Deactivate bouncing email addresses. Take one or more email addresses to deactivate from the command line, @@ -67,7 +70,7 @@ class Command(BaseCommand): self.stderr.write('No person is associated with <%s>\n' % (a, )) else: self.stderr.write('Address not found: <%s>\n' % (a, )) - with open('./failed', 'a') as failed: + with io.open('./failed', 'a') as failed: failed.write(messages[a].as_string(unixfrom=True)) failed.write('\n') diff --git a/ietf/person/migrations/0001_initial.py b/ietf/person/migrations/0001_initial.py index f04d7b4e4..66d405bbf 100644 --- a/ietf/person/migrations/0001_initial.py +++ b/ietf/person/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import datetime from django.conf import settings diff --git a/ietf/person/migrations/0002_auto_20180330_0808.py b/ietf/person/migrations/0002_auto_20180330_0808.py index 6c852f67c..3c4d7c9ab 100644 --- a/ietf/person/migrations/0002_auto_20180330_0808.py +++ b/ietf/person/migrations/0002_auto_20180330_0808.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.11 on 2018-03-30 08:08 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/person/migrations/0003_auto_20180504_1519.py b/ietf/person/migrations/0003_auto_20180504_1519.py index 7abd048e2..f24581584 100644 --- a/ietf/person/migrations/0003_auto_20180504_1519.py +++ b/ietf/person/migrations/0003_auto_20180504_1519.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.12 on 2018-05-04 15:19 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import datetime from django.conf import settings diff --git a/ietf/person/migrations/0004_populate_email_origin.py b/ietf/person/migrations/0004_populate_email_origin.py index 3d30f9abb..5a07e0b83 100644 --- a/ietf/person/migrations/0004_populate_email_origin.py +++ b/ietf/person/migrations/0004_populate_email_origin.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.12 on 2018-05-10 05:28 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys diff --git a/ietf/person/migrations/0005_populate_person_name_from_draft.py b/ietf/person/migrations/0005_populate_person_name_from_draft.py index 62caa3868..dc4cb829d 100644 --- a/ietf/person/migrations/0005_populate_person_name_from_draft.py +++ b/ietf/person/migrations/0005_populate_person_name_from_draft.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.12 on 2018-05-10 05:28 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys diff --git a/ietf/person/migrations/0006_auto_20180910_0719.py b/ietf/person/migrations/0006_auto_20180910_0719.py index cfdb07b6e..2a86ac351 100644 --- a/ietf/person/migrations/0006_auto_20180910_0719.py +++ b/ietf/person/migrations/0006_auto_20180910_0719.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.13 on 2018-09-10 07:19 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.conf import settings from django.db import migrations diff --git a/ietf/person/migrations/0007_auto_20180929_1303.py b/ietf/person/migrations/0007_auto_20180929_1303.py index c8440c6a5..310691e6d 100644 --- a/ietf/person/migrations/0007_auto_20180929_1303.py +++ b/ietf/person/migrations/0007_auto_20180929_1303.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.15 on 2018-09-29 13:03 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/person/migrations/0008_auto_20181014_1448.py b/ietf/person/migrations/0008_auto_20181014_1448.py index 8c0292974..a7cee4884 100644 --- a/ietf/person/migrations/0008_auto_20181014_1448.py +++ b/ietf/person/migrations/0008_auto_20181014_1448.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-10-14 14:48 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/person/migrations/0009_auto_20190118_0725.py b/ietf/person/migrations/0009_auto_20190118_0725.py index e3a1e4233..d96edbe59 100644 --- a/ietf/person/migrations/0009_auto_20190118_0725.py +++ b/ietf/person/migrations/0009_auto_20190118_0725.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.18 on 2019-01-18 07:25 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/person/models.py b/ietf/person/models.py index d0159c69c..6e40686f8 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -1,4 +1,8 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import datetime import email.utils @@ -7,16 +11,17 @@ import six import uuid from hashids import Hashids -from urlparse import urljoin +from six.moves.urllib.parse import urljoin from django.conf import settings - -from django.core.validators import validate_email -from django.core.exceptions import ObjectDoesNotExist -from django.db import models from django.contrib.auth.models import User +from django.core.exceptions import ObjectDoesNotExist +from django.core.validators import validate_email +from django.db import models from django.template.loader import render_to_string +from django.utils.encoding import python_2_unicode_compatible, smart_bytes from django.utils.text import slugify + from simple_history.models import HistoricalRecords import debug # pyflakes:ignore @@ -30,6 +35,7 @@ from ietf.utils import log from ietf.utils.models import ForeignKey, OneToOneField +@python_2_unicode_compatible class Person(models.Model): history = HistoricalRecords() user = OneToOneField(User, blank=True, null=True, on_delete=models.SET_NULL) @@ -47,7 +53,7 @@ class Person(models.Model): name_from_draft = models.CharField("Full Name (from submission)", null=True, max_length=255, editable=False, help_text="Name as found in a draft submission.") consent = models.NullBooleanField("I hereby give my consent to the use of the personal details I have provided (photo, bio, name, email) within the IETF Datatracker", null=True, default=None) - def __unicode__(self): + def __str__(self): return self.plain_name() def name_parts(self): return name_parts(self.name) @@ -79,11 +85,15 @@ class Person(models.Model): def plain_ascii(self): if not hasattr(self, '_cached_plain_ascii'): if self.ascii: - ascii = unidecode_name(self.ascii) + if isinstance(self.ascii, six.binary_type): + uname = six.text_type(self.ascii) + ascii = unidecode_name(uname) + else: + ascii = unidecode_name(self.ascii) else: ascii = unidecode_name(self.name) prefix, first, middle, last, suffix = name_parts(ascii) - self._cached_plain_ascii = u" ".join([first, last]) + self._cached_plain_ascii = " ".join([first, last]) return self._cached_plain_ascii def initials(self): return initials(self.ascii or self.name) @@ -96,7 +106,7 @@ class Person(models.Model): may be an object or the group acronym.""" if group: from ietf.group.models import Group - if isinstance(group, str) or isinstance(group, unicode): + if isinstance(group, six.string_types): group = Group.objects.get(acronym=group) e = Email.objects.filter(person=self, role__group=group, role__name=role_name) else: @@ -144,7 +154,7 @@ class Person(models.Model): def photo_name(self,thumb=False): hasher = Hashids(salt='Person photo name salt',min_length=5) _, first, _, last, _ = name_parts(self.ascii) - return u'%s-%s%s' % ( slugify(u"%s %s" % (first, last)), hasher.encode(self.id), '-th' if thumb else '' ) + return '%s-%s%s' % ( slugify("%s %s" % (first, last)), hasher.encode(self.id), '-th' if thumb else '' ) def has_drafts(self): from ietf.doc.models import Document @@ -225,6 +235,7 @@ class Person(models.Model): ct1['ascii'] = self.ascii return ct1 +@python_2_unicode_compatible class Alias(models.Model): """This is used for alternative forms of a name. This is the primary lookup point for names, and should always contain the @@ -247,11 +258,12 @@ class Alias(models.Model): send_mail_preformatted(None, msg) - def __unicode__(self): + def __str__(self): return self.name class Meta: verbose_name_plural = "Aliases" +@python_2_unicode_compatible class Email(models.Model): history = HistoricalRecords() address = models.CharField(max_length=64, primary_key=True, validators=[validate_email]) @@ -262,7 +274,7 @@ class Email(models.Model): active = models.BooleanField(default=True) # Old email addresses are *not* purged, as history # information points to persons through these - def __unicode__(self): + def __str__(self): return self.address or "Email object with id: %s"%self.pk def get_name(self): @@ -281,9 +293,9 @@ class Email(models.Model): Use self.formatted_email() for that. """ if self.person: - return u"%s <%s>" % (self.person.plain_name(), self.address) + return "%s <%s>" % (self.person.plain_name(), self.address) else: - return u"<%s>" % self.address + return "<%s>" % self.address def formatted_email(self): """ @@ -323,6 +335,7 @@ PERSON_API_KEY_ENDPOINTS = [ ("/api/meeting/session/video/url", "/api/meeting/session/video/url"), ] +@python_2_unicode_compatible class PersonalApiKey(models.Model): person = ForeignKey(Person, related_name='apikeys') endpoint = models.CharField(max_length=128, null=False, blank=False, choices=PERSON_API_KEY_ENDPOINTS) @@ -335,10 +348,8 @@ class PersonalApiKey(models.Model): @classmethod def validate_key(cls, s): import struct, hashlib, base64 - try: - key = base64.urlsafe_b64decode(six.binary_type(s)) - except TypeError: - return None + assert isinstance(s, six.binary_type) + key = base64.urlsafe_b64decode(s) id, salt, hash = struct.unpack(KEY_STRUCT, key) k = cls.objects.filter(id=id) if not k.exists(): @@ -346,6 +357,7 @@ class PersonalApiKey(models.Model): k = k.first() check = hashlib.sha256() for v in (str(id), str(k.person.id), k.created.isoformat(), k.endpoint, str(k.valid), salt, settings.SECRET_KEY): + v = smart_bytes(v) check.update(v) return k if check.digest() == hash else None @@ -355,12 +367,13 @@ class PersonalApiKey(models.Model): hash = hashlib.sha256() # Hash over: ( id, person, created, endpoint, valid, salt, secret ) for v in (str(self.id), str(self.person.id), self.created.isoformat(), self.endpoint, str(self.valid), self.salt, settings.SECRET_KEY): + v = smart_bytes(v) hash.update(v) - key = struct.pack(KEY_STRUCT, self.id, six.binary_type(self.salt), hash.digest()) - self._cached_hash = base64.urlsafe_b64encode(key) + key = struct.pack(KEY_STRUCT, self.id, bytes(self.salt), hash.digest()) + self._cached_hash = base64.urlsafe_b64encode(key).decode('ascii') return self._cached_hash - def __unicode__(self): + def __str__(self): return "%s (%s): %s ..." % (self.endpoint, self.created.strftime("%Y-%m-%d %H:%M"), self.hash()[:16]) PERSON_EVENT_CHOICES = [ @@ -369,14 +382,15 @@ PERSON_EVENT_CHOICES = [ ("email_address_deactivated", "Email address deactivated"), ] +@python_2_unicode_compatible class PersonEvent(models.Model): person = ForeignKey(Person) time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened") type = models.CharField(max_length=50, choices=PERSON_EVENT_CHOICES) desc = models.TextField() - def __unicode__(self): - return u"%s %s at %s" % (self.person.plain_name(), self.get_type_display().lower(), self.time) + def __str__(self): + return "%s %s at %s" % (self.person.plain_name(), self.get_type_display().lower(), self.time) class Meta: ordering = ['-time', '-id'] diff --git a/ietf/person/name.py b/ietf/person/name.py index 4c16256a5..1cfbd06fb 100644 --- a/ietf/person/name.py +++ b/ietf/person/name.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2011-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import re import unidecode @@ -8,7 +14,7 @@ def name_particle_match(name): return re.search(r" (af|al|Al|de|der|di|Di|du|el|El|Hadi|in 't|Le|st\.?|St\.?|ten|ter|van|van der|van 't|Van|von|von der|Von|zu) ", name) def name_parts(name): - prefix, first, middle, last, suffix = u"", u"", u"", u"", u"" + prefix, first, middle, last, suffix = "", "", "", "", "" if not name.strip(): return prefix, first, middle, last, suffix @@ -36,7 +42,7 @@ def name_parts(name): parts = parts[:-1] if len(parts) > 2: # Check if we have a surname with nobiliary particle - full = u" ".join(parts) + full = " ".join(parts) if full.upper() == full: full = full.lower() # adjust case for all-uppercase input # This is an incomplete list. Adjust as needed to handle known ietf @@ -48,7 +54,7 @@ def name_parts(name): if len(parts) > 2: first = parts[0] last = parts[-1] - middle = u" ".join(parts[1:-1]) + middle = " ".join(parts[1:-1]) elif len(parts) == 2: first, last = parts else: @@ -63,16 +69,16 @@ def initials(name): prefix, first, middle, last, suffix = name_parts(name) given = first if middle: - given += u" "+middle + given += " "+middle # Don't use non-word characters as initials. # Example: The Bulgarian transcribed name "'Rnest Balkanska" should not have an initial of "'". - given = re.sub('[^ .\w]', '', given) - initials = u" ".join([ n[0].upper()+'.' for n in given.split() ]) + given = re.sub(r'[^ .\w]', '', given) + initials = " ".join([ n[0].upper()+'.' for n in given.split() ]) return initials def plain_name(name): prefix, first, middle, last, suffix = name_parts(name) - return u" ".join( n for n in (first, last) if n) + return " ".join( n for n in (first, last) if n) def capfirst(s): # Capitalize the first word character, skipping non-word characters and @@ -127,7 +133,7 @@ def normalize_name(s): if __name__ == "__main__": import sys - name = u" ".join(sys.argv[1:]) - print name_parts(name) - print initials(name) + name = " ".join(sys.argv[1:]) + print(name_parts(name)) + print(initials(name)) diff --git a/ietf/person/resources.py b/ietf/person/resources.py index 049b3e104..91dc19d4f 100644 --- a/ietf/person/resources.py +++ b/ietf/person/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2014-11-13 23:53 + + from ietf.api import ModelResource from tastypie.fields import ToOneField from tastypie.constants import ALL, ALL_WITH_RELATIONS diff --git a/ietf/person/tests.py b/ietf/person/tests.py index 7224a7051..410f89cbd 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -1,10 +1,14 @@ +# Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import datetime -import json +import six + from pyquery import PyQuery -from StringIO import StringIO +from io import StringIO from django.urls import reverse as urlreverse import debug # pyflakes:ignore @@ -35,7 +39,7 @@ class PersonTests(TestCase): r = self.client.get(urlreverse("ietf.person.views.ajax_select2_search", kwargs={ "model_name": "email"}), dict(q=person.name)) self.assertEqual(r.status_code, 200) - data = json.loads(r.content) + data = r.json() self.assertEqual(data[0]["id"], person.email_address()) def test_default_email(self): @@ -68,28 +72,28 @@ class PersonTests(TestCase): self.assertEqual(r.status_code, 200) def test_name_methods(self): - person = PersonFactory(name=u"Dr. Jens F. Möller", ) + person = PersonFactory(name="Dr. Jens F. Möller", ) - self.assertEqual(person.name, u"Dr. Jens F. Möller" ) - self.assertEqual(person.ascii_name(), u"Dr. Jens F. Moller" ) - self.assertEqual(person.plain_name(), u"Jens Möller" ) - self.assertEqual(person.plain_ascii(), u"Jens Moller" ) - self.assertEqual(person.initials(), u"J. F.") - self.assertEqual(person.first_name(), u"Jens" ) - self.assertEqual(person.last_name(), u"Möller" ) + self.assertEqual(person.name, "Dr. Jens F. Möller" ) + self.assertEqual(person.ascii_name(), "Dr. Jens F. Moller" ) + self.assertEqual(person.plain_name(), "Jens Möller" ) + self.assertEqual(person.plain_ascii(), "Jens Moller" ) + self.assertEqual(person.initials(), "J. F.") + self.assertEqual(person.first_name(), "Jens" ) + self.assertEqual(person.last_name(), "Möller" ) - person = PersonFactory(name=u"å´å»ºå¹³") + person = PersonFactory(name="å´å»ºå¹³") # The following are probably incorrect because the given name should # be Jianping and the surname should be Wu ... # TODO: Figure out better handling for names with CJK characters. # Maybe use ietf.person.cjk.* - self.assertEqual(person.ascii_name(), u"Wu Jian Ping") + self.assertEqual(person.ascii_name(), "Wu Jian Ping") def test_duplicate_person_name(self): empty_outbox() p = PersonFactory(name="Föö Bär") PersonFactory(name=p.name) - self.assertTrue("possible duplicate" in unicode(outbox[0]["Subject"]).lower()) + self.assertTrue("possible duplicate" in six.text_type(outbox[0]["Subject"]).lower()) def test_merge(self): url = urlreverse("ietf.person.views.merge") @@ -244,7 +248,7 @@ class PersonUtilsTests(TestCase): nomcom = NomComFactory() position = PositionFactory(nomcom=nomcom) nominee = NomineeFactory(nomcom=nomcom, person=mars.get_chair().person) - feedback = FeedbackFactory(user=source, author=person, nomcom=nomcom) + feedback = FeedbackFactory(user=source, author=person.email().address, nomcom=nomcom) feedback.nominees.add(nominee) nomination = NominationFactory(nominee=nominee, user=source, position=position, comments=feedback) diff --git a/ietf/person/utils.py b/ietf/person/utils.py index e72926123..d496f8c70 100755 --- a/ietf/person/utils.py +++ b/ietf/person/utils.py @@ -1,7 +1,13 @@ -from __future__ import unicode_literals, print_function +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import os import pprint +import six import sys import syslog @@ -19,7 +25,7 @@ def merge_persons(source, target, file=sys.stdout, verbose=False): changes = [] # write log - syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER) + syslog.openlog(str(os.path.basename(__file__)), syslog.LOG_PID, syslog.LOG_USER) syslog.syslog("Merging person records {} => {}".format(source.pk,target.pk)) # handle primary emails @@ -49,13 +55,13 @@ def merge_persons(source, target, file=sys.stdout, verbose=False): objs, opts, user, admin_site, using) deletable_objects_summary = deletable_objects[1] if len(deletable_objects_summary) > 1: # should only inlcude one object (Person) - print("Not Deleting Person: {}({})".format(source.ascii,source.pk), file=file) - print("Related objects remain:", file=file) + six.print_("Not Deleting Person: {}({})".format(source.ascii,source.pk), file=file) + six.print_("Related objects remain:", file=file) pprint.pprint(deletable_objects[1], stream=file) success = False else: success = True - print("Deleting Person: {}({})".format(source.ascii,source.pk), file=file) + six.print_("Deleting Person: {}({})".format(source.ascii,source.pk), file=file) source.delete() return success, changes @@ -108,7 +114,7 @@ def move_related_objects(source, target, file, verbose=False): field_name = related_object.field.name queryset = getattr(source, accessor).all() if verbose: - print("Merging {}:{}".format(accessor,queryset.count()),file=file) + six.print_("Merging {}:{}".format(accessor,queryset.count()), file=file) kwargs = { field_name:target } queryset.update(**kwargs) diff --git a/ietf/person/views.py b/ietf/person/views.py index da7b2f960..59e0cc101 100644 --- a/ietf/person/views.py +++ b/ietf/person/views.py @@ -1,5 +1,11 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime -from StringIO import StringIO +from io import StringIO from django.contrib import messages from django.db.models import Q @@ -109,7 +115,7 @@ def merge(request): output = StringIO() success, changes = merge_persons(source, target, file=output) if success: - messages.success(request, u'Merged {} ({}) to {} ({}). {})'.format( + messages.success(request, 'Merged {} ({}) to {} ({}). {})'.format( source.name, source_id, target.name, target.id, changes)) else: messages.error(request, output) diff --git a/ietf/redirects/models.py b/ietf/redirects/models.py index d9b7b46df..6a3b702e5 100644 --- a/ietf/redirects/models.py +++ b/ietf/redirects/models.py @@ -1,9 +1,15 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import models +from django.utils.encoding import python_2_unicode_compatible from ietf.utils.models import ForeignKey +@python_2_unicode_compatible class Redirect(models.Model): """Mapping of CGI script to url. The "rest" is a sprintf-style string with %(param)s entries to insert @@ -22,8 +28,9 @@ class Redirect(models.Model): rest = models.CharField(max_length=100, blank=True) remove = models.CharField(max_length=50, blank=True) def __str__(self): - return "%s -> %s/%s" % (self.cgi, self.url, self.rest) + return "%s -> %s/%s" % (self.cgi, self.url, self.rest) +@python_2_unicode_compatible class Suffix(models.Model): """This is a "rest" and "remove" (see Redirect class) for requests with command=. @@ -31,10 +38,11 @@ class Suffix(models.Model): rest = models.CharField(max_length=100, blank=True) remove = models.CharField(max_length=50, blank=True) def __str__(self): - return "-> %s - %s" % (self.rest, self.remove) + return "-> %s - %s" % (self.rest, self.remove) class Meta: verbose_name_plural="Suffixes" +@python_2_unicode_compatible class Command(models.Model): """When a request comes in with a command= argument, the command is looked up in this table to see if there @@ -47,12 +55,12 @@ class Command(models.Model): script = ForeignKey(Redirect, related_name='commands', editable=False) suffix = ForeignKey(Suffix, null=True, blank=True) def __str__(self): - ret = "%s?command=%s" % (self.script.cgi, self.command) - if self.suffix_id: - ret += " %s" % (self.suffix) - return ret + ret = "%s?command=%s" % (self.script.cgi, self.command) + if self.suffix_id: + ret += " %s" % (self.suffix) + return ret class Meta: - unique_together = (("script", "command"), ) + unique_together = (("script", "command"), ) # changes done by convert-096.py:changed maxlength to max_length # removed core diff --git a/ietf/redirects/resources.py b/ietf/redirects/resources.py index d65726280..672edd958 100644 --- a/ietf/redirects/resources.py +++ b/ietf/redirects/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2014-11-13 23:53 + + from ietf.api import ModelResource from tastypie.fields import ToOneField from tastypie.constants import ALL, ALL_WITH_RELATIONS diff --git a/ietf/redirects/tests.py b/ietf/redirects/tests.py index 4b8aec2c5..0337c68b1 100644 --- a/ietf/redirects/tests.py +++ b/ietf/redirects/tests.py @@ -1,3 +1,5 @@ +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -31,6 +33,8 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import, print_function, unicode_literals + from ietf.utils.test_utils import split_url, TestCase import debug # pyflakes:ignore @@ -45,46 +49,47 @@ REDIRECT_TESTS = { # idindex/idtracker '/public/pidtracker.cgi?command=view_id&dTag=11171&rfc_flag=0': - '/idtracker/11171/', + ('/idtracker/11171/', ), '/public/idindex.cgi?command=do_search_id&filename=draft-mills-sntp-v4-00.txt': - '/drafts/?filename=draft-mills-sntp-v4-00.txt', + ('/drafts/?filename=draft-mills-sntp-v4-00.txt', ), '/public/idindex.cgi?command=do_search_id&filename=draft-ietf-isis-interoperable&search_button=SEARCH': - '/drafts/?search_button=SEARCH&filename=draft-ietf-isis-interoperable', + ('/drafts/?filename=draft-ietf-isis-interoperable&search_button=SEARCH', + '/drafts/?search_button=SEARCH&filename=draft-ietf-isis-interoperable'), '/public/idindex.cgi?command=do_search_id&filename=rfc0038.txt': - '/drafts/?filename=rfc0038.txt', + ('/drafts/?filename=rfc0038.txt', ), '/public/idindex.cgi?command=id_detail&id=7096': - '/drafts/7096/', + ('/drafts/7096/', ), '/public/idindex.cgi?command=view_related_docs&id=10845': - '/drafts/10845/related/', + ('/drafts/10845/related/', ), '/public/idindex.cgi?command=id_detail&filename=draft-l3vpn-as4octet-ext-community': - '/drafts/draft-l3vpn-as4octet-ext-community/', + ('/drafts/draft-l3vpn-as4octet-ext-community/', ), # non-ASCII parameter '/public/pidtracker.cgi?command=view_id&dTag=11171%D182&rfc_flag=0': - '/idtracker/', - '/idtracker/': '/doc/', + ('/idtracker/', ), + '/idtracker/': ('/doc/', ), # ipr '/public/ipr_disclosure.cgi': - '/ipr/about/', + ('/ipr/about/', ), '/public/ipr_detail_show.cgi?ipr_id=693': - '/ipr/693/', + ('/ipr/693/', ), # liaisons '/public/liaison_detail.cgi?detail_id=340': - '/liaison/340/', + ('/liaison/340/', ), # meeting '/public/meeting_agenda_html.cgi?meeting_num=72': - '/meeting/72/agenda.html', + ('/meeting/72/agenda.html', ), '/public/meeting_materials.cgi?meeting_num=76': - '/meeting/76/materials.html', + ('/meeting/76/materials.html', ), # RedirectTrailingPeriod middleware '/sitemap.xml.': - '/sitemap.xml' + ('/sitemap.xml', ), } @@ -99,7 +104,7 @@ class RedirectsTests(TestCase): location = response['Location'] if location.startswith("http://testserver/"): location = location[17:] - self.assertEqual(location, dst, (src, dst, location)) + self.assertIn(location, dst, (src, dst, location)) class MainUrlTests(TestCase): def test_urls(self): diff --git a/ietf/redirects/views.py b/ietf/redirects/views.py index 3f4bc8520..37d2b7645 100644 --- a/ietf/redirects/views.py +++ b/ietf/redirects/views.py @@ -1,4 +1,8 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from django.http import HttpResponsePermanentRedirect, Http404, BadHeaderError from django.shortcuts import get_object_or_404 @@ -10,7 +14,7 @@ from ietf.redirects.models import Redirect, Command def redirect(request, path="", script=""): if path: - script = path + "/" + script + script = path + "/" + script redir = get_object_or_404(Redirect, cgi=script) url = "/" + redir.url + "/" (rest, remove) = (redir.rest, redir.remove) @@ -24,70 +28,70 @@ def redirect(request, path="", script=""): else: rparam = request.GET for flag in redir.commands.all().filter(command__startswith='^'): - fc = flag.command[1:].split("^") - if len(fc) > 1: - if rparam.get('command') != fc[1]: - continue - if rparam.has_key(fc[0]): - remove_args.append(fc[0]) - num = re.match('(\d+)', rparam[fc[0]]) - if (num and int(num.group(1))) or (num is None): - cmd = flag - break + fc = flag.command[1:].split("^") + if len(fc) > 1: + if rparam.get('command') != fc[1]: + continue + if fc[0] in rparam: + remove_args.append(fc[0]) + num = re.match(r'(\d+)', rparam[fc[0]]) + if (num and int(num.group(1))) or (num is None): + cmd = flag + break # # If that search didn't result in a match, then look # for an exact match for the command= parameter. if cmd is None: - try: - cmd = redir.commands.all().get(command=rparam['command']) - except Command.DoesNotExist: - pass # it's ok, there's no more-specific request. - except KeyError: - pass # it's ok, request didn't have 'command'. - except: - pass # strange exception like the one described in - # http://merlot.tools.ietf.org/tools/ietfdb/ticket/179 ? - # just ignore the command string. + try: + cmd = redir.commands.all().get(command=rparam['command']) + except Command.DoesNotExist: + pass # it's ok, there's no more-specific request. + except KeyError: + pass # it's ok, request didn't have 'command'. + except: + pass # strange exception like the one described in + # http://merlot.tools.ietf.org/tools/ietfdb/ticket/179 ? + # just ignore the command string. if cmd is not None: - remove_args.append('command') - if cmd.url: - rest = cmd.url + "/" - else: - rest = "" - if cmd.suffix: - rest = rest + cmd.suffix.rest - remove = cmd.suffix.remove - else: - remove = "" + remove_args.append('command') + if cmd.url: + rest = cmd.url + "/" + else: + rest = "" + if cmd.suffix: + rest = rest + cmd.suffix.rest + remove = cmd.suffix.remove + else: + remove = "" try: # This throws exception (which gets caught below) if request # contains non-ASCII characters. The old scripts didn't support # non-ASCII characters anyway, so there's no need to handle # them fully correctly in these redirects. - url += str(rest % rparam) - url += "/" + (rest % rparam).encode('ascii') + url += (rest % rparam) + "/" except: - # rest had something in it that request didn't have, so just - # redirect to the root of the tool. - pass + # rest had something in it that request didn't have, so just + # redirect to the root of the tool. + pass # Be generous in what you accept: collapse multiple slashes url = re.sub(r'/+', '/', url) if remove: - url = re.sub(re.escape(remove) + "/?$", "", url) + url = re.sub(re.escape(remove) + "/?$", "", url) # If there is a dot in the last url segment, remove the # trailing slash. This is basically the inverse of the # APPEND_SLASH middleware. if '/' in url and '.' in url.split('/')[-2]: - url = url.rstrip('/') + url = url.rstrip('/') # Copy the GET arguments, remove all the ones we were # expecting and if there are any left, add them to the URL. get = request.GET.copy() remove_args += re.findall(r'%\(([^)]+)\)', rest) for arg in remove_args: - if get.has_key(arg): - get.pop(arg) + if arg in get: + get.pop(arg) if get: - url += '?' + get.urlencode() + url += '?' + get.urlencode() try: return HttpResponsePermanentRedirect(url) except BadHeaderError: diff --git a/ietf/release/tests.py b/ietf/release/tests.py index b03d85292..4f9fdded2 100644 --- a/ietf/release/tests.py +++ b/ietf/release/tests.py @@ -1,6 +1,8 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals from pyquery import PyQuery diff --git a/ietf/release/urls.py b/ietf/release/urls.py index d33fcbb73..34cb023e5 100644 --- a/ietf/release/urls.py +++ b/ietf/release/urls.py @@ -1,6 +1,8 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2015-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals from django.views.generic import TemplateView diff --git a/ietf/release/views.py b/ietf/release/views.py index faf560c64..742dcba4f 100644 --- a/ietf/release/views.py +++ b/ietf/release/views.py @@ -1,7 +1,10 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2012-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import re import json @@ -43,7 +46,7 @@ def get_coverage_data(): with gzip.open(settings.TEST_COVERAGE_MASTER_FILE, "rb") as file: coverage_data = json.load(file) else: - with open(settings.TEST_COVERAGE_MASTER_FILE) as file: + with io.open(settings.TEST_COVERAGE_MASTER_FILE) as file: coverage_data = json.load(file) cache.set(cache_key, coverage_data, 60*60*24) return coverage_data diff --git a/ietf/review/admin.py b/ietf/review/admin.py index 4fbe41059..7a36fec88 100644 --- a/ietf/review/admin.py +++ b/ietf/review/admin.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import simple_history from django.contrib import admin @@ -17,7 +23,7 @@ class ReviewerSettingsAdmin(simple_history.admin.SimpleHistoryAdmin): admin.site.register(ReviewerSettings, ReviewerSettingsAdmin) class ReviewSecretarySettingsAdmin(admin.ModelAdmin): - list_display = [u'id', 'team', 'person', 'remind_days_before_deadline'] + list_display = ['id', 'team', 'person', 'remind_days_before_deadline'] raw_id_fields = ['team', 'person'] admin.site.register(ReviewSecretarySettings, ReviewSecretarySettingsAdmin) diff --git a/ietf/review/mailarch.py b/ietf/review/mailarch.py index 025146f27..fc50f1075 100644 --- a/ietf/review/mailarch.py +++ b/ietf/review/mailarch.py @@ -1,15 +1,31 @@ +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + + # various utilities for working with the mailarch mail archive at # mailarchive.ietf.org -import datetime, tarfile, mailbox, tempfile, hashlib, base64, email.utils -import urllib -import urllib2, contextlib -import debug # pyflakes:ignore +import contextlib +import datetime +import tarfile +import mailbox +import tempfile +import hashlib +import base64 +import email.utils + +from six.moves.urllib.parse import urlencode +from six.moves.urllib.request import urlopen + import debug # pyflakes:ignore from pyquery import PyQuery from django.conf import settings +from django.utils.encoding import force_bytes def list_name_from_email(list_email): if not list_email.endswith("@ietf.org"): @@ -22,9 +38,9 @@ def hash_list_message_id(list_name, msgid): # https://www.mail-archive.com/faq.html#listserver except the list # name (without "@ietf.org") is used instead of the full address, # and rightmost "=" signs are (optionally) stripped - sha = hashlib.sha1(msgid) - sha.update(list_name) - return base64.urlsafe_b64encode(sha.digest()).rstrip("=") + sha = hashlib.sha1(force_bytes(msgid)) + sha.update(force_bytes(list_name)) + return base64.urlsafe_b64encode(sha.digest()).rstrip(b"=") def construct_query_urls(review_req, query=None): list_name = list_name_from_email(review_req.team.list_email) @@ -34,7 +50,7 @@ def construct_query_urls(review_req, query=None): if not query: query = review_req.doc.name - encoded_query = "?" + urllib.urlencode({ + encoded_query = "?" + urlencode({ "qdr": "c", # custom time frame "start_date": (datetime.date.today() - datetime.timedelta(days=180)).isoformat(), "email_list": list_name, @@ -63,7 +79,7 @@ def retrieve_messages_from_mbox(mbox_fileobj): mbox = mailbox.mbox(mbox_file.name, create=False) for msg in mbox: - content = u"" + content = "" for part in msg.walk(): if part.get_content_type() == "text/plain": @@ -93,7 +109,7 @@ def retrieve_messages(query_data_url): """Retrieve and return selected content from mailarch.""" res = [] - with contextlib.closing(urllib2.urlopen(query_data_url, timeout=15)) as fileobj: + with contextlib.closing(urlopen(query_data_url, timeout=15)) as fileobj: content_type = fileobj.info()["Content-type"] if not content_type.startswith("application/x-tar"): if content_type.startswith("text/html"): diff --git a/ietf/review/migrations/0001_initial.py b/ietf/review/migrations/0001_initial.py index 33928e808..a57ead202 100644 --- a/ietf/review/migrations/0001_initial.py +++ b/ietf/review/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import datetime from django.db import migrations, models diff --git a/ietf/review/migrations/0002_unavailableperiod_reason.py b/ietf/review/migrations/0002_unavailableperiod_reason.py index ac3f89973..6a2754388 100644 --- a/ietf/review/migrations/0002_unavailableperiod_reason.py +++ b/ietf/review/migrations/0002_unavailableperiod_reason.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.14 on 2018-07-23 15:11 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/review/migrations/0003_add_notify_ad_when.py b/ietf/review/migrations/0003_add_notify_ad_when.py index 8ad40f67a..3a950a413 100644 --- a/ietf/review/migrations/0003_add_notify_ad_when.py +++ b/ietf/review/migrations/0003_add_notify_ad_when.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-02 10:10 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import ietf.review.models diff --git a/ietf/review/migrations/0004_reviewteamsettings_secr_mail_alias.py b/ietf/review/migrations/0004_reviewteamsettings_secr_mail_alias.py index 3db6e7b66..91fbcf94a 100644 --- a/ietf/review/migrations/0004_reviewteamsettings_secr_mail_alias.py +++ b/ietf/review/migrations/0004_reviewteamsettings_secr_mail_alias.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-03 03:10 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/review/migrations/0005_set_secdir_notify_ad_when.py b/ietf/review/migrations/0005_set_secdir_notify_ad_when.py index 89421c849..ade5a5287 100644 --- a/ietf/review/migrations/0005_set_secdir_notify_ad_when.py +++ b/ietf/review/migrations/0005_set_secdir_notify_ad_when.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-02 10:20 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/review/migrations/0006_historicalreviewersettings.py b/ietf/review/migrations/0006_historicalreviewersettings.py index fe8704405..0f9bc75d8 100644 --- a/ietf/review/migrations/0006_historicalreviewersettings.py +++ b/ietf/review/migrations/0006_historicalreviewersettings.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-09 08:31 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.conf import settings from django.db import migrations, models diff --git a/ietf/review/migrations/0007_allow_notify_ad_when_to_be_blank.py b/ietf/review/migrations/0007_allow_notify_ad_when_to_be_blank.py index ef6abc7ff..91016a8ed 100644 --- a/ietf/review/migrations/0007_allow_notify_ad_when_to_be_blank.py +++ b/ietf/review/migrations/0007_allow_notify_ad_when_to_be_blank.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.17 on 2018-12-06 13:16 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/review/migrations/0008_remove_reviewrequest_old_id.py b/ietf/review/migrations/0008_remove_reviewrequest_old_id.py index f2bc08577..df107e5b0 100644 --- a/ietf/review/migrations/0008_remove_reviewrequest_old_id.py +++ b/ietf/review/migrations/0008_remove_reviewrequest_old_id.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.17 on 2019-01-03 12:34 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/review/migrations/0009_refactor_review_request.py b/ietf/review/migrations/0009_refactor_review_request.py index 766dc209e..78638239d 100644 --- a/ietf/review/migrations/0009_refactor_review_request.py +++ b/ietf/review/migrations/0009_refactor_review_request.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.18 on 2019-01-04 14:27 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/review/migrations/0010_populate_review_assignments.py b/ietf/review/migrations/0010_populate_review_assignments.py index d8f38f24b..3602645c4 100644 --- a/ietf/review/migrations/0010_populate_review_assignments.py +++ b/ietf/review/migrations/0010_populate_review_assignments.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.18 on 2019-01-04 14:34 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import sys @@ -37,8 +40,8 @@ def forward(apps, schema_editor): sys.stderr.write('\n') # introduce a newline before tqdm starts writing for request in tqdm(ReviewRequest.objects.exclude(unused_reviewer__isnull=True)): assignment_state = map_request_state_to_assignment_state(request.state_id) - if not (assignment_state in (u'assigned', u'accepted', u'completed', u'no-response', u'overtaken', u'part-completed', u'rejected', u'withdrawn', u'unknown')): - print ("Trouble with review_request",request.pk,"with state",request.state_id) + if not (assignment_state in ('assigned', 'accepted', 'completed', 'no-response', 'overtaken', 'part-completed', 'rejected', 'withdrawn', 'unknown')): + print(("Trouble with review_request",request.pk,"with state",request.state_id)) exit(-1) ReviewAssignment.objects.create( review_request = request, diff --git a/ietf/review/migrations/0011_review_document2_fk.py b/ietf/review/migrations/0011_review_document2_fk.py index a18e178da..f613cdb9b 100644 --- a/ietf/review/migrations/0011_review_document2_fk.py +++ b/ietf/review/migrations/0011_review_document2_fk.py @@ -1,7 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-08 11:58 -# Copyright The IETF Trust 2019, All Rights Reserved -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations import django.db.models.deletion diff --git a/ietf/review/migrations/0012_remove_old_document_field.py b/ietf/review/migrations/0012_remove_old_document_field.py index 0028b6bb8..56790f507 100644 --- a/ietf/review/migrations/0012_remove_old_document_field.py +++ b/ietf/review/migrations/0012_remove_old_document_field.py @@ -1,7 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-20 09:53 -# Copyright The IETF Trust 2019, All Rights Reserved -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/review/migrations/0013_rename_field_document2.py b/ietf/review/migrations/0013_rename_field_document2.py index a9ce514ea..da3c17ea0 100644 --- a/ietf/review/migrations/0013_rename_field_document2.py +++ b/ietf/review/migrations/0013_rename_field_document2.py @@ -1,7 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-21 05:31 -# Copyright The IETF Trust 2019, All Rights Reserved -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/review/migrations/0014_document_primary_key_cleanup.py b/ietf/review/migrations/0014_document_primary_key_cleanup.py index d54fb3a9c..c8ef25beb 100644 --- a/ietf/review/migrations/0014_document_primary_key_cleanup.py +++ b/ietf/review/migrations/0014_document_primary_key_cleanup.py @@ -1,7 +1,9 @@ +# Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-06-10 03:47 -# Copyright The IETF Trust 2019, All Rights Reserved -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations import django.db.models.deletion diff --git a/ietf/review/models.py b/ietf/review/models.py index 39373cd04..4cdb34297 100644 --- a/ietf/review/models.py +++ b/ietf/review/models.py @@ -1,10 +1,15 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import datetime from simple_history.models import HistoricalRecords from django.db import models +from django.utils.encoding import python_2_unicode_compatible from ietf.doc.models import Document from ietf.group.models import Group @@ -13,6 +18,7 @@ from ietf.name.models import ReviewTypeName, ReviewRequestStateName, ReviewResul from ietf.utils.validators import validate_regular_expression_string from ietf.utils.models import ForeignKey, OneToOneField +@python_2_unicode_compatible class ReviewerSettings(models.Model): """Keeps track of admin data associated with a reviewer in a team.""" history = HistoricalRecords() @@ -33,24 +39,26 @@ class ReviewerSettings(models.Model): remind_days_before_deadline = models.IntegerField(null=True, blank=True, help_text="To get an email reminder in case you forget to do an assigned review, enter the number of days before review deadline you want to receive it. Clear the field if you don't want a reminder.") expertise = models.TextField(verbose_name="Reviewer's expertise in this team's area", max_length=2048, blank=True, help_text="Describe the reviewer's expertise in this team's area", default='') - def __unicode__(self): - return u"{} in {}".format(self.person, self.team) + def __str__(self): + return "{} in {}".format(self.person, self.team) class Meta: verbose_name_plural = "reviewer settings" +@python_2_unicode_compatible class ReviewSecretarySettings(models.Model): """Keeps track of admin data associated with a secretary in a team.""" team = ForeignKey(Group, limit_choices_to=~models.Q(reviewteamsettings=None)) person = ForeignKey(Person) remind_days_before_deadline = models.IntegerField(null=True, blank=True, help_text="To get an email reminder in case a reviewer forgets to do an assigned review, enter the number of days before review deadline you want to receive it. Clear the field if you don't want a reminder.") - def __unicode__(self): - return u"{} in {}".format(self.person, self.team) + def __str__(self): + return "{} in {}".format(self.person, self.team) class Meta: verbose_name_plural = "review secretary settings" +@python_2_unicode_compatible class UnavailablePeriod(models.Model): team = ForeignKey(Group, limit_choices_to=~models.Q(reviewteamsettings=None)) person = ForeignKey(Person) @@ -78,9 +86,10 @@ class UnavailablePeriod(models.Model): else: return "future" - def __unicode__(self): - return u"{} is unavailable in {} {} - {}".format(self.person, self.team.acronym, self.start_date or "", self.end_date or "") + def __str__(self): + return "{} is unavailable in {} {} - {}".format(self.person, self.team.acronym, self.start_date or "", self.end_date or "") +@python_2_unicode_compatible class ReviewWish(models.Model): """Reviewer wishes to review a document when it becomes available for review.""" time = models.DateTimeField(default=datetime.datetime.now) @@ -88,24 +97,26 @@ class ReviewWish(models.Model): person = ForeignKey(Person) doc = ForeignKey(Document) - def __unicode__(self): - return u"{} wishes to review {} in {}".format(self.person, self.doc.name, self.team.acronym) + def __str__(self): + return "{} wishes to review {} in {}".format(self.person, self.doc.name, self.team.acronym) class Meta: verbose_name_plural = "review wishes" +@python_2_unicode_compatible class NextReviewerInTeam(models.Model): team = ForeignKey(Group, limit_choices_to=~models.Q(reviewteamsettings=None)) next_reviewer = ForeignKey(Person) - def __unicode__(self): - return u"{} next in {}".format(self.next_reviewer, self.team) + def __str__(self): + return "{} next in {}".format(self.next_reviewer, self.team) class Meta: verbose_name = "next reviewer in team setting" verbose_name_plural = "next reviewer in team settings" +@python_2_unicode_compatible class ReviewRequest(models.Model): """Represents a request for a review and the process it goes through.""" state = ForeignKey(ReviewRequestStateName) @@ -121,8 +132,8 @@ class ReviewRequest(models.Model): requested_rev = models.CharField(verbose_name="requested revision", max_length=16, blank=True, help_text="Fill in if a specific revision is to be reviewed, e.g. 02") comment = models.TextField(verbose_name="Requester's comments and instructions", max_length=2048, blank=True, help_text="Provide any additional information to show to the review team secretary and reviewer", default='') - def __unicode__(self): - return u"%s review on %s by %s %s" % (self.type, self.doc, self.team, self.state) + def __str__(self): + return "%s review on %s by %s %s" % (self.type, self.doc, self.team, self.state) def all_completed_assignments_for_doc(self): return ReviewAssignment.objects.filter(review_request__doc=self.doc, state__in=['completed','part-completed']) @@ -130,6 +141,7 @@ class ReviewRequest(models.Model): def request_closed_time(self): return self.doc.request_closed_time(self) or self.time +@python_2_unicode_compatible class ReviewAssignment(models.Model): """ One of possibly many reviews assigned in response to a ReviewRequest """ review_request = ForeignKey(ReviewRequest) @@ -142,8 +154,8 @@ class ReviewAssignment(models.Model): result = ForeignKey(ReviewResultName, blank=True, null=True) mailarch_url = models.URLField(blank=True, null = True) - def __unicode__(self): - return u"Assignment for %s (%s) : %s %s of %s" % (self.reviewer.person, self.state, self.review_request.team.acronym, self.review_request.type, self.review_request.doc) + def __str__(self): + return "Assignment for %s (%s) : %s %s of %s" % (self.reviewer.person, self.state, self.review_request.team.acronym, self.review_request.type, self.review_request.doc) def get_default_review_types(): @@ -152,6 +164,7 @@ def get_default_review_types(): def get_default_review_results(): return ReviewResultName.objects.filter(slug__in=['not-ready', 'right-track', 'almost-ready', 'ready-issues', 'ready-nits', 'ready']) +@python_2_unicode_compatible class ReviewTeamSettings(models.Model): """Holds configuration specific to groups that are review teams""" group = OneToOneField(Group) @@ -161,8 +174,8 @@ class ReviewTeamSettings(models.Model): notify_ad_when = models.ManyToManyField(ReviewResultName, related_name='reviewteamsettings_notify_ad_set', blank=True) secr_mail_alias = models.CharField(verbose_name="Email alias for all of the review team secretaries", max_length=255, blank=True, help_text="Email alias for all of the review team secretaries") - def __unicode__(self): - return u"%s" % (self.group.acronym,) + def __str__(self): + return "%s" % (self.group.acronym,) class Meta: verbose_name = "Review team settings" diff --git a/ietf/review/resources.py b/ietf/review/resources.py index 45599e7b1..a7850dcb8 100644 --- a/ietf/review/resources.py +++ b/ietf/review/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2016-06-14 04:21 PDT + + from tastypie.resources import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore @@ -83,7 +86,7 @@ class UnavailablePeriodResource(ModelResource): "start_date": ALL, "end_date": ALL, "availability": ALL, - "reason": ALL, + "reason": ALL, "team": ALL_WITH_RELATIONS, "person": ALL_WITH_RELATIONS, } diff --git a/ietf/review/tests.py b/ietf/review/tests.py new file mode 100644 index 000000000..5d599669a --- /dev/null +++ b/ietf/review/tests.py @@ -0,0 +1,25 @@ +# Copyright The IETF Trust 2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +from ietf.utils.test_utils import TestCase +from .mailarch import hash_list_message_id + +class HashTest(TestCase): + + def test_hash_list_message_id(self): + for list, msgid, hash in ( + ('ietf', '156182196167.12901.11966487185176024571@ietfa.amsl.com', b'lr6RtZ4TiVMZn1fZbykhkXeKhEk'), + ('codesprints', 'E1hNffl-0004RM-Dh@zinfandel.tools.ietf.org', b'N1nFHHUXiFWYtdzBgjtqzzILFHI'), + ('xml2rfc', '3A0F4CD6-451F-44E2-9DA4-28235C638588@rfc-editor.org', b'g6DN4SxJGDrlSuKsubwb6rRSePU'), + (u'ietf', u'156182196167.12901.11966487185176024571@ietfa.amsl.com',b'lr6RtZ4TiVMZn1fZbykhkXeKhEk'), + (u'codesprints', u'E1hNffl-0004RM-Dh@zinfandel.tools.ietf.org', b'N1nFHHUXiFWYtdzBgjtqzzILFHI'), + (u'xml2rfc', u'3A0F4CD6-451F-44E2-9DA4-28235C638588@rfc-editor.org',b'g6DN4SxJGDrlSuKsubwb6rRSePU'), + (b'ietf', b'156182196167.12901.11966487185176024571@ietfa.amsl.com',b'lr6RtZ4TiVMZn1fZbykhkXeKhEk'), + (b'codesprints', b'E1hNffl-0004RM-Dh@zinfandel.tools.ietf.org', b'N1nFHHUXiFWYtdzBgjtqzzILFHI'), + (b'xml2rfc', b'3A0F4CD6-451F-44E2-9DA4-28235C638588@rfc-editor.org',b'g6DN4SxJGDrlSuKsubwb6rRSePU'), + ): + self.assertEqual(hash, hash_list_message_id(list, msgid)) + diff --git a/ietf/review/utils.py b/ietf/review/utils.py index 3d2754423..cfff82e8c 100644 --- a/ietf/review/utils.py +++ b/ietf/review/utils.py @@ -1,9 +1,14 @@ -# -*- coding: utf-8 -*- # Copyright The IETF Trust 2016-2019, All Rights Reserved -from __future__ import unicode_literals, print_function +# -*- coding: utf-8 -*- -import datetime, re, itertools +from __future__ import absolute_import, print_function, unicode_literals + +import datetime +import itertools +import re +import six + from collections import defaultdict, namedtuple from django.db.models import Q, Max, F @@ -128,12 +133,12 @@ def reviewer_rotation_list(team, skip_unavailable=False, dont_skip=[]): reviewers_to_skip = set() unavailable_periods = current_unavailable_periods_for_reviewers(team) - for person_id, periods in unavailable_periods.iteritems(): + for person_id, periods in unavailable_periods.items(): if periods and person_id not in dont_skip: reviewers_to_skip.add(person_id) days_needed_for_reviewers = days_needed_to_fulfill_min_interval_for_reviewers(team) - for person_id, days_needed in days_needed_for_reviewers.iteritems(): + for person_id, days_needed in days_needed_for_reviewers.items(): if person_id not in dont_skip: reviewers_to_skip.add(person_id) @@ -154,7 +159,7 @@ def days_needed_to_fulfill_min_interval_for_reviewers(team): now = datetime.datetime.now() res = {} - for person_id, latest_assignment_time in latest_assignments.iteritems(): + for person_id, latest_assignment_time in latest_assignments.items(): if latest_assignment_time is not None: min_interval = min_intervals.get(person_id) if min_interval is None: @@ -294,11 +299,11 @@ def sum_raw_review_assignment_aggregations(raw_aggregations): for raw_aggr in raw_aggregations: i_state_dict, i_late_state_dict, i_result_dict, i_assignment_to_closure_days_list, i_assignment_to_closure_days_count = raw_aggr - for s, v in i_state_dict.iteritems(): + for s, v in i_state_dict.items(): state_dict[s] += v - for s, v in i_late_state_dict.iteritems(): + for s, v in i_late_state_dict.items(): late_state_dict[s] += v - for r, v in i_result_dict.iteritems(): + for r, v in i_result_dict.items(): result_dict[r] += v assignment_to_closure_days_list.extend(i_assignment_to_closure_days_list) @@ -501,7 +506,7 @@ def assign_review_request_to_reviewer(request, review_req, reviewer, add_skip=Fa if prev_team_reviews.exists(): msg = msg + '\n\nThis team has completed other reviews of this document:\n' for assignment in prev_team_reviews: - msg += u'%s %s -%s %s\n'% ( + msg += '%s %s -%s %s\n'% ( assignment.completed_on.strftime('%d %b %Y'), assignment.reviewer.person.ascii, assignment.reviewed_rev or assignment.review_request.requested_rev, @@ -688,7 +693,7 @@ def suggested_review_requests_for_team(team): # filter those with existing explicit requests existing_requests = defaultdict(list) - for r in ReviewRequest.objects.filter(doc__id__in=requests.iterkeys(), team=team): + for r in ReviewRequest.objects.filter(doc__id__in=iter(requests.keys()), team=team): existing_requests[r.doc_id].append(r) def blocks(existing, request): @@ -706,7 +711,7 @@ def suggested_review_requests_for_team(team): return any([no_review_document, no_review_rev, pending, request_closed, some_assignment_completed]) - res = [r for r in requests.itervalues() + res = [r for r in requests.values() if not any(blocks(e, r) for e in existing_requests[r.doc_id])] res.sort(key=lambda r: (r.deadline, r.doc_id), reverse=True) return res @@ -719,7 +724,7 @@ def extract_revision_ordered_review_assignments_for_documents_and_replaced(revie replaces = extract_complete_replaces_ancestor_mapping_for_docs(names) assignments_for_each_doc = defaultdict(list) - replacement_name_set = set(e for l in replaces.itervalues() for e in l) | names + replacement_name_set = set(e for l in replaces.values() for e in l) | names for r in ( review_assignment_queryset.filter(review_request__doc__name__in=replacement_name_set) .order_by("-reviewed_rev","-assigned_on", "-id").iterator()): assignments_for_each_doc[r.review_request.doc.name].append(r) @@ -767,7 +772,7 @@ def extract_revision_ordered_review_requests_for_documents_and_replaced(review_r replaces = extract_complete_replaces_ancestor_mapping_for_docs(names) requests_for_each_doc = defaultdict(list) - for r in review_request_queryset.filter(doc__name__in=set(e for l in replaces.itervalues() for e in l) | names).order_by("-time", "-id").iterator(): + for r in review_request_queryset.filter(doc__name__in=set(e for l in replaces.values() for e in l) | names).order_by("-time", "-id").iterator(): requests_for_each_doc[r.doc.name].append(r) # now collect in breadth-first order to keep the revision order intact @@ -959,9 +964,9 @@ def make_assignment_choices(email_queryset, review_req): if stats: explanations.append(", ".join(stats)) - label = unicode(e.person) + label = six.text_type(e.person) if explanations: - label = u"{}: {}".format(label, u"; ".join(explanations)) + label = "{}: {}".format(label, "; ".join(explanations)) ranking.append({ "email": e, @@ -1026,10 +1031,10 @@ def review_assignments_needing_secretary_reminder(remind_date): if (deadline - remind_date).days == remind_days: assignment_pks[a_pk] = secretary_role_pk - review_assignments = { a.pk: a for a in ReviewAssignment.objects.filter(pk__in=assignment_pks.keys()).select_related("reviewer", "reviewer__person", "state", "review_request__team") } - secretary_roles = { r.pk: r for r in Role.objects.filter(pk__in=assignment_pks.values()).select_related("email", "person") } + review_assignments = { a.pk: a for a in ReviewAssignment.objects.filter(pk__in=list(assignment_pks.keys())).select_related("reviewer", "reviewer__person", "state", "review_request__team") } + secretary_roles = { r.pk: r for r in Role.objects.filter(pk__in=list(assignment_pks.values())).select_related("email", "person") } - return [ (review_assignments[a_pk], secretary_roles[secretary_role_pk]) for a_pk, secretary_role_pk in assignment_pks.iteritems() ] + return [ (review_assignments[a_pk], secretary_roles[secretary_role_pk]) for a_pk, secretary_role_pk in assignment_pks.items() ] def email_secretary_reminder(review_request, secretary_role): team = review_request.team diff --git a/ietf/secr/announcement/forms.py b/ietf/secr/announcement/forms.py index 6aaca732d..ea8be460c 100644 --- a/ietf/secr/announcement/forms.py +++ b/ietf/secr/announcement/forms.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django import forms from ietf.group.models import Group, Role @@ -37,7 +43,7 @@ def get_from_choices(user): if nomcom_choices: addresses = list(addresses) + nomcom_choices - return zip(addresses, addresses) + return list(zip(addresses, addresses)) def get_nomcom_choices(user): @@ -58,7 +64,7 @@ def get_nomcom_choices(user): def get_to_choices(): - return zip(TO_LIST,TO_LIST) + return list(zip(TO_LIST,TO_LIST)) # --------------------------------------------- @@ -96,7 +102,7 @@ class AnnounceForm(forms.ModelForm): self.initial['reply_to'] = 'ietf@ietf.org' if self.hidden: - for key in self.fields.keys(): + for key in list(self.fields.keys()): self.fields[key].widget = forms.HiddenInput() def clean(self): diff --git a/ietf/secr/announcement/tests.py b/ietf/secr/announcement/tests.py index d7ba222eb..6200af090 100644 --- a/ietf/secr/announcement/tests.py +++ b/ietf/secr/announcement/tests.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.urls import reverse from pyquery import PyQuery @@ -94,8 +100,7 @@ class SubmitAnnouncementCase(TestCase): 'body':'This is a test.'} self.client.login(username="secretary", password="secretary+password") response = self.client.post(url,post_data) - self.assertEqual(response.status_code, 200) - self.assertTrue('Confirm Announcement' in response.content) + self.assertContains(response, 'Confirm Announcement') response = self.client.post(confirm_url,post_data,follow=True) self.assertRedirects(response, url) self.assertEqual(len(outbox),1) diff --git a/ietf/secr/console/tests.py b/ietf/secr/console/tests.py index 354be5fe0..75ddf483b 100644 --- a/ietf/secr/console/tests.py +++ b/ietf/secr/console/tests.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + """ This file demonstrates two different styles of tests (one doctest and one unittest). These will both pass when you run "manage.py test". @@ -12,7 +18,7 @@ class SimpleTest(TestCase): """ Tests that 1 + 1 always equals 2. """ - self.failUnlessEqual(1 + 1, 2) + self.assertEqual(1 + 1, 2) __test__ = {"doctest": """ Another way to test that 1 + 1 is equal to 2. diff --git a/ietf/secr/drafts/forms.py b/ietf/secr/drafts/forms.py index 581e91bbf..0dd7539a0 100644 --- a/ietf/secr/drafts/forms.py +++ b/ietf/secr/drafts/forms.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import re import os @@ -45,11 +51,11 @@ class DocumentField(forms.FileField): if self.filename: # validate filename if base[:-3] != self.filename: - raise forms.ValidationError, "Filename: %s doesn't match Draft filename." % base[:-3] + raise forms.ValidationError("Filename: %s doesn't match Draft filename." % base[:-3]) # validate revision next_revision = str(int(self.rev)+1).zfill(2) if base[-2:] != next_revision: - raise forms.ValidationError, "Expected revision # %s" % (next_revision) + raise forms.ValidationError("Expected revision # %s" % (next_revision)) return file @@ -217,7 +223,7 @@ class EmailForm(forms.Form): super(EmailForm, self).__init__(*args, **kwargs) if self.hidden: - for key in self.fields.keys(): + for key in list(self.fields.keys()): self.fields[key].widget = forms.HiddenInput() class ExtendForm(forms.Form): diff --git a/ietf/secr/drafts/tests_views.py b/ietf/secr/drafts/tests_views.py index c8f7fd725..ca0968737 100644 --- a/ietf/secr/drafts/tests_views.py +++ b/ietf/secr/drafts/tests_views.py @@ -1,4 +1,11 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime +import io import os import shutil from collections import OrderedDict @@ -60,8 +67,7 @@ class SecrDraftsTestCase(TestCase): url = urlreverse('ietf.secr.drafts.views.approvals') self.client.login(username="secretary", password="secretary+password") response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertTrue('draft-dummy' in response.content) + self.assertContains(response, 'draft-dummy') def test_edit(self): draft = WgDraftFactory(states=[('draft','active'),('draft-stream-ietf','wg-doc'),('draft-iesg','ad-eval')], shepherd=EmailFactory()) @@ -105,8 +111,7 @@ class SecrDraftsTestCase(TestCase): post = dict(filename='draft',state=1,submit='submit') response = self.client.post(url, post) - self.assertEqual(response.status_code, 200) - self.assertTrue(draft.name in response.content) + self.assertContains(response, draft.name) def test_view(self): draft = WgDraftFactory() @@ -131,7 +136,7 @@ class SecrDraftsTestCase(TestCase): def test_resurrect(self): draft = WgDraftFactory() path = os.path.join(self.repository_dir, draft.filename_with_rev()) - with open(path, 'w') as file: + with io.open(path, 'w') as file: file.write('test') expire_draft(draft) email_url = urlreverse('ietf.secr.drafts.views.email', kwargs={'id':draft.name}) + "?action=resurrect" @@ -141,8 +146,7 @@ class SecrDraftsTestCase(TestCase): subject = 'Resurrection of %s' % draft.get_base_name() self.client.login(username="secretary", password="secretary+password") response = self.client.get(email_url) - self.assertEqual(response.status_code, 200) - self.assertTrue('Drafts - Email' in response.content) + self.assertContains(response, 'Drafts - Email') q = PyQuery(response.content) self.assertEqual(q("#id_subject").val(), subject) post_data = { @@ -154,8 +158,7 @@ class SecrDraftsTestCase(TestCase): 'submit': 'Save' } response = self.client.post(confirm_url, post_data) - self.assertEqual(response.status_code, 200) - self.assertTrue('Drafts - Confirm' in response.content) + self.assertContains(response, 'Drafts - Confirm') self.assertEqual(response.context['email']['subject'], subject) response = self.client.post(do_action_url, post_data) self.assertRedirects(response, view_url) @@ -193,8 +196,7 @@ class SecrDraftsTestCase(TestCase): response = self.client.post(url, extend_data) self.assertRedirects(response, email_url + '?' + urlencode(extend_data)) response = self.client.post(confirm_url, post_data) - self.assertEqual(response.status_code, 200) - self.assertTrue('Drafts - Confirm' in response.content) + self.assertContains(response, 'Drafts - Confirm') self.assertEqual(response.context['email']['subject'], subject) response = self.client.post(do_action_url, post_data) self.assertRedirects(response, view_url) @@ -227,8 +229,7 @@ class SecrDraftsTestCase(TestCase): response = self.client.post(url, withdraw_data) self.assertRedirects(response, email_url + '?' + urlencode(withdraw_data)) response = self.client.post(confirm_url, post_data) - self.assertEqual(response.status_code, 200) - self.assertTrue('Drafts - Confirm' in response.content) + self.assertContains(response, 'Drafts - Confirm') self.assertEqual(response.context['email']['subject'], subject) response = self.client.post(do_action_url, post_data) self.assertRedirects(response, view_url) diff --git a/ietf/secr/drafts/views.py b/ietf/secr/drafts/views.py index f5c343b1b..e6a3bfe27 100644 --- a/ietf/secr/drafts/views.py +++ b/ietf/secr/drafts/views.py @@ -1,8 +1,12 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import glob +import io import os import shutil from dateutil.parser import parse @@ -53,7 +57,7 @@ def handle_uploaded_file(f): ''' Save uploaded draft files to temporary directory ''' - destination = open(os.path.join(settings.IDSUBMIT_MANUAL_STAGING_DIR, f.name), 'wb+') + destination = io.open(os.path.join(settings.IDSUBMIT_MANUAL_STAGING_DIR, f.name), 'wb+') for chunk in f.chunks(): destination.write(chunk) destination.close() @@ -259,7 +263,7 @@ def authors(request, id): authors = draft.documentauthor_set.all() if authors: - order = authors.aggregate(Max('order')).values()[0] + 1 + order = list(authors.aggregate(Max('order')).values())[0] + 1 else: order = 1 DocumentAuthor.objects.create(document=draft, person=person, email=email, affiliation=affiliation, country=country, order=order) @@ -418,7 +422,7 @@ def email(request, id): # other problems with get_email_initial try: form = EmailForm(initial=get_email_initial(draft,action=action,input=data)) - except Exception, e: + except Exception as e: return render(request, 'drafts/error.html', { 'error': e},) return render(request, 'drafts/email.html', { diff --git a/ietf/secr/groups/tests.py b/ietf/secr/groups/tests.py index 8de85c501..c49d95cd1 100644 --- a/ietf/secr/groups/tests.py +++ b/ietf/secr/groups/tests.py @@ -1,4 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.urls import reverse from ietf.utils.test_utils import TestCase from ietf.group.models import Group @@ -26,9 +31,7 @@ class GroupsTest(TestCase): post_data = {'group_acronym':group.acronym,'submit':'Search'} self.client.login(username="secretary", password="secretary+password") response = self.client.post(url,post_data,follow=True) - #assert False, response.content - self.assertEqual(response.status_code, 200) - self.assertTrue(group.acronym in response.content) + self.assertContains(response, group.acronym) # ------- Test Add -------- # def test_add_button(self): @@ -48,8 +51,7 @@ class GroupsTest(TestCase): 'submit':'Save'} self.client.login(username="secretary", password="secretary+password") response = self.client.post(url,post_data) - self.assertEqual(response.status_code, 200) - self.assertTrue('This field is required' in response.content) + self.assertContains(response, 'This field is required') def test_add_group_dupe(self): group = GroupFactory() @@ -65,9 +67,7 @@ class GroupsTest(TestCase): 'submit':'Save'} self.client.login(username="secretary", password="secretary+password") response = self.client.post(url,post_data) - #print response.content - self.assertEqual(response.status_code, 200) - self.assertTrue('Group with this Acronym already exists' in response.content) + self.assertContains(response, 'Group with this Acronym already exists') def test_add_group_success(self): area = GroupFactory(type_id='area') @@ -113,7 +113,7 @@ class GroupsTest(TestCase): self.client.login(username="secretary", password="secretary+password") response = self.client.post(url,post_data,follow=True) self.assertRedirects(response, target) - self.assertTrue('changed successfully' in response.content) + self.assertContains(response, 'changed successfully') def test_edit_non_wg_group(self): parent_sdo = GroupFactory.create(type_id='sdo',state_id='active') @@ -133,7 +133,7 @@ class GroupsTest(TestCase): self.client.login(username="secretary", password="secretary+password") response = self.client.post(url,post_data,follow=True) self.assertRedirects(response, target) - self.assertTrue('changed successfully' in response.content) + self.assertContains(response, 'changed successfully') # ------- Test People -------- # def test_people_delete(self): @@ -161,4 +161,4 @@ class GroupsTest(TestCase): self.client.login(username="secretary", password="secretary+password") response = self.client.post(url,post_data,follow=True) self.assertRedirects(response, url) - self.assertTrue('added successfully' in response.content) + self.assertContains(response, 'added successfully') diff --git a/ietf/secr/meetings/blue_sheets.py b/ietf/secr/meetings/blue_sheets.py index fe1639103..49d8755ca 100644 --- a/ietf/secr/meetings/blue_sheets.py +++ b/ietf/secr/meetings/blue_sheets.py @@ -1,6 +1,15 @@ -from django.conf import settings +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- -''' + +from __future__ import absolute_import, print_function, unicode_literals + +import io + +from django.conf import settings +from django.utils.encoding import force_bytes + +r''' RTF quick reference (from Word2007RTFSpec9.doc): \fs24 : sets the font size to 24 half points \header : header on all pages @@ -16,9 +25,9 @@ RTF quick reference (from Word2007RTFSpec9.doc): ''' def create_blue_sheets(meeting, groups): - file = open(settings.SECR_BLUE_SHEET_PATH, 'w') + file = io.open(settings.SECR_BLUE_SHEET_PATH, 'wb') - header = '''{\\rtf1\\ansi\\ansicpg1252\\uc1 \\deff0\\deflang1033\\deflangfe1033 + header = b'''{\\rtf1\\ansi\\ansicpg1252\\uc1 \\deff0\\deflang1033\\deflangfe1033 {\\fonttbl{\\f0\\froman\\fcharset0\\fprq2{\\*\\panose 02020603050405020304}Times New Roman;}} {\\colortbl;\\red0\\green0\\blue0;\\red0\\green0\\blue255;\\red0\\green255\\blue255;\\red0\\green255\\blue0; \\red255\\green0\\blue255;\\red255\\green0\\blue0;\\red255\\green255\\blue0;\\red255\\green255\\blue255; @@ -31,7 +40,7 @@ def create_blue_sheets(meeting, groups): file.write(header) for group in groups: - group_header = ''' {\\header \\pard\\plain \\s15\\nowidctlpar\\widctlpar\\tqc\\tx4320\\tqr\\tx8640\\adjustright \\fs20\\cgrid + group_header = b''' {\\header \\pard\\plain \\s15\\nowidctlpar\\widctlpar\\tqc\\tx4320\\tqr\\tx8640\\adjustright \\fs20\\cgrid { Mailing List: %s \\tab\\tab Meeting # %s %s (%s) \\par } \\pard \\s15\\nowidctlpar\\widctlpar\\tqc\\tx4320\\tqr\\tx8640\\adjustright {\\b\\fs24 @@ -59,30 +68,31 @@ def create_blue_sheets(meeting, groups): \\par } \\pard \\fi-90\\li90\\nowidctlpar\\widctlpar\\adjustright {\\fs16 -''' % (group.list_email, - meeting.number, - group.acronym, - group.type, - meeting.number, - group.acronym, - group.type, - meeting.number, - group.name, - group.list_email) +''' % (force_bytes(group.list_email), + force_bytes(meeting.number), + force_bytes(group.acronym), + force_bytes(group.type), + force_bytes(meeting.number), + force_bytes(group.acronym), + force_bytes(group.type), + force_bytes(meeting.number), + force_bytes(group.name), + force_bytes(group.list_email), + ) file.write(group_header) for x in range(1,117): - line = '''\\par %s._________________________________________________ \\tab _____________________________________________________ + line = b'''\\par %s._________________________________________________ \\tab _____________________________________________________ \\par - ''' % x + ''' % force_bytes(x) file.write(line) - footer = '''} + footer = b'''} \\pard \\nowidctlpar\\widctlpar\\adjustright {\\fs16 \\sect } \\sectd \\pgnrestart\\linex0\\endnhere\\titlepg\\sectdefaultcl ''' file.write(footer) - file.write('\n}') + file.write(b'\n}') file.close() diff --git a/ietf/secr/meetings/tests.py b/ietf/secr/meetings/tests.py index 897ce6a5e..5082cb6b9 100644 --- a/ietf/secr/meetings/tests.py +++ b/ietf/secr/meetings/tests.py @@ -1,8 +1,14 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import os import shutil from pyquery import PyQuery -from StringIO import StringIO +from io import StringIO import debug # pyflakes:ignore @@ -153,7 +159,7 @@ class SecrMeetingTestCase(TestCase): def test_notifications(self): "Test Notifications" meeting = make_meeting_test_data() - url = reverse('ietf.secr.meetings.views.notifications',kwargs={'meeting_id':42}) + url = reverse('ietf.secr.meetings.views.notifications',kwargs={'meeting_id':72}) self.client.login(username="secretary", password="secretary+password") response = self.client.get(url) self.assertEqual(response.status_code, 200) @@ -186,7 +192,7 @@ class SecrMeetingTestCase(TestCase): def test_meetings_rooms(self): meeting = make_meeting_test_data() - url = reverse('ietf.secr.meetings.views.rooms',kwargs={'meeting_id':42,'schedule_name':'test-agenda'}) + url = reverse('ietf.secr.meetings.views.rooms',kwargs={'meeting_id':72,'schedule_name':'test-agenda'}) self.client.login(username="secretary", password="secretary+password") response = self.client.get(url) self.assertEqual(response.status_code, 200) @@ -212,7 +218,7 @@ class SecrMeetingTestCase(TestCase): def test_meetings_times(self): make_meeting_test_data() - url = reverse('ietf.secr.meetings.views.times',kwargs={'meeting_id':42,'schedule_name':'test-agenda'}) + url = reverse('ietf.secr.meetings.views.times',kwargs={'meeting_id':72,'schedule_name':'test-agenda'}) self.client.login(username="secretary", password="secretary+password") response = self.client.get(url) self.assertEqual(response.status_code, 200) @@ -223,7 +229,7 @@ class SecrMeetingTestCase(TestCase): 'name':'Test Morning Session' }, follow=True) self.assertRedirects(response, url) - self.assertTrue('Test Morning Session' in response.content) + self.assertContains(response, 'Test Morning Session') def test_meetings_times_delete(self): meeting = make_meeting_test_data() @@ -251,7 +257,7 @@ class SecrMeetingTestCase(TestCase): meeting = make_meeting_test_data() timeslot = TimeSlot.objects.filter(meeting=meeting,type='session').first() url = reverse('ietf.secr.meetings.views.times_edit',kwargs={ - 'meeting_id':42, + 'meeting_id':72, 'schedule_name':'test-agenda', 'time':timeslot.time.strftime("%Y:%m:%d:%H:%M") }) @@ -267,7 +273,7 @@ class SecrMeetingTestCase(TestCase): def test_meetings_nonsession(self): make_meeting_test_data() - url = reverse('ietf.secr.meetings.views.non_session',kwargs={'meeting_id':42,'schedule_name':'test-agenda'}) + url = reverse('ietf.secr.meetings.views.non_session',kwargs={'meeting_id':72,'schedule_name':'test-agenda'}) self.client.login(username="secretary", password="secretary+password") response = self.client.get(url) self.assertEqual(response.status_code, 200) @@ -276,7 +282,7 @@ class SecrMeetingTestCase(TestCase): meeting = make_meeting_test_data() room = meeting.room_set.first() group = Group.objects.get(acronym='secretariat') - url = reverse('ietf.secr.meetings.views.non_session',kwargs={'meeting_id':42,'schedule_name':'test-agenda'}) + url = reverse('ietf.secr.meetings.views.non_session',kwargs={'meeting_id':72,'schedule_name':'test-agenda'}) self.client.login(username="secretary", password="secretary+password") response = self.client.post(url, { 'day':'1', @@ -297,7 +303,7 @@ class SecrMeetingTestCase(TestCase): def test_meetings_nonsession_add_invalid(self): make_meeting_test_data() group = Group.objects.get(acronym='secretariat') - url = reverse('ietf.secr.meetings.views.non_session',kwargs={'meeting_id':42,'schedule_name':'test-agenda'}) + url = reverse('ietf.secr.meetings.views.non_session',kwargs={'meeting_id':72,'schedule_name':'test-agenda'}) self.client.login(username="secretary", password="secretary+password") response = self.client.post(url, { 'day':'1', @@ -309,14 +315,14 @@ class SecrMeetingTestCase(TestCase): 'group':group.pk, }) self.assertEqual(response.status_code, 200) - self.assertTrue('invalid format' in response.content) + self.assertContains(response, 'invalid format') def test_meetings_nonsession_edit(self): meeting = make_meeting_test_data() session = meeting.session_set.exclude(name='').first() # get first non-session session timeslot = session.official_timeslotassignment().timeslot - url = reverse('ietf.secr.meetings.views.non_session_edit',kwargs={'meeting_id':42,'schedule_name':meeting.agenda.name,'slot_id':timeslot.pk}) - redirect_url = reverse('ietf.secr.meetings.views.non_session',kwargs={'meeting_id':42,'schedule_name':'test-agenda'}) + url = reverse('ietf.secr.meetings.views.non_session_edit',kwargs={'meeting_id':72,'schedule_name':meeting.agenda.name,'slot_id':timeslot.pk}) + redirect_url = reverse('ietf.secr.meetings.views.non_session',kwargs={'meeting_id':72,'schedule_name':'test-agenda'}) new_time = timeslot.time + datetime.timedelta(days=1) self.client.login(username="secretary", password="secretary+password") response = self.client.get(url) diff --git a/ietf/secr/proceedings/migrations/0001_initial.py b/ietf/secr/proceedings/migrations/0001_initial.py index cb55ea290..48d6b6264 100644 --- a/ietf/secr/proceedings/migrations/0001_initial.py +++ b/ietf/secr/proceedings/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/secr/proceedings/models.py b/ietf/secr/proceedings/models.py index 7ddf6d6fe..e7d24e7c8 100644 --- a/ietf/secr/proceedings/models.py +++ b/ietf/secr/proceedings/models.py @@ -1,7 +1,14 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import os from django.conf import settings from django.db import models +from django.utils.encoding import python_2_unicode_compatible from ietf.meeting.models import Meeting @@ -56,6 +63,7 @@ class InterimMeeting(Meeting): else: return '' +@python_2_unicode_compatible class Registration(models.Model): rsn = models.AutoField(primary_key=True) fname = models.CharField(max_length=255) @@ -63,7 +71,7 @@ class Registration(models.Model): company = models.CharField(max_length=255) country = models.CharField(max_length=2) - def __unicode__(self): + def __str__(self): return "%s %s" % (self.fname, self.lname) class Meta: db_table = 'registrations' diff --git a/ietf/secr/proceedings/proc_utils.py b/ietf/secr/proceedings/proc_utils.py index a25777b68..6c2b9c384 100644 --- a/ietf/secr/proceedings/proc_utils.py +++ b/ietf/secr/proceedings/proc_utils.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + ''' proc_utils.py @@ -10,7 +13,7 @@ import datetime import os import re import subprocess -from urllib import urlencode +from six.moves.urllib.parse import urlencode import debug # pyflakes:ignore diff --git a/ietf/secr/proceedings/tests.py b/ietf/secr/proceedings/tests.py index 66cb0b873..f4d21ee6b 100644 --- a/ietf/secr/proceedings/tests.py +++ b/ietf/secr/proceedings/tests.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import debug # pyflakes:ignore +import io import json import os import shutil @@ -54,7 +58,7 @@ class VideoRecordingTestCase(TestCase): def test_get_urls_from_json(self): path = os.path.join(settings.BASE_DIR, "../test/data/youtube-playlistitems.json") - with open(path) as f: + with io.open(path) as f: doc = json.load(f) urls = _get_urls_from_json(doc) self.assertEqual(len(urls),2) @@ -87,7 +91,7 @@ class RecordingTestCase(TestCase): self.client.login(username="secretary", password="secretary+password") response = self.client.post(url,data,follow=True) self.assertEqual(response.status_code, 200) - self.assertTrue(group.acronym in response.content) + self.assertContains(response, group.acronym) # now test edit doc = session.materials.filter(type='recording').first() @@ -95,7 +99,7 @@ class RecordingTestCase(TestCase): url = reverse('ietf.secr.proceedings.views.recording_edit', kwargs={'meeting_num':meeting.number,'name':doc.name}) response = self.client.post(url,dict(external_url=external_url),follow=True) self.assertEqual(response.status_code, 200) - self.assertTrue(external_url in response.content) + self.assertContains(response, external_url) def test_import_audio_files(self): session = SessionFactory(status_id='sched',meeting__type_id='ietf') @@ -110,7 +114,7 @@ class RecordingTestCase(TestCase): path = os.path.join(settings.MEETING_RECORDINGS_DIR,'ietf' + timeslot.meeting.number,filename) if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) - with open(path, "w") as f: + with io.open(path, "w") as f: f.write('dummy') def get_filename_for_timeslot(self, timeslot): @@ -121,7 +125,7 @@ class RecordingTestCase(TestCase): date=timeslot.time.strftime('%Y%m%d-%H%M')) def test_import_audio_files_shared_timeslot(self): - meeting = MeetingFactory(type_id='ietf',number='42') + meeting = MeetingFactory(type_id='ietf',number='72') mars_session = SessionFactory(meeting=meeting,status_id='sched',group__acronym='mars') ames_session = SessionFactory(meeting=meeting,status_id='sched',group__acronym='ames') scheduled = SessionStatusName.objects.get(slug='sched') @@ -135,8 +139,8 @@ class RecordingTestCase(TestCase): import_audio_files(meeting) doc = mars_session.materials.filter(type='recording').first() self.assertTrue(doc in ames_session.materials.all()) - self.assertTrue(doc.docalias.filter(name='recording-42-mars-1')) - self.assertTrue(doc.docalias.filter(name='recording-42-ames-1')) + self.assertTrue(doc.docalias.filter(name='recording-72-mars-1')) + self.assertTrue(doc.docalias.filter(name='recording-72-ames-1')) def test_normalize_room_name(self): self.assertEqual(normalize_room_name('Test Room'),'testroom') @@ -149,7 +153,7 @@ class RecordingTestCase(TestCase): self.assertEqual(get_timeslot_for_filename(name),timeslot) def test_get_or_create_recording_document(self): - session = SessionFactory(meeting__type_id='ietf', meeting__number=42, group__acronym='mars') + session = SessionFactory(meeting__type_id='ietf', meeting__number=72, group__acronym='mars') # test create filename = 'ietf42-testroom-20000101-0800.mp3' @@ -167,17 +171,17 @@ class RecordingTestCase(TestCase): self.assertEqual(doc,doc2) def test_create_recording(self): - session = SessionFactory(meeting__type_id='ietf', meeting__number=42, group__acronym='mars') + session = SessionFactory(meeting__type_id='ietf', meeting__number=72, group__acronym='mars') filename = 'ietf42-testroomt-20000101-0800.mp3' url = settings.IETF_AUDIO_URL + 'ietf{}/{}'.format(session.meeting.number, filename) doc = create_recording(session, url) - self.assertEqual(doc.name,'recording-42-mars-1') + self.assertEqual(doc.name,'recording-72-mars-1') self.assertEqual(doc.group,session.group) self.assertEqual(doc.external_url,url) self.assertTrue(doc in session.materials.all()) def test_get_next_sequence(self): - session = SessionFactory(meeting__type_id='ietf', meeting__number=42, group__acronym='mars') + session = SessionFactory(meeting__type_id='ietf', meeting__number=72, group__acronym='mars') meeting = session.meeting group = session.group sequence = get_next_sequence(group,meeting,'recording') diff --git a/ietf/secr/proceedings/utils.py b/ietf/secr/proceedings/utils.py index 2ae0ead9f..9b8c7f258 100644 --- a/ietf/secr/proceedings/utils.py +++ b/ietf/secr/proceedings/utils.py @@ -1,5 +1,7 @@ +# Copyright The IETF Trust 2016-2019, All Rights Reserved import glob +import io import os from django.conf import settings @@ -34,7 +36,7 @@ def handle_upload_file(file,filename,meeting,subdir, request=None, encoding=None for f in old_files: os.remove(f) - destination = open(os.path.join(path,filename), 'wb+') + destination = io.open(os.path.join(path,filename), 'wb+') if extension in settings.MEETING_VALID_MIME_TYPE_EXTENSIONS['text/html']: file.open() text = file.read() diff --git a/ietf/secr/proceedings/views.py b/ietf/secr/proceedings/views.py index 571d159c2..4180dd3d1 100644 --- a/ietf/secr/proceedings/views.py +++ b/ietf/secr/proceedings/views.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import glob import itertools @@ -276,7 +282,7 @@ def recording_edit(request, meeting_num, name): by=request.user.person, doc=recording, rev=recording.rev, - desc=u'Changed URL to %s' % recording.external_url, + desc='Changed URL to %s' % recording.external_url, ) recording.save_with_history([e]) diff --git a/ietf/secr/roles/tests.py b/ietf/secr/roles/tests.py index c4cdf88cf..e70d5e563 100644 --- a/ietf/secr/roles/tests.py +++ b/ietf/secr/roles/tests.py @@ -1,4 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.urls import reverse from ietf.utils.test_utils import TestCase @@ -47,7 +52,7 @@ class SecrRolesMainTestCase(TestCase): self.client.login(username="secretary", password="secretary+password") response = self.client.post(url,post_data,follow=True) self.assertRedirects(response, target) - self.assertTrue('added successfully' in response.content) + self.assertContains(response, 'added successfully') def test_roles_add_no_group(self): person = Person.objects.get(name='Areað Irector') @@ -60,4 +65,4 @@ class SecrRolesMainTestCase(TestCase): self.client.login(username="secretary", password="secretary+password") response = self.client.post(url,post_data,follow=True) self.assertEqual(response.status_code, 200) - self.assertTrue('You must select a group' in response.content) + self.assertContains(response, 'You must select a group') diff --git a/ietf/secr/rolodex/forms.py b/ietf/secr/rolodex/forms.py index eafcd5147..ab094bdca 100644 --- a/ietf/secr/rolodex/forms.py +++ b/ietf/secr/rolodex/forms.py @@ -1,6 +1,8 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals import re diff --git a/ietf/secr/sreq/forms.py b/ietf/secr/sreq/forms.py index af36026b0..36a804042 100644 --- a/ietf/secr/sreq/forms.py +++ b/ietf/secr/sreq/forms.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django import forms import debug # pyflakes:ignore @@ -98,7 +104,7 @@ class SessionForm(forms.Form): self.fields['third_session'].initial = True if self.hidden: - for key in self.fields.keys(): + for key in list(self.fields.keys()): self.fields[key].widget = forms.HiddenInput() self.fields['resources'].widget = forms.MultipleHiddenInput() diff --git a/ietf/secr/sreq/tests.py b/ietf/secr/sreq/tests.py index ae180eb24..3d01dc0c5 100644 --- a/ietf/secr/sreq/tests.py +++ b/ietf/secr/sreq/tests.py @@ -1,12 +1,17 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- -from django.urls import reverse + +from __future__ import absolute_import, print_function, unicode_literals + import datetime +import six + +from django.urls import reverse import debug # pyflakes:ignore -from ietf.utils.test_utils import TestCase, unicontent +from ietf.utils.test_utils import TestCase from ietf.group.factories import GroupFactory, RoleFactory from ietf.meeting.models import Session, ResourceAssociation from ietf.meeting.factories import MeetingFactory, SessionFactory @@ -149,7 +154,7 @@ class SubmitRequestCase(TestCase): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q('#session-request-form')),1) - self.assertTrue('You must enter a length for all sessions' in unicontent(r)) + self.assertContains(r, 'You must enter a length for all sessions') def test_request_notification(self): meeting = MeetingFactory(type_id='ietf', date=datetime.date.today()) @@ -179,14 +184,14 @@ class SubmitRequestCase(TestCase): r = self.client.post(url,post_data) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertTrue('Confirm' in unicode(q("title"))) + self.assertTrue('Confirm' in six.text_type(q("title"))) # confirm post_data['submit'] = 'Submit' r = self.client.post(confirm_url,post_data) self.assertRedirects(r, reverse('ietf.secr.sreq.views.main')) self.assertEqual(len(outbox),len_before+1) notification = outbox[-1] - notification_payload = unicode(notification.get_payload(decode=True),"utf-8","replace") + notification_payload = six.text_type(notification.get_payload(decode=True),"utf-8","replace") session = Session.objects.get(meeting=meeting,group=group) self.assertEqual(session.resources.count(),1) self.assertEqual(session.people_constraints.count(),1) @@ -250,11 +255,11 @@ class NotMeetingCase(TestCase): # This is a sign of a problem - a get shouldn't have a side-effect like this one does self.assertEqual(r.status_code, 200) - self.assertTrue('A message was sent to notify not having a session' in unicontent(r)) + self.assertContains(r, 'A message was sent to notify not having a session') r = self.client.get(url,follow=True) self.assertEqual(r.status_code, 200) - self.assertTrue('is already marked as not meeting' in unicontent(r)) + self.assertContains(r, 'is already marked as not meeting') self.assertEqual(len(outbox),1) self.assertTrue('Not having a session' in outbox[0]['Subject']) diff --git a/ietf/secr/sreq/views.py b/ietf/secr/sreq/views.py index c4bf30c81..04a2ec4ba 100644 --- a/ietf/secr/sreq/views.py +++ b/ietf/secr/sreq/views.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime from django.conf import settings @@ -545,7 +548,7 @@ def new(request, acronym): # the "previous" querystring causes the form to be returned # pre-populated with data from last meeeting's session request - elif request.method == 'GET' and request.GET.has_key('previous'): + elif request.method == 'GET' and 'previous' in request.GET: previous_meeting = Meeting.objects.get(number=str(int(meeting.number) - 1)) previous_sessions = Session.objects.filter(meeting=previous_meeting,group=group).exclude(status__in=('notmeet','deleted')).order_by('id') if not previous_sessions: diff --git a/ietf/secr/telechat/tests.py b/ietf/secr/telechat/tests.py index 5b7245191..0b6aa524e 100644 --- a/ietf/secr/telechat/tests.py +++ b/ietf/secr/telechat/tests.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime from pyquery import PyQuery @@ -76,9 +79,9 @@ class SecrTelechatTestCase(TestCase): self.client.login(username="secretary", password="secretary+password") response = self.client.get(url) self.assertEqual(response.status_code, 200) - self.assertTrue("Has downref: Yes" in response.content) - self.assertTrue("Add rfc6666" in response.content) - self.assertTrue("to downref registry" in response.content) + self.assertContains(response, "Has downref: Yes") + self.assertContains(response, "Add rfc6666") + self.assertContains(response, "to downref registry") def test_doc_detail_draft_invalid(self): '''Test using a document not on telechat agenda''' @@ -88,7 +91,7 @@ class SecrTelechatTestCase(TestCase): self.client.login(username="secretary", password="secretary+password") response = self.client.get(url, follow=True) self.assertRedirects(response, reverse('ietf.secr.telechat.views.doc', kwargs={'date':date})) - self.assertTrue('not on the Telechat agenda' in response.content) + self.assertContains(response, 'not on the Telechat agenda') def test_doc_detail_charter(self): by=Person.objects.get(name="(System)") diff --git a/ietf/secr/telechat/views.py b/ietf/secr/telechat/views.py index c845d26ed..d78dde653 100644 --- a/ietf/secr/telechat/views.py +++ b/ietf/secr/telechat/views.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime from django.contrib import messages @@ -43,7 +49,7 @@ def get_doc_list(agenda): Document objects in the order they appear in the agenda sections 1-3. ''' docs = [] - for num, section in sorted(agenda['sections'].iteritems()): + for num, section in sorted(agenda['sections'].items()): if "docs" in section: docs.extend(section["docs"]) @@ -96,16 +102,16 @@ def get_section_header(doc, agenda): split = num.split(".") - for i in xrange(num.count(".")): + for i in range(num.count(".")): parent_num = ".".join(split[:i + 1]) parent = agenda["sections"].get(parent_num) if parent: if "." not in parent_num: parent_num += "." - header.append(u"%s %s" % (parent_num, parent["title"])) + header.append("%s %s" % (parent_num, parent["title"])) section = agenda["sections"][num] - header.append(u"%s %s" % (num, section["title"])) + header.append("%s %s" % (num, section["title"])) count = '%s of %s' % (section["docs"].index(doc) + 1, len(section["docs"])) header.append(count) @@ -116,7 +122,7 @@ def get_first_doc(agenda): ''' This function takes an agenda dictionary and returns the first document in the agenda ''' - for num, section in sorted(agenda['sections'].iteritems()): + for num, section in sorted(agenda['sections'].items()): if "docs" in section and section["docs"]: return section["docs"][0] diff --git a/ietf/secr/utils/ams_utils.py b/ietf/secr/utils/ams_utils.py index efd688c54..d8e3c2a62 100644 --- a/ietf/secr/utils/ams_utils.py +++ b/ietf/secr/utils/ams_utils.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.conf import settings from ietf.person.models import Person @@ -32,8 +38,8 @@ def get_last_revision(filename): """ files = glob.glob(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR,filename) + '-??.txt') if files: - sorted_files = sorted(files) - return get_revision(sorted_files[-1]) + sorted_files = sorted(files) + return get_revision(sorted_files[-1]) else: raise Exception('last revision not found in archive') diff --git a/ietf/secr/utils/document.py b/ietf/secr/utils/document.py index 72ffbda31..216958b70 100644 --- a/ietf/secr/utils/document.py +++ b/ietf/secr/utils/document.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + def get_full_path(doc): ''' Returns for name of file on disk with full path. This should really be a method on doc diff --git a/ietf/secr/utils/group.py b/ietf/secr/utils/group.py index 593f280d9..6fe69a56b 100644 --- a/ietf/secr/utils/group.py +++ b/ietf/secr/utils/group.py @@ -1,7 +1,11 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + # Python imports +import io import os # Django imports @@ -29,7 +33,7 @@ def get_charter_text(group): ''' charter = group.charter path = os.path.join(settings.CHARTER_PATH, '%s-%s.txt' % (charter.canonical_name(), charter.rev)) - f = file(path,'r') + f = io.open(path,'r') text = f.read() f.close() @@ -99,7 +103,7 @@ def groups_by_session(user, meeting, types=None): if not types: types = GroupFeatures.objects.filter(has_meetings=True).values_list('type', flat=True) - groups_session = filter(lambda g: g.type_id in types, groups_session) - groups_no_session = filter(lambda g: g.type_id in types, groups_no_session) + groups_session = [x for x in groups_session if x.type_id in types] + groups_no_session = [x for x in groups_no_session if x.type_id in types] return groups_session, groups_no_session diff --git a/ietf/secr/utils/test.py b/ietf/secr/utils/test.py index 191b3a986..3fc791923 100644 --- a/ietf/secr/utils/test.py +++ b/ietf/secr/utils/test.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + ''' Functions to aid unit testing ''' @@ -9,7 +15,7 @@ def reset(): me = Person.objects.get(name='Ryan Cross') me.role_set.all().delete() Role.objects.create(person=me,email_id='rcross@amsl.com',name_id='secr',group_id=4) - print me.role_set.all() + print(me.role_set.all()) def copy_roles(person): '''Copy the roles of person''' @@ -17,4 +23,4 @@ def copy_roles(person): me.role_set.all().delete() for role in person.role_set.all(): Role.objects.create(person=me,email_id='rcross@amsl.com',name=role.name,group=role.group) - print me.role_set.all() \ No newline at end of file + print(me.role_set.all()) \ No newline at end of file diff --git a/ietf/settings.py b/ietf/settings.py index 7ee32218a..a5cd92941 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2007-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + # Django settings for ietf project. # BASE_DIR and "settings_local" are from # http://code.djangoproject.com/wiki/SplitSettings @@ -23,11 +26,17 @@ warnings.filterwarnings("ignore", message="You passed a bytestring as `filenames warnings.filterwarnings("ignore", message="django.forms.extras is deprecated.", module="bootstrap3") warnings.filterwarnings("ignore", message="defusedxml.lxml is no longer supported and will be removed in a future release.", module="tastypie") warnings.filterwarnings("ignore", message="Duplicate index '.*' defined on the table") +# Warnings found under Python 3.7: +warnings.filterwarnings("ignore", message="Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated") +warnings.filterwarnings("ignore", message="'U' mode is deprecated", module="docutils.io") +warnings.filterwarnings("ignore", message="'U' mode is deprecated", module="xml2rfc") +warnings.filterwarnings("ignore", message="'U' mode is deprecated", module="site") +warnings.filterwarnings("ignore", message="Flags not at the start of the expression", module="genshi") try: import syslog - syslog.openlog("datatracker", syslog.LOG_PID, syslog.LOG_USER) + syslog.openlog(str("datatracker"), syslog.LOG_PID, syslog.LOG_USER) except ImportError: pass @@ -388,7 +397,6 @@ INSTALLED_APPS = ( 'django.contrib.sites', 'django.contrib.staticfiles', # External apps - 'anora', 'bootstrap3', 'corsheaders', 'django_markup', @@ -822,6 +830,7 @@ MEETING_VALID_MIME_TYPE_EXTENSIONS = { INTERNET_DRAFT_DAYS_TO_EXPIRE = 185 FLOORPLAN_MEDIA_DIR = 'floor' +FLOORPLAN_DIR = os.path.join(MEDIA_ROOT, FLOORPLAN_MEDIA_DIR) # ============================================================================== @@ -885,8 +894,7 @@ BADNESS_TOOBIG = 100 BADNESS_MUCHTOOBIG = 500 # do not run SELENIUM tests by default -SELENIUM_TESTS = False -SELENIUM_TESTS_ONLY = False +SKIP_SELENIUM = True # Set debug apps in settings_local.DEV_APPS @@ -1034,13 +1042,13 @@ UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS = 30 API_KEY_TYPE="ES256" # EC / P=256 -API_PUBLIC_KEY_PEM = """ +API_PUBLIC_KEY_PEM = b""" -----BEGIN PUBLIC KEY----- MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEqVojsaofDJScuMJN+tshumyNM5ME garzVPqkVovmF6yE7IJ/dv4FcV+QKCtJ/rOS8e36Y8ZAEVYuukhes0yZ1w== -----END PUBLIC KEY----- """ -API_PRIVATE_KEY_PEM = """ +API_PRIVATE_KEY_PEM = b""" -----BEGIN PRIVATE KEY----- MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgoI6LJkopKq8XrHi9 QqGQvE4A83TFYjqLz+8gULYecsqhRANCAASpWiOxqh8MlJy4wk362yG6bI0zkwSB @@ -1057,7 +1065,7 @@ for app in INSTALLED_APPS: if app.startswith('ietf'): app_settings_file = os.path.join(BASE_DIR, '../', app.replace('.', os.sep), "settings.py") if os.path.exists(app_settings_file): - exec "from %s import *" % (app+".settings") + exec("from %s import *" % (app+".settings")) # Add DEV_APPS to INSTALLED_APPS INSTALLED_APPS += DEV_APPS diff --git a/ietf/settings_releasetest.py b/ietf/settings_releasetest.py index 86826736d..421e6f851 100644 --- a/ietf/settings_releasetest.py +++ b/ietf/settings_releasetest.py @@ -1,10 +1,17 @@ +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + + # Standard settings except we use SQLite, this is useful for speeding # up tests that depend on the test database, try for instance: # # ./manage.py test --settings=settings_sqlitetest doc.ChangeStateTestCase # -from settings import * # pyflakes:ignore +from .settings import * # pyflakes:ignore # Workaround to avoid spending minutes stepping through the migrations in # every test run. The result of this is to use the 'syncdb' way of creating diff --git a/ietf/settings_sqlitetest.py b/ietf/settings_sqlitetest.py index f6f471ee6..9657f7775 100644 --- a/ietf/settings_sqlitetest.py +++ b/ietf/settings_sqlitetest.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + # Standard settings except we use SQLite and skip migrations, this is # useful for speeding up tests that depend on the test database, try # for instance: @@ -6,7 +12,7 @@ # import os -from settings import * # pyflakes:ignore +from ietf.settings import * # pyflakes:ignore import debug # pyflakes:ignore debug.debug = True diff --git a/ietf/settings_testcrawl.py b/ietf/settings_testcrawl.py index 967e3f5b7..ce1f4f8b3 100644 --- a/ietf/settings_testcrawl.py +++ b/ietf/settings_testcrawl.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + # Standard settings except we enable caching like in the production # environment, this is useful for speeding up the test crawl, try for # instance: @@ -5,8 +11,8 @@ # bin/test-crawl --settings=ietf.settings_testcrawl # -from settings import * # pyflakes:ignore -from settings import TEMPLATES +from .settings import * # pyflakes:ignore +from .settings import TEMPLATES TEMPLATES[0]['OPTIONS']['loaders'] = ( ('django.template.loaders.cached.Loader', ( diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py index 5946abc0c..0d170eeb3 100755 --- a/ietf/stats/backfill_data.py +++ b/ietf/stats/backfill_data.py @@ -1,8 +1,10 @@ #!/usr/bin/env python # Copyright The IETF Trust 2017-2019, All Rights Reserved +# -*- coding: utf-8 -*- -from __future__ import print_function, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals +import io import sys import os import os.path @@ -16,7 +18,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py") if os.path.exists(virtualenv_activation): - execfile(virtualenv_activation, dict(__file__=virtualenv_activation)) + exec(compile(io.open(virtualenv_activation, "rb").read(), virtualenv_activation, 'exec'), dict(__file__=virtualenv_activation)) import django django.setup() @@ -44,7 +46,7 @@ if args.document: docs_qs = docs_qs.filter(docalias__name=args.document) ts = time.strftime("%Y-%m-%d_%H:%M%z") -logfile = open('backfill-authorstats-%s.log'%ts, 'w') +logfile = io.open('backfill-authorstats-%s.log'%ts, 'w') print("Writing log to %s" % os.path.abspath(logfile.name)) def say(msg): @@ -84,7 +86,7 @@ for doc in docs_qs.prefetch_related("docalias", "formal_languages", "documentaut say("Skipping %s, no txt file found at %s" % (doc.name, path)) continue - with open(path, 'rb') as f: + with io.open(path, 'rb') as f: say("\nProcessing %s" % doc.name) sys.stdout.flush() d = Draft(unicode(f.read()), path) diff --git a/ietf/stats/management/commands/fetch_meeting_attendance.py b/ietf/stats/management/commands/fetch_meeting_attendance.py index bc511d34f..5078c7cee 100644 --- a/ietf/stats/management/commands/fetch_meeting_attendance.py +++ b/ietf/stats/management/commands/fetch_meeting_attendance.py @@ -1,3 +1,4 @@ +# Copyright The IETF Trust 2017-2019, All Rights Reserved # Copyright 2016 IETF Trust import datetime @@ -12,7 +13,7 @@ from ietf.stats.utils import get_meeting_registration_data logtag = __name__.split('.')[-1] logname = "user.log" -syslog.openlog(logtag, syslog.LOG_PID, syslog.LOG_USER) +syslog.openlog(str(logtag), syslog.LOG_PID, syslog.LOG_USER) class Command(BaseCommand): help = "Fetch meeting attendee figures from ietf.org/registration/attendees." diff --git a/ietf/stats/migrations/0001_initial.py b/ietf/stats/migrations/0001_initial.py index a83449658..cc96985e8 100644 --- a/ietf/stats/migrations/0001_initial.py +++ b/ietf/stats/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/ietf/stats/models.py b/ietf/stats/models.py index ac2dfb034..924ff962e 100644 --- a/ietf/stats/models.py +++ b/ietf/stats/models.py @@ -1,6 +1,11 @@ -# Copyright The IETF Trust 2017, All Rights Reserved +# Copyright The IETF Trust 2017-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import models +from django.utils.encoding import python_2_unicode_compatible import debug # pyflakes:ignore @@ -10,6 +15,7 @@ from ietf.person.models import Person from ietf.utils.models import ForeignKey +@python_2_unicode_compatible class AffiliationAlias(models.Model): """Records that alias should be treated as name for statistical purposes.""" @@ -17,8 +23,8 @@ class AffiliationAlias(models.Model): alias = models.CharField(max_length=255, help_text="Note that aliases will be matched case-insensitive and both before and after some clean-up.", unique=True) name = models.CharField(max_length=255) - def __unicode__(self): - return u"{} -> {}".format(self.alias, self.name) + def __str__(self): + return "{} -> {}".format(self.alias, self.name) def save(self, *args, **kwargs): self.alias = self.alias.lower() @@ -27,14 +33,16 @@ class AffiliationAlias(models.Model): class Meta: verbose_name_plural = "affiliation aliases" +@python_2_unicode_compatible class AffiliationIgnoredEnding(models.Model): """Records that ending should be stripped from the affiliation for statistical purposes.""" ending = models.CharField(max_length=255, help_text="Regexp with ending, e.g. 'Inc\\.?' - remember to escape .!") - def __unicode__(self): + def __str__(self): return self.ending +@python_2_unicode_compatible class CountryAlias(models.Model): """Records that alias should be treated as country for statistical purposes.""" @@ -42,12 +50,13 @@ class CountryAlias(models.Model): alias = models.CharField(max_length=255, help_text="Note that lower-case aliases are matched case-insensitive while aliases with at least one uppercase letter is matched case-sensitive. So 'United States' is best entered as 'united states' so it both matches 'United States' and 'United states' and 'UNITED STATES', whereas 'US' is best entered as 'US' so it doesn't accidentally match an ordinary word like 'us'.") country = ForeignKey(CountryName, max_length=255) - def __unicode__(self): - return u"{} -> {}".format(self.alias, self.country.name) + def __str__(self): + return "{} -> {}".format(self.alias, self.country.name) class Meta: verbose_name_plural = "country aliases" +@python_2_unicode_compatible class MeetingRegistration(models.Model): """Registration attendee records from the IETF registration system""" meeting = ForeignKey(Meeting) @@ -58,5 +67,5 @@ class MeetingRegistration(models.Model): person = ForeignKey(Person, blank=True, null=True) email = models.EmailField(blank=True, null=True) - def __unicode__(self): - return u"{} {}".format(self.first_name, self.last_name) + def __str__(self): + return "{} {}".format(self.first_name, self.last_name) diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py index 6d7afd9f5..f7bae4a25 100644 --- a/ietf/stats/resources.py +++ b/ietf/stats/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2017-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the makeresources management command 2017-02-15 10:10 PST + + from tastypie.resources import ModelResource from tastypie.fields import ToManyField # pyflakes:ignore from tastypie.constants import ALL, ALL_WITH_RELATIONS # pyflakes:ignore diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index f0c6888ae..d4247e305 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import datetime from mock import patch @@ -12,7 +15,7 @@ import debug # pyflakes:ignore from django.urls import reverse as urlreverse from django.contrib.auth.models import User -from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent +from ietf.utils.test_utils import login_testing_unauthorized, TestCase import ietf.stats.views from ietf.submit.models import Submission @@ -158,7 +161,7 @@ class StatisticsTests(TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertTrue("United States" in unicontent(r)) + self.assertContains(r, "United States") def test_review_stats(self): reviewer = PersonFactory() @@ -214,7 +217,7 @@ class StatisticsTests(TestCase): '''Test function to get reg data. Confirm leading/trailing spaces stripped''' response = Response() response.status_code = 200 - response._content = '[{"LastName":"Smith ","FirstName":" John","Company":"ABC","Country":"US","Email":"john.doe@example.us"}]' + response._content = b'[{"LastName":"Smith ","FirstName":" John","Company":"ABC","Country":"US","Email":"john.doe@example.us"}]' mock_get.return_value = response meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96") get_meeting_registration_data(meeting) @@ -226,7 +229,7 @@ class StatisticsTests(TestCase): def test_get_meeting_registration_data_user_exists(self, mock_get): response = Response() response.status_code = 200 - response._content = '[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US","Email":"john.doe@example.us"}]' + response._content = b'[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US","Email":"john.doe@example.us"}]' email = "john.doe@example.us" user = User.objects.create(username=email) user.save() diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index 94fbe6951..7929ecadd 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -1,13 +1,19 @@ +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + from django.conf import settings from ietf.stats import views from ietf.utils.urls import url urlpatterns = [ - url("^$", views.stats_index), - url("^document/(?:(?Pauthors|pages|words|format|formlang|author/(?:documents|affiliation|country|continent|citations|hindex)|yearly/(?:affiliation|country|continent))/)?$", views.document_stats), - url("^knowncountries/$", views.known_countries_list), - url("^meeting/(?P\d+)/(?Pcountry|continent)/$", views.meeting_stats), - url("^meeting/(?:(?Poverview|country|continent)/)?$", views.meeting_stats), - url("^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), + url(r"^$", views.stats_index), + url(r"^document/(?:(?Pauthors|pages|words|format|formlang|author/(?:documents|affiliation|country|continent|citations|hindex)|yearly/(?:affiliation|country|continent))/)?$", views.document_stats), + url(r"^knowncountries/$", views.known_countries_list), + url(r"^meeting/(?P\d+)/(?Pcountry|continent)/$", views.meeting_stats), + url(r"^meeting/(?:(?Poverview|country|continent)/)?$", views.meeting_stats), + url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), ] diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py index 1a2f5d971..fa8f23f1b 100644 --- a/ietf/stats/utils.py +++ b/ietf/stats/utils.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2017-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import re import requests from collections import defaultdict @@ -86,7 +89,7 @@ def get_aliased_affiliations(affiliations): # now we just need to pick the most popular uppercase/lowercase # spelling for each affiliation with more than one - for similar_affiliations in affiliations_with_case_spellings.itervalues(): + for similar_affiliations in affiliations_with_case_spellings.values(): if len(similar_affiliations) > 1: most_popular = sorted(similar_affiliations, key=affiliation_sort_key, reverse=True)[0] for affiliation in similar_affiliations: @@ -115,8 +118,8 @@ def get_aliased_countries(countries): return possible_alias known_re_aliases = { - re.compile(u"\\b{}\\b".format(re.escape(alias))): name - for alias, name in known_aliases.iteritems() + re.compile("\\b{}\\b".format(re.escape(alias))): name + for alias, name in known_aliases.items() } # specific hack: check for zip codes from the US since in the @@ -130,7 +133,7 @@ def get_aliased_countries(countries): t = t.strip() if t: return t - return u"" + return "" known_countries = set(CountryName.objects.values_list("name", flat=True)) @@ -180,7 +183,7 @@ def get_aliased_countries(countries): # country name anywhere country_lower = country.lower() found = False - for alias_re, name in known_re_aliases.iteritems(): + for alias_re, name in known_re_aliases.items(): if alias_re.search(country) or alias_re.search(country_lower): res[original_country] = name found = True diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 94e7a30e4..bd9b4210d 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -1,6 +1,9 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved # -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + import os import calendar import datetime @@ -43,11 +46,11 @@ def stats_index(request): return render(request, "stats/index.html") def generate_query_string(query_dict, overrides): - query_part = u"" + query_part = "" if query_dict or overrides: d = query_dict.copy() - for k, v in overrides.iteritems(): + for k, v in overrides.items(): if type(v) in (list, tuple): if not v: if k in d: @@ -55,14 +58,14 @@ def generate_query_string(query_dict, overrides): else: d.setlist(k, v) else: - if v is None or v == u"": + if v is None or v == "": if k in d: del d[k] else: d[k] = v if d: - query_part = u"?" + d.urlencode() + query_part = "?" + d.urlencode() return query_part @@ -86,7 +89,7 @@ def add_url_to_choices(choices, url_builder): def put_into_bin(value, bin_size): if value is None: - return (value, value) + return (0, '') v = (value // bin_size) * bin_size return (v, "{} - {}".format(v, v + bin_size - 1)) @@ -94,13 +97,13 @@ def put_into_bin(value, bin_size): def prune_unknown_bin_with_known(bins): # remove from the unknown bin all authors within the # named/known bins - all_known = { n for b, names in bins.iteritems() if b for n in names } + all_known = { n for b, names in bins.items() if b for n in names } bins[""] = [name for name in bins[""] if name not in all_known] if not bins[""]: del bins[""] def count_bins(bins): - return len({ n for b, names in bins.iteritems() if b for n in names }) + return len({ n for b, names in bins.items() if b for n in names }) def add_labeled_top_series_from_bins(chart_data, bins, limit): """Take bins on the form (x, label): [name1, name2, ...], figure out @@ -108,13 +111,13 @@ def add_labeled_top_series_from_bins(chart_data, bins, limit): them into sorted series like [(x1, len(names1)), (x2, len(names2)), ...].""" aggregated_bins = defaultdict(set) xs = set() - for (x, label), names in bins.iteritems(): + for (x, label), names in bins.items(): xs.add(x) aggregated_bins[label].update(names) xs = list(sorted(xs)) - sorted_bins = sorted(aggregated_bins.iteritems(), key=lambda t: len(t[1]), reverse=True) + sorted_bins = sorted(aggregated_bins.items(), key=lambda t: len(t[1]), reverse=True) top = [ label for label, names in list(sorted_bins)[:limit]] for label in top: @@ -136,7 +139,7 @@ def document_stats(request, stats_type=None): "stats_type": stats_type if stats_type_override is Ellipsis else stats_type_override, } - return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) + return urlreverse(document_stats, kwargs={ k: v for k, v in kwargs.items() if v is not None }) + generate_query_string(request.GET, get_overrides) # the length limitation is to keep the key shorter than memcached's limit # of 250 after django has added the key_prefix and key_version parameters @@ -238,8 +241,8 @@ def document_stats(request, stats_type=None): total_docs = docalias_qs.values_list("docs__name").distinct().count() - def generate_canonical_names(docalias_qs): - for doc_id, ts in itertools.groupby(docalias_qs.order_by("docs__name"), lambda t: t[0]): + def generate_canonical_names(values): + for doc_id, ts in itertools.groupby(values.order_by("docs__name"), lambda a: a[0]): chosen = None for t in ts: if chosen is None: @@ -249,7 +252,6 @@ def document_stats(request, stats_type=None): chosen = t elif t[1].startswith("draft") and not chosen[1].startswith("rfc"): chosen = t - yield chosen if stats_type == "authors": @@ -258,10 +260,10 @@ def document_stats(request, stats_type=None): bins = defaultdict(set) for name, canonical_name, author_count in generate_canonical_names(docalias_qs.values_list("docs__name", "name").annotate(Count("docs__documentauthor"))): - bins[author_count].add(canonical_name) + bins[author_count or 0].add(canonical_name) series_data = [] - for author_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): + for author_count, names in sorted(bins.items(), key=lambda t: t[0]): percentage = len(names) * 100.0 / (total_docs or 1) series_data.append((author_count, percentage)) table_data.append((author_count, percentage, len(names), list(names)[:names_limit])) @@ -274,10 +276,10 @@ def document_stats(request, stats_type=None): bins = defaultdict(set) for name, canonical_name, pages in generate_canonical_names(docalias_qs.values_list("docs__name", "name", "docs__pages")): - bins[pages].add(canonical_name) + bins[pages or 0].add(canonical_name) series_data = [] - for pages, names in sorted(bins.iteritems(), key=lambda t: t[0]): + for pages, names in sorted(bins.items(), key=lambda t: t[0]): percentage = len(names) * 100.0 / (total_docs or 1) if pages is not None: series_data.append((pages, len(names))) @@ -296,7 +298,7 @@ def document_stats(request, stats_type=None): bins[put_into_bin(words, bin_size)].add(canonical_name) series_data = [] - for (value, words), names in sorted(bins.iteritems(), key=lambda t: t[0][0]): + for (value, words), names in sorted(bins.items(), key=lambda t: t[0][0]): percentage = len(names) * 100.0 / (total_docs or 1) if words is not None: series_data.append((value, len(names))) @@ -349,7 +351,7 @@ def document_stats(request, stats_type=None): bins[ext.upper()].add(canonical_name) series_data = [] - for fmt, names in sorted(bins.iteritems(), key=lambda t: t[0]): + for fmt, names in sorted(bins.items(), key=lambda t: t[0]): percentage = len(names) * 100.0 / (total_docs or 1) series_data.append((fmt, len(names))) @@ -363,10 +365,10 @@ def document_stats(request, stats_type=None): bins = defaultdict(set) for name, canonical_name, formal_language_name in generate_canonical_names(docalias_qs.values_list("docs__name", "name", "docs__formal_languages__name")): - bins[formal_language_name].add(canonical_name) + bins[formal_language_name or ""].add(canonical_name) series_data = [] - for formal_language, names in sorted(bins.iteritems(), key=lambda t: t[0]): + for formal_language, names in sorted(bins.items(), key=lambda t: t[0]): percentage = len(names) * 100.0 / (total_docs or 1) if formal_language is not None: series_data.append((formal_language, len(names))) @@ -412,12 +414,12 @@ def document_stats(request, stats_type=None): person_qs = Person.objects.filter(person_filters) for name, document_count in person_qs.values_list("name").annotate(Count("documentauthor")): - bins[document_count].add(name) + bins[document_count or 0].add(name) total_persons = count_bins(bins) series_data = [] - for document_count, names in sorted(bins.iteritems(), key=lambda t: t[0]): + for document_count, names in sorted(bins.items(), key=lambda t: t[0]): percentage = len(names) * 100.0 / (total_persons or 1) series_data.append((document_count, percentage)) plain_names = [ plain_name(n) for n in names ] @@ -450,7 +452,7 @@ def document_stats(request, stats_type=None): total_persons = count_bins(bins) series_data = [] - for affiliation, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + for affiliation, names in sorted(bins.items(), key=lambda t: t[0].lower()): percentage = len(names) * 100.0 / (total_persons or 1) if affiliation: series_data.append((affiliation, len(names))) @@ -462,7 +464,7 @@ def document_stats(request, stats_type=None): chart_data.append({ "data": series_data }) - for alias, name in sorted(aliases.iteritems(), key=lambda t: t[1]): + for alias, name in sorted(aliases.items(), key=lambda t: t[1]): alias_data.append((name, alias)) elif stats_type == "author/country": @@ -485,7 +487,7 @@ def document_stats(request, stats_type=None): countries = { c.name: c for c in CountryName.objects.all() } eu_name = "EU" - eu_countries = { c for c in countries.itervalues() if c.in_eu } + eu_countries = { c for c in countries.values() if c.in_eu } for name, country in name_country_set: country_name = aliases.get(country, country) @@ -499,7 +501,7 @@ def document_stats(request, stats_type=None): total_persons = count_bins(bins) series_data = [] - for country, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + for country, names in sorted(bins.items(), key=lambda t: t[0].lower()): percentage = len(names) * 100.0 / (total_persons or 1) if country: series_data.append((country, len(names))) @@ -511,7 +513,7 @@ def document_stats(request, stats_type=None): chart_data.append({ "data": series_data }) - for alias, country_name in aliases.iteritems(): + for alias, country_name in aliases.items(): alias_data.append((country_name, alias, countries.get(country_name))) alias_data.sort() @@ -541,7 +543,7 @@ def document_stats(request, stats_type=None): total_persons = count_bins(bins) series_data = [] - for continent, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + for continent, names in sorted(bins.items(), key=lambda t: t[0].lower()): percentage = len(names) * 100.0 / (total_persons or 1) if continent: series_data.append((continent, len(names))) @@ -563,12 +565,12 @@ def document_stats(request, stats_type=None): person_qs = Person.objects.filter(person_filters) for name, citations in person_qs.values_list("name").annotate(Count("documentauthor__document__docalias__relateddocument")): - bins[citations].add(name) + bins[citations or 0].add(name) total_persons = count_bins(bins) series_data = [] - for citations, names in sorted(bins.iteritems(), key=lambda t: t[0], reverse=True): + for citations, names in sorted(bins.items(), key=lambda t: t[0], reverse=True): percentage = len(names) * 100.0 / (total_persons or 1) series_data.append((citations, percentage)) plain_names = [ plain_name(n) for n in names ] @@ -589,12 +591,12 @@ def document_stats(request, stats_type=None): values = person_qs.values_list("name", "documentauthor__document").annotate(Count("documentauthor__document__docalias__relateddocument")) for name, ts in itertools.groupby(values.order_by("name"), key=lambda t: t[0]): h_index = compute_hirsch_index([citations for _, document, citations in ts]) - bins[h_index].add(name) + bins[h_index or 0].add(name) total_persons = count_bins(bins) series_data = [] - for citations, names in sorted(bins.iteritems(), key=lambda t: t[0], reverse=True): + for citations, names in sorted(bins.items(), key=lambda t: t[0], reverse=True): percentage = len(names) * 100.0 / (total_persons or 1) series_data.append((citations, percentage)) plain_names = [ plain_name(n) for n in names ] @@ -674,7 +676,7 @@ def document_stats(request, stats_type=None): countries = { c.name: c for c in CountryName.objects.all() } eu_name = "EU" - eu_countries = { c for c in countries.itervalues() if c.in_eu } + eu_countries = { c for c in countries.values() if c.in_eu } bins = defaultdict(set) @@ -770,7 +772,7 @@ def meeting_stats(request, num=None, stats_type=None): if number is not None: kwargs["num"] = number - return urlreverse(meeting_stats, kwargs={ k: v for k, v in kwargs.iteritems() if v is not None }) + generate_query_string(request.GET, get_overrides) + return urlreverse(meeting_stats, kwargs={ k: v for k, v in kwargs.items() if v is not None }) + generate_query_string(request.GET, get_overrides) cache_key = ("stats:meeting_stats:%s:%s:%s" % (num, stats_type, slugify(request.META.get('QUERY_STRING',''))))[:228] data = cache.get(cache_key) @@ -808,7 +810,7 @@ def meeting_stats(request, num=None, stats_type=None): } def reg_name(r): - return email.utils.formataddr(((r.first_name + u" " + r.last_name).strip(), r.email)) + return email.utils.formataddr(((r.first_name + " " + r.last_name).strip(), r.email)) if meeting and any(stats_type == t[0] for t in possible_stats_types): attendees = MeetingRegistration.objects.filter(meeting=meeting) @@ -835,7 +837,7 @@ def meeting_stats(request, num=None, stats_type=None): total_attendees = count_bins(bins) series_data = [] - for country, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + for country, names in sorted(bins.items(), key=lambda t: t[0].lower()): percentage = len(names) * 100.0 / (total_attendees or 1) if country: series_data.append((country, len(names))) @@ -869,7 +871,7 @@ def meeting_stats(request, num=None, stats_type=None): total_attendees = count_bins(bins) series_data = [] - for continent, names in sorted(bins.iteritems(), key=lambda t: t[0].lower()): + for continent, names in sorted(bins.items(), key=lambda t: t[0].lower()): percentage = len(names) * 100.0 / (total_attendees or 1) if continent: series_data.append((continent, len(names))) @@ -903,14 +905,14 @@ def meeting_stats(request, num=None, stats_type=None): bins[meeting_number].add(name) series_data = {} - for continent in continents.keys(): + for continent in list(continents.keys()): series_data[continent] = [] for m in meetings: country = CountryName.objects.get(slug=m.country) url = build_meeting_stats_url(number=m.number, stats_type_override="country") - for continent in continents.keys(): + for continent in list(continents.keys()): if continent == country.continent.name: d = { "name": "IETF {} - {}, {}".format(int(m.number), m.city, country), @@ -928,7 +930,7 @@ def meeting_stats(request, num=None, stats_type=None): table_data.append((m, url, m.attendees, country)) - for continent in continents.keys(): + for continent in list(continents.keys()): # series_data[continent].sort(key=lambda t: t[0]["x"]) chart_data.append( { "name": continent, "data": series_data[continent] }) diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index dd03abccf..5036680ae 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -1,14 +1,18 @@ -# Copyright The IETF Trust 2016, All Rights Reserved -from __future__ import unicode_literals, print_function +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import re -import sys -from xym import xym import shutil +import six +import sys import tempfile -import StringIO +from xym import xym from django.conf import settings import debug # pyflakes:ignore @@ -80,6 +84,8 @@ class DraftIdnitsChecker(object): cmd = "%s %s %s" % (settings.IDSUBMIT_IDNITS_BINARY, self.options, path) code, out, err = pipe(cmd) + out = out.decode() + err = err.decode() if code != 0 or out == "": message = "idnits error: %s:\n Error %s: %s" %( cmd, code, err) log(message) @@ -87,7 +93,7 @@ class DraftIdnitsChecker(object): else: message = out - if re.search("\s+Summary:\s+0\s+|No nits found", out): + if re.search(r"\s+Summary:\s+0\s+|No nits found", out): passed = True else: passed = False @@ -142,19 +148,23 @@ class DraftYangChecker(object): # This places the yang models as files in workdir saved_stdout = sys.stdout saved_stderr = sys.stderr - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() + sys.stdout = six.StringIO() + sys.stderr = six.StringIO() extractor.extract_yang_model(file.readlines()) model_list = extractor.get_extracted_models(False, True) out = sys.stdout.getvalue() err = sys.stderr.getvalue() - sys.stdout = saved_stdout - sys.stderr = saved_stderr # signature change in xym: except Exception as exc: + sys.stdout = saved_stdout + sys.stderr = saved_stderr msg = "Exception when running xym on %s: %s" % (name, exc) log(msg) + raise return None, msg, 0, 0, info + finally: + sys.stdout = saved_stdout + sys.stderr = saved_stderr if not model_list: # Found no yang models, don't deliver any YangChecker result return None, "", 0, 0, info @@ -196,7 +206,7 @@ class DraftYangChecker(object): settings.SUBMIT_YANG_IANA_MODEL_DIR, ]) if os.path.exists(path): - with open(path) as file: + with io.open(path) as file: text = file.readlines() # pyang cmd_template = settings.SUBMIT_PYANG_COMMAND @@ -204,6 +214,8 @@ class DraftYangChecker(object): cmd_version = VersionInfo.objects.get(command=command).version cmd = cmd_template.format(libs=modpath, model=path) code, out, err = pipe(cmd) + out = out.decode() + err = err.decode() if code > 0 or len(err.strip()) > 0 : error_lines = err.splitlines() assertion('len(error_lines) > 0') @@ -235,6 +247,8 @@ class DraftYangChecker(object): cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, ) code, out, err = pipe(cmd) + out = out.decode() + err = err.decode() if code > 0 or len(err.strip()) > 0: err_lines = err.splitlines() for line in err_lines: diff --git a/ietf/submit/forms.py b/ietf/submit/forms.py index cd99c9439..d9b6dab2e 100644 --- a/ietf/submit/forms.py +++ b/ietf/submit/forms.py @@ -1,10 +1,19 @@ +# Copyright The IETF Trust 2011-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import re import datetime import email import pytz -import xml2rfc +import six import tempfile +import xml2rfc + from email.utils import formataddr from unidecode import unidecode @@ -12,6 +21,7 @@ from django import forms from django.conf import settings from django.utils.html import mark_safe from django.urls import reverse as urlreverse +from django.utils.encoding import force_str import debug # pyflakes:ignore @@ -32,7 +42,7 @@ from ietf.submit.parsers.xml_parser import XMLParser from ietf.utils.draft import Draft class SubmissionBaseUploadForm(forms.Form): - xml = forms.FileField(label=u'.xml format', required=True) + xml = forms.FileField(label='.xml format', required=True) def __init__(self, request, *args, **kwargs): super(SubmissionBaseUploadForm, self).__init__(*args, **kwargs) @@ -107,9 +117,9 @@ class SubmissionBaseUploadForm(forms.Form): if not f: return f - parsed_info = parser_class(f).critical_parse() - if parsed_info.errors: - raise forms.ValidationError(parsed_info.errors) + self.parsed_info = parser_class(f).critical_parse() + if self.parsed_info.errors: + raise forms.ValidationError(self.parsed_info.errors) return f @@ -139,7 +149,7 @@ class SubmissionBaseUploadForm(forms.Form): # over to the xml parser. XXX FIXME: investigate updating # xml2rfc to be able to work with file handles to in-memory # files. - with open(tfn, 'wb+') as tf: + with io.open(tfn, 'wb+') as tf: for chunk in xml_file.chunks(): tf.write(chunk) os.environ["XML_LIBRARY"] = settings.XML_LIBRARY @@ -184,10 +194,10 @@ class SubmissionBaseUploadForm(forms.Form): self.revision = None self.filename = draftname self.title = self.xmlroot.findtext('front/title').strip() - if type(self.title) is unicode: + if type(self.title) is six.text_type: self.title = unidecode(self.title) self.abstract = (self.xmlroot.findtext('front/abstract') or '').strip() - if type(self.abstract) is unicode: + if type(self.abstract) is six.text_type: self.abstract = unidecode(self.abstract) author_info = self.xmlroot.findall('front/author') for author in author_info: @@ -214,7 +224,7 @@ class SubmissionBaseUploadForm(forms.Form): bytes = txt_file.read() txt_file.seek(0) try: - text = bytes.decode('utf8') + text = bytes.decode(self.parsed_info.charset) except UnicodeDecodeError as e: raise forms.ValidationError('Failed decoding the uploaded file: "%s"' % str(e)) # @@ -304,7 +314,7 @@ class SubmissionBaseUploadForm(forms.Form): else: name_parts = name.split("-") if len(name_parts) < 3: - raise forms.ValidationError(u"The draft name \"%s\" is missing a third part, please rename it" % name) + raise forms.ValidationError("The draft name \"%s\" is missing a third part, please rename it" % name) if name.startswith('draft-ietf-') or name.startswith("draft-irtf-"): @@ -339,10 +349,10 @@ class SubmissionBaseUploadForm(forms.Form): return None class SubmissionManualUploadForm(SubmissionBaseUploadForm): - xml = forms.FileField(label=u'.xml format', required=False) # xml field with required=False instead of True - txt = forms.FileField(label=u'.txt format', required=False) - pdf = forms.FileField(label=u'.pdf format', required=False) - ps = forms.FileField(label=u'.ps format', required=False) + xml = forms.FileField(label='.xml format', required=False) # xml field with required=False instead of True + txt = forms.FileField(label='.txt format', required=False) + pdf = forms.FileField(label='.pdf format', required=False) + ps = forms.FileField(label='.ps format', required=False) def __init__(self, request, *args, **kwargs): super(SubmissionManualUploadForm, self).__init__(request, *args, **kwargs) @@ -368,7 +378,7 @@ class SubmissionAutoUploadForm(SubmissionBaseUploadForm): class NameEmailForm(forms.Form): name = forms.CharField(required=True) - email = forms.EmailField(label=u'Email address', required=True) + email = forms.EmailField(label='Email address', required=True) def __init__(self, *args, **kwargs): super(NameEmailForm, self).__init__(*args, **kwargs) @@ -392,7 +402,7 @@ class AuthorForm(NameEmailForm): class SubmitterForm(NameEmailForm): #Fields for secretariat only - approvals_received = forms.BooleanField(label=u'Approvals received', required=False, initial=False) + approvals_received = forms.BooleanField(label='Approvals received', required=False, initial=False) def cleaned_line(self): line = self.cleaned_data["name"] @@ -422,13 +432,13 @@ class ReplacesForm(forms.Form): class EditSubmissionForm(forms.ModelForm): title = forms.CharField(required=True, max_length=255) - rev = forms.CharField(label=u'Revision', max_length=2, required=True) + rev = forms.CharField(label='Revision', max_length=2, required=True) document_date = forms.DateField(required=True) pages = forms.IntegerField(required=True) formal_languages = forms.ModelMultipleChoiceField(queryset=FormalLanguageName.objects.filter(used=True), widget=forms.CheckboxSelectMultiple, required=False) abstract = forms.CharField(widget=forms.Textarea, required=True, strip=False) - note = forms.CharField(label=mark_safe(u'Comment to the Secretariat'), widget=forms.Textarea, required=False, strip=False) + note = forms.CharField(label=mark_safe('Comment to the Secretariat'), widget=forms.Textarea, required=False, strip=False) class Meta: model = Submission @@ -507,7 +517,7 @@ class SubmissionEmailForm(forms.Form): '''Returns a ietf.message.models.Message object''' self.message_text = self.cleaned_data['message'] try: - message = email.message_from_string(self.message_text) + message = email.message_from_string(force_str(self.message_text)) except Exception as e: self.add_error('message', e) return None diff --git a/ietf/submit/mail.py b/ietf/submit/mail.py index dafbd9e9e..1014ea6ea 100644 --- a/ietf/submit/mail.py +++ b/ietf/submit/mail.py @@ -1,4 +1,8 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import re import email @@ -12,6 +16,7 @@ from django.urls import reverse as urlreverse from django.core.validators import ValidationError from django.contrib.sites.models import Site from django.template.loader import render_to_string +from django.utils.encoding import force_text, force_str import debug # pyflakes:ignore @@ -31,13 +36,13 @@ def send_submission_confirmation(request, submission, chair_notice=False): from_email = settings.IDSUBMIT_FROM_EMAIL (to_email, cc) = gather_address_lists('sub_confirmation_requested',submission=submission) - confirm_url = settings.IDTRACKER_BASE_URL + urlreverse('ietf.submit.views.confirm_submission', kwargs=dict(submission_id=submission.pk, auth_token=generate_access_token(submission.auth_key))) + confirmation_url = settings.IDTRACKER_BASE_URL + urlreverse('ietf.submit.views.confirm_submission', kwargs=dict(submission_id=submission.pk, auth_token=generate_access_token(submission.auth_key))) status_url = settings.IDTRACKER_BASE_URL + urlreverse('ietf.submit.views.submission_status', kwargs=dict(submission_id=submission.pk, access_token=submission.access_token())) send_mail(request, to_email, from_email, subject, 'submit/confirm_submission.txt', { 'submission': submission, - 'confirm_url': confirm_url, + 'confirmation_url': confirmation_url, 'status_url': status_url, 'chair_notice': chair_notice, }, @@ -82,7 +87,7 @@ def send_approval_request_to_group(request, submission): return all_addrs def send_manual_post_request(request, submission, errors): - subject = u'Manual Post Requested for %s' % submission.name + subject = 'Manual Post Requested for %s' % submission.name from_email = settings.IDSUBMIT_FROM_EMAIL (to_email,cc) = gather_address_lists('sub_manual_post_requested',submission=submission) checker = DraftIdnitsChecker(options=[]) # don't use the default --submitcheck limitation @@ -172,7 +177,7 @@ def get_reply_to(): address with "plus addressing" using a random string. Guaranteed to be unique""" local,domain = get_base_submission_message_address().split('@') while True: - rand = base64.urlsafe_b64encode(os.urandom(12)) + rand = force_text(base64.urlsafe_b64encode(os.urandom(12))) address = "{}+{}@{}".format(local,rand,domain) q = Message.objects.filter(reply_to=address) if not q: @@ -185,7 +190,7 @@ def process_response_email(msg): a matching value in the reply_to field, associated to a submission. Create a Message object for the incoming message and associate it to the original message via new SubmissionEvent""" - message = email.message_from_string(msg) + message = email.message_from_string(force_str(msg)) to = message.get('To') # exit if this isn't a response we're interested in (with plus addressing) @@ -234,7 +239,7 @@ def process_response_email(msg): save_submission_email_attachments(submission_email_event, parts) - log(u"Received submission email from %s" % msg.frm) + log("Received submission email from %s" % msg.frm) return msg diff --git a/ietf/submit/management/commands/manualpost_email.py b/ietf/submit/management/commands/manualpost_email.py index 710f4a0f9..10d3ed3d5 100644 --- a/ietf/submit/management/commands/manualpost_email.py +++ b/ietf/submit/management/commands/manualpost_email.py @@ -1,3 +1,10 @@ +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import sys from django.core.management.base import BaseCommand, CommandError @@ -7,7 +14,7 @@ from ietf.submit.mail import process_response_email import debug # pyflakes:ignore class Command(BaseCommand): - help = (u"Process incoming manual post email responses") + help = ("Process incoming manual post email responses") def add_arguments(self, parser): parser.add_argument('--email-file', dest='email', help='File containing email (default: stdin)') @@ -19,7 +26,7 @@ class Command(BaseCommand): if not email: msg = sys.stdin.read() else: - msg = open(email, "r").read() + msg = io.open(email, "r").read() try: process_response_email(msg) diff --git a/ietf/submit/migrations/0001_initial.py b/ietf/submit/migrations/0001_initial.py index 1d85ccba4..a7b039aa5 100644 --- a/ietf/submit/migrations/0001_initial.py +++ b/ietf/submit/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import datetime from django.db import migrations, models diff --git a/ietf/submit/migrations/0002_submission_document2_fk.py b/ietf/submit/migrations/0002_submission_document2_fk.py index 153a34559..debd1d52c 100644 --- a/ietf/submit/migrations/0002_submission_document2_fk.py +++ b/ietf/submit/migrations/0002_submission_document2_fk.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-08 11:58 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations import django.db.models.deletion diff --git a/ietf/submit/migrations/0003_remove_old_document_field.py b/ietf/submit/migrations/0003_remove_old_document_field.py index 8a5cf2104..fe4d880ef 100644 --- a/ietf/submit/migrations/0003_remove_old_document_field.py +++ b/ietf/submit/migrations/0003_remove_old_document_field.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-25 06:44 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/submit/migrations/0004_rename_field_document2.py b/ietf/submit/migrations/0004_rename_field_document2.py index cb8e32f15..b1bdbdcec 100644 --- a/ietf/submit/migrations/0004_rename_field_document2.py +++ b/ietf/submit/migrations/0004_rename_field_document2.py @@ -1,7 +1,9 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.20 on 2019-05-25 06:46 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations diff --git a/ietf/submit/models.py b/ietf/submit/models.py index 6e6b09c2d..04f7400bf 100644 --- a/ietf/submit/models.py +++ b/ietf/submit/models.py @@ -1,8 +1,15 @@ +# Copyright The IETF Trust 2011-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import datetime import email +import jsonfield from django.db import models -import jsonfield +from django.utils.encoding import python_2_unicode_compatible import debug # pyflakes:ignore @@ -23,6 +30,7 @@ def parse_email_line(line): name, addr = email.utils.parseaddr(line) if '@' in line else (line, '') return dict(name=name, email=addr) +@python_2_unicode_compatible class Submission(models.Model): state = ForeignKey(DraftSubmissionStateName) remote_ip = models.CharField(max_length=100, blank=True) @@ -54,8 +62,8 @@ class Submission(models.Model): draft = ForeignKey(Document, null=True, blank=True) - def __unicode__(self): - return u"%s-%s" % (self.name, self.rev) + def __str__(self): + return "%s-%s" % (self.name, self.rev) def submitter_parsed(self): return parse_email_line(self.submitter) @@ -70,6 +78,7 @@ class Submission(models.Model): checks = [ self.checks.filter(checker=c).latest('time') for c in self.checks.values_list('checker', flat=True).distinct() ] return checks +@python_2_unicode_compatible class SubmissionCheck(models.Model): time = models.DateTimeField(default=datetime.datetime.now) submission = ForeignKey(Submission, related_name='checks') @@ -81,42 +90,45 @@ class SubmissionCheck(models.Model): items = jsonfield.JSONField(null=True, blank=True, default='{}') symbol = models.CharField(max_length=64, default='') # - def __unicode__(self): + def __str__(self): return "%s submission check: %s: %s" % (self.checker, 'Passed' if self.passed else 'Failed', self.message[:48]+'...') def has_warnings(self): return self.warnings != '[]' def has_errors(self): return self.errors != '[]' +@python_2_unicode_compatible class SubmissionEvent(models.Model): submission = ForeignKey(Submission) time = models.DateTimeField(default=datetime.datetime.now) by = ForeignKey(Person, null=True, blank=True) desc = models.TextField() - def __unicode__(self): - return u"%s %s by %s at %s" % (self.submission.name, self.desc, self.by.plain_name() if self.by else "(unknown)", self.time) + def __str__(self): + return "%s %s by %s at %s" % (self.submission.name, self.desc, self.by.plain_name() if self.by else "(unknown)", self.time) class Meta: ordering = ("-time", "-id") +@python_2_unicode_compatible class Preapproval(models.Model): """Pre-approved draft submission name.""" name = models.CharField(max_length=255, db_index=True) by = ForeignKey(Person) time = models.DateTimeField(default=datetime.datetime.now) - def __unicode__(self): + def __str__(self): return self.name +@python_2_unicode_compatible class SubmissionEmailEvent(SubmissionEvent): message = ForeignKey(Message, null=True, blank=True,related_name='manualevents') msgtype = models.CharField(max_length=25) in_reply_to = ForeignKey(Message, null=True, blank=True,related_name='irtomanual') - def __unicode__(self): - return u"%s %s by %s at %s" % (self.submission.name, self.desc, self.by.plain_name() if self.by else "(unknown)", self.time) + def __str__(self): + return "%s %s by %s at %s" % (self.submission.name, self.desc, self.by.plain_name() if self.by else "(unknown)", self.time) class Meta: ordering = ['-time', '-id'] diff --git a/ietf/submit/parsers/base.py b/ietf/submit/parsers/base.py index 94c928a87..b92fabdb7 100644 --- a/ietf/submit/parsers/base.py +++ b/ietf/submit/parsers/base.py @@ -1,3 +1,8 @@ +# Copyright The IETF Trust 2011-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import re import magic @@ -62,21 +67,37 @@ class FileParser(object): regexp = re.compile(r'&|\|\/|;|\*|\s|\$') chars = regexp.findall(name) if chars: - self.parsed_info.add_error(u'Invalid characters were found in the name of the file which was just submitted: %s' % ', '.join(set(chars))) + self.parsed_info.add_error('Invalid characters were found in the name of the file which was just submitted: %s' % ', '.join(set(chars))) def parse_max_size(self): max_size = settings.IDSUBMIT_MAX_DRAFT_SIZE[self.ext] if self.fd.size > max_size: - self.parsed_info.add_error(u'File size is larger than the permitted maximum of %s' % filesizeformat(max_size)) + self.parsed_info.add_error('File size is larger than the permitted maximum of %s' % filesizeformat(max_size)) self.parsed_info.metadata.file_size = self.fd.size def parse_filename_extension(self): if not self.fd.name.lower().endswith('.'+self.ext): - self.parsed_info.add_error(u'Expected the %s file to have extension ".%s", found the name "%s"' % (self.ext.upper(), self.ext, self.fd.name)) + self.parsed_info.add_error('Expected the %s file to have extension ".%s", found the name "%s"' % (self.ext.upper(), self.ext, self.fd.name)) def parse_file_type(self): self.fd.file.seek(0) - content = self.fd.file.read() - mimetype = magic.from_buffer(content, mime=True) + content = self.fd.file.read(64*1024) + if hasattr(magic, "open"): + m = magic.open(magic.MAGIC_MIME) + m.load() + filetype = m.buffer(content) + else: + m = magic.Magic() + m.cookie = magic.magic_open(magic.MAGIC_NONE | magic.MAGIC_MIME | magic.MAGIC_MIME_ENCODING) + magic.magic_load(m.cookie, None) + filetype = m.from_buffer(content) + if ';' in filetype and 'charset=' in filetype: + mimetype, charset = re.split('; *charset=', filetype) + else: + mimetype = re.split(';', filetype)[0] + charset = 'utf-8' if not mimetype in self.mimetypes: - self.parsed_info.add_error(u'Expected an %s file of type "%s", found one of type "%s"' % (self.ext.upper(), '" or "'.join(self.mimetypes), mimetype)) + self.parsed_info.add_error('Expected an %s file of type "%s", found one of type "%s"' % (self.ext.upper(), '" or "'.join(self.mimetypes), mimetype)) + self.parsed_info.mimetype = mimetype + self.parsed_info.charset = charset + \ No newline at end of file diff --git a/ietf/submit/parsers/plain_parser.py b/ietf/submit/parsers/plain_parser.py index afb65233f..c7b0538af 100644 --- a/ietf/submit/parsers/plain_parser.py +++ b/ietf/submit/parsers/plain_parser.py @@ -1,5 +1,12 @@ +# Copyright The IETF Trust 2011-2019, All Rights Reserved + + +from __future__ import absolute_import, print_function, unicode_literals + import re +import debug # pyflakes:ignore + from ietf.submit.parsers.base import FileParser @@ -14,58 +21,51 @@ class PlainParser(FileParser): # no other file parsing is recommended def critical_parse(self): super(PlainParser, self).critical_parse() - self.parse_file_charset() + self.check_file_charset() self.parse_name() return self.parsed_info - def parse_file_charset(self): - import magic - self.fd.file.seek(0) - content = self.fd.file.read() - if hasattr(magic, "open"): - m = magic.open(magic.MAGIC_MIME) - m.load() - filetype = m.buffer(content) - else: - m = magic.Magic() - m.cookie = magic.magic_open(magic.MAGIC_NONE | magic.MAGIC_MIME | magic.MAGIC_MIME_ENCODING) - magic.magic_load(m.cookie, None) - filetype = m.from_buffer(content) - if not 'ascii' in filetype and not 'utf-8' in filetype: + def check_file_charset(self): + charset = self.parsed_info.charset + if not charset in ['us-ascii', 'utf-8',]: self.parsed_info.add_error('A plain text ASCII document is required. ' 'Found an unexpected encoding: "%s". ' - 'You probably have one or more non-ascii characters in your file.' % filetype + 'You probably have one or more non-ascii characters in your file.' % charset ) + if self.fd.charset and charset != self.fd.charset: + self.parsed_info.add_error("Unexpected charset mismatch: upload: %s, libmagic: %s" % (self.fd.charset, charset)) + def parse_name(self): self.fd.file.seek(0) - draftre = re.compile('(draft-\S+)') - revisionre = re.compile('.*-(\d+)$') + draftre = re.compile(r'(draft-\S+)') + revisionre = re.compile(r'.*-(\d+)$') limit = 80 - while limit: - limit -= 1 - line = self.fd.readline() - match = draftre.search(line) - if not match: - continue - name = match.group(1) - name = re.sub('^[^\w]+', '', name) - name = re.sub('[^\w]+$', '', name) - name = re.sub('\.txt$', '', name) - extra_chars = re.sub('[0-9a-z\-]', '', name) - if extra_chars: - if len(extra_chars) == 1: - self.parsed_info.add_error((u'The document name on the first page, "%s", contains a disallowed character with byte code: %s ' % (name.decode('utf-8','replace'), ord(extra_chars[0]))) + - u'(see https://www.ietf.org/id-info/guidelines.html#naming for details).') + if self.parsed_info.charset in ['us-ascii', 'utf-8']: + while limit: + limit -= 1 + line = self.fd.readline().decode(self.parsed_info.charset) + match = draftre.search(line) + if not match: + continue + name = match.group(1) + name = re.sub(r'^[^\w]+', '', name) + name = re.sub(r'[^\w]+$', '', name) + name = re.sub(r'\.txt$', '', name) + extra_chars = re.sub(r'[0-9a-z\-]', '', name) + if extra_chars: + if len(extra_chars) == 1: + self.parsed_info.add_error(('The document name on the first page, "%s", contains a disallowed character with byte code: %s ' % (name.decode('utf-8','replace'), ord(extra_chars[0]))) + + '(see https://www.ietf.org/id-info/guidelines.html#naming for details).') + else: + self.parsed_info.add_error(('The document name on the first page, "%s", contains disallowed characters with byte codes: %s ' % (name.decode('utf-8','replace'), (', '.join([ str(ord(c)) for c in extra_chars] )))) + + '(see https://www.ietf.org/id-info/guidelines.html#naming for details).') + match_revision = revisionre.match(name) + if match_revision: + self.parsed_info.metadata.rev = match_revision.group(1) else: - self.parsed_info.add_error((u'The document name on the first page, "%s", contains disallowed characters with byte codes: %s ' % (name.decode('utf-8','replace'), (', '.join([ str(ord(c)) for c in extra_chars] )))) + - u'(see https://www.ietf.org/id-info/guidelines.html#naming for details).') - match_revision = revisionre.match(name) - if match_revision: - self.parsed_info.metadata.rev = match_revision.group(1) - else: - self.parsed_info.add_error(u'The name found on the first page of the document does not contain a revision: "%s"' % (name.decode('utf-8','replace'),)) - name = re.sub('-\d+$', '', name) - self.parsed_info.metadata.name = name - return - self.parsed_info.add_error('The first page of the document does not contain a legitimate name that start with draft-*') + self.parsed_info.add_error('The name found on the first page of the document does not contain a revision: "%s"' % (name.decode('utf-8','replace'),)) + name = re.sub(r'-\d+$', '', name) + self.parsed_info.metadata.name = name + return + self.parsed_info.add_error('The first page of the document does not contain a legitimate name that starts with draft-*') diff --git a/ietf/submit/resources.py b/ietf/submit/resources.py index 66a82cdf8..aed899717 100644 --- a/ietf/submit/resources.py +++ b/ietf/submit/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2014-11-13 23:53 + + from ietf.api import ModelResource from tastypie.fields import ToOneField, ToManyField from tastypie.constants import ALL, ALL_WITH_RELATIONS diff --git a/ietf/submit/templatetags/submit_tags.py b/ietf/submit/templatetags/submit_tags.py index 13e93abf7..514e8a7f4 100644 --- a/ietf/submit/templatetags/submit_tags.py +++ b/ietf/submit/templatetags/submit_tags.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2011-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import os from django import template @@ -19,14 +25,14 @@ def show_submission_files(context, submission): continue result.append({'ext': '%s' % ext[1:], 'exists': exists, - 'url': '%s%s-%s%s' % (settings.IDSUBMIT_STAGING_URL, submission.name, submission.rev, ext)}) + 'url': '%s%s-%s%s' % (settings.IDSUBMIT_STAGING_URL, submission.name, submission.rev, ext)}) return {'files': result} @register.filter def two_pages_decorated_with_errors(submission, errors): pages = submission.first_two_pages or '' - if 'rev' not in errors.keys(): + if 'rev' not in list(errors.keys()): return mark_safe('
%s
' % escape(pages)) result = '
\n'
     for line in pages.split('\n'):
diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py
index 08c03dbfa..d87d73885 100644
--- a/ietf/submit/tests.py
+++ b/ietf/submit/tests.py
@@ -1,20 +1,25 @@
 # Copyright The IETF Trust 2011-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
-from __future__ import unicode_literals, print_function
+
+
+from __future__ import absolute_import, print_function, unicode_literals
 
 import datetime
 import email
+import io
 import os
 import re
 import shutil
+import six
 import sys
 
 
-from StringIO import StringIO
+from io import StringIO
 from pyquery import PyQuery
 
 from django.conf import settings
 from django.urls import reverse as urlreverse
+from django.utils.encoding import force_str, force_text
 
 import debug                            # pyflakes:ignore
 
@@ -33,15 +38,15 @@ from ietf.person.models import Person
 from ietf.person.factories import UserFactory, PersonFactory
 from ietf.submit.models import Submission, Preapproval
 from ietf.submit.mail import add_submission_email, process_response_email
-from ietf.utils.mail import outbox, empty_outbox
+from ietf.utils.mail import outbox, empty_outbox, get_payload
 from ietf.utils.models import VersionInfo
-from ietf.utils.test_utils import login_testing_unauthorized, unicontent, TestCase
+from ietf.utils.test_utils import login_testing_unauthorized, TestCase
 from ietf.utils.draft import Draft
 
 
 def submission_file(name, rev, group, format, templatename, author=None, email=None, title=None, year=None, ascii=True):
     # construct appropriate text draft
-    f = open(os.path.join(settings.BASE_DIR, "submit", templatename))
+    f = io.open(os.path.join(settings.BASE_DIR, "submit", templatename))
     template = f.read()
     f.close()
 
@@ -145,7 +150,7 @@ class SubmitTests(TestCase):
 
         status_url = r["Location"]
         for format in formats:
-            self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.%s" % (name, rev, format))))
+            self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, format))))
         self.assertEqual(Submission.objects.filter(name=name).count(), 1)
         submission = Submission.objects.get(name=name)
         if len(submission.authors) != 1:
@@ -185,17 +190,18 @@ class SubmitTests(TestCase):
 
         return r
 
-    def extract_confirm_url(self, confirm_email):
-        # dig out confirm_email link
-        msg = confirm_email.get_payload(decode=True)
+    def extract_confirmation_url(self, confirmation_email):
+        # dig out confirmation_email link
+        charset = confirmation_email.get_content_charset()
+        msg = confirmation_email.get_payload(decode=True).decode(charset)
         line_start = "http"
-        confirm_url = None
+        confirmation_url = None
         for line in msg.split("\n"):
             if line.strip().startswith(line_start):
-                confirm_url = line.strip()
-        self.assertTrue(confirm_url)
+                confirmation_url = line.strip()
+        self.assertTrue(confirmation_url)
 
-        return confirm_url
+        return confirmation_url
 
     def submit_new_wg(self, formats):
         # submit new -> supply submitter info -> approve
@@ -274,8 +280,8 @@ class SubmitTests(TestCase):
         self.assertEqual(new_revision.type, "new_revision")
         self.assertEqual(new_revision.by.name, author.name)
         self.assertTrue(draft.latest_event(type="added_suggested_replaces"))
-        self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))))
-        self.assertTrue(os.path.exists(os.path.join(self.repository_dir, u"%s-%s.txt" % (name, rev))))
+        self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))))
+        self.assertTrue(os.path.exists(os.path.join(self.repository_dir, "%s-%s.txt" % (name, rev))))
         self.assertEqual(draft.type_id, "draft")
         self.assertEqual(draft.stream_id, "ietf")
         self.assertTrue(draft.expires >= datetime.datetime.now() + datetime.timedelta(days=settings.INTERNET_DRAFT_DAYS_TO_EXPIRE - 1))
@@ -289,17 +295,17 @@ class SubmitTests(TestCase):
         self.assertEqual(draft.relations_that_doc("possibly-replaces").count(), 1)
         self.assertTrue(draft.relations_that_doc("possibly-replaces").first().target, sug_replaced_alias)
         self.assertEqual(len(outbox), mailbox_before + 5)
-        self.assertIn((u"I-D Action: %s" % name), outbox[-4]["Subject"])
-        self.assertIn(author.ascii, unicode(outbox[-4]))
-        self.assertIn((u"I-D Action: %s" % name), outbox[-3]["Subject"])
-        self.assertIn(author.ascii, unicode(outbox[-3]))
+        self.assertIn(("I-D Action: %s" % name), outbox[-4]["Subject"])
+        self.assertIn(author.ascii, get_payload(outbox[-4]))
+        self.assertIn(("I-D Action: %s" % name), outbox[-3]["Subject"])
+        self.assertIn(author.ascii, get_payload(outbox[-3]))
         self.assertIn("New Version Notification",outbox[-2]["Subject"])
-        self.assertIn(name, unicode(outbox[-2]))
-        self.assertIn("mars", unicode(outbox[-2]))
+        self.assertIn(name, get_payload(outbox[-2]))
+        self.assertIn("mars", get_payload(outbox[-2]))
         # Check "Review of suggested possible replacements for..." mail
         self.assertIn("review", outbox[-1]["Subject"].lower())
-        self.assertIn(name, unicode(outbox[-1]))
-        self.assertIn(sug_replaced_alias.name, unicode(outbox[-1]))
+        self.assertIn(name, get_payload(outbox[-1]))
+        self.assertIn(sug_replaced_alias.name, get_payload(outbox[-1]))
         self.assertIn("ames-chairs@", outbox[-1]["To"].lower())
         self.assertIn("mars-chairs@", outbox[-1]["To"].lower())
 
@@ -367,7 +373,7 @@ class SubmitTests(TestCase):
         prev_author = draft.documentauthor_set.all()[0]
         if change_authors:
             # Make it such that one of the previous authors has an invalid email address
-            bogus_person, bogus_email = ensure_person_email_info_exists(u'Bogus Person', None, draft.name)
+            bogus_person, bogus_email = ensure_person_email_info_exists('Bogus Person', None, draft.name)
             DocumentAuthor.objects.create(document=draft, person=bogus_person, email=bogus_email, order=draft.documentauthor_set.latest('order').order+1)
 
         # Set the revision needed tag
@@ -379,7 +385,7 @@ class SubmitTests(TestCase):
 
         # write the old draft in a file so we can check it's moved away
         old_rev = draft.rev
-        with open(os.path.join(self.repository_dir, "%s-%s.txt" % (name, old_rev)), 'w') as f:
+        with io.open(os.path.join(self.repository_dir, "%s-%s.txt" % (name, old_rev)), 'w') as f:
             f.write("a" * 2000)
 
         old_docevents = list(draft.docevent_set.all())
@@ -393,7 +399,7 @@ class SubmitTests(TestCase):
         status_url = r["Location"]
         r = self.client.get(status_url)
         self.assertEqual(r.status_code, 200)
-        self.assertTrue("The submission is pending approval by the authors" in unicontent(r))
+        self.assertContains(r, "The submission is pending approval by the authors")
 
         self.assertEqual(len(outbox), mailbox_before + 1)
         confirm_email = outbox[-1]
@@ -407,7 +413,7 @@ class SubmitTests(TestCase):
         self.assertTrue("unknown-email-" not in confirm_email["To"])
         if change_authors:
             # Since authors changed, ensure chairs are copied (and that the message says why)
-            self.assertTrue("chairs have been copied" in unicode(confirm_email))
+            self.assertTrue("chairs have been copied" in six.text_type(confirm_email))
             if group_type in ['wg','rg','ag']:
                 self.assertTrue("mars-chairs@" in confirm_email["To"].lower())
             elif group_type == 'area':
@@ -417,19 +423,19 @@ class SubmitTests(TestCase):
             if stream_type=='ise':
                self.assertTrue("rfc-ise@" in confirm_email["To"].lower())
         else:
-            self.assertNotIn("chairs have been copied", unicode(confirm_email))
+            self.assertNotIn("chairs have been copied", six.text_type(confirm_email))
             self.assertNotIn("mars-chairs@", confirm_email["To"].lower())
 
-        confirm_url = self.extract_confirm_url(confirm_email)
+        confirmation_url = self.extract_confirmation_url(confirm_email)
 
         # go to confirm page
-        r = self.client.get(confirm_url)
+        r = self.client.get(confirmation_url)
         q = PyQuery(r.content)
         self.assertEqual(len(q('[type=submit]:contains("Confirm")')), 1)
 
         # confirm
         mailbox_before = len(outbox)
-        r = self.client.post(confirm_url, {'action':'confirm'})
+        r = self.client.post(confirmation_url, {'action':'confirm'})
         self.assertEqual(r.status_code, 302)
 
         new_docevents = draft.docevent_set.exclude(pk__in=[event.pk for event in old_docevents])
@@ -473,8 +479,8 @@ class SubmitTests(TestCase):
 
         self.assertTrue(not os.path.exists(os.path.join(self.repository_dir, "%s-%s.txt" % (name, old_rev))))
         self.assertTrue(os.path.exists(os.path.join(self.archive_dir, "%s-%s.txt" % (name, old_rev))))
-        self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))))
-        self.assertTrue(os.path.exists(os.path.join(self.repository_dir, u"%s-%s.txt" % (name, rev))))
+        self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))))
+        self.assertTrue(os.path.exists(os.path.join(self.repository_dir, "%s-%s.txt" % (name, rev))))
         self.assertEqual(draft.type_id, "draft")
         if stream_type == 'ietf':
             self.assertEqual(draft.stream_id, "ietf")
@@ -484,23 +490,23 @@ class SubmitTests(TestCase):
         self.assertEqual(len(authors), 1)
         self.assertIn(author, [ a.person for a in authors ])
         self.assertEqual(len(outbox), mailbox_before + 3)
-        self.assertTrue((u"I-D Action: %s" % name) in outbox[-3]["Subject"])
-        self.assertTrue((u"I-D Action: %s" % name) in draft.message_set.order_by("-time")[0].subject)
-        self.assertTrue(author.ascii in unicode(outbox[-3]))
+        self.assertTrue(("I-D Action: %s" % name) in outbox[-3]["Subject"])
+        self.assertTrue(("I-D Action: %s" % name) in draft.message_set.order_by("-time")[0].subject)
+        self.assertTrue(author.ascii in get_payload(outbox[-3]))
         self.assertTrue("i-d-announce@" in outbox[-3]['To'])
         self.assertTrue("New Version Notification" in outbox[-2]["Subject"])
-        self.assertTrue(name in unicode(outbox[-2]))
+        self.assertTrue(name in get_payload(outbox[-2]))
         interesting_address = {'ietf':'mars', 'irtf':'irtf-chair', 'iab':'iab-chair', 'ise':'rfc-ise'}[draft.stream_id]
-        self.assertTrue(interesting_address in unicode(outbox[-2]))
+        self.assertTrue(interesting_address in force_text(outbox[-2].as_string()))
         if draft.stream_id == 'ietf':
-            self.assertTrue(draft.ad.role_email("ad").address in unicode(outbox[-2]))
-            self.assertTrue(ballot_position.ad.role_email("ad").address in unicode(outbox[-2]))
+            self.assertTrue(draft.ad.role_email("ad").address in force_text(outbox[-2].as_string()))
+            self.assertTrue(ballot_position.ad.role_email("ad").address in force_text(outbox[-2].as_string()))
         self.assertTrue("New Version Notification" in outbox[-1]["Subject"])
-        self.assertTrue(name in unicode(outbox[-1]))
+        self.assertTrue(name in get_payload(outbox[-1]))
         r = self.client.get(urlreverse('ietf.doc.views_search.recent_drafts'))
         self.assertEqual(r.status_code, 200)
-        self.assertIn(draft.name,  unicontent(r))
-        self.assertIn(draft.title, unicontent(r))
+        self.assertContains(r, draft.name)
+        self.assertContains(r, draft.title)
 
 
     def test_submit_existing_txt(self):
@@ -547,7 +553,7 @@ class SubmitTests(TestCase):
         status_url = r["Location"]
         r = self.client.get(status_url)
         self.assertEqual(r.status_code, 200)
-        self.assertTrue("The submission is pending email authentication" in unicontent(r))
+        self.assertContains(r, "The submission is pending email authentication")
 
         self.assertEqual(len(outbox), mailbox_before + 1)
         confirm_email = outbox[-1]
@@ -556,18 +562,18 @@ class SubmitTests(TestCase):
         # both submitter and author get email
         self.assertTrue(author.email().address.lower() in confirm_email["To"])
         self.assertTrue("submitter@example.com" in confirm_email["To"])
-        self.assertFalse("chairs have been copied" in unicode(confirm_email))
+        self.assertFalse("chairs have been copied" in six.text_type(confirm_email))
 
-        confirm_url = self.extract_confirm_url(outbox[-1])
+        confirmation_url = self.extract_confirmation_url(outbox[-1])
 
         # go to confirm page
-        r = self.client.get(confirm_url)
+        r = self.client.get(confirmation_url)
         q = PyQuery(r.content)
         self.assertEqual(len(q('[type=submit]:contains("Confirm")')), 1)
 
         # confirm
         mailbox_before = len(outbox)
-        r = self.client.post(confirm_url, {'action':'confirm'})
+        r = self.client.post(confirmation_url, {'action':'confirm'})
         self.assertEqual(r.status_code, 302)
 
         draft = Document.objects.get(docalias__name=name)
@@ -598,25 +604,25 @@ class SubmitTests(TestCase):
         replaced_alias = draft.docalias.first()
         r = self.supply_extra_metadata(name, status_url, "Submitter Name", "author@example.com", replaces=str(replaced_alias.pk))
         self.assertEqual(r.status_code, 200)
-        self.assertTrue('cannot replace itself' in unicontent(r))
+        self.assertContains(r, 'cannot replace itself')
         replaced_alias = DocAlias.objects.get(name='draft-ietf-random-thing')
         r = self.supply_extra_metadata(name, status_url, "Submitter Name", "author@example.com", replaces=str(replaced_alias.pk))
         self.assertEqual(r.status_code, 200)
-        self.assertTrue('cannot replace an RFC' in unicontent(r))
+        self.assertContains(r, 'cannot replace an RFC')
         replaced_alias.document.set_state(State.objects.get(type='draft-iesg',slug='approved'))
         replaced_alias.document.set_state(State.objects.get(type='draft',slug='active'))
         r = self.supply_extra_metadata(name, status_url, "Submitter Name", "author@example.com", replaces=str(replaced_alias.pk))
         self.assertEqual(r.status_code, 200)
-        self.assertTrue('approved by the IESG and cannot' in unicontent(r))
+        self.assertContains(r, 'approved by the IESG and cannot')
         r = self.supply_extra_metadata(name, status_url, "Submitter Name", "author@example.com", replaces='')
         self.assertEqual(r.status_code, 302)
         status_url = r["Location"]
         r = self.client.get(status_url)
         self.assertEqual(len(outbox), mailbox_before + 1)
-        confirm_url = self.extract_confirm_url(outbox[-1])
-        self.assertFalse("chairs have been copied" in unicode(outbox[-1]))
+        confirmation_url = self.extract_confirmation_url(outbox[-1])
+        self.assertFalse("chairs have been copied" in str(outbox[-1]))
         mailbox_before = len(outbox)
-        r = self.client.post(confirm_url, {'action':'confirm'})
+        r = self.client.post(confirmation_url, {'action':'confirm'})
         self.assertEqual(r.status_code, 302)
         self.assertEqual(len(outbox), mailbox_before+3)
         draft = Document.objects.get(docalias__name=name)
@@ -624,9 +630,8 @@ class SubmitTests(TestCase):
         self.assertEqual(draft.relateddocument_set.filter(relationship_id='replaces').count(), replaces_count)
         #
         r = self.client.get(urlreverse('ietf.doc.views_search.recent_drafts'))
-        self.assertEqual(r.status_code, 200)
-        self.assertIn(draft.name,  unicontent(r))
-        self.assertIn(draft.title, unicontent(r))
+        self.assertContains(r, draft.name)
+        self.assertContains(r, draft.title)
 
     def test_submit_cancel_confirmation(self):
         ad=Person.objects.get(user__username='ad')
@@ -642,9 +647,9 @@ class SubmitTests(TestCase):
         status_url = r["Location"]
         r = self.client.get(status_url)
         self.assertEqual(len(outbox), mailbox_before + 1)
-        confirm_url = self.extract_confirm_url(outbox[-1])
+        confirmation_url = self.extract_confirmation_url(outbox[-1])
         mailbox_before = len(outbox)
-        r = self.client.post(confirm_url, {'action':'cancel'})
+        r = self.client.post(confirmation_url, {'action':'cancel'})
         self.assertEqual(r.status_code, 302)
         self.assertEqual(len(outbox), mailbox_before)
         draft = Document.objects.get(docalias__name=name)
@@ -696,7 +701,7 @@ class SubmitTests(TestCase):
 
         # cancel
         r = self.client.post(status_url, dict(action=action))
-        self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))))
+        self.assertTrue(not os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))))
 
     def test_edit_submission_and_force_post(self):
         # submit -> edit
@@ -805,7 +810,7 @@ class SubmitTests(TestCase):
         # search status page
         r = self.client.get(urlreverse("ietf.submit.views.search_submission"))
         self.assertEqual(r.status_code, 200)
-        self.assertTrue("submission status" in unicontent(r))
+        self.assertContains(r, "submission status")
 
         # search
         r = self.client.post(urlreverse("ietf.submit.views.search_submission"), dict(name=name))
@@ -819,8 +824,7 @@ class SubmitTests(TestCase):
 
         # status page as unpriviliged => no edit button
         r = self.client.get(unprivileged_status_url)
-        self.assertEqual(r.status_code, 200)
-        self.assertTrue(("submission status of %s" % name) in unicontent(r).lower())
+        self.assertContains(r, "Submission status of %s" % name)
         q = PyQuery(r.content)
         adjust_button = q('[type=submit]:contains("Adjust")')
         self.assertEqual(len(adjust_button), 0)
@@ -886,15 +890,15 @@ class SubmitTests(TestCase):
 
         self.assertEqual(Submission.objects.filter(name=name).count(), 1)
 
-        self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))))
-        self.assertTrue(name in open(os.path.join(self.staging_dir, u"%s-%s.txt" % (name, rev))).read())
-        self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))))
-        self.assertTrue(name in open(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))).read())
-        self.assertTrue('' in open(os.path.join(self.staging_dir, u"%s-%s.xml" % (name, rev))).read())
-        self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.pdf" % (name, rev))))
-        self.assertTrue('This is PDF' in open(os.path.join(self.staging_dir, u"%s-%s.pdf" % (name, rev))).read())
-        self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.ps" % (name, rev))))
-        self.assertTrue('This is PostScript' in open(os.path.join(self.staging_dir, u"%s-%s.ps" % (name, rev))).read())
+        self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))))
+        self.assertTrue(name in io.open(os.path.join(self.staging_dir, "%s-%s.txt" % (name, rev))).read())
+        self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))))
+        self.assertTrue(name in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read())
+        self.assertTrue('' in io.open(os.path.join(self.staging_dir, "%s-%s.xml" % (name, rev))).read())
+        self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev))))
+        self.assertTrue('This is PDF' in io.open(os.path.join(self.staging_dir, "%s-%s.pdf" % (name, rev))).read())
+        self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.ps" % (name, rev))))
+        self.assertTrue('This is PostScript' in io.open(os.path.join(self.staging_dir, "%s-%s.ps" % (name, rev))).read())
 
     def test_expire_submissions(self):
         s = Submission.objects.create(name="draft-ietf-mars-foo",
@@ -925,10 +929,10 @@ class SubmitTests(TestCase):
 
     def test_help_pages(self):
         r = self.client.get(urlreverse("ietf.submit.views.note_well"))
-        self.assertEquals(r.status_code, 200)
+        self.assertEqual(r.status_code, 200)
 
         r = self.client.get(urlreverse("ietf.submit.views.tool_instructions"))
-        self.assertEquals(r.status_code, 200)
+        self.assertEqual(r.status_code, 200)
         
     def test_blackout_access(self):
         # get
@@ -1016,7 +1020,7 @@ class SubmitTests(TestCase):
 
         # submit
         #author = PersonFactory(name=u"Jörgen Nilsson".encode('latin1'))
-        user = UserFactory(first_name=u"Jörgen", last_name=u"Nilsson")
+        user = UserFactory(first_name="Jörgen", last_name="Nilsson")
         author = PersonFactory(user=user)
 
         file, __ = submission_file(name, rev, group, "txt", "test_submission.nonascii", author=author, ascii=False)
@@ -1051,7 +1055,7 @@ class SubmitTests(TestCase):
         r = self.client.get(status_url)
         q = PyQuery(r.content)
         #
-        self.assertContains(r, u'The yang validation returned 1 error')
+        self.assertContains(r, 'The yang validation returned 1 error')
         #
         m = q('#yang-validation-message').text()
         for command in ['xym', 'pyang', 'yanglint']:
@@ -1176,7 +1180,7 @@ Please submit my draft at http://test.com/mydraft.txt
 
 Thank you
 """.format(datetime.datetime.now().ctime())
-        message = email.message_from_string(message_string)
+        message = email.message_from_string(force_str(message_string))
         submission, submission_email_event = (
             add_submission_email(request=None,
                                  remote_ip ="192.168.0.1",
@@ -1259,7 +1263,7 @@ ZSBvZiBsaW5lcyAtIGJ1dCBpdCBjb3VsZCBiZSBhIGRyYWZ0Cg==
 --------------090908050800030909090207--
 """.format(frm, datetime.datetime.now().ctime())
 
-        message = email.message_from_string(message_string)
+        message = email.message_from_string(force_str(message_string))
         submission, submission_email_event = (
             add_submission_email(request=None,
                                  remote_ip ="192.168.0.1",
@@ -1553,7 +1557,7 @@ Subject: test
 
         status_url = r["Location"]
         for format in formats:
-            self.assertTrue(os.path.exists(os.path.join(self.staging_dir, u"%s-%s.%s" % (name, rev, format))))
+            self.assertTrue(os.path.exists(os.path.join(self.staging_dir, "%s-%s.%s" % (name, rev, format))))
         self.assertEqual(Submission.objects.filter(name=name).count(), 1)
         submission = Submission.objects.get(name=name)
         self.assertTrue(all([ c.passed!=False for c in submission.checks.all() ]))
@@ -1690,7 +1694,7 @@ class RefsTests(TestCase):
 
         group = None
         file, __ = submission_file('draft-some-subject', '00', group, 'txt', "test_submission.txt", )
-        draft = Draft(file.read().decode('utf-8'), file.name)
+        draft = Draft(file.read(), file.name)
         refs = draft.get_refs()
         self.assertEqual(refs['rfc2119'], 'norm')
         self.assertEqual(refs['rfc8174'], 'norm')
diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index 816742684..37ddc1f2b 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -1,7 +1,11 @@
 # Copyright The IETF Trust 2011-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
 
+
+from __future__ import absolute_import, print_function, unicode_literals
+
 import datetime
+import io
 import os
 import re
 import six                              # pyflakes:ignore
@@ -202,7 +206,7 @@ def post_submission(request, submission, approvedDesc):
     submitter_parsed = submission.submitter_parsed()
     if submitter_parsed["name"] and submitter_parsed["email"]:
         submitter, _ = ensure_person_email_info_exists(submitter_parsed["name"], submitter_parsed["email"], submission.name)
-        submitter_info = u'%s <%s>' % (submitter_parsed["name"], submitter_parsed["email"])
+        submitter_info = '%s <%s>' % (submitter_parsed["name"], submitter_parsed["email"])
     else:
         submitter = system
         submitter_info = system.name
@@ -460,7 +464,7 @@ def ensure_person_email_info_exists(name, email, docname):
         person.name = name
         person.name_from_draft = name
         log.assertion('isinstance(person.name, six.text_type)')
-        person.ascii = unidecode_name(person.name).decode('ascii')
+        person.ascii = unidecode_name(person.name)
         person.save()
     else:
         person.name_from_draft = name
@@ -472,7 +476,7 @@ def ensure_person_email_info_exists(name, email, docname):
     else:
         # we're in trouble, use a fake one
         active = False
-        addr = u"unknown-email-%s" % person.plain_ascii().replace(" ", "-")
+        addr = "unknown-email-%s" % person.plain_ascii().replace(" ", "-")
 
     try:
         email = person.email_set.get(address=addr)
@@ -525,7 +529,7 @@ def cancel_submission(submission):
 
 def rename_submission_files(submission, prev_rev, new_rev):
     from ietf.submit.forms import SubmissionManualUploadForm
-    for ext in SubmissionManualUploadForm.base_fields.keys():
+    for ext in list(SubmissionManualUploadForm.base_fields.keys()):
         source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, prev_rev, ext))
         dest = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, new_rev, ext))
         if os.path.exists(source):
@@ -533,7 +537,7 @@ def rename_submission_files(submission, prev_rev, new_rev):
 
 def move_files_to_repository(submission):
     from ietf.submit.forms import SubmissionManualUploadForm
-    for ext in SubmissionManualUploadForm.base_fields.keys():
+    for ext in list(SubmissionManualUploadForm.base_fields.keys()):
         source = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext))
         dest = os.path.join(settings.IDSUBMIT_REPOSITORY_PATH, '%s-%s.%s' % (submission.name, submission.rev, ext))
         if os.path.exists(source):
@@ -597,12 +601,9 @@ def expire_submission(submission, by):
 
     SubmissionEvent.objects.create(submission=submission, by=by, desc="Cancelled expired submission")
 
-def get_draft_meta(form):
-    authors = []
+def save_files(form):
     file_name = {}
-    abstract = None
-    file_size = None
-    for ext in form.fields.keys():
+    for ext in list(form.fields.keys()):
         if not ext in form.formats:
             continue
         f = form.cleaned_data[ext]
@@ -611,10 +612,16 @@ def get_draft_meta(form):
 
         name = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.%s' % (form.filename, form.revision, ext))
         file_name[ext] = name
-        with open(name, 'wb+') as destination:
+        with io.open(name, 'wb+') as destination:
             for chunk in f.chunks():
                 destination.write(chunk)
+    return file_name
 
+def get_draft_meta(form, saved_files):
+    authors = []
+    file_name = saved_files
+    abstract = None
+    file_size = None
     if form.cleaned_data['xml']:
         if not ('txt' in form.cleaned_data and form.cleaned_data['txt']):
             file_name['txt'] = os.path.join(settings.IDSUBMIT_STAGING_PATH, '%s-%s.txt' % (form.filename, form.revision))
@@ -641,8 +648,8 @@ def get_draft_meta(form):
         # Some meta-information, such as the page-count, can only
         # be retrieved from the generated text file.  Provide a
         # parsed draft object to get at that kind of information.
-        with open(file_name['txt']) as txt_file:
-            form.parsed_draft = Draft(txt_file.read().decode('utf8'), txt_file.name)
+        with io.open(file_name['txt']) as txt_file:
+            form.parsed_draft = Draft(txt_file.read(), txt_file.name)
 
     else:
         file_size = form.cleaned_data['txt'].size
@@ -665,9 +672,9 @@ def get_draft_meta(form):
 
             def turn_into_unicode(s):
                 if s is None:
-                    return u""
+                    return ""
 
-                if isinstance(s, unicode):
+                if isinstance(s, six.text_type):
                     return s
                 else:
                     try:
@@ -762,8 +769,8 @@ def send_confirmation_emails(request, submission, requires_group_approval, requi
 
         sent_to = send_approval_request_to_group(request, submission)
 
-        desc = "sent approval email to group chairs: %s" % u", ".join(sent_to)
-        docDesc = u"Request for posting approval emailed to group chairs: %s" % u", ".join(sent_to)
+        desc = "sent approval email to group chairs: %s" % ", ".join(sent_to)
+        docDesc = "Request for posting approval emailed to group chairs: %s" % ", ".join(sent_to)
 
     else:
         group_authors_changed = False
@@ -783,11 +790,11 @@ def send_confirmation_emails(request, submission, requires_group_approval, requi
         sent_to = send_submission_confirmation(request, submission, chair_notice=group_authors_changed)
 
         if submission.state_id == "aut-appr":
-            desc = u"sent confirmation email to previous authors: %s" % u", ".join(sent_to)
-            docDesc = "Request for posting confirmation emailed to previous authors: %s" % u", ".join(sent_to)
+            desc = "sent confirmation email to previous authors: %s" % ", ".join(sent_to)
+            docDesc = "Request for posting confirmation emailed to previous authors: %s" % ", ".join(sent_to)
         else:
-            desc = u"sent confirmation email to submitter and authors: %s" % u", ".join(sent_to)
-            docDesc = "Request for posting confirmation emailed to submitter and authors: %s" % u", ".join(sent_to)
+            desc = "sent confirmation email to submitter and authors: %s" % ", ".join(sent_to)
+            docDesc = "Request for posting confirmation emailed to submitter and authors: %s" % ", ".join(sent_to)
     return sent_to, desc, docDesc
 
     
diff --git a/ietf/submit/views.py b/ietf/submit/views.py
index a08c39035..733794115 100644
--- a/ietf/submit/views.py
+++ b/ietf/submit/views.py
@@ -1,6 +1,9 @@
-# Copyright The IETF Trust 2007-2019, All Rights Reserved
+# Copyright The IETF Trust 2011-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
 
+
+from __future__ import absolute_import, print_function, unicode_literals
+
 import re
 import base64
 import datetime
@@ -31,18 +34,19 @@ from ietf.submit.models import (Submission, Preapproval,
 from ietf.submit.utils import ( approvable_submissions_for_user, preapprovals_for_user,
     recently_approved_by_user, validate_submission, create_submission_event, docevent_from_submission,
     post_submission, cancel_submission, rename_submission_files, remove_submission_files, get_draft_meta,
-    get_submission, fill_in_submission, apply_checkers, send_confirmation_emails )
+    get_submission, fill_in_submission, apply_checkers, send_confirmation_emails, save_files )
 from ietf.stats.utils import clean_country_name
 from ietf.utils.accesstoken import generate_access_token
 from ietf.utils.log import log
-from ietf.utils.mail import send_mail_message
+from ietf.utils.mail import parseaddr, send_mail_message
 
 def upload_submission(request):
     if request.method == 'POST':
         try:
             form = SubmissionManualUploadForm(request, data=request.POST, files=request.FILES)
             if form.is_valid():
-                authors, abstract, file_name, file_size = get_draft_meta(form)
+                saved_files = save_files(form)
+                authors, abstract, file_name, file_size = get_draft_meta(form, saved_files)
 
                 submission = get_submission(form)
                 try:
@@ -87,7 +91,7 @@ def api_submit(request):
     if request.method == 'GET':
         return render(request, 'submit/api_submit_info.html')
     elif request.method == 'POST':
-        e = None
+        exception = None
         try:
             form = SubmissionAutoUploadForm(request, data=request.POST, files=request.FILES)
             if form.is_valid():
@@ -101,7 +105,8 @@ def api_submit(request):
                 if not hasattr(user, 'person'):
                     return err(400, "No person with username %s" % username)
 
-                authors, abstract, file_name, file_size = get_draft_meta(form)
+                saved_files = save_files(form)
+                authors, abstract, file_name, file_size = get_draft_meta(form, saved_files)
                 for a in authors:
                     if not a['email']:
                         raise ValidationError("Missing email address for author %s" % a)
@@ -132,7 +137,7 @@ def api_submit(request):
                 requires_prev_authors_approval = Document.objects.filter(name=submission.name)
 
                 sent_to, desc, docDesc = send_confirmation_emails(request, submission, requires_group_approval, requires_prev_authors_approval)
-                msg = u"Set submitter to \"%s\" and %s" % (submission.submitter, desc)
+                msg = "Set submitter to \"%s\" and %s" % (submission.submitter, desc)
                 create_submission_event(request, submission, msg)
                 docevent_from_submission(request, submission, docDesc, who="(System)")
 
@@ -142,14 +147,17 @@ def api_submit(request):
             else:
                 raise ValidationError(form.errors)
         except IOError as e:
+            exception = e
             return err(500, "IO Error: %s" % str(e))
         except ValidationError as e:
+            exception = e
             return err(400, "Validation Error: %s" % str(e))
         except Exception as e:
+            exception = e
             raise
             return err(500, "Exception: %s" % str(e))            
         finally:
-            if e and submission:
+            if exception and submission:
                 remove_submission_files(submission)
                 submission.delete()
     else:
@@ -213,8 +221,10 @@ def submission_status(request, submission_id, access_token=None):
 
     # Begin common code chunk
     addrs = gather_address_lists('sub_confirmation_requested',submission=submission)
-    confirmation_list = addrs.to
-    confirmation_list.extend(addrs.cc)
+    addresses = addrs.to
+    addresses.extend(addrs.cc)
+    # Convert from RFC 2822 format if needed
+    confirmation_list = [ "%s <%s>" % parseaddr(a) for a in addresses ]
 
     requires_group_approval = (submission.rev == '00'
         and submission.group and submission.group.features.req_subm_approval
@@ -229,7 +239,7 @@ def submission_status(request, submission_id, access_token=None):
     if submission.state_id == "cancel":
         message = ('error', 'This submission has been cancelled, modification is no longer possible.')
     elif submission.state_id == "auth":
-        message = ('success', u'The submission is pending email authentication. An email has been sent to: %s' % ", ".join(confirmation_list))
+        message = ('success', 'The submission is pending email authentication. An email has been sent to: %s' % ", ".join(confirmation_list))
     elif submission.state_id == "grp-appr":
         message = ('success', 'The submission is pending approval by the group chairs.')
     elif submission.state_id == "aut-appr":
@@ -262,12 +272,12 @@ def submission_status(request, submission_id, access_token=None):
                     # go directly to posting submission
                     docevent_from_submission(request, submission, desc="Uploaded new revision")
 
-                    desc = u"Secretariat manually posting. Approvals already received"
+                    desc = "Secretariat manually posting. Approvals already received"
                     post_submission(request, submission, desc)
                     create_submission_event(request, submission, desc)
                 else:
                     sent_to, desc, docDesc = send_confirmation_emails(request, submission, requires_group_approval, requires_prev_authors_approval)
-                    msg = u"Set submitter to \"%s\", replaces to %s and %s" % (
+                    msg = "Set submitter to \"%s\", replaces to %s and %s" % (
                         submission.submitter,
                         ", ".join(prettify_std_name(r.name) for r in replaces) if replaces else "(none)",
                         desc)
@@ -288,9 +298,9 @@ def submission_status(request, submission_id, access_token=None):
         elif action == "sendfullurl" and submission.state_id not in ("cancel", "posted"):
             sent_to = send_full_url(request, submission)
 
-            message = ('success', u'An email has been sent with the full access URL to: %s' % u",".join(confirmation_list))
+            message = ('success', 'An email has been sent with the full access URL to: %s' % ",".join(confirmation_list))
 
-            create_submission_event(request, submission, u"Sent full access URL to: %s" % u", ".join(sent_to))
+            create_submission_event(request, submission, "Sent full access URL to: %s" % ", ".join(sent_to))
 
         elif action == "cancel" and submission.state.next_states.filter(slug="cancel"):
             if not can_cancel:
@@ -418,7 +428,7 @@ def edit_submission(request, submission_id, access_token=None):
             ]
 
             if changed_fields:
-                desc = u"Edited %s and sent request for manual post" % u", ".join(changed_fields)
+                desc = "Edited %s and sent request for manual post" % ", ".join(changed_fields)
             else:
                 desc = "Sent request for manual post"
 
diff --git a/ietf/sync/iana.py b/ietf/sync/iana.py
index 23c88c1dd..972f2dda6 100644
--- a/ietf/sync/iana.py
+++ b/ietf/sync/iana.py
@@ -1,17 +1,28 @@
+# Copyright The IETF Trust 2012-2019, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+
+from __future__ import absolute_import, print_function, unicode_literals
+
 import base64
 import datetime
 import email
 import json
 import re
-import urllib2
 
-from django.utils.http import urlquote
+from six.moves.urllib.request import Request, urlopen
+
 from django.conf import settings
+from django.utils.encoding import force_str
+from django.utils.http import urlquote
+
+import debug                            # pyflakes:ignore
 
 from ietf.doc.mails import email_state_changed
 from ietf.doc.models import Document, DocEvent, State, StateDocEvent, StateType
 from ietf.doc.utils import add_state_change_event
 from ietf.person.models import Person
+from ietf.utils.mail import parseaddr
 from ietf.utils.timezone import local_timezone_to_utc, email_time_to_local_timezone, utc_to_local_timezone
 
 
@@ -19,7 +30,7 @@ from ietf.utils.timezone import local_timezone_to_utc, email_time_to_local_timez
 #CHANGES_URL = "https://datatracker.dev.icann.org:8080/data-tracker/changes"
 
 def fetch_protocol_page(url):
-    f = urllib2.urlopen(settings.IANA_SYNC_PROTOCOLS_URL)
+    f = urlopen(settings.IANA_SYNC_PROTOCOLS_URL)
     text = f.read()
     f.close()
     return text
@@ -63,12 +74,12 @@ def update_rfc_log_from_protocol_page(rfc_names, rfc_must_published_later_than):
 def fetch_changes_json(url, start, end):
     url += "?start=%s&end=%s" % (urlquote(local_timezone_to_utc(start).strftime("%Y-%m-%d %H:%M:%S")),
                                  urlquote(local_timezone_to_utc(end).strftime("%Y-%m-%d %H:%M:%S")))
-    request = urllib2.Request(url)
+    request = Request(url)
     # HTTP basic auth
     username = "ietfsync"
     password = settings.IANA_SYNC_PASSWORD
     request.add_header("Authorization", "Basic %s" % base64.encodestring("%s:%s" % (username, password)).replace("\n", ""))
-    f = urllib2.urlopen(request)
+    f = urlopen(request)
     text = f.read()
     f.close()
     return text
@@ -215,11 +226,11 @@ def update_history_with_changes(changes, send_email=True):
 
 def find_document_name(text):
     prefixes = ['draft','conflict-review','status-change','charter']
-    leading_delimiter_re = '(? 0:
@@ -503,7 +508,7 @@ def update_docs_from_rfc_index(data, skip_older_than_date=None):
                 rev=doc.rev,
                 by=system,
                 type="sync_from_rfc_editor",
-                desc=u"Received changes through RFC Editor sync (%s)" % u", ".join(changes),
+                desc="Received changes through RFC Editor sync (%s)" % ", ".join(changes),
             ))
 
             doc.save_with_history(events)
@@ -517,7 +522,7 @@ def post_approved_draft(url, name):
     the data from the Datatracker and start processing it. Returns
     response and error (empty string if no error)."""
 
-    request = urllib2.Request(url)
+    request = Request(url)
     request.add_header("Content-type", "application/x-www-form-urlencoded")
     request.add_header("Accept", "text/plain")
     # HTTP basic auth
@@ -531,7 +536,7 @@ def post_approved_draft(url, name):
     log("Posting RFC-Editor notifcation of approved draft '%s' to '%s'" % (name, url))
     text = error = ""
     try:
-        f = urllib2.urlopen(request, data=urllib.urlencode({ 'draft': name }), timeout=20)
+        f = urlopen(request, data=urlencode({ 'draft': name }), timeout=20)
         text = f.read()
         status_code = f.getcode()
         f.close()
@@ -547,6 +552,6 @@ def post_approved_draft(url, name):
         # catch everything so we don't leak exceptions, convert them
         # into string instead
         log("Exception on RFC-Editor notification for draft '%s': '%s'" % (name, e))
-        error = unicode(e)
+        error = six.text_type(e)
 
     return text, error
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index fd254ab0c..df2d33a53 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -1,15 +1,21 @@
 # Copyright The IETF Trust 2012-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
 
+
+from __future__ import absolute_import, print_function, unicode_literals
+
 import os
+import io
 import json
 import datetime
-import StringIO
+import quopri
 import shutil
 
 from django.conf import settings
 from django.urls import reverse as urlreverse
 
+import debug                            # pyflakes:ignore
+
 from ietf.doc.factories import WgDraftFactory
 from ietf.doc.models import Document, DocAlias, DocEvent, DeletedEvent, DocTagName, RelatedDocument, State, StateDocEvent
 from ietf.doc.utils import add_state_change_event
@@ -17,7 +23,7 @@ from ietf.group.factories import GroupFactory
 from ietf.person.models import Person
 from ietf.sync import iana, rfceditor
 from ietf.utils.mail import outbox, empty_outbox
-from ietf.utils.test_utils import login_testing_unauthorized, unicontent
+from ietf.utils.test_utils import login_testing_unauthorized
 from ietf.utils.test_utils import TestCase
 
 
@@ -136,9 +142,11 @@ class IANASyncTests(TestCase):
     def test_iana_review_mail(self):
         draft = WgDraftFactory()
 
-        subject_template = u'Subject: [IANA #12345] Last Call: <%(draft)s-%(rev)s.txt> (Long text) to Informational RFC'
-        msg_template = u"""From: "%(person)s via RT" 
+        subject_template = 'Subject: [IANA #12345] Last Call: <%(draft)s-%(rev)s.txt> (Long text) to Informational RFC'
+        msg_template = """From: %(fromaddr)s
 Date: Thu, 10 May 2012 12:00:0%(rtime)d +0000
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=utf-8
 %(subject)s
 
 (BEGIN IANA %(tag)s%(embedded_name)s)
@@ -172,20 +180,22 @@ ICANN
                     if embedded_name or not 'Vacuous' in subject: 
                     
                         rtime = 7*subjects.index(subject) + 5*tags.index(tag) + embedded_names.index(embedded_name)
-                        msg = msg_template % dict(person=Person.objects.get(user__username="iana").name,
+                        person=Person.objects.get(user__username="iana")
+                        fromaddr = person.email().formatted_email()
+                        msg = msg_template % dict(person=quopri.encodestring(person.name.encode('utf-8')),
+                                                  fromaddr=fromaddr,
                                                   draft=draft.name,
                                                   rev=draft.rev,
                                                   tag=tag,
                                                   rtime=rtime,
                                                   subject=subject,
                                                   embedded_name=embedded_name,)
-    
                         doc_name, review_time, by, comment = iana.parse_review_email(msg.encode('utf-8'))
     
                         self.assertEqual(doc_name, draft.name)
                         self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 5, 0, rtime))
                         self.assertEqual(by, Person.objects.get(user__username="iana"))
-                        self.assertTrue("there are no IANA Actions" in comment.replace("\n", ""))
+                        self.assertIn("there are no IANA Actions", comment.replace("\n", ""))
     
                         events_before = DocEvent.objects.filter(doc=draft, type="iana_review").count()
                         iana.add_review_comment(doc_name, review_time, by, comment)
@@ -205,7 +215,7 @@ ICANN
         login_testing_unauthorized(self, "secretary", url)
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-        self.assertTrue("new changes at" in unicontent(r))
+        self.assertContains(r, "new changes at")
 
         # we don't actually try posting as that would trigger a real run
         
@@ -226,7 +236,7 @@ class RFCSyncTests(TestCase):
         settings.INTERNET_DRAFT_ARCHIVE_DIR = self.save_archive_dir
 
     def write_draft_file(self, name, size):
-        with open(os.path.join(self.id_dir, name), 'w') as f:
+        with io.open(os.path.join(self.id_dir, name), 'w') as f:
             f.write("a" * size)
 
     def test_rfc_index(self):
@@ -306,7 +316,7 @@ class RFCSyncTests(TestCase):
                        area=doc.group.parent.acronym,
                        group=doc.group.acronym)
 
-        data = rfceditor.parse_index(StringIO.StringIO(t))
+        data = rfceditor.parse_index(io.StringIO(t))
         self.assertEqual(len(data), 1)
 
         rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract = data[0]
@@ -387,7 +397,7 @@ class RFCSyncTests(TestCase):
                               group=draft.group.name,
                               ref="draft-ietf-test")
 
-        drafts, warnings = rfceditor.parse_queue(StringIO.StringIO(t))
+        drafts, warnings = rfceditor.parse_queue(io.StringIO(t))
         self.assertEqual(len(drafts), 1)
         self.assertEqual(len(warnings), 0)
 
@@ -430,7 +440,7 @@ class DiscrepanciesTests(TestCase):
         doc.set_state(State.objects.get(used=True, type="draft-iesg", slug="ann"))
 
         r = self.client.get(urlreverse("ietf.sync.views.discrepancies"))
-        self.assertTrue(doc.name in unicontent(r))
+        self.assertContains(r, doc.name)
 
         # draft with IANA state "In Progress" but RFC Editor state not IANA
         doc = Document.objects.create(name="draft-ietf-test2", type_id="draft")
@@ -439,7 +449,7 @@ class DiscrepanciesTests(TestCase):
         doc.set_state(State.objects.get(used=True, type="draft-rfceditor", slug="auth"))
 
         r = self.client.get(urlreverse("ietf.sync.views.discrepancies"))
-        self.assertTrue(doc.name in unicontent(r))
+        self.assertContains(r, doc.name)
 
         # draft with IANA state "Waiting on RFC Editor" or "RFC-Ed-Ack"
         # but RFC Editor state is IANA
@@ -449,7 +459,7 @@ class DiscrepanciesTests(TestCase):
         doc.set_state(State.objects.get(used=True, type="draft-rfceditor", slug="iana"))
 
         r = self.client.get(urlreverse("ietf.sync.views.discrepancies"))
-        self.assertTrue(doc.name in unicontent(r))
+        self.assertContains(r, doc.name)
 
         # draft with state other than "RFC Ed Queue" or "RFC Published"
         # that are in RFC Editor or IANA queues
@@ -458,7 +468,7 @@ class DiscrepanciesTests(TestCase):
         doc.set_state(State.objects.get(used=True, type="draft-rfceditor", slug="auth"))
 
         r = self.client.get(urlreverse("ietf.sync.views.discrepancies"))
-        self.assertTrue(doc.name in unicontent(r))
+        self.assertContains(r, doc.name)
 
 class RFCEditorUndoTests(TestCase):
     def test_rfceditor_undo(self):
@@ -480,7 +490,7 @@ class RFCEditorUndoTests(TestCase):
         # get
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
-        self.assertTrue(e2.doc.name in unicontent(r))
+        self.assertContains(r, e2.doc.name)
 
         # delete e2
         deleted_before = DeletedEvent.objects.count()
diff --git a/ietf/templates/doc/document_conflict_review.html b/ietf/templates/doc/document_conflict_review.html
index aa804d208..8e379890f 100644
--- a/ietf/templates/doc/document_conflict_review.html
+++ b/ietf/templates/doc/document_conflict_review.html
@@ -135,7 +135,7 @@
   {% if not snapshot and user|has_role:"Area Director,Secretariat" %}
     {% if request.user|has_role:"Secretariat" %}
       {% if doc.get_state_slug == 'appr-reqnopub-pend' or doc.get_state_slug == 'appr-noprob-pend' %}
-        Approve conflict review
+        Approve conflict review
       {% endif %}
     {% endif %}
   {% endif %}
diff --git a/ietf/templates/submit/confirm_submission.txt b/ietf/templates/submit/confirm_submission.txt
index 29174d52c..87e0f50f6 100644
--- a/ietf/templates/submit/confirm_submission.txt
+++ b/ietf/templates/submit/confirm_submission.txt
@@ -10,7 +10,7 @@ The chairs have been copied since this is a group document whose author list has
 {%endif%}
 Please follow this link to the page where you can confirm the posting:
 
-{{ confirm_url }}
+{{ confirmation_url }}
 
 
 Best regards,
diff --git a/ietf/utils/accesstoken.py b/ietf/utils/accesstoken.py
index da07c9a3a..a6ba65935 100644
--- a/ietf/utils/accesstoken.py
+++ b/ietf/utils/accesstoken.py
@@ -1,10 +1,18 @@
+# Copyright The IETF Trust 2013-2019, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+
+from __future__ import absolute_import, print_function, unicode_literals
+
 import time, random, hashlib
 
 from django.conf import settings
+from django.utils.encoding import force_bytes, force_text
+
 
 def generate_random_key(max_length=32):
     """Generate a random access token."""
-    return hashlib.sha256(settings.SECRET_KEY + ("%.16f" % time.time()) + ("%.16f" % random.random())).hexdigest()[:max_length]
+    return hashlib.sha256(force_bytes(settings.SECRET_KEY) + (b"%.16f" % time.time()) + (b"%.16f" % random.random())).hexdigest()[:max_length]
 
 def generate_access_token(key, max_length=32):
     """Make an access token out of key."""
@@ -12,4 +20,4 @@ def generate_access_token(key, max_length=32):
     # we hash it with the private key to make sure only we can
     # generate and use the final token - so storing the key in the
     # database is safe
-    return hashlib.sha256(settings.SECRET_KEY + key).hexdigest()[:max_length]
+    return force_text(hashlib.sha256(force_bytes(settings.SECRET_KEY) + force_bytes(key)).hexdigest()[:max_length])
diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py
index 94c0a9c96..ec141b78c 100644
--- a/ietf/utils/admin.py
+++ b/ietf/utils/admin.py
@@ -1,4 +1,14 @@
+# Copyright The IETF Trust 2011-2019, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import six
+
 from django.contrib import admin
+from django.utils.encoding import force_text
+
 from ietf.utils.models import VersionInfo
 
 def name(obj):
@@ -8,10 +18,10 @@ def name(obj):
         if callable(obj.name):
             name = obj.name()
         else:
-            name = unicode(obj.name)
+            name = force_text(obj.name)
         if name:
             return name
-    return unicode(obj)
+    return six.text_type(obj)
     
 def admin_link(field, label=None, ordering="", display=name, suffix=""):
     if not label:
@@ -39,15 +49,15 @@ def admin_link(field, label=None, ordering="", display=name, suffix=""):
             app = obj._meta.app_label
             model = obj.__class__.__name__.lower()
             id = obj.pk
-            chunks += [ u'%(display)s' %
+            chunks += [ '%(display)s' %
                 {'app':app, "model": model, "id":id, "display": display(obj), "suffix":suffix, } ]
-        return u", ".join(chunks)
+        return ", ".join(chunks)
     _link.allow_tags = True
     _link.short_description = label
     _link.admin_order_field = ordering
     return _link
 
-from models import DumpInfo
+from .models import DumpInfo
 class DumpInfoAdmin(admin.ModelAdmin):
     list_display = ['date', 'host', 'tz']
     list_filter = ['date']
diff --git a/ietf/utils/aliases.py b/ietf/utils/aliases.py
index bc9a71a12..6c31e9ec2 100644
--- a/ietf/utils/aliases.py
+++ b/ietf/utils/aliases.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# Copyright The IETF Trust 2013-2019, All Rights Reserved
 # -*- coding: utf-8 -*-
 # -*- Python -*-
 #
@@ -6,12 +7,17 @@
 #
 # Author: Markus Stenberg 
 #
+
+
+from __future__ import absolute_import, print_function, unicode_literals
+
 """
 
 Mailing list alias dumping utilities
 
 """
 
+
 from django.conf import settings
 
 import debug                            # pyflakes:ignore
@@ -38,7 +44,7 @@ def rewrite_address_list(l):
     h = {}
     for address in l:
         #address = address.strip()
-        if h.has_key(address): continue
+        if address in h: continue
         h[address] = True
         yield address
 
@@ -46,12 +52,12 @@ def dump_sublist(afile, vfile, alias, adomains, vdomain, emails):
     if not emails:
         return emails
     # Nones in the list should be skipped
-    emails = filter(None, emails)
+    emails = [_f for _f in emails if _f]
 
     # Make sure emails are sane and eliminate the Nones again for
     # non-sane ones
     emails = [rewrite_email_address(e) for e in emails]
-    emails = filter(None, emails)
+    emails = [_f for _f in emails if _f]
 
     # And we'll eliminate the duplicates too but preserve order
     emails = list(rewrite_address_list(emails))
@@ -71,7 +77,7 @@ def dump_sublist(afile, vfile, alias, adomains, vdomain, emails):
         # If there's unicode in email address, something is badly
         # wrong and we just silently punt
         # XXX - is there better approach?
-        print '# Error encoding', alias, repr(emails)
+        print('# Error encoding', alias, repr(emails))
         return []
     return emails
 
diff --git a/ietf/utils/bootstrap.py b/ietf/utils/bootstrap.py
index 3fb57a9e8..b4e169f5a 100644
--- a/ietf/utils/bootstrap.py
+++ b/ietf/utils/bootstrap.py
@@ -1,9 +1,15 @@
+# Copyright The IETF Trust 2016-2019, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+
+from __future__ import absolute_import, print_function, unicode_literals
+
 import bootstrap3.renderers
 
 class SeparateErrorsFromHelpTextFieldRenderer(bootstrap3.renderers.FieldRenderer):
     def append_to_field(self, html):
         if self.field_help:
-            html += u'
{}
'.format(self.field_help) + html += '
{}
'.format(self.field_help) for e in self.field_errors: - html += u'
{}
'.format(e) + html += '
{}
'.format(e) return html diff --git a/ietf/utils/decorators.py b/ietf/utils/decorators.py index 6cce65a6e..ff91a2860 100644 --- a/ietf/utils/decorators.py +++ b/ietf/utils/decorators.py @@ -1,4 +1,8 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import datetime @@ -8,6 +12,7 @@ from django.conf import settings from django.contrib.auth import login from django.http import HttpResponse from django.shortcuts import render +from django.utils.encoding import force_bytes import debug # pyflakes:ignore @@ -49,7 +54,7 @@ def require_api_key(f, request, *args, **kwargs): if not hash: return err(400, "Missing apikey parameter") # Check hash - key = PersonalApiKey.validate_key(hash) + key = PersonalApiKey.validate_key(force_bytes(hash)) if not key: return err(400, "Invalid apikey") # Check endpoint @@ -79,7 +84,7 @@ def require_api_key(f, request, *args, **kwargs): def _memoize(func, self, *args, **kwargs): ''''Memoize wrapper for instance methouds. Use @lru_cache for functions.''' if kwargs: # frozenset is used to ensure hashability - key = args, frozenset(kwargs.items()) + key = args, frozenset(list(kwargs.items())) else: key = args # instance method, set up cache if needed diff --git a/ietf/utils/draft.py b/ietf/utils/draft.py index fa12a0458..adf1468ae 100755 --- a/ietf/utils/draft.py +++ b/ietf/utils/draft.py @@ -1,11 +1,17 @@ #!/usr/bin/python +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- # -*- python -*- + + +from __future__ import absolute_import, print_function, unicode_literals + """ NAME - %(program)s - Extract meta-information from an IETF draft. + %(program)s - Extract meta-information from an IETF draft. SYNOPSIS - %(program)s [OPTIONS] DRAFTLIST_FILE + %(program)s [OPTIONS] DRAFTLIST_FILE DESCRIPTION Extract information about authors' names and email addresses, @@ -16,30 +22,31 @@ DESCRIPTION %(options)s AUTHOR - Written by Henrik Levkowetz, + Written by Henrik Levkowetz, COPYRIGHT - Copyright 2008 Henrik Levkowetz + Copyright 2008 Henrik Levkowetz - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or (at - your option) any later version. There is NO WARRANTY; not even the - implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - PURPOSE. See the GNU General Public License for more details. + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or (at + your option) any later version. There is NO WARRANTY; not even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + PURPOSE. See the GNU General Public License for more details. """ -from __future__ import unicode_literals -from __future__ import print_function + + import datetime import getopt +import io import os import os.path import re -import stat import six +import stat import sys import time @@ -106,7 +113,7 @@ def _err(string): # ---------------------------------------------------------------------- def _gettext(file): - file = open(file) + file = io.open(file) text = file.read() file.close() @@ -190,7 +197,7 @@ class Draft(): name, __ = base.split(".", 1) else: name = base - revmatch = re.search("\d\d$", name) + revmatch = re.search(r"\d\d$", name) if revmatch: filename = name[:-3] revision = name[-2:] @@ -242,36 +249,36 @@ class Draft(): for line in self.rawlines: linecount += 1 line = line.rstrip() - if re.search("\[?page [0-9ivx]+\]?[ \t\f]*$", line, re.I): + if re.search(r"\[?page [0-9ivx]+\]?[ \t\f]*$", line, re.I): pages, page, newpage = endpage(pages, page, newpage, line) continue - if re.search("\f", line, re.I): + if re.search(r"\f", line, re.I): pages, page, newpage = begpage(pages, page, newpage) continue - if re.search("^ *Internet.Draft.+ .+[12][0-9][0-9][0-9] *$", line, re.I): + if re.search(r"^ *Internet.Draft.+ .+[12][0-9][0-9][0-9] *$", line, re.I): pages, page, newpage = begpage(pages, page, newpage, line) continue # if re.search("^ *Internet.Draft +", line, re.I): # newpage = True # continue - if re.search("^ *Draft.+[12][0-9][0-9][0-9] *$", line, re.I): + if re.search(r"^ *Draft.+[12][0-9][0-9][0-9] *$", line, re.I): pages, page, newpage = begpage(pages, page, newpage, line) continue - if re.search("^RFC[ -]?[0-9]+.*( +)[12][0-9][0-9][0-9]$", line, re.I): + if re.search(r"^RFC[ -]?[0-9]+.*( +)[12][0-9][0-9][0-9]$", line, re.I): pages, page, newpage = begpage(pages, page, newpage, line) continue - if re.search("^draft-[-a-z0-9_.]+.*[0-9][0-9][0-9][0-9]$", line, re.I): + if re.search(r"^draft-[-a-z0-9_.]+.*[0-9][0-9][0-9][0-9]$", line, re.I): pages, page, newpage = endpage(pages, page, newpage, line) continue - if linecount > 15 and re.search(".{58,}(Jan|Feb|Mar|March|Apr|April|May|Jun|June|Jul|July|Aug|Sep|Oct|Nov|Dec) (19[89][0-9]|20[0-9][0-9]) *$", line, re.I): + if linecount > 15 and re.search(r".{58,}(Jan|Feb|Mar|March|Apr|April|May|Jun|June|Jul|July|Aug|Sep|Oct|Nov|Dec) (19[89][0-9]|20[0-9][0-9]) *$", line, re.I): pages, page, newpage = begpage(pages, page, newpage, line) continue - if newpage and re.search("^ *draft-[-a-z0-9_.]+ *$", line, re.I): + if newpage and re.search(r"^ *draft-[-a-z0-9_.]+ *$", line, re.I): pages, page, newpage = begpage(pages, page, newpage, line) continue - if re.search("^[^ \t]+", line): + if re.search(r"^[^ \t]+", line): sentence = True - if re.search("[^ \t]", line): + if re.search(r"[^ \t]", line): if newpage: # 36 is a somewhat arbitrary count for a 'short' line shortthis = len(line.strip()) < 36 # 36 is a somewhat arbitrary count for a 'short' line @@ -299,7 +306,7 @@ class Draft(): # ---------------------------------------------------------------------- def get_pagecount(self): if self._pagecount == None: - label_pages = len(re.findall("\[page [0-9ixldv]+\]", self.text, re.I)) + label_pages = len(re.findall(r"\[page [0-9ixldv]+\]", self.text, re.I)) count_pages = len(self.pages) if label_pages > count_pages/2: self._pagecount = label_pages @@ -342,7 +349,7 @@ class Draft(): def get_status(self): if self._status == None: for line in self.lines[:10]: - status_match = re.search("^\s*Intended [Ss]tatus:\s*(.*?) ", line) + status_match = re.search(r"^\s*Intended [Ss]tatus:\s*(.*?) ", line) if status_match: self._status = status_match.group(1) break @@ -415,8 +422,8 @@ class Draft(): def get_abstract(self): if self._abstract: return self._abstract - abstract_re = re.compile('^(\s*)abstract', re.I) - header_re = re.compile("^(\s*)([0-9]+\.? |Appendix|Status of|Table of|Full Copyright|Copyright|Intellectual Property|Acknowled|Author|Index|Disclaimer).*", re.I) + abstract_re = re.compile(r'^(\s*)abstract', re.I) + header_re = re.compile(r"^(\s*)([0-9]+\.? |Appendix|Status of|Table of|Full Copyright|Copyright|Intellectual Property|Acknowled|Author|Index|Disclaimer).*", re.I) begin = False abstract = [] abstract_indent = 0 @@ -445,7 +452,7 @@ class Draft(): def _check_abstract_indent(self, abstract, indent): - indentation_re = re.compile('^(\s)*') + indentation_re = re.compile(r'^(\s)*') indent_lines = [] for line in abstract.split('\n'): if line: @@ -806,7 +813,7 @@ class Draft(): _debug( "Cut: '%s'" % form[beg:end]) author_match = re.search(authpat, columns[col].strip()).group(1) _debug( "AuthMatch: '%s'" % (author_match,)) - if re.search('\(.*\)$', author_match.strip()): + if re.search(r'\(.*\)$', author_match.strip()): author_match = author_match.rsplit('(',1)[0].strip() if author_match in companies_seen: companies[i] = authors[i] @@ -886,7 +893,7 @@ class Draft(): # for a in authors: # if a and a not in companies_seen: # _debug("Search for: %s"%(r"(^|\W)"+re.sub("\.? ", ".* ", a)+"(\W|$)")) - authmatch = [ a for a in authors[i+1:] if a and not a.lower() in companies_seen and (re.search((r"(?i)(^|\W)"+re.sub("[. ]+", ".*", a)+"(\W|$)"), line.strip()) or acronym_match(a, line.strip()) )] + authmatch = [ a for a in authors[i+1:] if a and not a.lower() in companies_seen and (re.search((r"(?i)(^|\W)"+re.sub(r"[. ]+", ".*", a)+r"(\W|$)"), line.strip()) or acronym_match(a, line.strip()) )] if authmatch: _debug(" ? Other author or company ? : %s" % authmatch) @@ -914,9 +921,9 @@ class Draft(): column = l.replace('\t', 8 * ' ')[max(0, beg - 1):end].strip() except: column = l - column = re.sub(" *(?:\(at\)| | at ) *", "@", column) - column = re.sub(" *(?:\(dot\)| | dot ) *", ".", column) - column = re.sub("&cisco.com", "@cisco.com", column) + column = re.sub(r" *(?:\(at\)| | at ) *", "@", column) + column = re.sub(r" *(?:\(dot\)| | dot ) *", ".", column) + column = re.sub(r"&cisco.com", "@cisco.com", column) column = column.replace("\xa0", " ") return column @@ -1002,13 +1009,13 @@ class Draft(): def get_title(self): if self._title: return self._title - match = re.search('(?:\n\s*\n\s*)((.+\n){0,2}(.+\n*))(\s+').replace('<', '<' ).replace('&', '&') -@keep_lazy(six.text_type) +@keep_lazy(str) def remove_tags(html, tags): """Returns the given HTML sanitized, and with the given tags removed.""" allowed = set(acceptable_tags) - set([ t.lower() for t in tags ]) @@ -54,7 +59,7 @@ class Cleaner(lxml.html.clean.Cleaner): # Copied from lxml 4.2.0 and modified to insert charset meta: def clean_html(self, html): result_type = type(html) - if isinstance(html, basestring): + if isinstance(html, six.string_types): doc = lxml.html.fromstring(html) else: doc = copy.deepcopy(html) diff --git a/ietf/utils/log.py b/ietf/utils/log.py index d59874bd0..f250aaccb 100644 --- a/ietf/utils/log.py +++ b/ietf/utils/log.py @@ -1,9 +1,14 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import sys import logging import inspect import os.path +import six import traceback try: @@ -28,23 +33,27 @@ def getclass(frame): def getcaller(): parent, pfile, pline, pfunction, lines, index = inspect.stack()[2] - moduleinfo = inspect.getmoduleinfo(pfile) - pmodule = moduleinfo[0] if moduleinfo else None + pmodule = inspect.getmodulename(pfile) pclass = getclass(parent) return (pmodule, pclass, pfunction, pfile, pline) -def log(msg): +def log(msg, e=None): "Uses syslog by preference. Logs the given calling point and message." global logfunc def _flushfunc(): pass _logfunc = logfunc if settings.SERVER_MODE == 'test': - return +## Comment in when debugging for instance test smtp server failures: +# if e: +# _logfunc = debug.say +# _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) +# else: + return elif settings.DEBUG == True: _logfunc = debug.say _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) - if isinstance(msg, unicode): + if isinstance(msg, six.text_type): msg = msg.encode('unicode_escape') try: mod, cls, func, file, line = getcaller() diff --git a/ietf/utils/mail.py b/ietf/utils/mail.py index 7ebe32e88..4289a8a1f 100644 --- a/ietf/utils/mail.py +++ b/ietf/utils/mail.py @@ -1,7 +1,14 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import copy import datetime +import logging +import re +import six import smtplib import sys import textwrap @@ -23,6 +30,7 @@ from django.core.exceptions import ImproperlyConfigured, ValidationError from django.core.validators import validate_email from django.template.loader import render_to_string from django.template import Context,RequestContext +from django.utils.encoding import force_text, force_str, force_bytes import debug # pyflakes:ignore @@ -49,12 +57,12 @@ def empty_outbox(): outbox[:] = [] def add_headers(msg): - if not(msg.has_key('Message-ID')): - msg['Message-ID'] = make_msgid('idtracker') - if not(msg.has_key('Date')): - msg['Date'] = formatdate(time.time(), True) - if not(msg.has_key('From')): - msg['From'] = settings.DEFAULT_FROM_EMAIL + if not('Message-ID' in msg): + msg['Message-ID'] = make_msgid('idtracker') + if not('Date' in msg): + msg['Date'] = formatdate(time.time(), True) + if not('From' in msg): + msg['From'] = settings.DEFAULT_FROM_EMAIL return msg class SMTPSomeRefusedRecipients(smtplib.SMTPException): @@ -90,7 +98,7 @@ def send_smtp(msg, bcc=None): addrlist += [bcc] to = [addr for name, addr in getaddresses(addrlist) if ( addr != '' and not addr.startswith('unknown-email-') )] if not to: - log(u"No addressees for email from '%s', subject '%s'. Nothing sent." % (frm, msg.get('Subject', '[no subject]'))) + log("No addressees for email from '%s', subject '%s'. Nothing sent." % (frm, msg.get('Subject', '[no subject]'))) else: if test_mode: outbox.append(msg) @@ -113,12 +121,12 @@ def send_smtp(msg, bcc=None): # advertise the AUTH capability. server.ehlo() server.login(settings.EMAIL_HOST_USER, settings.EMAIL_HOST_PASSWORD) - unhandled = server.sendmail(frm, to, msg.as_string()) + unhandled = server.sendmail(frm, to, force_bytes(msg.as_string())) if unhandled != {}: raise SMTPSomeRefusedRecipients(message="%d addresses were refused"%len(unhandled),original_msg=msg,refusals=unhandled) except Exception as e: # need to improve log message - log(u"Exception while trying to send email from '%s' to %s subject '%s'" % (frm, to, msg.get('Subject', '[no subject]'))) + log("Exception while trying to send email from '%s' to %s subject '%s'" % (frm, to, msg.get('Subject', '[no subject]')), e=e) if isinstance(e, smtplib.SMTPException): e.original_msg=msg raise @@ -129,8 +137,8 @@ def send_smtp(msg, bcc=None): server.quit() except smtplib.SMTPServerDisconnected: pass - subj = msg.get('Subject', u'[no subject]') - log(u"sent email from '%s' to %s id %s subject '%s'" % (frm, to, msg.get('Message-ID', u''), subj)) + subj = force_text(msg.get('Subject', '[no subject]')) + log("sent email from '%s' to %s id %s subject '%s'" % (frm, to, msg.get('Message-ID', ''), subj)) def copy_email(msg, to, toUser=False, originalBcc=None): ''' @@ -145,11 +153,11 @@ def copy_email(msg, to, toUser=False, originalBcc=None): # django settings if debugging? # Should this be a template? if settings.SERVER_MODE == 'production': - explanation = "This is a copy of a message sent from the I-D tracker." + explanation = "This is a copy of a message sent from the I-D tracker." elif settings.SERVER_MODE == 'test' and toUser: - explanation = "The attached message was generated by an instance of the tracker\nin test mode. It is being sent to you because you, or someone acting\non your behalf, is testing the system. If you do not recognize\nthis action, please accept our apologies and do not be concerned as\nthe action is being taken in a test context." + explanation = "The attached message was generated by an instance of the tracker\nin test mode. It is being sent to you because you, or someone acting\non your behalf, is testing the system. If you do not recognize\nthis action, please accept our apologies and do not be concerned as\nthe action is being taken in a test context." else: - explanation = "The attached message would have been sent, but the tracker is in %s mode.\nIt was not sent to anybody." % settings.SERVER_MODE + explanation = "The attached message would have been sent, but the tracker is in %s mode.\nIt was not sent to anybody." % settings.SERVER_MODE if originalBcc: explanation += ("\nIn addition to the destinations derived from the header below, the message would have been sent Bcc to %s" % originalBcc) new.attach(MIMEText(explanation + "\n\n")) @@ -166,7 +174,7 @@ def mail_context(request): return RequestContext(request) else: return Context() - + def send_mail(request, to, frm, subject, template, context, *args, **kwargs): ''' Send an email to the destination [list], with the given return @@ -178,7 +186,7 @@ def send_mail(request, to, frm, subject, template, context, *args, **kwargs): return send_mail_text(request, to, frm, subject, txt, *args, **kwargs) def encode_message(txt): - assert isinstance(txt, unicode) + assert isinstance(txt, six.text_type) return MIMEText(txt.encode('utf-8'), 'plain', 'UTF-8') def send_mail_text(request, to, frm, subject, txt, cc=None, extra=None, toUser=False, bcc=None, copy=True): @@ -217,10 +225,13 @@ def formataddr(addrtuple): address field. Does what's needed, and returns a string value suitable for use in a To: or Cc: email header field. """ - name, addr = addrtuple - if name and not isascii(name): - name = str(Header(name, 'utf-8')) - return simple_formataddr((name, addr)) + if six.PY2: + name, addr = addrtuple + if name and not isascii(name): + name = str(Header(name, 'utf-8')) + return simple_formataddr((name, addr)) + else: + return simple_formataddr(addrtuple) def parseaddr(addr): """ @@ -230,7 +241,8 @@ def parseaddr(addr): which case a 2-tuple of ('', '') is returned. """ - addr = u''.join( [ s.decode(m) if m else unicode(s) for (s,m) in decode_header(addr) ] ) + + addr = ''.join( [ ( s.decode(m) if m else s.decode()) if isinstance(s, six.binary_type) else s for (s,m) in decode_header(addr) ] ) name, addr = simple_parseaddr(addr) return name, addr @@ -256,7 +268,7 @@ def condition_message(to, frm, subject, msg, cc, extra): else: extra['Reply-To'] = [frm, ] frm = on_behalf_of(frm) - msg['From'] = frm + msg['From'] = frm # The following is a hack to avoid an issue with how the email module (as of version 4.0.3) # breaks lines when encoding header fields with anything other than the us-ascii codec. @@ -269,6 +281,7 @@ def condition_message(to, frm, subject, msg, cc, extra): if name: to_hdr.append('"%s"' % name) to_hdr.append("<%s>," % addr) + # Please note: The following .encode() does _not_ take a charset argument to_str = to_hdr.encode() if to_str and to_str[-1] == ',': to_str=to_str[:-1] @@ -277,14 +290,14 @@ def condition_message(to, frm, subject, msg, cc, extra): msg['To'] = to_str if cc: - msg['Cc'] = cc + msg['Cc'] = cc msg['Subject'] = subject msg['X-Test-IDTracker'] = (settings.SERVER_MODE == 'production') and 'no' or 'yes' msg['X-IETF-IDTracker'] = ietf.__version__ msg['Auto-Submitted'] = "auto-generated" msg['Precedence'] = "bulk" if extra: - for k, v in extra.items(): + for k, v in extra.items(): if v: assertion('len(list(set(v))) == len(v)') try: @@ -309,7 +322,7 @@ def send_mail_mime(request, to, frm, subject, msg, cc=None, extra=None, toUser=F """Send MIME message with content already filled in.""" condition_message(to, frm, subject, msg, cc, extra) - + # start debug server with python -m smtpd -n -c DebuggingServer localhost:2025 # then put USING_DEBUG_EMAIL_SERVER=True and EMAIL_HOST='localhost' # and EMAIL_PORT=2025 in settings_local.py @@ -330,12 +343,12 @@ def send_mail_mime(request, to, frm, subject, msg, cc=None, extra=None, toUser=F show_that_mail_was_sent(request,'Email was sent',msg,bcc) elif settings.SERVER_MODE == 'test': - if toUser: - copy_email(msg, to, toUser=True, originalBcc=bcc) - elif request and request.COOKIES.has_key( 'testmailcc' ): - copy_email(msg, request.COOKIES[ 'testmailcc' ],originalBcc=bcc) + if toUser: + copy_email(msg, to, toUser=True, originalBcc=bcc) + elif request and 'testmailcc' in request.COOKIES: + copy_email(msg, request.COOKIES[ 'testmailcc' ],originalBcc=bcc) try: - copy_to = settings.EMAIL_COPY_TO + copy_to = settings.EMAIL_COPY_TO except AttributeError: copy_to = "ietf.tracker.archive+%s@gmail.com" % settings.SERVER_MODE if copy_to and (copy or not production) and not (test_mode or debugging): # if we're running automated tests, this copy is just annoying @@ -352,10 +365,10 @@ def send_mail_mime(request, to, frm, subject, msg, cc=None, extra=None, toUser=F def parse_preformatted(preformatted, extra={}, override={}): """Parse preformatted string containing mail with From:, To:, ...,""" - msg = message_from_string(preformatted.encode("utf-8")) + msg = message_from_string(force_str(preformatted)) msg.set_charset('UTF-8') - for k, v in override.iteritems(): + for k, v in override.items(): if k in msg: del msg[k] if v: @@ -368,7 +381,7 @@ def parse_preformatted(preformatted, extra={}, override={}): headers = copy.copy(msg) # don't modify the message for key in ['To', 'From', 'Subject', 'Bcc']: del headers[key] - for k in headers.keys(): + for k in list(headers.keys()): v = headers.get_all(k, []) if k in extra: ev = extra[k] @@ -398,7 +411,7 @@ def parse_preformatted(preformatted, extra={}, override={}): bcc = msg['Bcc'] del msg['Bcc'] - for v in extra.values(): + for v in list(extra.values()): assertion('len(list(set(v))) == len(v)') return (msg, extra, bcc) @@ -408,7 +421,7 @@ def send_mail_preformatted(request, preformatted, extra={}, override={}): extra headers as needed).""" (msg, extra, bcc) = parse_preformatted(preformatted, extra, override) - txt = msg.get_payload().decode(str(msg.get_charset())) + txt = get_payload(msg) send_mail_text(request, msg['To'], msg["From"], msg["Subject"], txt, extra=extra, bcc=bcc) return msg @@ -427,7 +440,7 @@ def send_mail_message(request, message, extra={}): def exception_components(e): # See if it's a non-smtplib exception that we faked - if len(e.args)==1 and isinstance(e.args[0],dict) and e.args[0].has_key('really'): + if len(e.args)==1 and isinstance(e.args[0],dict) and 'really' in e.args[0]: orig = e.args[0] extype = orig['really'] tb = orig['tb'] @@ -440,10 +453,10 @@ def exception_components(e): def log_smtp_exception(e): (extype, value, tb) = exception_components(e) - log("SMTP Exception: %s : %s" % (extype,value)) + log("SMTP Exception: %s : %s" % (extype,value), e) if isinstance(e,SMTPSomeRefusedRecipients): - log(" SomeRefused: %s"%(e.summary_refusals())) - log(" Traceback: %s" % tb) + log(" SomeRefused: %s"%(e.summary_refusals()), e) + log(" Traceback: %s" % tb, e) return (extype, value, tb) def build_warning_message(request, e): @@ -522,6 +535,7 @@ def is_valid_email(address): except ValidationError: return False +logger = logging.getLogger('django') def get_email_addresses_from_text(text): """ @@ -533,5 +547,23 @@ def get_email_addresses_from_text(text): Returns a list of properly formatted email address strings. """ - return [ formataddr(e) for e in getaddresses([text, ]) ] + def valid(email): + name, addr = email + try: + validate_email(addr) + return True + except ValidationError: + logger.error('Bad data: get_email_addresses_from_text() got an ' + 'invalid email address tuple: {email}, in "{text}".'.format(email=email, text=text)) + return False + # whitespace normalization -- getaddresses doesn't do this + text = re.sub(r'(?u)\s+', ' ', text) + return [ formataddr(e) for e in getaddresses([text, ]) if valid(e) ] + +def get_payload(msg, decode=False): + if six.PY2: + return msg.get_payload(decode=decode).decode(msg.get_content_charset('utf-8')) + else: + return msg.get_payload(decode=decode) + \ No newline at end of file diff --git a/ietf/utils/management/commands/check_referential_integrity.py b/ietf/utils/management/commands/check_referential_integrity.py index 66176a386..8d409afac 100644 --- a/ietf/utils/management/commands/check_referential_integrity.py +++ b/ietf/utils/management/commands/check_referential_integrity.py @@ -1,3 +1,10 @@ +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import six from tqdm import tqdm @@ -25,18 +32,18 @@ class Command(BaseCommand): debug.pprint('dir(field)') raise if verbosity > 1: - print " %s -> %s.%s" % (field.name,foreign_model.__module__,foreign_model.__name__), + six.print_(" %s -> %s.%s" % (field.name,foreign_model.__module__,foreign_model.__name__), end=' ') used = set(field.model.objects.values_list(field.name,flat=True)) used.discard(None) exists = set(foreign_model.objects.values_list('pk',flat=True)) if verbosity > 1: if used - exists: - print " ** Bad key values:",list(used - exists) + six.print_(" ** Bad key values:",list(used - exists)) else: - print " ok" + six.print_(" ok") else: if used - exists: - print "\n%s.%s.%s -> %s.%s ** Bad key values:" % (model.__module__,model.__name__,field.name,foreign_model.__module__,foreign_model.__name__),list(used - exists) + six.print_("\n%s.%s.%s -> %s.%s ** Bad key values:" % (model.__module__,model.__name__,field.name,foreign_model.__module__,foreign_model.__name__),list(used - exists)) def check_reverse_field(field): try: @@ -49,33 +56,33 @@ class Command(BaseCommand): foreign_field_name = field.remote_field.name foreign_accessor_name = field.remote_field.get_accessor_name() if verbosity > 1: - print " %s <- %s -> %s.%s" % (field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), + six.print_(" %s <- %s -> %s.%s" % (field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), end=' ') try: used = set(foreign_model.objects.values_list(foreign_field_name, flat=True)) except FieldError: try: used = set(foreign_model.objects.values_list(foreign_accessor_name, flat=True)) except FieldError: - print " ** Warning: could not find reverse name for %s.%s -> %s.%s" % (field.model.__module__, field.model.__name__, foreign_model.__name__, foreign_field_name), + six.print_(" ** Warning: could not find reverse name for %s.%s -> %s.%s" % (field.model.__module__, field.model.__name__, foreign_model.__name__, foreign_field_name), end=' ') used.discard(None) exists = set(field.model.objects.values_list('pk',flat=True)) if verbosity > 1: if used - exists: - print " ** Bad key values:\n ",list(used - exists) + six.print_(" ** Bad key values:\n ",list(used - exists)) else: - print " ok" + six.print_(" ok") else: if used - exists: - print "\n%s.%s <- %s -> %s.%s ** Bad key values:\n " % (field.model.__module__, field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), list(used - exists) + six.print_("\n%s.%s <- %s -> %s.%s ** Bad key values:\n " % (field.model.__module__, field.model.__name__, field.remote_field.through._meta.db_table, foreign_model.__module__, foreign_model.__name__), list(used - exists)) for conf in tqdm([ c for c in apps.get_app_configs() if c.name.startswith('ietf.')], desc='apps', disable=verbose): if verbosity > 1: - print "Checking", conf.name + six.print_("Checking", conf.name) for model in tqdm(list(conf.get_models()), desc='models', disable=verbose): if model._meta.proxy: continue if verbosity > 1: - print " %s.%s" % (model.__module__,model.__name__) + six.print_(" %s.%s" % (model.__module__,model.__name__)) for field in [f for f in model._meta.fields if isinstance(f, (ForeignKey, OneToOneField)) ]: check_field(field) for field in [f for f in model._meta.many_to_many ]: diff --git a/ietf/utils/management/commands/coverage_changes.py b/ietf/utils/management/commands/coverage_changes.py index c119f9f46..9058fc790 100644 --- a/ietf/utils/management/commands/coverage_changes.py +++ b/ietf/utils/management/commands/coverage_changes.py @@ -1,15 +1,19 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals -import os -import json -import codecs import gzip +import io +import json +import os +import six from difflib import ndiff from django.conf import settings from django.core.management.base import BaseCommand, CommandError -from django.utils.six import string_types import debug # pyflakes:ignore @@ -57,14 +61,14 @@ class Command(BaseCommand): valid_sections = ['template', 'url', 'code'] def read_coverage(self, filename, version=None): - if isinstance(filename, string_types): + if isinstance(filename, six.string_types): try: if filename.endswith(".gz"): file = gzip.open(filename, "rb") else: - file = codecs.open(filename, "r", encoding="utf-8") + file = io.open(filename, "r", encoding="utf-8") except IOError as e: - self.stderr.write(u"%s" % e) + self.stderr.write("%s" % e) exit(1) else: file = filename @@ -87,8 +91,8 @@ class Command(BaseCommand): lcoverage = latest_coverage[section]["covered"] lformat = latest_coverage[section].get("format", 1) # - mkeys = mcoverage.keys() - lkeys = lcoverage.keys() + mkeys = list(mcoverage.keys()) + lkeys = list(lcoverage.keys()) # keys = list(lkeys) keys.sort() @@ -181,7 +185,7 @@ class Command(BaseCommand): lcoverage = latest_coverage[section]["covered"] lformat = latest_coverage[section].get("format", 1) # - lkeys = lcoverage.keys() + lkeys = list(lcoverage.keys()) # keys = list(lkeys) keys.sort() diff --git a/ietf/utils/management/commands/create_group_wikis.py b/ietf/utils/management/commands/create_group_wikis.py index c7d239a0a..cf6d947d7 100644 --- a/ietf/utils/management/commands/create_group_wikis.py +++ b/ietf/utils/management/commands/create_group_wikis.py @@ -1,9 +1,15 @@ -# Copyright 2016 IETF Trust +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import os import copy -import syslog +import io import pkg_resources +import six +import syslog from trac.core import TracError from trac.env import Environment @@ -24,7 +30,7 @@ from ietf.utils.pipe import pipe logtag = __name__.split('.')[-1] logname = "user.log" -syslog.openlog(logtag, syslog.LOG_PID, syslog.LOG_USER) +syslog.openlog(str(logtag), syslog.LOG_PID, syslog.LOG_USER) class Command(BaseCommand): help = "Create group wikis for WGs, RGs and Areas which don't have one." @@ -59,7 +65,10 @@ class Command(BaseCommand): else: self.note("Running %s %s ..." % (os.path.basename(cmd), " ".join(quoted_args))) command = [ cmd, ] + list(args) + command = ' '.join(command).encode('utf-8') code, out, err = pipe(command) + out = out.decode('utf-8') + err = err.decode('utf-8') msg = None if code != 0: msg = "Error %s: %s when executing '%s'" % (code, err, " ".join(command)) @@ -81,9 +90,8 @@ class Command(BaseCommand): return msg err, out= self.svn_admin_cmd("create", svn ) if err: - msg = "Error %s creating svn repository %s:\n %s" % (err, svn, out) - self.log(msg) - return msg + self.log(err) + return err return "" # --- trac --- @@ -130,8 +138,8 @@ class Command(BaseCommand): name = unicode_unquote(name.encode('utf-8')) if os.path.isfile(filename): self.note(" Adding page %s" % name) - with open(filename) as file: - text = file.read().decode('utf-8') + with io.open(filename, encoding='utf-8') as file: + text = file.read() self.add_wiki_page(env, name, text) def add_custom_wiki_pages(self, group, env): @@ -201,7 +209,7 @@ class Command(BaseCommand): sect, key, val = options[i] val = val.format(**group.__dict__) options[i] = sect, key, val - # Try to creat ethe environment, remove unwanted defaults, and add + # Try to create the environment, remove unwanted defaults, and add # custom pages and settings. if self.dummy_run: self.note("Would create Trac for group '%s' at %s" % (group.acronym, group.trac_dir)) @@ -320,13 +328,13 @@ class Command(BaseCommand): self.svn_dir_pattern = options.get('svn_dir_pattern', settings.TRAC_SVN_DIR_PATTERN) self.group_list = options.get('group_list', None) self.dummy_run = options.get('dummy_run', False) - self.wiki_dir_pattern = os.path.join(settings.BASE_DIR, '..', self.wiki_dir_pattern) + self.wiki_dir_pattern = os.path.join(str(settings.BASE_DIR), str('..'), self.wiki_dir_pattern) self.svn_dir_pattern = os.path.join(settings.BASE_DIR, '..', self.svn_dir_pattern) if not self.group_list is None: self.group_list = self.group_list.split('.') - if isinstance(self.verbosity, (type(""), type(u""))) and self.verbosity.isdigit(): + if isinstance(self.verbosity, six.string_types) and self.verbosity.isdigit(): self.verbosity = int(self.verbosity) if self.dummy_run and self.verbosity < 2: diff --git a/ietf/utils/management/commands/download_extras.py b/ietf/utils/management/commands/download_extras.py new file mode 100644 index 000000000..74bfc1e1b --- /dev/null +++ b/ietf/utils/management/commands/download_extras.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import subprocess + +from textwrap import dedent + +from django.core.management.base import BaseCommand +from django.conf import settings + +import debug # pyflakes:ignore + +class Command(BaseCommand): + """ + Download extra files (photos, floorplans, ...) + """ + + help = dedent(__doc__).strip() + + def handle(self, *filenames, **options): + for src, dst in ( + ('rsync.ietf.org::dev.media/', settings.MEDIA_ROOT), ): + if src and dst: + if not dst.endswith(os.pathsep): + dst += os.pathsep + subprocess.call(('rsync', '-auz', '--info=progress2', src, dst)) + \ No newline at end of file diff --git a/ietf/utils/management/commands/dumprelated.py b/ietf/utils/management/commands/dumprelated.py index 48583f0aa..60a7ea70b 100644 --- a/ietf/utils/management/commands/dumprelated.py +++ b/ietf/utils/management/commands/dumprelated.py @@ -1,3 +1,10 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import warnings from collections import OrderedDict @@ -152,7 +159,7 @@ class Command(BaseCommand): Collate the objects to be serialized. If count_only is True, just count the number of objects to be serialized. """ - models = serializers.sort_dependencies(app_list.items()) + models = serializers.sort_dependencies(list(app_list.items())) for model in models: if model in excluded_models: continue @@ -186,7 +193,7 @@ class Command(BaseCommand): if (output and self.stdout.isatty() and options['verbosity'] > 0): progress_output = self.stdout object_count = sum(get_objects(count_only=True)) - stream = open(output, 'w') if output else None + stream = io.open(output, 'w') if output else None try: serializers.serialize( format, get_objects(), indent=indent, diff --git a/ietf/utils/management/commands/import_htpasswd.py b/ietf/utils/management/commands/import_htpasswd.py index 6093dfb08..2a8a2e5b7 100644 --- a/ietf/utils/management/commands/import_htpasswd.py +++ b/ietf/utils/management/commands/import_htpasswd.py @@ -1,3 +1,5 @@ +# Copyright The IETF Trust 2014-2019, All Rights Reserved +import io import sys from textwrap import dedent @@ -6,7 +8,7 @@ from django.contrib.auth.models import User from django.core.management.base import BaseCommand def import_htpasswd_file(filename, verbosity=1, overwrite=False): - with open(filename) as file: + with io.open(filename) as file: for line in file: if not ':' in line: raise ValueError('Found a line without colon separator in the htpassword file %s:'+ diff --git a/ietf/utils/management/commands/loadrelated.py b/ietf/utils/management/commands/loadrelated.py index 31d24dfb2..78da0e33e 100644 --- a/ietf/utils/management/commands/loadrelated.py +++ b/ietf/utils/management/commands/loadrelated.py @@ -1,6 +1,8 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals import gzip import os @@ -26,7 +28,7 @@ import debug # pyflakes:ignore from ietf.community.models import notify_events class Command(loaddata.Command): - help = (u""" + help = (""" Load a fixture of related objects to the database. The fixture is expected to contain a set of related objects, created with the 'dumprelated' management diff --git a/ietf/utils/management/commands/makefixture.py b/ietf/utils/management/commands/makefixture.py index 907ec3185..fbffb061f 100644 --- a/ietf/utils/management/commands/makefixture.py +++ b/ietf/utils/management/commands/makefixture.py @@ -1,4 +1,10 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- # From https://github.com/ericholscher/django-test-utils/blob/master/test_utils/management/commands/makefixture.py + + +from __future__ import absolute_import, print_function, unicode_literals + """ "Make fixture" command. @@ -38,6 +44,9 @@ python manage.py makefixture --format=xml --indent=4 YourModel[3] AnotherModel a #known issues: #no support for generic relations #no support for one-to-one relations + +import six + from django.core import serializers from django.core.management.base import CommandError from django.core.management.base import LabelCommand @@ -101,7 +110,7 @@ class Command(LabelCommand): objects = [] for model, slice in models: - if isinstance(slice, basestring) and slice: + if isinstance(slice, six.string_types) and slice: objects.extend(model._default_manager.filter(pk__exact=slice)) elif not slice or type(slice) is list: items = model._default_manager.all() @@ -132,7 +141,7 @@ class Command(LabelCommand): related = [] for obj in objects: if DEBUG: - print "Adding %s[%s]" % (model_name(obj), obj.pk) + print("Adding %s[%s]" % (model_name(obj), obj.pk)) # follow forward relation fields for f in obj.__class__._meta.fields + obj.__class__._meta.many_to_many: if isinstance(f, ForeignKey): @@ -150,7 +159,7 @@ class Command(LabelCommand): try: return serializers.serialize(format, all, indent=indent) - except Exception, e: + except Exception as e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e) diff --git a/ietf/utils/management/commands/populate_yang_model_dirs.py b/ietf/utils/management/commands/populate_yang_model_dirs.py index 28631c74f..5ea74fa31 100644 --- a/ietf/utils/management/commands/populate_yang_model_dirs.py +++ b/ietf/utils/management/commands/populate_yang_model_dirs.py @@ -1,12 +1,15 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved -from __future__ import print_function, unicode_literals +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import os import sys import time from pathlib2 import Path -from StringIO import StringIO +from io import StringIO from textwrap import dedent from xym import xym diff --git a/ietf/utils/management/commands/pyflakes.py b/ietf/utils/management/commands/pyflakes.py index 376df9986..bf9c4b17e 100644 --- a/ietf/utils/management/commands/pyflakes.py +++ b/ietf/utils/management/commands/pyflakes.py @@ -1,4 +1,9 @@ +# Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + from __future__ import absolute_import + import ast import os from pyflakes import checker, messages @@ -7,6 +12,8 @@ import sys from django.conf import settings from django.core.management.base import BaseCommand +import debug # pyflakes:ignore + # BlackHole, PySyntaxError and checking based on # https://github.com/patrys/gedit-pyflakes-plugin.git class BlackHole(object): @@ -50,7 +57,7 @@ def check(codeString, filename, verbosity=1): try: with BlackHole(): tree = ast.parse(codeString, filename) - except SyntaxError, e: + except SyntaxError as e: return [PySyntaxError(filename, e.lineno, e.offset, e.text)] else: # Okay, it's syntactically valid. Now parse it into an ast and check @@ -63,12 +70,13 @@ def check(codeString, filename, verbosity=1): if lines[message.lineno-1].find('pyflakes:ignore') < 0] # honour pyflakes: - messages.sort(lambda a, b: cmp(a.lineno, b.lineno)) + messages.sort(key=lambda x: x.lineno) if verbosity > 0: if len(messages): sys.stderr.write('F') else: sys.stderr.write('.') + sys.stderr.flush() if verbosity > 1: sys.stderr.write(" %s\n" % filename) return messages @@ -81,8 +89,8 @@ def checkPath(filename, verbosity): @return: the number of warnings printed """ try: - return check(file(filename, 'U').read() + '\n', filename, verbosity) - except IOError, msg: + return check(open(filename).read() + '\n', filename, verbosity) + except IOError as msg: return ["%s: %s" % (filename, msg.args[1])] except TypeError: pass @@ -97,7 +105,7 @@ def checkPaths(filenames, verbosity): try: warnings.extend(checkPath(os.path.join(dirpath, filename), verbosity)) except TypeError as e: - print("Exception while processing dirpath=%s, filename=%s: %s" % (dirpath, filename,e )) + print("Exception while processing dirpath=%s, filename=%s: %s" % (dirpath, filename, e )) raise else: warnings.extend(checkPath(arg, verbosity)) @@ -114,11 +122,11 @@ class Command(BaseCommand): filenames = getattr(settings, 'PYFLAKES_DEFAULT_ARGS', ['.']) verbosity = int(options.get('verbosity')) warnings = checkPaths(filenames, verbosity=verbosity) - print "" + print("") for warning in warnings: - print warning + print(warning) if warnings: - print 'Total warnings: %d' % len(warnings) + print('Total warnings: %d' % len(warnings)) raise SystemExit(1) diff --git a/ietf/utils/management/commands/run_yang_model_checks.py b/ietf/utils/management/commands/run_yang_model_checks.py index 5d3135237..3fed40a44 100644 --- a/ietf/utils/management/commands/run_yang_model_checks.py +++ b/ietf/utils/management/commands/run_yang_model_checks.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2017-2019, All Rights Reserved -from __future__ import print_function, unicode_literals +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import json diff --git a/ietf/utils/management/commands/send_apikey_usage_emails.py b/ietf/utils/management/commands/send_apikey_usage_emails.py index 2718ef02a..f8f923021 100644 --- a/ietf/utils/management/commands/send_apikey_usage_emails.py +++ b/ietf/utils/management/commands/send_apikey_usage_emails.py @@ -1,6 +1,8 @@ +# Copyright The IETF Trust 2017-2019, All Rights Reserved # -*- coding: utf-8 -*- -# Copyright The IETF Trust 2017, All Rights Reserved -from __future__ import print_function, unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals import datetime diff --git a/ietf/utils/management/commands/send_gdpr_consent_request.py b/ietf/utils/management/commands/send_gdpr_consent_request.py index 80b6e174e..9a50399dc 100644 --- a/ietf/utils/management/commands/send_gdpr_consent_request.py +++ b/ietf/utils/management/commands/send_gdpr_consent_request.py @@ -1,6 +1,8 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals import datetime import time @@ -14,7 +16,7 @@ from ietf.person.models import Person, PersonEvent from ietf.utils.mail import send_mail class Command(BaseCommand): - help = (u""" + help = (""" Send GDPR consent request emails to persons who have not indicated consent to having their personal information stored. Each send is logged as a PersonEvent. @@ -97,7 +99,7 @@ class Command(BaseCommand): 'person': person, 'settings': settings, }, ) - e = PersonEvent.objects.create(person=person, type='gdpr_notice_email', + PersonEvent.objects.create(person=person, type='gdpr_notice_email', desc="Sent GDPR notice email to %s with confirmation deadline %s" % (to, date)) time.sleep(delay) diff --git a/ietf/utils/management/commands/tests.py b/ietf/utils/management/commands/tests.py index 0ce1127e1..807141703 100644 --- a/ietf/utils/management/commands/tests.py +++ b/ietf/utils/management/commands/tests.py @@ -1,9 +1,17 @@ +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os +import six import tempfile from django.core.management import call_command from django.test import TestCase -from django.utils.six import StringIO +#from io import StringIO import debug # pyflakes:ignore @@ -77,12 +85,12 @@ class CoverageChangeTestCase(TestCase): } """ mfh, master = tempfile.mkstemp(suffix='.json') - with open(master, "w") as file: + with io.open(master, "w") as file: file.write(master_txt) lfh, latest = tempfile.mkstemp(suffix='.json') - with open(latest, "w") as file: + with io.open(latest, "w") as file: file.write(latest_txt) - output = StringIO() + output = six.StringIO() call_command('coverage_changes', master, latest, stdout=output) text = output.getvalue() os.unlink(master) diff --git a/ietf/utils/management/commands/update_community_list_index.py b/ietf/utils/management/commands/update_community_list_index.py index cecf68e8a..299038388 100644 --- a/ietf/utils/management/commands/update_community_list_index.py +++ b/ietf/utils/management/commands/update_community_list_index.py @@ -1,6 +1,8 @@ # Copyright The IETF Trust 2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + + +from __future__ import absolute_import, print_function, unicode_literals from django.core.management.base import BaseCommand @@ -11,7 +13,7 @@ from ietf.community.models import SearchRule from ietf.community.utils import reset_name_contains_index_for_rule class Command(BaseCommand): - help = (u""" + help = (""" Update the index tables for stored regex-based document search rules. """) diff --git a/ietf/utils/management/commands/update_external_command_info.py b/ietf/utils/management/commands/update_external_command_info.py index 04949951c..7e63d7c65 100644 --- a/ietf/utils/management/commands/update_external_command_info.py +++ b/ietf/utils/management/commands/update_external_command_info.py @@ -1,4 +1,8 @@ -from __future__ import print_function, unicode_literals +# Copyright The IETF Trust 2017-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import sys diff --git a/ietf/utils/markup_txt.py b/ietf/utils/markup_txt.py index fd71a517f..6e680fd81 100644 --- a/ietf/utils/markup_txt.py +++ b/ietf/utils/markup_txt.py @@ -1,3 +1,6 @@ +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- +# # Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -30,31 +33,17 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import re -import six -import string +import six # pyflakes:ignore from django.utils.html import escape from ietf.utils import log from ietf.utils.text import wordwrap -def markup_ascii(content, width=None): - log.unreachable('2017-12-08') - if six.PY2: - assert isinstance(content, basestring) - # at this point, "content" is normal string - # fix most common non-ASCII characters - t1 = string.maketrans("\x91\x92\x93\x94\x95\x96\x97\xc6\xe8\xe9", "\'\'\"\"o--\'ee") - # map everything except printable ASCII, TAB, LF, FF to "?" - t2 = string.maketrans('','') - t3 = "?"*9 + "\t\n?\f" + "?"*19 + t2[32:127] + "?"*129 - t4 = t1.translate(t3) - content = content.translate(t4) - else: - log.assertion('six.PY2') - return markup(content.decode('ascii'), width) - def markup(content, width=None): log.assertion('isinstance(content, six.text_type)') # normalize line endings to LF only @@ -73,11 +62,11 @@ def markup(content, width=None): # expand tabs + escape content = escape(content.expandtabs()) - content = re.sub("\n(.+\[Page \d+\])\n\f\n(.+)\n", """\n\g<1>\n\g<2>\n""", content) - content = re.sub("\n(.+\[Page \d+\])\n\s*$", """\n\g<1>\n""", content) + content = re.sub(r"\n(.+\[Page \d+\])\n\f\n(.+)\n", r"""\n\g<1>\n\g<2>\n""", content) + content = re.sub(r"\n(.+\[Page \d+\])\n\s*$", r"""\n\g<1>\n""", content) # remove remaining FFs (to be valid XHTML) content = content.replace("\f","\n") - content = re.sub("\n\n([0-9]+\\.|[A-Z]\\.[0-9]|Appendix|Status of|Abstract|Table of|Full Copyright|Copyright|Intellectual Property|Acknowled|Author|Index)(.*)(?=\n\n)", """\n\n\g<1>\g<2>""", content) + content = re.sub(r"\n\n([0-9]+\\.|[A-Z]\\.[0-9]|Appendix|Status of|Abstract|Table of|Full Copyright|Copyright|Intellectual Property|Acknowled|Author|Index)(.*)(?=\n\n)", r"""\n\n\g<1>\g<2>""", content) return "
" + content + "
\n" diff --git a/ietf/utils/migrations/0001_initial.py b/ietf/utils/migrations/0001_initial.py index 0d4130201..d9f9002e4 100644 --- a/ietf/utils/migrations/0001_initial.py +++ b/ietf/utils/migrations/0001_initial.py @@ -1,6 +1,9 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Generated by Django 1.11.10 on 2018-02-20 10:52 -from __future__ import unicode_literals + + +from __future__ import absolute_import, print_function, unicode_literals from django.db import migrations, models diff --git a/ietf/utils/ordereddict.py b/ietf/utils/ordereddict.py index 94cc04e6c..bdc911efd 100644 --- a/ietf/utils/ordereddict.py +++ b/ietf/utils/ordereddict.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + def insert_after_in_ordered_dict(dictionary, key, value, after): """There's no "insert" in ordered dict so simulate it instead by re-adding entries. Obviously that's not ideal, but for small dicts the @@ -5,7 +11,7 @@ def insert_after_in_ordered_dict(dictionary, key, value, after): dictionary[key] = value reorder = False - l = dictionary.items() # don't mutate the dict while looping + l = list(dictionary.items()) # don't mutate the dict while looping for k, v in l: if reorder and k != key: del dictionary[k] diff --git a/ietf/utils/pdf.py b/ietf/utils/pdf.py index c863d4db1..c4d39601c 100644 --- a/ietf/utils/pdf.py +++ b/ietf/utils/pdf.py @@ -1,13 +1,20 @@ +# Copyright The IETF Trust 2015-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import io import re def pdf_pages(filename): """Return number of pages in PDF.""" try: - infile = open(filename, "r") + infile = io.open(filename, "r") except IOError: return 0 for line in infile: - m = re.match('\] /Count ([0-9]+)',line) + m = re.match(r'\] /Count ([0-9]+)',line) if m: return int(m.group(1)) return 0 diff --git a/ietf/utils/pipe.py b/ietf/utils/pipe.py index 845bc901e..e61fce373 100644 --- a/ietf/utils/pipe.py +++ b/ietf/utils/pipe.py @@ -1,27 +1,33 @@ +# Copyright The IETF Trust 2010-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + # Simplified interface to os.popen3() def pipe(cmd, str=None): - from popen2 import Popen3 as Popen + from subprocess import Popen, PIPE bufsize = 4096 MAX = 65536*16 if str and len(str) > 4096: # XXX: Hardcoded Linux 2.4, 2.6 pipe buffer size bufsize = len(str) - pipe = Popen(cmd, True, bufsize) + pipe = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=bufsize, shell=True) if not str is None: - pipe.tochild.write(str) - pipe.tochild.close() + pipe.stdin.write(str) + pipe.stdin.close() - out = "" - err = "" + out = b"" + err = b"" while True: - str = pipe.fromchild.read() + str = pipe.stdout.read() if str: out += str code = pipe.poll() - if code > -1: - err = pipe.childerr.read() + if code != None: + err = pipe.stderr.read() break if len(out) >= MAX: err = "Output exceeds %s bytes and has been truncated" % MAX diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py index c1f115d05..6d61c5e2e 100644 --- a/ietf/utils/resources.py +++ b/ietf/utils/resources.py @@ -1,5 +1,8 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2014-11-13 05:39 + + from ietf.api import ModelResource from tastypie.fields import CharField from tastypie.constants import ALL diff --git a/ietf/utils/templatetags/htmlfilters.py b/ietf/utils/templatetags/htmlfilters.py index 4bae6da2a..80ad13e78 100644 --- a/ietf/utils/templatetags/htmlfilters.py +++ b/ietf/utils/templatetags/htmlfilters.py @@ -1,6 +1,9 @@ -# Copyright the IETF Trust 2017, All Rights Reserved +# Copyright The IETF Trust 2017-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals -from __future__ import unicode_literals from django.template.library import Library from django.template.defaultfilters import stringfilter diff --git a/ietf/utils/templatetags/textfilters.py b/ietf/utils/templatetags/textfilters.py index 6c8102d5e..ad4d6cb64 100644 --- a/ietf/utils/templatetags/textfilters.py +++ b/ietf/utils/templatetags/textfilters.py @@ -1,4 +1,8 @@ -from __future__ import unicode_literals +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import bleach diff --git a/ietf/utils/test_data.py b/ietf/utils/test_data.py index 038b4f82d..3fb3b4386 100644 --- a/ietf/utils/test_data.py +++ b/ietf/utils/test_data.py @@ -1,7 +1,9 @@ -# Copyright The IETF Trust 2017-2019, All Rights Reserved +# Copyright The IETF Trust 2011-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals + +from __future__ import absolute_import, print_function, unicode_literals + import datetime @@ -98,7 +100,7 @@ def make_immutable_base_data(): # one area area = create_group(name="Far Future", acronym="farfut", type_id="area", parent=ietf) - create_person(area, "ad", name=u"Areað Irector", username="ad", email_address="aread@example.org") + create_person(area, "ad", name="Areað Irector", username="ad", email_address="aread@example.org") # second area opsarea = create_group(name="Operations", acronym="ops", type_id="area", parent=ietf) @@ -338,7 +340,7 @@ def make_test_data(): # meeting Meeting.objects.create( - number="42", + number="72", type_id="ietf", date=datetime.date.today() + datetime.timedelta(days=180), city="New York", @@ -421,7 +423,7 @@ def make_review_data(doc): u = User.objects.create(username="reviewer") u.set_password("reviewer+password") u.save() - reviewer = Person.objects.create(name=u"Some Réviewer", ascii="Some Reviewer", user=u) + reviewer = Person.objects.create(name="Some Réviewer", ascii="Some Reviewer", user=u) email = Email.objects.create(address="reviewer@example.com", person=reviewer, origin=u.username) for team in (team1, team2, team3): @@ -444,14 +446,14 @@ def make_review_data(doc): u = User.objects.create(username="reviewsecretary") u.set_password("reviewsecretary+password") u.save() - reviewsecretary = Person.objects.create(name=u"Réview Secretary", ascii="Review Secretary", user=u) + reviewsecretary = Person.objects.create(name="Réview Secretary", ascii="Review Secretary", user=u) reviewsecretary_email = Email.objects.create(address="reviewsecretary@example.com", person=reviewsecretary, origin=u.username) Role.objects.create(name_id="secr", person=reviewsecretary, email=reviewsecretary_email, group=team1) u = User.objects.create(username="reviewsecretary3") u.set_password("reviewsecretary3+password") u.save() - reviewsecretary3 = Person.objects.create(name=u"Réview Secretary3", ascii="Review Secretary3", user=u) + reviewsecretary3 = Person.objects.create(name="Réview Secretary3", ascii="Review Secretary3", user=u) reviewsecretary3_email = Email.objects.create(address="reviewsecretary3@example.com", person=reviewsecretary, origin=u.username) Role.objects.create(name_id="secr", person=reviewsecretary3, email=reviewsecretary3_email, group=team3) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 67eacab25..d146ad75a 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -1,7 +1,6 @@ -# Copyright The IETF Trust 2007-2019, All Rights Reserved +# Copyright The IETF Trust 2009-2019, All Rights Reserved # -*- coding: utf-8 -*- - - +# # Portion Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -35,6 +34,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import, print_function, unicode_literals + +import io import re import os import sys @@ -44,7 +46,6 @@ import pytz import importlib import socket import datetime -import codecs import gzip import unittest import factory.random @@ -93,14 +94,14 @@ def safe_create_test_db(self, verbosity, *args, **kwargs): global test_database_name, old_create keepdb = kwargs.get('keepdb', False) if not keepdb: - print " Creating test database..." + print(" Creating test database...") if settings.DATABASES["default"]["ENGINE"] == 'django.db.backends.mysql': settings.DATABASES["default"]["OPTIONS"] = settings.DATABASE_TEST_OPTIONS - print " Using OPTIONS: %s" % settings.DATABASES["default"]["OPTIONS"] + print(" Using OPTIONS: %s" % settings.DATABASES["default"]["OPTIONS"]) test_database_name = old_create(self, 0, *args, **kwargs) if settings.GLOBAL_TEST_FIXTURES: - print " Loading global test fixtures: %s" % ", ".join(settings.GLOBAL_TEST_FIXTURES) + print(" Loading global test fixtures: %s" % ", ".join(settings.GLOBAL_TEST_FIXTURES)) loadable = [f for f in settings.GLOBAL_TEST_FIXTURES if "." not in f] call_command('loaddata', *loadable, verbosity=int(verbosity)-1, commit=False, database="default") @@ -120,7 +121,7 @@ def safe_destroy_test_db(*args, **kwargs): keepdb = kwargs.get('keepdb', False) if not keepdb: if settings.DATABASES["default"]["NAME"] != test_database_name: - print ' NOT SAFE; Changing settings.DATABASES["default"]["NAME"] from %s to %s' % (settings.DATABASES["default"]["NAME"], test_database_name) + print(' NOT SAFE; Changing settings.DATABASES["default"]["NAME"] from %s to %s' % (settings.DATABASES["default"]["NAME"], test_database_name)) settings.DATABASES["default"]["NAME"] = test_database_name return old_destroy(*args, **kwargs) @@ -228,7 +229,7 @@ def save_test_results(failures, test_labels): # results and avoid re-running tests if we've alread run them with OK # result after the latest code changes: topdir = os.path.dirname(os.path.dirname(settings.BASE_DIR)) - tfile = codecs.open(os.path.join(topdir,".testresult"), "a", encoding='utf-8') + tfile = io.open(os.path.join(topdir,".testresult"), "a", encoding='utf-8') timestr = time.strftime("%Y-%m-%d %H:%M:%S") if failures: tfile.write("%s FAILED (failures=%s)\n" % (timestr, failures)) @@ -239,7 +240,6 @@ def save_test_results(failures, test_labels): tfile.write("%s OK\n" % (timestr, )) tfile.close() - def set_coverage_checking(flag=True): global template_coverage_collection global code_coverage_collection @@ -267,7 +267,7 @@ class CoverageReporter(Reporter): analysis = self.coverage._analyze(fr) nums = analysis.numbers missing_nums = sorted(analysis.missing) - with open(analysis.filename) as file: + with io.open(analysis.filename) as file: lines = file.read().splitlines() missing_lines = [ lines[l-1] for l in missing_nums ] result["covered"][fr.relative_filename()] = (nums.n_statements, nums.pc_covered/100.0, missing_nums, missing_lines) @@ -296,11 +296,11 @@ class CoverageTest(unittest.TestCase): latest_coverage_version = self.runner.coverage_master["version"] master_data = self.runner.coverage_master[latest_coverage_version][test] - master_missing = [ k for k,v in master_data["covered"].items() if not v ] + master_missing = [ k for k,v in list(master_data["covered"].items()) if not v ] master_coverage = master_data["coverage"] test_data = self.runner.coverage_data[test] - test_missing = [ k for k,v in test_data["covered"].items() if not v ] + test_missing = [ k for k,v in list(test_data["covered"].items()) if not v ] test_coverage = test_data["coverage"] # Assert coverage failure only if we're running the full test suite -- if we're @@ -527,7 +527,7 @@ class IetfTestRunner(DiscoverRunner): with gzip.open(self.coverage_file, "rb") as file: self.coverage_master = json.load(file) else: - with codecs.open(self.coverage_file, encoding='utf-8') as file: + with io.open(self.coverage_file, encoding='utf-8') as file: self.coverage_master = json.load(file) self.coverage_data = { "time": datetime.datetime.now(pytz.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), @@ -564,7 +564,7 @@ class IetfTestRunner(DiscoverRunner): self.code_coverage_checker.start() if settings.SITE_ID != 1: - print " Changing SITE_ID to '1' during testing." + print(" Changing SITE_ID to '1' during testing.") settings.SITE_ID = 1 if settings.TEMPLATES[0]['OPTIONS']['string_if_invalid'] != '': @@ -572,7 +572,7 @@ class IetfTestRunner(DiscoverRunner): settings.TEMPLATES[0]['OPTIONS']['string_if_invalid'] = '' if settings.INTERNAL_IPS: - print " Changing INTERNAL_IPS to '[]' during testing." + print(" Changing INTERNAL_IPS to '[]' during testing.") settings.INTERNAL_IPS = [] assert not settings.IDTRACKER_BASE_URL.endswith('/') @@ -586,7 +586,7 @@ class IetfTestRunner(DiscoverRunner): ietf.utils.mail.SMTP_ADDR['port'] = base + offset self.smtpd_driver = SMTPTestServerDriver((ietf.utils.mail.SMTP_ADDR['ip4'],ietf.utils.mail.SMTP_ADDR['port']),None) self.smtpd_driver.start() - print(" Running an SMTP test server on %(ip4)s:%(port)s to catch outgoing email." % ietf.utils.mail.SMTP_ADDR) + print((" Running an SMTP test server on %(ip4)s:%(port)s to catch outgoing email." % ietf.utils.mail.SMTP_ADDR)) break except socket.error: pass @@ -594,9 +594,9 @@ class IetfTestRunner(DiscoverRunner): maybe_create_svn_symlinks(settings) if os.path.exists(settings.UTILS_TEST_RANDOM_STATE_FILE): - print " Loading factory-boy random state from %s" % settings.UTILS_TEST_RANDOM_STATE_FILE + print(" Loading factory-boy random state from %s" % settings.UTILS_TEST_RANDOM_STATE_FILE) else: - print " Saving factory-boy random state to %s" % settings.UTILS_TEST_RANDOM_STATE_FILE + print(" Saving factory-boy random state to %s" % settings.UTILS_TEST_RANDOM_STATE_FILE) with open(settings.UTILS_TEST_RANDOM_STATE_FILE, 'w') as f: s = factory.random.get_random_state() json.dump(s, f) @@ -614,16 +614,16 @@ class IetfTestRunner(DiscoverRunner): coverage_latest = {} coverage_latest["version"] = "latest" coverage_latest["latest"] = self.coverage_data - with codecs.open(latest_coverage_file, "w", encoding='utf-8') as file: + with open(latest_coverage_file, "w") as file: json.dump(coverage_latest, file, indent=2, sort_keys=True) if self.save_version_coverage: self.coverage_master["version"] = self.save_version_coverage self.coverage_master[self.save_version_coverage] = self.coverage_data if self.coverage_file.endswith('.gz'): - with gzip.open(self.coverage_file, "wb") as file: + with gzip.open(self.coverage_file, "wt", encoding='ascii') as file: json.dump(self.coverage_master, file, sort_keys=True) else: - with codecs.open(self.coverage_file, "w", encoding="utf-8") as file: + with open(self.coverage_file, "w") as file: json.dump(self.coverage_master, file, indent=2, sort_keys=True) super(IetfTestRunner, self).teardown_test_environment(**kwargs) @@ -709,7 +709,7 @@ class IetfTestRunner(DiscoverRunner): if self.run_full_test_suite: print("Test coverage data:") else: - print("Test coverage for this test run across the related app%s (%s):" % (("s" if len(self.test_apps)>1 else ""), ", ".join(self.test_apps))) + print(("Test coverage for this test run across the related app%s (%s):" % (("s" if len(self.test_apps)>1 else ""), ", ".join(self.test_apps)))) for test in ["template", "url", "code"]: latest_coverage_version = self.coverage_master["version"] @@ -724,19 +724,19 @@ class IetfTestRunner(DiscoverRunner): test_coverage = test_data["coverage"] if self.run_full_test_suite: - print(" %8s coverage: %6.2f%% (%s: %6.2f%%)" % - (test.capitalize(), test_coverage*100, latest_coverage_version, master_coverage*100, )) + print((" %8s coverage: %6.2f%% (%s: %6.2f%%)" % + (test.capitalize(), test_coverage*100, latest_coverage_version, master_coverage*100, ))) else: - print(" %8s coverage: %6.2f%%" % - (test.capitalize(), test_coverage*100, )) + print((" %8s coverage: %6.2f%%" % + (test.capitalize(), test_coverage*100, ))) - print(""" + print((""" Per-file code and template coverage and per-url-pattern url coverage data for the latest test run has been written to %s. Per-statement code coverage data has been written to '.coverage', readable by the 'coverage' program. - """.replace(" ","") % (settings.TEST_COVERAGE_LATEST_FILE)) + """.replace(" ","") % (settings.TEST_COVERAGE_LATEST_FILE))) save_test_results(failures, test_labels) diff --git a/ietf/utils/test_smtpserver.py b/ietf/utils/test_smtpserver.py index 58227ee2d..b74797e52 100644 --- a/ietf/utils/test_smtpserver.py +++ b/ietf/utils/test_smtpserver.py @@ -1,6 +1,15 @@ +# Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import smtpd import threading import asyncore +import six + +import debug # pyflakes:ignore class AsyncCoreLoopThread(object): @@ -27,19 +36,29 @@ class AsyncCoreLoopThread(object): class SMTPTestChannel(smtpd.SMTPChannel): +# mail_options = ['BODY=8BITMIME', 'SMTPUTF8'] + def smtp_RCPT(self, arg): - if not self._SMTPChannel__mailfrom: - self.push('503 Error: need MAIL command') + if (six.PY2 and not self._SMTPChannel__mailfrom) or (six.PY3 and not self.mailfrom): + self.push(str('503 Error: need MAIL command')) return - address = self._SMTPChannel__getaddr('TO:', arg) if arg else None + if six.PY2: + address = self._SMTPChannel__getaddr('TO:', arg) if arg else None + else: + arg = self._strip_command_keyword('TO:', arg) + address, __ = self._getaddr(arg) if not address: - self.push('501 Syntax: RCPT TO:
') + self.push(str('501 Syntax: RCPT TO:
')) return if "poison" in address: - self.push('550 Error: Not touching that') - return - self._SMTPChannel__rcpttos.append(address) - self.push('250 Ok') + self.push(str('550 Error: Not touching that')) + return + if six.PY2: + self._SMTPChannel__rcpttos.append(address) + else: + self.rcpt_options = [] + self.rcpttos.append(address) + self.push(str('250 Ok')) class SMTPTestServer(smtpd.SMTPServer): @@ -57,7 +76,7 @@ class SMTPTestServer(smtpd.SMTPServer): #channel = SMTPTestChannel(self, conn, addr) SMTPTestChannel(self, conn, addr) - def process_message(self, peer, mailfrom, rcpttos, data): + def process_message(self, peer, mailfrom, rcpttos, data, mail_options=[], rcpt_options=[]): self.inbox.append(data) diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index 4cb5020e6..fc5df7ef6 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -1,5 +1,6 @@ -# Copyright The IETF Trust 2007, All Rights Reserved - +# Copyright The IETF Trust 2009-2019, All Rights Reserved +# -*- coding: utf-8 -*- +# # Portion Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen # @@ -32,28 +33,36 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import, print_function, unicode_literals + import os import re import email import html5lib +import six import sys -import urllib2 + +from six.moves.urllib.parse import unquote from unittest.util import strclass from bs4 import BeautifulSoup import django.test from django.conf import settings +from django.utils.encoding import python_2_unicode_compatible from django.utils.text import slugify import debug # pyflakes:ignore +from ietf.utils.mail import get_payload + real_database_name = settings.DATABASES["default"]["NAME"] def split_url(url): if "?" in url: url, args = url.split("?", 1) - args = dict([ map(urllib2.unquote,arg.split("=", 1)) for arg in args.split("&") if "=" in arg ]) + args = dict([ list(map(unquote,arg.split("=", 1))) for arg in args.split("&") if "=" in arg ]) else: args = {} return url, args @@ -73,7 +82,7 @@ def unicontent(r): def textcontent(r): text = BeautifulSoup(r.content, 'lxml').get_text() - text = re.sub('(\n\s+){2,}', '\n\n', text) + text = re.sub(r'(\n\s+){2,}', '\n\n', text) return text def reload_db_objects(*objects): @@ -92,6 +101,7 @@ class ReverseLazyTest(django.test.TestCase): response = self.client.get('/ipr/update/') self.assertRedirects(response, "/ipr/", status_code=301) +@python_2_unicode_compatible class TestCase(django.test.TestCase): """ Does basically the same as django.test.TestCase, but adds asserts for html5 validation. @@ -123,6 +133,8 @@ class TestCase(django.test.TestCase): def tempdir(self, label): slug = slugify(self.__class__.__name__.replace('.','-')) dirname = "tmp-{label}-{slug}-dir".format(**locals()) + if 'VIRTUAL_ENV' in os.environ: + dirname = os.path.join(os.environ['VIRTUAL_ENV'], dirname) path = os.path.abspath(dirname) if not os.path.exists(path): os.mkdir(path) @@ -145,7 +157,7 @@ class TestCase(django.test.TestCase): errors = [html.tostring(n).decode() for n in PyQuery(response.content)(error_css_selector)] if errors: - explanation = u"{} != {}\nGot form back with errors:\n----\n".format(response.status_code, 302) + u"----\n".join(errors) + explanation = "{} != {}\nGot form back with errors:\n----\n".format(response.status_code, 302) + "----\n".join(errors) self.assertEqual(response.status_code, 302, explanation) self.assertEqual(response.status_code, 302) @@ -163,7 +175,8 @@ class TestCase(django.test.TestCase): if subject: mlist = [ m for m in mlist if subject in m["Subject"] ] if text: - mlist = [ m for m in mlist if text in m.get_payload(decode=True) ] + assert isinstance(text, six.text_type) + mlist = [ m for m in mlist if text in get_payload(m) ] if count and len(mlist) != count: sys.stderr.write("Wrong count in assertMailboxContains(). The complete mailbox contains %s emails:\n\n" % len(mailbox)) for m in mailbox: @@ -175,6 +188,6 @@ class TestCase(django.test.TestCase): self.assertGreater(len(mlist), 0) def __str__(self): - return "%s (%s.%s)" % (self._testMethodName, strclass(self.__class__),self._testMethodName) + return u"%s (%s.%s)" % (self._testMethodName, strclass(self.__class__),self._testMethodName) diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 3f4f255c1..8c778cb2b 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -1,18 +1,21 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved # -*- coding: utf-8 -*- -from __future__ import unicode_literals, print_function + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os.path -import types import shutil +import six +import types from email.mime.image import MIMEImage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from fnmatch import fnmatch from importlib import import_module -from pipe import pipe -from StringIO import StringIO +from .pipe import pipe from textwrap import dedent from unittest import skipIf from tempfile import mkdtemp @@ -26,6 +29,7 @@ from django.template.defaulttags import URLNode from django.template.loader import get_template from django.templatetags.static import StaticNode from django.urls import reverse as urlreverse +from django.utils.encoding import force_text import debug # pyflakes:ignore @@ -36,7 +40,7 @@ from ietf.submit.tests import submission_file from ietf.utils.bower_storage import BowerStorageFinder from ietf.utils.draft import Draft, getmeta from ietf.utils.log import unreachable, assertion -from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox +from ietf.utils.mail import send_mail_preformatted, send_mail_text, send_mail_mime, outbox, get_payload from ietf.utils.test_runner import get_template_paths, set_coverage_checking from ietf.utils.test_utils import TestCase @@ -68,7 +72,7 @@ body self.assertSameEmail(recv['Cc'], 'cc1@example.com, cc2@example.com') self.assertSameEmail(recv['Bcc'], None) self.assertEqual(recv['Subject'], 'subject') - self.assertEqual(recv.get_payload(), 'body\n') + self.assertEqual(get_payload(recv), 'body\n') override = { 'To': 'oto1@example.net, oto2@example.net', @@ -98,7 +102,7 @@ body self.assertSameEmail(recv['Cc'], ', occ2@example.net') self.assertSameEmail(recv['Bcc'], None) self.assertEqual(recv['Subject'], 'osubject') - self.assertEqual(recv.get_payload(), 'body\n') + self.assertEqual(get_payload(recv), 'body\n') extra = {'Fuzz': [ 'bucket' ]} send_mail_preformatted(request=None, preformatted=msg, extra=extra, override={}) @@ -115,7 +119,7 @@ class TestSMTPServer(TestCase): def test_address_rejected(self): def send_simple_mail(to): - send_mail_text(None, to=to, frm=None, subject="Test for rejection", txt=u"dummy body") + send_mail_text(None, to=to, frm=None, subject="Test for rejection", txt="dummy body") len_before = len(outbox) send_simple_mail('good@example.com,poison@example.com') @@ -131,7 +135,7 @@ class TestSMTPServer(TestCase): def send_complex_mail(to): msg = MIMEMultipart() - textpart= MIMEText(dedent(u"""\ + textpart= MIMEText(dedent("""\ Sometimes people send mail with things like “smart quotes†in them. Sometimes they have attachments with pictures. """),_charset='utf-8') @@ -139,7 +143,7 @@ class TestSMTPServer(TestCase): img = MIMEImage(b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00\x10\x08\x06\x00\x00\x00\x1f\xf3\xffa\x00\x00\x02\x88IDATx\xda\xa5\x93\xcbO\x13Q\x14\xc6\xbf\xb9\xd32}L\x9fZ\x06\x10Q\x90\x10\x85b\x89\xfe\x01\x06BXK"\xd4hB\xdc\xa0\x06q\xe1c% H1l\xd0\x8dbT6\x1a5\x91\x12#K\x891\xf2\x07\x98\xc8[L\x1ay\xa8@\xdb\xd0\xd2\xe9\x83N;\xbdc\x1f\x11\x03\x04\x17zW\'_\xce\xf9\xdd\xef\x9c\x9c\xc3\xe0?\x1f\xb3S\xf8\xfe\xba\xc2Be\xa9]m\xd6\x9e\xe6x\xde\x9e\xd1\xa4HdF\x0e\xc5G\x89\x8a{X\xec\xfc\x1a\xdc\x1307X\xd4$T\nC\xc6\xfc|\x13\x8d\xa6\x00\xe5O\x16\xd1\xb3\x10}\xbe\x90w\xce\xdbZyeed\x17\xc03(4\x15\x9d(s\x13\xca!\x10\xa6\xb0\x1a\xb6\x9b\x0b\x84\x95\xb4F@\x89\x84\xe5O\x0b\xcdG\xaf\xae\x8dl\x01V\x9f\x1d\xb4q\x16\xde\xa33[\x8d\xe3\x93)\xdc\x7f\x9b\xc4\xf3\x1b\x1c,|\xaex#\n\xb4\x0cH\xb8\xd6\xa8F\xad\x83El# \xc6\x83\xb1\xf2\xa2\x0bK\xfe,`y\xd0\xe6*\x03\xaes\x0c\xea\xaas.\xc6g\x14t\xbcR\xd0P\x03t;\tX\x15\x83\xd5\xf9\xc5\xbe\x926_W6\xe3\xe7ca\xc2Z,82\xb1\xeb\r\x8b\xb1)\x82\xde\xa6\x14\xea\xec4\x0b\xf88K\xd0\xe5fQ_M\xd1s&\x95k\xe9\x87w\xf2\xc0eoM\x16\xe0\x1b*\xdc4XM\x9aL\xfca\x8e\xc5\xbd1\x0e//\xc6`\xd5\xe7Z\x08\xa6[8\xffT\x87\xeb\r\x12\xea\xabr\x80p \x14\xcfo]\xd5f\x01k\x8fl\x9bF3\xaf\xf9=\xb0X\x82\x81.O\xd96\xc4\x9d\x9a\xb8\x11\x89\x17\xb4\xf9s\x80\xe5\x01\xc3\xc4\xfe}FG\\\x064\xaa\xbf/\x0eM3\x92i\x13\xe1\x908Yr3\x9ck\xe1[\xbf\xd6%X\xf4\x9d\xef=z$(\xc1\xa9\xc3Q\xf0\x1c\xddV(\xa7\x18Ly9L\xafq8{\\D0\x14\xbd{\xe4V\xac3\x0bX\xe8\xd7\xdb\xb4,\xf5\x18\xb4j\xe3\xf8\xa2\x1e/\xa6\xac`\x18\x06\x02\x9f\x84\x8a\xa4\x07\x16c\xb1\xbe\xc9\xa2\xf6P\x04-\x8e\x00\x12\xc9\x84(&\xd9\xf2\x8a\x8e\x88\x7fk[\xbet\xe75\x0bzf\x98cI\xd6\xe6\xfc\xba\x06\xd3~\x1d\x12\xe9\x9fK\xcd\x12N\x16\xc4\xa0UQH)\x8a\x95\x08\x9c\xf6^\xc9\xbdk\x95\xe7o\xab\x9c&\xb5\xf2\x84W3\xa6\x9dG\x92\x19_$\xa9\x84\xd6%r\xc9\xde\x97\x1c\xde\xf3\x98\x96\xee\xb0\x16\x99\xd2v\x15\x94\xc6<\xc2Te\xb4\x04Ufe\x85\x8c2\x84<(\xeb\x91\xf7>\xa6\x7fy\xbf\x00\x96T\xff\x11\xf7\xd8R\xb9\x00\x00\x00\x00IEND\xaeB`\x82') msg.attach(img) - send_mail_mime(request=None, to=to, frm=settings.DEFAULT_FROM_EMAIL, subject=u'Ñто Ñложно', msg=msg, cc=None, extra=None) + send_mail_mime(request=None, to=to, frm=settings.DEFAULT_FROM_EMAIL, subject='Ñто Ñложно', msg=msg, cc=None, extra=None) len_before = len(outbox) send_complex_mail('good@example.com') @@ -157,12 +161,12 @@ def get_callbacks(urllist): callbacks.update(get_callbacks(entry.url_patterns)) else: if hasattr(entry, '_callback_str'): - callbacks.add(unicode(entry._callback_str)) + callbacks.add(force_text(entry._callback_str)) if (hasattr(entry, 'callback') and entry.callback and type(entry.callback) in [types.FunctionType, types.MethodType ]): callbacks.add("%s.%s" % (entry.callback.__module__, entry.callback.__name__)) if hasattr(entry, 'name') and entry.name: - callbacks.add(unicode(entry.name)) + callbacks.add(force_text(entry.name)) # There are some entries we don't handle here, mostly clases # (such as Feed subclasses) @@ -275,6 +279,7 @@ class TemplateChecksTestCase(TestCase): r = self.client.get(url) self.assertTemplateUsed(r, '500.html') +@skipIf(True, "Trac not available for Python3 as of 14 Jul 2019") @skipIf(skip_wiki_glue_testing, skip_message) class TestWikiGlueManagementCommand(TestCase): @@ -297,14 +302,18 @@ class TestWikiGlueManagementCommand(TestCase): set_coverage_checking(True) def test_wiki_create_output(self): - for type in ['wg','rg','ag','area']: - GroupFactory(type_id=type) + for group_type in ['wg','rg','ag','area']: + GroupFactory(type_id=group_type) groups = Group.objects.filter( type__slug__in=['wg','rg','ag','area'], state__slug='active' ).order_by('acronym') - out = StringIO() - err = StringIO() + out = six.StringIO() + err = six.StringIO() + debug.type('self.wiki_dir_pattern') + debug.show('self.wiki_dir_pattern') + debug.type('self.svn_dir_pattern') + debug.show('self.svn_dir_pattern') call_command('create_group_wikis', stdout=out, stderr=err, verbosity=2, wiki_dir_pattern=self.wiki_dir_pattern, svn_dir_pattern=self.svn_dir_pattern, @@ -426,7 +435,7 @@ class DraftTests(TestCase): def test_get_meta(self): tempdir = mkdtemp() filename = os.path.join(tempdir,self.draft.source) - with open(filename,'w') as file: + with io.open(filename,'w') as file: file.write(self.draft.text) self.assertEqual(getmeta(filename)['docdeststatus'],'Informational') shutil.rmtree(tempdir) diff --git a/ietf/utils/tests_restapi.py b/ietf/utils/tests_restapi.py index 93f0df687..b8eb591b6 100644 --- a/ietf/utils/tests_restapi.py +++ b/ietf/utils/tests_restapi.py @@ -1,4 +1,11 @@ -from __future__ import print_function +# Copyright The IETF Trust 2014-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + +import six +import sys import debug debug.debug = True @@ -19,12 +26,16 @@ class RestApi(ResourceTestCaseMixin, TestCase): """ # print(' fetching %s' % resource) r = self.api_client.get(resource, format=format) - if format == 'json': - self.assertValidJSONResponse(r) - elif format == 'xml': - self.assertValidXMLResponse(r) - else: - raise Exception("Unknown format found when testing the RestApi: %s" % (format, )) + try: + if format == 'json': + self.assertValidJSONResponse(r) + elif format == 'xml': + self.assertValidXMLResponse(r) + else: + raise Exception("Unknown format found when testing the RestApi: %s" % (format, )) + except Exception: + sys.stderr.write(" * Exception for resource: %s, format: %s\n" % (resource, format)) + raise data = self.deserialize(r) for name in data: if 'list_endpoint' in data[name]: @@ -73,6 +84,6 @@ class RestApi(ResourceTestCaseMixin, TestCase): for doc in doclist: for key in doc: value = doc[key] - if isinstance(value, basestring) and value.startswith('%s/'%apitop): + if isinstance(value, six.string_types) and value.startswith('%s/'%apitop): self.api_client.get(value, format='json') diff --git a/ietf/utils/texescape.py b/ietf/utils/texescape.py index 0c73fea96..623c192db 100644 --- a/ietf/utils/texescape.py +++ b/ietf/utils/texescape.py @@ -1,6 +1,11 @@ +# Copyright The IETF Trust 2018-2019, All Rights Reserved # -*- coding: utf-8 -*- # Copied from https://github.com/sphinx-doc/sphinx/blob/master/sphinx/util/texescape.py # Copyright and license as indicated in the original and below + + +from __future__ import absolute_import, print_function, unicode_literals + """ sphinx.util.texescape ~~~~~~~~~~~~~~~~~~~~~ @@ -11,7 +16,7 @@ :license: BSD, see LICENSE for details. """ -from __future__ import unicode_literals + tex_replacements = [ # map TeX special chars diff --git a/ietf/utils/text.py b/ietf/utils/text.py index 30a38891d..3faa978fb 100644 --- a/ietf/utils/text.py +++ b/ietf/utils/text.py @@ -1,19 +1,22 @@ -from __future__ import unicode_literals +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import re +import six import textwrap -import types import unicodedata from django.utils.functional import keep_lazy -from django.utils import six from django.utils.safestring import mark_safe import debug # pyflakes:ignore -from texescape import init as texescape_init, tex_escape_map +from .texescape import init as texescape_init, tex_escape_map -@keep_lazy(six.text_type) +@keep_lazy(str) def xslugify(value): """ Converts to ASCII. Converts spaces to hyphens. Removes characters that @@ -22,8 +25,8 @@ def xslugify(value): (I.e., does the same as slugify, but also converts slashes to dashes.) """ value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') - value = re.sub('[^\w\s/-]', '', value).strip().lower() - return mark_safe(re.sub('[-\s/]+', '-', value)) + value = re.sub(r'[^\w\s/-]', '', value).strip().lower() + return mark_safe(re.sub(r'[-\s/]+', '-', value)) def strip_prefix(text, prefix): if text.startswith(prefix): @@ -57,7 +60,7 @@ def fill(text, width): def wordwrap(text, width=80): """Wraps long lines without loosing the formatting and indentation of short lines""" - if not isinstance(text, (types.StringType,types.UnicodeType)): + if not isinstance(text, six.string_types): return text width = int(width) # ensure we have an int, if this is used as a template filter text = re.sub(" *\r\n", "\n", text) # get rid of DOS line endings diff --git a/ietf/utils/textupload.py b/ietf/utils/textupload.py index 574342d32..6dc010d56 100644 --- a/ietf/utils/textupload.py +++ b/ietf/utils/textupload.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + import re from django.core.exceptions import ValidationError @@ -11,12 +17,12 @@ def get_cleaned_text_file_content(uploaded_file): django.core.exceptions.ValidationError exceptions.""" if not uploaded_file: - return u"" + return "" if uploaded_file.size and uploaded_file.size > 10 * 1000 * 1000: raise ValidationError("Text file too large (size %s)." % uploaded_file.size) - content = "".join(uploaded_file.chunks()) + content = b"".join(uploaded_file.chunks()) # try to fixup encoding import magic @@ -33,18 +39,17 @@ def get_cleaned_text_file_content(uploaded_file): if not filetype.startswith("text"): raise ValidationError("Uploaded file does not appear to be a text file.") - match = re.search("charset=([\w-]+)", filetype) + match = re.search(r"charset=([\w-]+)", filetype) if not match: raise ValidationError("File has unknown encoding.") encoding = match.group(1) - if "ascii" not in encoding: - try: - content = content.decode(encoding) - except Exception as e: - raise ValidationError("Error decoding file (%s). Try submitting with UTF-8 encoding or remove non-ASCII characters." % str(e)) + try: + content = content.decode(encoding) + except Exception as e: + raise ValidationError("Error decoding file (%s). Try submitting with UTF-8 encoding or remove non-ASCII characters." % str(e)) # turn line-endings into Unix style content = content.replace("\r\n", "\n").replace("\r", "\n") - return content.encode("utf-8") + return content diff --git a/ietf/utils/urls.py b/ietf/utils/urls.py index d122fceb2..a6a3abe9d 100644 --- a/ietf/utils/urls.py +++ b/ietf/utils/urls.py @@ -1,7 +1,12 @@ -# Copyright The IETF Trust 2016, All Rights Reserved +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals -import six import debug # pyflakes:ignore +import six + from inspect import isclass from django.conf.urls import url as django_url diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py index f2a6b319e..8ad5a6466 100644 --- a/ietf/utils/validators.py +++ b/ietf/utils/validators.py @@ -1,6 +1,8 @@ -# -*- python -*- -# Copyright The IETF Trust 2007, All Rights Reserved -from __future__ import unicode_literals +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals import os import re @@ -77,8 +79,8 @@ def validate_mime_type(file, valid): mime_type, encoding = get_mime_type(raw) # work around mis-identification of text where a line has 'virtual' as # the first word: - if mime_type == 'text/x-c++' and re.search('(?m)^virtual\s', raw): - mod = raw.replace(str('virtual'), str(' virtual')) + if mime_type == 'text/x-c++' and re.search(br'(?m)^virtual\s', raw): + mod = raw.replace(b'virtual', b' virtual') mime_type, encoding = get_mime_type(mod) if valid and not mime_type in valid: raise ValidationError('Found content with unexpected mime type: %s. Expected one of %s.' % diff --git a/ietf/virtualenv-manage.py b/ietf/virtualenv-manage.py index df6477fd9..6cfe31e06 100755 --- a/ietf/virtualenv-manage.py +++ b/ietf/virtualenv-manage.py @@ -1,5 +1,11 @@ #!/usr/bin/env python +# Copyright The IETF Trust 2016-2019, All Rights Reserved +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +import io import os import sys @@ -11,7 +17,7 @@ os.chdir(path) # Virtualenv support virtualenv_activation = os.path.join(path, "env", "bin", "activate_this.py") if os.path.exists(virtualenv_activation): - execfile(virtualenv_activation, dict(__file__=virtualenv_activation)) + exec(compile(io.open(virtualenv_activation, "rb").read(), virtualenv_activation, 'exec'), dict(__file__=virtualenv_activation)) else: raise RuntimeError("Could not find the expected virtual python environment.") diff --git a/ietf/wsgi.py b/ietf/wsgi.py index d82a32446..cacc8ed60 100644 --- a/ietf/wsgi.py +++ b/ietf/wsgi.py @@ -1,3 +1,9 @@ +# Copyright The IETF Trust 2013-2019, All Rights Reserved +# -*- coding: utf-8 -*- + + +from __future__ import absolute_import, print_function, unicode_literals + """ WSGI configuration for the datatracker. @@ -25,12 +31,12 @@ WSGIPythonEggs /var/www/.python-eggs/ WSGIScriptAlias / /srv/www/ietfdb/ietf/wsgi.py - AuthType Digest - AuthName "IETF" - AuthUserFile /var/local/loginmgr/digest - AuthGroupFile /var/local/loginmgr/groups - AuthDigestDomain http://tools.ietf.org/ - Require valid-user + AuthType Digest + AuthName "IETF" + AuthUserFile /var/local/loginmgr/digest + AuthGroupFile /var/local/loginmgr/groups + AuthDigestDomain http://tools.ietf.org/ + Require valid-user ---- @@ -38,19 +44,20 @@ WSGIPythonEggs /var/www/.python-eggs/ """ +import io import os import sys import syslog path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -syslog.openlog("datatracker", syslog.LOG_PID, syslog.LOG_USER) +syslog.openlog(str("datatracker"), syslog.LOG_PID, syslog.LOG_USER) # Virtualenv support virtualenv_activation = os.path.join(path, "env", "bin", "activate_this.py") if os.path.exists(virtualenv_activation): syslog.syslog("Starting datatracker wsgi with virtualenv %s" % os.path.dirname(os.path.dirname(virtualenv_activation))) - execfile(virtualenv_activation, dict(__file__=virtualenv_activation)) + exec(compile(io.open(virtualenv_activation, "rb").read(), virtualenv_activation, 'exec'), dict(__file__=virtualenv_activation)) else: syslog.syslog("Starting datatracker wsgi without virtualenv") diff --git a/pyzmail/__init__.py b/pyzmail/__init__.py new file mode 100644 index 000000000..f6c8854ab --- /dev/null +++ b/pyzmail/__init__.py @@ -0,0 +1,21 @@ +# +# pyzmail/__init__.py +# (c) Alain Spineux +# http://www.magiksys.net/pyzmail +# Released under LGPL + +from . import utils +from .generate import compose_mail, send_mail, send_mail2 +from .parse import email_address_re, PyzMessage, PzMessage, decode_text +from .parse import message_from_string, message_from_file +from .parse import message_from_bytes, message_from_binary_file # python >= 3.2 +from .version import __version__ + +# to help epydoc to display functions available from top of the package +__all__= [ 'compose_mail', 'send_mail', 'send_mail2', 'email_address_re', \ + 'PyzMessage', 'PzMessage', 'decode_text', '__version__', + 'utils', 'generate', 'parse', 'version', + 'message_from_string','message_from_file', + 'message_from_binary_file', 'message_from_bytes', # python >= 3.2 + ] + diff --git a/pyzmail/generate.py b/pyzmail/generate.py new file mode 100644 index 000000000..76fadc76d --- /dev/null +++ b/pyzmail/generate.py @@ -0,0 +1,506 @@ +# +# pyzmail/generate.py +# (c) Alain Spineux +# http://www.magiksys.net/pyzmail +# Released under LGPL + +""" +Useful functions to compose and send emails. + +For short: + +>>> payload, mail_from, rcpt_to, msg_id=compose_mail((u'Me', 'me@foo.com'), +... [(u'Him', 'him@bar.com')], u'the subject', 'iso-8859-1', ('Hello world', 'us-ascii'), +... attachments=[('attached', 'text', 'plain', 'text.txt', 'us-ascii')]) +... #doctest: +SKIP +>>> error=send_mail(payload, mail_from, rcpt_to, 'localhost', smtp_port=25) +... #doctest: +SKIP +""" + +import os, sys +import time +import base64 +import smtplib, socket +import email +import email.encoders +import email.header +import email.utils +import email.mime +import email.mime.base +import email.mime.text +import email.mime.image +import email.mime.multipart + +from . import utils + +def format_addresses(addresses, header_name=None, charset=None): + """ + Convert a list of addresses into a MIME-compliant header for a From, To, Cc, + or any other I{address} related field. + This mixes the use of email.utils.formataddr() and email.header.Header(). + + @type addresses: list + @param addresses: list of addresses, can be a mix of string a tuple of the form + C{[ 'address@domain', (u'Name', 'name@domain'), ...]}. + If C{u'Name'} contains non us-ascii characters, it must be a + unicode string or encoded using the I{charset} argument. + @type header_name: string or None + @keyword header_name: the name of the header. Its length is used to limit + the length of the first line of the header according the RFC's + requirements. (not very important, but it's better to match the + requirements when possible) + @type charset: str + @keyword charset: the encoding charset for non unicode I{name} and a B{hint} + for encoding of unicode string. In other words, + if the I{name} of an address in a byte string containing non + I{us-ascii} characters, then C{name.decode(charset)} + must generate the expected result. If a unicode string + is used instead, charset will be tried to encode the + string, if it fail, I{utf-8} will be used. + With B{Python 3.x} I{charset} is no more a hint and an exception will + be raised instead of using I{utf-8} has a fall back. + @rtype: str + @return: the encoded list of formated addresses separated by commas, + ready to use as I{Header} value. + + >>> print format_addresses([('John', 'john@foo.com') ], 'From', 'us-ascii').encode() + John + >>> print format_addresses([(u'l\\xe9o', 'leo@foo.com') ], 'To', 'iso-8859-1').encode() + =?iso-8859-1?q?l=E9o?= + >>> print format_addresses([(u'l\\xe9o', 'leo@foo.com') ], 'To', 'us-ascii').encode() + ... # don't work in 3.X because charset is more than a hint + ... #doctest: +SKIP + =?utf-8?q?l=C3=A9o?= + >>> # because u'l\xe9o' cannot be encoded into us-ascii, utf8 is used instead + >>> print format_addresses([('No\\xe9', 'noe@f.com'), (u'M\\u0101ori', 'maori@b.com') ], 'Cc', 'iso-8859-1').encode() + ... # don't work in 3.X because charset is more than a hint + ... #doctest: +SKIP + =?iso-8859-1?q?No=E9?= , =?utf-8?b?TcSBb3Jp?= + >>> # 'No\xe9' is already encoded into iso-8859-1, but u'M\\u0101ori' cannot be encoded into iso-8859-1 + >>> # then utf8 is used here + >>> print format_addresses(['a@bar.com', ('John', 'john@foo.com') ], 'From', 'us-ascii').encode() + a@bar.com , John + """ + header=email.header.Header(charset=charset, header_name=header_name) + for i, address in enumerate(addresses): + if i!=0: + # add separator between addresses + header.append(',', charset='us-ascii') + + try: + name, addr=address + except ValueError: + # address is not a tuple, their is no name, only email address + header.append(address, charset='us-ascii') + else: + # check if address name is a unicode or byte string in "pure" us-ascii + if utils.is_usascii(name): + # name is a us-ascii byte string, i can use formataddr + formated_addr=email.utils.formataddr((name, addr)) + # us-ascii must be used and not default 'charset' + header.append(formated_addr, charset='us-ascii') + else: + # this is not as "pure" us-ascii string + # Header will use "RFC2047" to encode the address name + # if name is byte string, charset will be used to decode it first + header.append(name) + # here us-ascii must be used and not default 'charset' + header.append('<%s>' % (addr,), charset='us-ascii') + + return header + + +def build_mail(text, html=None, attachments=[], embeddeds=[]): + """ + Generate the core of the email message regarding the parameters. + The structure of the MIME email may vary, but the general one is as follow:: + + multipart/mixed (only if attachments are included) + | + +-- multipart/related (only if embedded contents are included) + | | + | +-- multipart/alternative (only if text AND html are available) + | | | + | | +-- text/plain (text version of the message) + | | +-- text/html (html version of the message) + | | + | +-- image/gif (where to include embedded contents) + | + +-- application/msword (where to add attachments) + + @param text: the text version of the message, under the form of a tuple: + C{(encoded_content, encoding)} where I{encoded_content} is a byte string + encoded using I{encoding}. + I{text} can be None if the message has no text version. + @type text: tuple or None + @keyword html: the HTML version of the message, under the form of a tuple: + C{(encoded_content, encoding)} where I{encoded_content} is a byte string + encoded using I{encoding} + I{html} can be None if the message has no HTML version. + @type html: tuple or None + @keyword attachments: the list of attachments to include into the mail, in the + form [(data, maintype, subtype, filename, charset), ..] where : + - I{data} : is the raw data, or a I{charset} encoded string for 'text' + content. + - I{maintype} : is a MIME main type like : 'text', 'image', 'application' .... + - I{subtype} : is a MIME sub type of the above I{maintype} for example : + 'plain', 'png', 'msword' for respectively 'text/plain', 'image/png', + 'application/msword'. + - I{filename} this is the filename of the attachment, it must be a + 'us-ascii' string or a tuple of the form + C{(encoding, language, encoded_filename)} + following the RFC2231 requirement, for example + C{('iso-8859-1', 'fr', u'r\\xe9pertoir.png'.encode('iso-8859-1'))} + - I{charset} : if I{maintype} is 'text', then I{data} must be encoded + using this I{charset}. It can be None for non 'text' content. + @type attachments: list + @keyword embeddeds: is a list of documents embedded inside the HTML or text + version of the message. It is similar to the I{attachments} list, + but I{filename} is replaced by I{content_id} that is related to + the B{cid} reference into the HTML or text version of the message. + @type embeddeds: list + @rtype: inherit from email.Message + @return: the message in a MIME object + + >>> mail=build_mail(('Hello world', 'us-ascii'), attachments=[('attached', 'text', 'plain', 'text.txt', 'us-ascii')]) + >>> mail.set_boundary('===limit1==') + >>> print mail.as_string(unixfrom=False) + Content-Type: multipart/mixed; boundary="===limit1==" + MIME-Version: 1.0 + + --===limit1== + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + + Hello world + --===limit1== + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename="text.txt" + + attached + --===limit1==-- + """ + + main=text_part=html_part=None + if text: + content, charset=text + main=text_part=email.mime.text.MIMEText(content, 'plain', charset) + + if html: + content, charset=html + main=html_part=email.mime.text.MIMEText(content, 'html', charset) + + if not text_part and not html_part: + main=text_part=email.mime.text.MIMEText('', 'plain', 'us-ascii') + elif text_part and html_part: + # need to create a multipart/alternative to include text and html version + main=email.mime.multipart.MIMEMultipart('alternative', None, [text_part, html_part]) + + if embeddeds: + related=email.mime.multipart.MIMEMultipart('related') + related.attach(main) + for part in embeddeds: + if not isinstance(part, email.mime.base.MIMEBase): + data, maintype, subtype, content_id, charset=part + if (maintype=='text'): + part=email.mime.text.MIMEText(data, subtype, charset) + else: + part=email.mime.base.MIMEBase(maintype, subtype) + part.set_payload(data) + email.encoders.encode_base64(part) + part.add_header('Content-ID', '<'+content_id+'>') + part.add_header('Content-Disposition', 'inline') + related.attach(part) + main=related + + if attachments: + mixed=email.mime.multipart.MIMEMultipart('mixed') + mixed.attach(main) + for part in attachments: + if not isinstance(part, email.mime.base.MIMEBase): + data, maintype, subtype, filename, charset=part + if (maintype=='text'): + part=email.mime.text.MIMEText(data, subtype, charset) + else: + part=email.mime.base.MIMEBase(maintype, subtype) + part.set_payload(data) + email.encoders.encode_base64(part) + part.add_header('Content-Disposition', 'attachment', filename=filename) + mixed.attach(part) + main=mixed + + return main + +def complete_mail(message, sender, recipients, subject, default_charset, cc=[], bcc=[], message_id_string=None, date=None, headers=[]): + """ + Fill in the From, To, Cc, Subject, Date and Message-Id I{headers} of + one existing message regarding the parameters. + + @type message:email.Message + @param message: the message to fill in + @type sender: tuple + @param sender: a tuple of the form (u'Sender Name', 'sender.address@domain.com') + @type recipients: list + @param recipients: a list of addresses. Address can be tuple or string like + expected by L{format_addresses()}, for example: C{[ 'address@dmain.com', + (u'Recipient Name', 'recipient.address@domain.com'), ... ]} + @type subject: str + @param subject: The subject of the message, can be a unicode string or a + string encoded using I{default_charset} encoding. Prefert unicode to + byte string here. + @type default_charset: str + @param default_charset: The default charset for this email. Arguments + that are non unicode string are supposed to be encoded using this charset. + This I{charset} will be used has an hint when encoding mail content. + @type cc: list + @keyword cc: The I{carbone copy} addresses. Same format as the I{recipients} + argument. + @type bcc: list + @keyword bcc: The I{blind carbone copy} addresses. Same format as the I{recipients} + argument. + @type message_id_string: str or None + @keyword message_id_string: if None, don't append any I{Message-ID} to the + mail, let the SMTP do the job, else use the string to generate a unique + I{ID} using C{email.utils.make_msgid()}. The generated value is + returned as last argument. For example use the name of your application. + @type date: int or None + @keyword date: utc time in second from the epoch or None. If None then + use curent time C{time.time()} instead. + @type headers: list of tuple + @keyword headers: a list of C{(field, value)} tuples to fill in the mail + header fields. Values are encoded using I{default_charset}. + @rtype: tuple + @return: B{(payload, mail_from, rcpt_to, msg_id)} + - I{payload} (str) is the content of the email, generated from the message + - I{mail_from} (str) is the address of the sender to pass to the SMTP host + - I{rcpt_to} (list) is a list of the recipients addresses to pass to the SMTP host + of the form C{[ 'a@b.com', c@d.com', ]}. This combine all recipients, + I{carbone copy} addresses and I{blind carbone copy} addresses. + - I{msg_id} (None or str) None if message_id_string==None else the generated value for + the message-id. If not None, this I{Message-ID} is already written + into the payload. + + >>> import email.mime.text + >>> msg=email.mime.text.MIMEText('The text.', 'plain', 'us-ascii') + >>> # I could use build_mail() instead + >>> payload, mail_from, rcpt_to, msg_id=complete_mail(msg, ('Me', 'me@foo.com'), + ... [ ('Him', 'him@bar.com'), ], 'Non unicode subject', 'iso-8859-1', + ... cc=['her@bar.com',], date=1313558269, headers=[('User-Agent', u'pyzmail'), ]) + >>> print payload + ... # 3.X encode User-Agent: using 'iso-8859-1' even if it contains only us-asccii + ... # doctest: +ELLIPSIS + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + From: Me + To: Him + Cc: her@bar.com + Subject: =?iso-8859-1?q?Non_unicode_subject?= + Date: ... + User-Agent: ...pyzmail... + + The text. + >>> print 'mail_from=%r rcpt_to=%r' % (mail_from, rcpt_to) + mail_from='me@foo.com' rcpt_to=['him@bar.com', 'her@bar.com'] + """ + def getaddr(address): + if isinstance(address, tuple): + return address[1] + else: + return address + + mail_from=getaddr(sender[1]) + rcpt_to=list(map(getaddr, recipients)) + rcpt_to.extend(list(map(getaddr, cc))) + rcpt_to.extend(list(map(getaddr, bcc))) + + message['From'] = format_addresses([ sender, ], header_name='from', charset=default_charset) + if recipients: + message['To'] = format_addresses(recipients, header_name='to', charset=default_charset) + if cc: + message['Cc'] = format_addresses(cc, header_name='cc', charset=default_charset) + message['Subject'] = email.header.Header(subject, default_charset) + if date: + utc_from_epoch=date + else: + utc_from_epoch=time.time() + message['Date'] = email.utils.formatdate(utc_from_epoch, localtime=True) + + if message_id_string: + msg_id=message['Message-Id']=email.utils.make_msgid(message_id_string) + else: + msg_id=None + + for field, value in headers: + message[field]=email.header.Header(value, default_charset) + + payload=message.as_string() + + return payload, mail_from, rcpt_to, msg_id + +def compose_mail(sender, recipients, subject, default_charset, text, html=None, attachments=[], embeddeds=[], cc=[], bcc=[], message_id_string=None, date=None, headers=[]): + """ + Compose an email regarding the arguments. Call L{build_mail()} and + L{complete_mail()} at once. + + Read the B{parameters} descriptions of both functions L{build_mail()} and L{complete_mail()}. + + Returned value is the same as for L{build_mail()} and L{complete_mail()}. + You can pass the returned values to L{send_mail()} or L{send_mail2()}. + + @rtype: tuple + @return: B{(payload, mail_from, rcpt_to, msg_id)} + + >>> payload, mail_from, rcpt_to, msg_id=compose_mail((u'Me', 'me@foo.com'), [(u'Him', 'him@bar.com')], u'the subject', 'iso-8859-1', ('Hello world', 'us-ascii'), attachments=[('attached', 'text', 'plain', 'text.txt', 'us-ascii')]) + """ + message=build_mail(text, html, attachments, embeddeds) + return complete_mail(message, sender, recipients, subject, default_charset, cc, bcc, message_id_string, date, headers) + + +def send_mail2(payload, mail_from, rcpt_to, smtp_host, smtp_port=25, smtp_mode='normal', smtp_login=None, smtp_password=None): + """ + Send the message to a SMTP host. Look at the L{send_mail()} documentation. + L{send_mail()} call this function and catch all exceptions to convert them + into a user friendly error message. The returned value + is always a dictionary. It can be empty if all recipients have been + accepted. + + @rtype: dict + @return: This function return the value returnd by C{smtplib.SMTP.sendmail()} + or raise the same exceptions. + + This method will return normally if the mail is accepted for at least one + recipient. Otherwise it will raise an exception. That is, if this + method does not raise an exception, then someone should get your mail. + If this method does not raise an exception, it returns a dictionary, + with one entry for each recipient that was refused. Each entry contains a + tuple of the SMTP error code and the accompanying error message sent by the server. + + @raise smtplib.SMTPException: Look at the standard C{smtplib.SMTP.sendmail()} documentation. + + """ + if smtp_mode=='ssl': + smtp=smtplib.SMTP_SSL(smtp_host, smtp_port) + else: + smtp=smtplib.SMTP(smtp_host, smtp_port) + if smtp_mode=='tls': + smtp.starttls() + + if smtp_login and smtp_password: + if sys.version_info<(3, 0): + # python 2.x + # login and password must be encoded + # because HMAC used in CRAM_MD5 require non unicode string + smtp.login(smtp_login.encode('utf-8'), smtp_password.encode('utf-8')) + else: + #python 3.x + smtp.login(smtp_login, smtp_password) + try: + ret=smtp.sendmail(mail_from, rcpt_to, payload) + finally: + try: + smtp.quit() + except Exception as e: + pass + + return ret + +def send_mail(payload, mail_from, rcpt_to, smtp_host, smtp_port=25, smtp_mode='normal', smtp_login=None, smtp_password=None): + """ + Send the message to a SMTP host. Handle SSL, TLS and authentication. + I{payload}, I{mail_from} and I{rcpt_to} can come from values returned by + L{complete_mail()}. This function call L{send_mail2()} but catch all + exceptions and return friendly error message instead. + + @type payload: str + @param payload: the mail content. + @type mail_from: str + @param mail_from: the sender address, for example: C{'me@domain.com'}. + @type rcpt_to: list + @param rcpt_to: The list of the recipient addresses in the form + C{[ 'a@b.com', c@d.com', ]}. No names here, only email addresses. + @type smtp_host: str + @param smtp_host: the IP address or the name of the SMTP host. + @type smtp_port: int + @keyword smtp_port: the port to connect to on the SMTP host. Default is C{25}. + @type smtp_mode: str + @keyword smtp_mode: the way to connect to the SMTP host, can be: + C{'normal'}, C{'ssl'} or C{'tls'}. default is C{'normal'} + @type smtp_login: str or None + @keyword smtp_login: If authentication is required, this is the login. + Be carefull to I{UTF8} encode your login if it contains + non I{us-ascii} characters. + @type smtp_password: str or None + @keyword smtp_password: If authentication is required, this is the password. + Be carefull to I{UTF8} encode your password if it + contains non I{us-ascii} characters. + + @rtype: dict or str + @return: This function return a dictionary of failed recipients + or a string with an error message. + + If all recipients have been accepted the dictionary is empty. If the + returned value is a string, none of the recipients will get the message. + + The dictionary is exactly of the same sort as + smtplib.SMTP.sendmail() returns with one entry for each recipient that + was refused. Each entry contains a tuple of the SMTP error code and + the accompanying error message sent by the server. + + Example: + + >>> send_mail('Subject: hello\\n\\nmessage', 'a@foo.com', [ 'b@bar.com', ], 'localhost') #doctest: +SKIP + {} + + Here is how to use the returned value:: + if isinstance(ret, dict): + if ret: + print 'failed' recipients: + for recipient, (code, msg) in ret.iteritems(): + print 'code=%d recipient=%s\terror=%s' % (code, recipient, msg) + else: + print 'success' + else: + print 'Error:', ret + + To use your GMail account to send your mail:: + smtp_host='smtp.gmail.com' + smtp_port=587 + smtp_mode='tls' + smtp_login='your.gmail.addresse@gmail.com' + smtp_password='your.gmail.password' + + Use your GMail address for the sender ! + + """ + + error=dict() + try: + ret=send_mail2(payload, mail_from, rcpt_to, smtp_host, smtp_port, smtp_mode, smtp_login, smtp_password) + except (socket.error, ) as e: + error='server %s:%s not responding: %s' % (smtp_host, smtp_port, e) + except smtplib.SMTPAuthenticationError as e: + error='authentication error: %s' % (e, ) + except smtplib.SMTPRecipientsRefused as e: + # code, error=e.recipients[recipient_addr] + error='all recipients refused: '+', '.join(list(e.recipients.keys())) + except smtplib.SMTPSenderRefused as e: + # e.sender, e.smtp_code, e.smtp_error + error='sender refused: %s' % (e.sender, ) + except smtplib.SMTPDataError as e: + error='SMTP protocol mismatch: %s' % (e, ) + except smtplib.SMTPHeloError as e: + error="server didn't reply properly to the HELO greeting: %s" % (e, ) + except smtplib.SMTPException as e: + error='SMTP error: %s' % (e, ) +# except Exception, e: +# raise # unknown error + else: + # failed addresses and error messages + error=ret + + return error + diff --git a/pyzmail/parse.py b/pyzmail/parse.py new file mode 100644 index 000000000..da5c10469 --- /dev/null +++ b/pyzmail/parse.py @@ -0,0 +1,817 @@ +# +# pyzmail/parse.py +# (c) Alain Spineux +# http://www.magiksys.net/pyzmail +# Released under LGPL + +""" +Useful functions to parse emails + +@var email_address_re: a regex that match well formed email address (from perlfaq9) +@undocumented: atom_rfc2822 +@undocumented: atom_posfix_restricted +@undocumented: atom +@undocumented: dot_atom +@undocumented: local +@undocumented: domain_lit +@undocumented: domain +@undocumented: addr_spec +""" + +import re +import io +import email +import email.errors +import email.header +import email.message +import mimetypes + +from .utils import * + +# email address REGEX matching the RFC 2822 spec from perlfaq9 +# my $atom = qr{[a-zA-Z0-9_!#\$\%&'*+/=?\^`{}~|\-]+}; +# my $dot_atom = qr{$atom(?:\.$atom)*}; +# my $quoted = qr{"(?:\\[^\r\n]|[^\\"])*"}; +# my $local = qr{(?:$dot_atom|$quoted)}; +# my $domain_lit = qr{\[(?:\\\S|[\x21-\x5a\x5e-\x7e])*\]}; +# my $domain = qr{(?:$dot_atom|$domain_lit)}; +# my $addr_spec = qr{$local\@$domain}; +# +# Python's translation +atom_rfc2822=r"[a-zA-Z0-9_!#\$\%&'*+/=?\^`{}~|\-]+" +atom_posfix_restricted=r"[a-zA-Z0-9_#\$&'*+/=?\^`{}~|\-]+" # without '!' and '%' +atom=atom_rfc2822 +dot_atom=atom + r"(?:\." + atom + ")*" +quoted=r'"(?:\\[^\r\n]|[^\\"])*"' +local="(?:" + dot_atom + "|" + quoted + ")" +domain_lit=r"\[(?:\\\S|[\x21-\x5a\x5e-\x7e])*\]" +domain="(?:" + dot_atom + "|" + domain_lit + ")" +addr_spec=local + "\@" + domain +# and the result +email_address_re=re.compile('^'+addr_spec+'$') + +class MailPart: + """ + Data related to a mail part (aka message content, attachment or + embedded content in an email) + + @type charset: str or None + @ivar charset: the encoding of the I{get_payload()} content if I{type} is 'text/*' + and charset has been specified in the message + @type content_id: str or None + @ivar content_id: the MIME Content-ID if specified in the message. + @type description: str or None + @ivar description: the MIME Content-Description if specified in the message. + @type disposition: str or None + @ivar disposition: C{None}, C{'inline'} or C{'attachment'} depending + the MIME Content-Disposition value + @type filename: unicode or None + @ivar filename: the name of the file, if specified in the message. + @type part: inherit from email.mime.base.MIMEBase + @ivar part: the related part inside the message. + @type is_body: str or None + @ivar is_body: None if this part is not the mail content itself (an + attachment or embedded content), C{'text/plain'} if this part is the + text content or C{'text/html'} if this part is the HTML version. + @type sanitized_filename: str or None + @ivar sanitized_filename: This field is filled by L{PyzMessage} to store + a valid unique filename related or not with the original filename. + @type type: str + @ivar type: the MIME type, like 'text/plain', 'image/png', 'application/msword' ... + """ + + def __init__(self, part, filename=None, type=None, charset=None, content_id=None, description=None, disposition=None, sanitized_filename=None, is_body=None): + """ + Create an mail part and initialize all attributes + """ + self.part=part # original python part + self.filename=filename # filename in unicode (if any) + self.type=type # the mime-type + self.charset=charset # the charset (if any) + self.description=description # if any + self.disposition=disposition # 'inline', 'attachment' or None + self.sanitized_filename=sanitized_filename # cleanup your filename here (TODO) + self.is_body=is_body # usually in (None, 'text/plain' or 'text/html') + self.content_id=content_id # if any + if self.content_id: + # strip '<>' to ease search and replace in "root" content (TODO) + if self.content_id.startswith('<') and self.content_id.endswith('>'): + self.content_id=self.content_id[1:-1] + + def get_payload(self): + """ + decode and return part payload. if I{type} is 'text/*' and I{charset} + not C{None}, be careful to take care of the text encoding. Use + something like C{part.get_payload().decode(part.charset)} + """ + + payload=None + if self.type.startswith('message/'): + # I don't use msg.as_string() because I want to use mangle_from_=False + if sys.version_info<(3, 0): + # python 2.x + from email.generator import Generator + fp = io.StringIO() + g = Generator(fp, mangle_from_=False) + g.flatten(self.part, unixfrom=False) + payload=fp.getvalue() + else: + # support only for python >= 3.2 + from email.generator import BytesGenerator + import io + fp = io.BytesIO() + g = BytesGenerator(fp, mangle_from_=False) + g.flatten(self.part, unixfrom=False) + payload=fp.getvalue() + + else: + payload=self.part.get_payload(decode=True) + return payload + + def __repr__(self): + st='MailPart<' + if self.is_body: + st+='*' + st+=self.type + if self.charset: + st+=' charset='+self.charset + if self.filename: + st+=' filename='+self.filename + if self.content_id: + st+=' content_id='+self.content_id + st+=' len=%d' % (len(self.get_payload()), ) + st+='>' + return st + + + +_line_end_re=re.compile('\r\n|\n\r|\n|\r') + +def _friendly_header(header): + """ + Convert header returned by C{email.message.Message.get()} into a + user friendly string. + + Py3k C{email.message.Message.get()} return C{header.Header()} with charset + set to C{charset.UNKNOWN8BIT} when the header contains invalid characters, + else it return I{str} as Python 2.X does + + @type header: str or email.header.Header + @param header: the header to convert into a user friendly string + + @rtype: str + @returns: the converter header + """ + + save=header + if isinstance(header, email.header.Header): + header=str(header) + + return re.sub(_line_end_re, ' ', header) + +def decode_mail_header(value, default_charset='us-ascii'): + """ + Decode a header value into a unicode string. + Works like a more smarter python + C{u"".join(email.header.decode_header()} function + + @type value: str + @param value: the value of the header. + @type default_charset: str + @keyword default_charset: if one charset used in the header (multiple charset + can be mixed) is unknown, then use this charset instead. + + >>> decode_mail_header('=?iso-8859-1?q?Courrier_=E8lectronique_en_Fran=E7ais?=') + u'Courrier \\xe8lectronique en Fran\\xe7ais' + """ + +# value=_friendly_header(value) + try: + headers=email.header.decode_header(value) + except email.errors.HeaderParseError: + # this can append in email.base64mime.decode(), for example for this value: + # '=?UTF-8?B?15HXmdeh15jXqNeVINeY15DXpteUINeTJ9eV16jXlSDXkdeg15XXldeUINem15PXpywg15TXptei16bXldei15nXnSDXqdecINek15zXmdeZ?==?UTF-8?B?157XldeR15nXnCwg157Xldek16Ig157Xl9eV15wg15HXodeV15bXnyDXk9ec15DXnCDXldeh15gg157Xl9eR16rXldeqINep15wg15HXmdeQ?==?UTF-8?B?15zXmNeZ?=' + # then return a sanitized ascii string + # TODO: some improvements are possible here, but a failure here is + # unlikely + return value.encode('us-ascii', 'replace').decode('us-ascii') + else: + for i, (text, charset) in enumerate(headers): + # python 3.x + # email.header.decode_header('a') -> [('a', None)] + # email.header.decode_header('a =?ISO-8859-1?Q?foo?= b') + # --> [(b'a', None), (b'foo', 'iso-8859-1'), (b'b', None)] + # in Py3 text is sometime str and sometime byte :-( + # python 2.x + # email.header.decode_header('a') -> [('a', None)] + # email.header.decode_header('a =?ISO-8859-1?Q?foo?= b') + # --> [('a', None), ('foo', 'iso-8859-1'), ('b', None)] + if (charset is None and sys.version_info>=(3, 0)): + # Py3 + if isinstance(text, str): + # convert Py3 string into bytes string to be sure their is no + # non us-ascii chars and because next line expect byte string + text=text.encode('us-ascii', 'replace') + try: + headers[i]=text.decode(charset or 'us-ascii', 'replace') + except LookupError: + # if the charset is unknown, force default + headers[i]=text.decode(default_charset, 'replace') + + return "".join(headers) + +def get_mail_addresses(message, header_name): + """ + retrieve all email addresses from one message header + + @type message: email.message.Message + @param message: the email message + @type header_name: str + @param header_name: the name of the header, can be 'from', 'to', 'cc' or + any other header containing one or more email addresses + @rtype: list + @returns: a list of the addresses in the form of tuples + C{[(u'Name', 'addresse@domain.com'), ...]} + + >>> import email + >>> import email.mime.text + >>> msg=email.mime.text.MIMEText('The text.', 'plain', 'us-ascii') + >>> msg['From']=email.email.utils.formataddr(('Me', 'me@foo.com')) + >>> msg['To']=email.email.utils.formataddr(('A', 'a@foo.com'))+', '+email.email.utils.formataddr(('B', 'b@foo.com')) + >>> print msg.as_string(unixfrom=False) + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + From: Me + To: A , B + + The text. + >>> get_mail_addresses(msg, 'from') + [(u'Me', 'me@foo.com')] + >>> get_mail_addresses(msg, 'to') + [(u'A', 'a@foo.com'), (u'B', 'b@foo.com')] + """ + addrs=email.utils.getaddresses([ _friendly_header(h) for h in message.get_all(header_name, [])]) + for i, (addr_name, addr) in enumerate(addrs): + if not addr_name and addr: + # only one string! Is it the address or the address name ? + # use the same for both and see later + addr_name=addr + + if is_usascii(addr): + # address must be ascii only and must match address regex + if not email_address_re.match(addr): + addr='' + else: + addr='' + addrs[i]=(decode_mail_header(addr_name), addr) + return addrs + +def get_filename(part): + """ + Find the filename of a mail part. Many MUA send attachments with the + filename in the I{name} parameter of the I{Content-type} header instead + of in the I{filename} parameter of the I{Content-Disposition} header. + + @type part: inherit from email.mime.base.MIMEBase + @param part: the mail part + @rtype: None or unicode + @returns: the filename or None if not found + + >>> import email.mime.image + >>> attach=email.mime.image.MIMEImage('data', 'png') + >>> attach.add_header('Content-Disposition', 'attachment', filename='image.png') + >>> get_filename(attach) + u'image.png' + >>> print attach.as_string(unixfrom=False) + Content-Type: image/png + MIME-Version: 1.0 + Content-Transfer-Encoding: base64 + Content-Disposition: attachment; filename="image.png" + + ZGF0YQ== + >>> import email.mime.text + >>> attach=email.mime.text.MIMEText('The text.', 'plain', 'us-ascii') + >>> attach.add_header('Content-Disposition', 'attachment', filename=('iso-8859-1', 'fr', u'Fran\\xe7ais.txt'.encode('iso-8859-1'))) + >>> get_filename(attach) + u'Fran\\xe7ais.txt' + >>> print attach.as_string(unixfrom=False) + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: attachment; filename*="iso-8859-1'fr'Fran%E7ais.txt" + + The text. + """ + filename=part.get_param('filename', None, 'content-disposition') + if not filename: + filename=part.get_param('name', None) # default is 'content-type' + + if filename: + if isinstance(filename, tuple): + # RFC 2231 must be used to encode parameters inside MIME header + filename=email.utils.collapse_rfc2231_value(filename).strip() + else: + # But a lot of MUA erroneously use RFC 2047 instead of RFC 2231 + # in fact anybody missuse RFC2047 here !!! + filename=decode_mail_header(filename) + + return filename + +def _search_message_content(contents, part): + """ + recursive search of message content (text or HTML) inside + the structure of the email. Used by L{search_message_content()} + + @type contents: dict + @param contents: contents already found in parents or brothers I{parts}. + The dictionary will be completed as and when. key is the MIME type of the part. + @type part: inherit email.mime.base.MIMEBase + @param part: the part of the mail to look inside recursively. + """ + type=part.get_content_type() + if part.is_multipart(): # type.startswith('multipart/'): + # explore only True 'multipart/*' + # because 'messages/rfc822' are 'multipart/*' too but + # must not be explored here + if type=='multipart/related': + # the first part or the one pointed by start + start=part.get_param('start', None) + related_type=part.get_param('type', None) + for i, subpart in enumerate(part.get_payload()): + if (not start and i==0) or (start and start==subpart.get('Content-Id')): + _search_message_content(contents, subpart) + return + elif type=='multipart/alternative': + # all parts are candidates and latest is the best + for subpart in part.get_payload(): + _search_message_content(contents, subpart) + elif type in ('multipart/report', 'multipart/signed'): + # only the first part is candidate + try: + subpart=part.get_payload()[0] + except IndexError: + return + else: + _search_message_content(contents, subpart) + return + + elif type=='multipart/encrypted': + # the second part is the good one, but we need to de-crypt it + # using the first part. Do nothing + return + + else: + # unknown types must be handled as 'multipart/mixed' + # This is the peace of code that could probably be improved, + # I use a heuristic : if not already found, use first valid non + # 'attachment' parts found + for subpart in part.get_payload(): + tmp_contents=dict() + _search_message_content(tmp_contents, subpart) + for k, v in tmp_contents.items(): + if not subpart.get_param('attachment', None, 'content-disposition')=='': + # if not an attachment, initiate value if not already found + contents.setdefault(k, v) + return + else: + contents[part.get_content_type().lower()]=part + return + + return + +def search_message_content(mail): + """ + search of message content (text or HTML) inside + the structure of the mail. This function is used by L{get_mail_parts()} + to set the C{is_body} part of the L{MailPart}s + + @type mail: inherit from email.message.Message + @param mail: the message to search in. + @rtype: dict + @returns: a dictionary of the form C{{'text/plain': text_part, 'text/html': html_part}} + where text_part and html_part inherite from C{email.mime.text.MIMEText} + and are respectively the I{text} and I{HTML} version of the message content. + One part can be missing. The dictionay can aven be empty if none of the + parts math the requirements to be considered as the content. + """ + contents=dict() + _search_message_content(contents, mail) + return contents + +def get_mail_parts(msg): + """ + return a list of all parts of the message as a list of L{MailPart}. + Retrieve parts attributes to fill in L{MailPart} object. + + @type msg: inherit email.message.Message + @param msg: the message + @rtype: list + @returns: list of mail parts + + >>> import email.mime.multipart + >>> msg=email.mime.multipart.MIMEMultipart(boundary='===limit1==') + >>> import email.mime.text + >>> txt=email.mime.text.MIMEText('The text.', 'plain', 'us-ascii') + >>> msg.attach(txt) + >>> import email.mime.image + >>> image=email.mime.image.MIMEImage('data', 'png') + >>> image.add_header('Content-Disposition', 'attachment', filename='image.png') + >>> msg.attach(image) + >>> print msg.as_string(unixfrom=False) + Content-Type: multipart/mixed; boundary="===limit1==" + MIME-Version: 1.0 + + --===limit1== + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + + The text. + --===limit1== + Content-Type: image/png + MIME-Version: 1.0 + Content-Transfer-Encoding: base64 + Content-Disposition: attachment; filename="image.png" + + ZGF0YQ== + --===limit1==-- + >>> parts=get_mail_parts(msg) + >>> parts + [MailPart<*text/plain charset=us-ascii len=9>, MailPart] + >>> # the star "*" means this is the mail content, not an attachment + >>> parts[0].get_payload().decode(parts[0].charset) + u'The text.' + >>> parts[1].filename, len(parts[1].get_payload()) + (u'image.png', 4) + + """ + mailparts=[] + + # retrieve messages of the email + contents=search_message_content(msg) + # reverse contents dict + parts=dict((v,k) for k, v in contents.items()) + + # organize the stack to handle deep first search + stack=[ msg, ] + while stack: + part=stack.pop(0) + type=part.get_content_type() + if type.startswith('message/'): + # ('message/delivery-status', 'message/rfc822', 'message/disposition-notification'): + # I don't want to explore the tree deeper her and just save source using msg.as_string() + # but I don't use msg.as_string() because I want to use mangle_from_=False + filename='message.eml' + mailparts.append(MailPart(part, filename=filename, type=type, charset=part.get_param('charset'), description=part.get('Content-Description'))) + elif part.is_multipart(): + # insert new parts at the beginning of the stack (deep first search) + stack[:0]=part.get_payload() + else: + charset=part.get_param('charset') + filename=get_filename(part) + + disposition=None + if part.get_param('inline', None, 'content-disposition')=='': + disposition='inline' + elif part.get_param('attachment', None, 'content-disposition')=='': + disposition='attachment' + + mailparts.append(MailPart(part, filename=filename, type=type, charset=charset, content_id=part.get('Content-Id'), description=part.get('Content-Description'), disposition=disposition, is_body=parts.get(part, False))) + + return mailparts + + +def decode_text(payload, charset, default_charset): + """ + Try to decode text content by trying multiple charset until success. + First try I{charset}, else try I{default_charset} finally + try popular charsets in order : ascii, utf-8, utf-16, windows-1252, cp850 + If all fail then use I{default_charset} and replace wrong characters + + @type payload: str + @param payload: the content to decode + @type charset: str or None + @param charset: the first charset to try if != C{None} + @type default_charset: str or None + @param default_charset: the second charset to try if != C{None} + + @rtype: tuple + @returns: a tuple of the form C{(payload, charset)} + - I{payload}: this is the decoded payload if charset is not None and + payload is a unicode string + - I{charset}: the charset that was used to decode I{payload} If charset is + C{None} then something goes wrong: if I{payload} is unicode then + invalid characters have been replaced and the used charset is I{default_charset} + else, if I{payload} is still byte string then nothing has been done. + + + """ + for chset in [ charset, default_charset, 'ascii', 'utf-8', 'utf-16', 'windows-1252', 'cp850' ]: + if chset: + try: + return payload.decode(chset), chset + except UnicodeError: + pass + + if default_charset: + return payload.decode(chset, 'replace'), None + + return payload, None + +class PyzMessage(email.message.Message): + """ + Inherit from email.message.Message. Combine L{get_mail_parts()}, + L{get_mail_addresses()} and L{decode_mail_header()} into a + B{convenient} object to access mail contents and attributes. + This class also B{sanitize} part filenames. + + @type mailparts: list of L{MailPart} + @ivar mailparts: list of L{MailPart} objects composing the email, I{text_part} + and I{html_part} are part of this list as are other attachements and embedded + contents. + @type text_part: L{MailPart} or None + @ivar text_part: the L{MailPart} object that contains the I{text} + version of the message, None if the mail has not I{text} content. + @type html_part: L{MailPart} or None + @ivar html_part: the L{MailPart} object that contains the I{HTML} + version of the message, None if the mail has not I{HTML} content. + + @note: Sample: + + >>> raw='''Content-Type: text/plain; charset="us-ascii" + ... MIME-Version: 1.0 + ... Content-Transfer-Encoding: 7bit + ... Subject: The subject + ... From: Me + ... To: A , B + ... + ... The text. + ... ''' + >>> msg=PyzMessage.factory(raw) + >>> print 'Subject: %r' % (msg.get_subject(), ) + Subject: u'The subject' + >>> print 'From: %r' % (msg.get_address('from'), ) + From: (u'Me', 'me@foo.com') + >>> print 'To: %r' % (msg.get_addresses('to'), ) + To: [(u'A', 'a@foo.com'), (u'B', 'b@foo.com')] + >>> print 'Cc: %r' % (msg.get_addresses('cc'), ) + Cc: [] + >>> for mailpart in msg.mailparts: + ... print ' %sfilename=%r sanitized_filename=%r type=%s charset=%s desc=%s size=%d' % ('*'if mailpart.is_body else ' ', mailpart.filename, mailpart.sanitized_filename, mailpart.type, mailpart.charset, mailpart.part.get('Content-Description'), 0 if mailpart.get_payload()==None else len(mailpart.get_payload())) + ... if mailpart.is_body=='text/plain': + ... payload, used_charset=decode_text(mailpart.get_payload(), mailpart.charset, None) + ... print ' >', payload.split('\\n')[0] + ... + *filename=None sanitized_filename='text.txt' type=text/plain charset=us-ascii desc=None size=10 + > The text. + """ + + @staticmethod + def smart_parser(input): + """ + Use the appropriate parser and return a email.message.Message object + (this is not a L{PyzMessage} object) + + @type input: string, file, bytes, binary_file or email.message.Message + @param input: the source of the message + @rtype: email.message.Message + @returns: the message + """ + if isinstance(input, email.message.Message): + return input + + if sys.version_info<(3, 0): + # python 2.x + if isinstance(input, str): + return email.message_from_string(input) + elif hasattr(input, 'read') and hasattr(input, 'readline'): + return email.message_from_file(input) + else: + raise ValueError('input must be a string, a file or a Message') + else: + # python 3.x + if isinstance(input, str): + return email.message_from_string(input) + elif isinstance(input, bytes): + # python >= 3.2 only + return email.message_from_bytes(input) + elif hasattr(input, 'read') and hasattr(input, 'readline'): + if hasattr(input, 'encoding'): + # python >= 3.2 only + return email.message_from_file(input) + else: + return email.message_from_binary_file(input) + else: + raise ValueError('input must be a string a bytes, a file or a Message') + + @staticmethod + def factory(input): + """ + Use the appropriate parser and return a L{PyzMessage} object + see L{smart_parser} + @type input: string, file, bytes, binary_file or email.message.Message + @param input: the source of the message + @rtype: L{PyzMessage} + @returns: the L{PyzMessage} message + """ + return PyzMessage(PyzMessage.smart_parser(input)) + + + def __init__(self, message): + """ + Initialize the object with data coming from I{message}. + + @type message: inherit email.message.Message + @param message: The message + """ + if not isinstance(message, email.message.Message): + raise ValueError("message must inherit from email.message.Message use PyzMessage.factory() instead") + self.__dict__.update(message.__dict__) + + self.mailparts=get_mail_parts(self) + self.text_part=None + self.html_part=None + + filenames=[] + for part in self.mailparts: + ext=mimetypes.guess_extension(part.type) + if not ext: + # default to .bin + ext='.bin' + elif ext=='.ksh': + # guess_extension() is not very accurate, .txt is more + # appropriate than .ksh + ext='.txt' + + sanitized_filename=sanitize_filename(part.filename, part.type.split('/', 1)[0], ext) + sanitized_filename=handle_filename_collision(sanitized_filename, filenames) + filenames.append(sanitized_filename.lower()) + part.sanitized_filename=sanitized_filename + + if part.is_body=='text/plain': + self.text_part=part + + if part.is_body=='text/html': + self.html_part=part + + def get_addresses(self, name): + """ + return the I{name} header value as an list of addresses tuple as + returned by L{get_mail_addresses()} + + @type name: str + @param name: the name of the header to read value from: 'to', 'cc' are + valid I{name} here. + @rtype: tuple + @returns: a tuple of the form C{('Sender Name', 'sender.address@domain.com')} + or C{('', '')} if no header match that I{name}. + """ + return get_mail_addresses(self, name) + + def get_address(self, name): + """ + return the I{name} header value as an address tuple as returned by + L{get_mail_addresses()} + + @type name: str + @param name: the name of the header to read value from: : C{'from'} can + be used to return the sender address. + @rtype: list of tuple + @returns: a list of tuple of the form C{[('Recipient Name', 'recipient.address@domain.com'), ...]} + or an empty list if no header match that I{name}. + """ + value=get_mail_addresses(self, name) + if value: + return value[0] + else: + return ('', '') + + def get_subject(self, default=''): + """ + return the RFC2047 decoded subject. + + @type default: any + @param default: The value to return if the message has no I{Subject} + @rtype: unicode + @returns: the subject or C{default} + """ + return self.get_decoded_header('subject', default) + + def get_decoded_header(self, name, default=''): + """ + return decoded header I{name} using RFC2047. Always use this function + to access header, because any header can contain invalid characters + and this function sanitize the string and avoid unicode exception later + in your program. + EVEN for date, I already saw a "Center box bar horizontal" instead + of a minus character. + + @type name: str + @param name: the name of the header to read value from. + @type default: any + @param default: The value to return if the I{name} field don't exist + in this message. + @rtype: unicode + @returns: the value of the header having that I{name} or C{default} if no + header have that name. + """ + value=self.get(name) + if value==None: + value=default + else: + value=decode_mail_header(value) + return value + +class PzMessage(PyzMessage): + """ + Old name and interface for PyzMessage. + B{Deprecated} + """ + + def __init__(self, input): + """ + Initialize the object with data coming from I{input}. + + @type input: str or file or email.message.Message + @param input: used as the raw content for the email, can be a string, + a file object or an email.message.Message object. + """ + PyzMessage.__init__(self, self.smart_parser(input)) + + +def message_from_string(s, *args, **kws): + """ + Parse a string into a L{PyzMessage} object model. + @type s: str + @param s: the input string + @rtype: L{PyzMessage} + @return: the L{PyzMessage} object + """ + return PyzMessage(email.message_from_string(s, *args, **kws)) + +def message_from_file(fp, *args, **kws): + """ + Read a file and parse its contents into a L{PyzMessage} object model. + @type fp: text_file + @param fp: the input file (must be open in text mode if Python >= 3.0) + @rtype: L{PyzMessage} + @return: the L{PyzMessage} object + """ + return PyzMessage(email.message_from_file(fp, *args, **kws)) + +def message_from_bytes(s, *args, **kws): + """ + Parse a bytes string into a L{PyzMessage} object model. + B{(Python >= 3.2)} + @type s: bytes + @param s: the input bytes string + @rtype: L{PyzMessage} + @return: the L{PyzMessage} object + """ + return PyzMessage(email.message_from_bytes(s, *args, **kws)) + +def message_from_binary_file(fp, *args, **kws): + """ + Read a binary file and parse its contents into a L{PyzMessage} object model. + B{(Python >= 3.2)} + @type fp: binary_file + @param fp: the input file, must be open in binary mode + @rtype: L{PyzMessage} + @return: the L{PyzMessage} object + """ + return PyzMessage(email.message_from_binary_file(fp, *args, **kws)) + + +if __name__ == "__main__": + import sys + + if len(sys.argv)<=1: + print('usage : %s filename' % sys.argv[0]) + print('read an email from file and display a resume of its content') + sys.exit(1) + + msg=PyzMessage.factory(open(sys.argv[1], 'rb')) + + print('Subject: %r' % (msg.get_subject(), )) + print('From: %r' % (msg.get_address('from'), )) + print('To: %r' % (msg.get_addresses('to'), )) + print('Cc: %r' % (msg.get_addresses('cc'), )) + print('Date: %r' % (msg.get_decoded_header('date', ''), )) + print('Message-Id: %r' % (msg.get_decoded_header('message-id', ''), )) + + for mailpart in msg.mailparts: + # dont forget to be careful to sanitize 'filename' and be carefull + # for filename collision, to before to save : + print(' %sfilename=%r type=%s charset=%s desc=%s size=%d' % ('*'if mailpart.is_body else ' ', mailpart.filename, mailpart.type, mailpart.charset, mailpart.part.get('Content-Description'), 0 if mailpart.get_payload()==None else len(mailpart.get_payload()))) + + if mailpart.is_body=='text/plain': + # print first 3 lines + payload, used_charset=decode_text(mailpart.get_payload(), mailpart.charset, None) + for line in payload.split('\n')[:3]: + # be careful console can be unable to display unicode characters + if line: + print(' >', line) + + + diff --git a/form_utils/__init__.py b/pyzmail/tests/__init__.py similarity index 100% rename from form_utils/__init__.py rename to pyzmail/tests/__init__.py diff --git a/pyzmail/tests/test_both.py b/pyzmail/tests/test_both.py new file mode 100644 index 000000000..2607ed8d7 --- /dev/null +++ b/pyzmail/tests/test_both.py @@ -0,0 +1,99 @@ +import unittest +import pyzmail +from pyzmail.generate import * +from pyzmail.parse import * + +class TestBoth(unittest.TestCase): + + def setUp(self): + pass + + def test_compose_and_parse(self): + """test generate and parse""" + + sender=('Me', 'me@foo.com') + recipients=[('Him', 'him@bar.com'), 'just@me.com'] + subject='Le sujet en Fran\xe7ais' + text_content='Bonjour aux Fran\xe7ais' + prefered_encoding='iso-8859-1' + text_encoding='iso-8859-1' + attachments=[('attached content', 'text', 'plain', 'textfile1.txt', 'us-ascii'), + ('Fran\xe7ais', 'text', 'plain', 'textfile2.txt', 'iso-8859-1'), + ('Fran\xe7ais', 'text', 'plain', 'textfile3.txt', 'iso-8859-1'), + (b'image', 'image', 'jpg', 'imagefile.jpg', None), + ] + embeddeds=[('embedded content', 'text', 'plain', 'embedded', 'us-ascii'), + (b'picture', 'image', 'png', 'picture', None), + ] + headers=[ ('X-extra', 'extra value'), ('X-extra2', "Seconde ent\xe8te"), ('X-extra3', 'last extra'),] + + message_id_string='pyzmail' + date=1313558269 + + payload, mail_from, rcpt_to, msg_id=pyzmail.compose_mail(\ + sender, \ + recipients, \ + subject, \ + prefered_encoding, \ + (text_content, text_encoding), \ + html=None, \ + attachments=attachments, \ + embeddeds=embeddeds, \ + headers=headers, \ + message_id_string=message_id_string, \ + date=date\ + ) + + msg=PyzMessage.factory(payload) + + self.assertEqual(sender, msg.get_address('from')) + self.assertEqual(recipients[0], msg.get_addresses('to')[0]) + self.assertEqual(recipients[1], msg.get_addresses('to')[1][1]) + self.assertEqual(subject, msg.get_subject()) + self.assertEqual(subject, msg.get_decoded_header('subject')) + + # try to handle different timezone carefully + mail_date=list(email.utils.parsedate(msg.get_decoded_header('date'))) + self.assertEqual(mail_date[:6], list(time.localtime(date))[:6]) + + self.assertNotEqual(msg.get('message-id').find(message_id_string), -1) + for name, value in headers: + self.assertEqual(value, msg.get_decoded_header(name)) + + for mailpart in msg.mailparts: + if mailpart.is_body: + self.assertEqual(mailpart.content_id, None) + self.assertEqual(mailpart.filename, None) + self.assertEqual(type(mailpart.sanitized_filename), str) + if mailpart.type=='text/plain': + self.assertEqual(mailpart.get_payload(), text_content.encode(text_encoding)) + else: + self.fail('found unknown body part') + else: + if mailpart.filename: + lst=attachments + self.assertEqual(mailpart.filename, mailpart.sanitized_filename) + self.assertEqual(mailpart.content_id, None) + elif mailpart.content_id: + lst=embeddeds + self.assertEqual(mailpart.filename, None) + else: + self.fail('found unknown part') + + found=False + for attach in lst: + found=(mailpart.filename and attach[3]==mailpart.filename) \ + or (mailpart.content_id and attach[3]==mailpart.content_id) + if found: + break + + if found: + self.assertEqual(mailpart.type, attach[1]+'/'+attach[2]) + payload=mailpart.get_payload() + if attach[1]=='text' and attach[4] and isinstance(attach[0], str): + payload=payload.decode(attach[4]) + self.assertEqual(payload, attach[0]) + else: + self.fail('found unknown attachment') + + diff --git a/pyzmail/tests/test_generate.py b/pyzmail/tests/test_generate.py new file mode 100644 index 000000000..7afdebc59 --- /dev/null +++ b/pyzmail/tests/test_generate.py @@ -0,0 +1,30 @@ +import unittest, doctest +import pyzmail +from pyzmail.generate import * + +class TestGenerate(unittest.TestCase): + + def setUp(self): + pass + + def test_format_addresses(self): + """test format_addresse""" + self.assertEqual('foo@example.com', str(format_addresses([ 'foo@example.com', ]))) + self.assertEqual('Foo ', str(format_addresses([ ('Foo', 'foo@example.com'), ]))) + # notice the space around the comma + self.assertEqual('foo@example.com , bar@example.com', str(format_addresses([ 'foo@example.com', 'bar@example.com']))) + # notice the space around the comma + self.assertEqual('Foo , Bar ', str(format_addresses([ ('Foo', 'foo@example.com'), ( 'Bar', 'bar@example.com')]))) + +# Add doctest +def load_tests(loader, tests, ignore): + # this works with python 2.7 and 3.x + tests.addTests(doctest.DocTestSuite(pyzmail.generate)) + return tests + +def additional_tests(): + # Add doctest for python 2.6 and below + if sys.version_info<(2, 7): + return doctest.DocTestSuite(pyzmail.generate) + else: + return unittest.TestSuite() diff --git a/pyzmail/tests/test_parse.py b/pyzmail/tests/test_parse.py new file mode 100644 index 000000000..f7a5adb9e --- /dev/null +++ b/pyzmail/tests/test_parse.py @@ -0,0 +1,290 @@ +import unittest, doctest +import pyzmail +from pyzmail.parse import * + + +class Msg: + """mimic a email.Message""" + def __init__(self, value): + self.value=value + + def get_all(self, header_name, default): + if self.value: + return [self.value, ] + else: + return [] + +class TestParse(unittest.TestCase): + + def setUp(self): + pass + + def test_decode_mail_header(self): + """test decode_mail_header()""" + self.assertEqual(decode_mail_header(''), '') + self.assertEqual(decode_mail_header('hello'), 'hello') + self.assertEqual(decode_mail_header('hello '), 'hello ') + self.assertEqual(decode_mail_header('=?iso-8859-1?q?Courrier_=E8lectronique_Fran=E7ais?='), 'Courrier \xe8lectronique Fran\xe7ais') + self.assertEqual(decode_mail_header('=?utf8?q?Courrier_=C3=A8lectronique_Fran=C3=A7ais?='), 'Courrier \xe8lectronique Fran\xe7ais') + self.assertEqual(decode_mail_header('=?utf-8?b?RnJhbsOnYWlz?='), 'Fran\xe7ais') + self.assertEqual(decode_mail_header('=?iso-8859-1?q?Courrier_=E8lectronique_?= =?utf8?q?Fran=C3=A7ais?='), 'Courrier \xe8lectronique Fran\xe7ais') + self.assertEqual(decode_mail_header('=?iso-8859-1?q?Courrier_=E8lectronique_?= =?utf-8?b?RnJhbsOnYWlz?='), 'Courrier \xe8lectronique Fran\xe7ais') + self.assertEqual(decode_mail_header('h_subject_q_iso_8858_1 : =?ISO-8859-1?Q?Fran=E7ais=E20accentu=E9?= !'), 'h_subject_q_iso_8858_1 :Fran\xe7ais\xe20accentu\xe9!') + + def test_get_mail_addresses(self): + """test get_mail_addresses()""" + self.assertEqual([ ('foo@example.com', 'foo@example.com') ], get_mail_addresses(Msg('foo@example.com'), 'to')) + self.assertEqual([ ('Foo', 'foo@example.com'), ], get_mail_addresses(Msg('Foo '), 'to')) + # notice the space around the comma + self.assertEqual([ ('foo@example.com', 'foo@example.com'), ('bar@example.com', 'bar@example.com')], get_mail_addresses(Msg('foo@example.com , bar@example.com'), 'to')) + self.assertEqual([ ('Foo', 'foo@example.com'), ( 'Bar', 'bar@example.com')], get_mail_addresses(Msg('Foo , Bar '), 'to')) + self.assertEqual([ ('Foo', 'foo@example.com'), ('bar@example.com', 'bar@example.com')], get_mail_addresses(Msg('Foo , bar@example.com'), 'to')) + self.assertEqual([ ('Mr Foo', 'foo@example.com'), ('bar@example.com', 'bar@example.com')], get_mail_addresses(Msg('Mr\nFoo , bar@example.com'), 'to')) + + self.assertEqual([ ('Beno\xeet', 'benoit@example.com')], get_mail_addresses(Msg('=?utf-8?q?Beno=C3=AEt?= '), 'to')) + + # address already encoded into utf8 (bad) + address='Ant\xf3nio Foo '.encode('utf8') + if sys.version_info<(3, 0): + self.assertEqual([('Ant\ufffd\ufffdnio Foo', 'a.foo@example.com')], get_mail_addresses(Msg(address), 'to')) + else: + # Python 3.2 return header when surrogate characters are used in header + self.assertEqual([('Ant??nio Foo', 'a.foo@example.com'), ], get_mail_addresses(Msg(email.header.Header(address, charset=email.charset.UNKNOWN8BIT, header_name='to')), 'to')) + + def test_get_filename(self): + """test get_filename()""" + import email.mime.image + + filename='Fran\xe7ais.png' + if sys.version_info<(3, 0): + encoded_filename=filename.encode('iso-8859-1') + else: + encoded_filename=filename + + payload=b'data' + attach=email.mime.image.MIMEImage(payload, 'png') + attach.add_header('Content-Disposition', 'attachment', filename='image.png') + self.assertEqual('image.png', get_filename(attach)) + + attach=email.mime.image.MIMEImage(payload, 'png') + attach.add_header('Content-Disposition', 'attachment', filename=('iso-8859-1', 'fr', encoded_filename)) + self.assertEqual('Fran\xe7ais.png', get_filename(attach)) + + attach=email.mime.image.MIMEImage(payload, 'png') + attach.set_param('name', 'image.png') + self.assertEqual('image.png', get_filename(attach)) + + attach=email.mime.image.MIMEImage(payload, 'png') + attach.set_param('name', ('iso-8859-1', 'fr', encoded_filename)) + self.assertEqual('Fran\xe7ais.png', get_filename(attach)) + + attach=email.mime.image.MIMEImage(payload, 'png') + attach.add_header('Content-Disposition', 'attachment', filename='image.png') + attach.set_param('name', 'image_wrong.png') + self.assertEqual('image.png', get_filename(attach)) + + def test_get_mailparts(self): + """test get_mailparts()""" + import email.mime.multipart + import email.mime.text + import email.mime.image + msg=email.mime.multipart.MIMEMultipart(boundary='===limit1==') + txt=email.mime.text.MIMEText('The text.', 'plain', 'us-ascii') + msg.attach(txt) + image=email.mime.image.MIMEImage(b'data', 'png') + image.add_header('Content-Disposition', 'attachment', filename='image.png') + image.add_header('Content-Description', 'the description') + image.add_header('Content-ID', '') + msg.attach(image) + + raw=msg.as_string(unixfrom=False) + expected_raw="""Content-Type: multipart/mixed; boundary="===limit1==" +MIME-Version: 1.0 + +--===limit1== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +The text. +--===limit1== +Content-Type: image/png +MIME-Version: 1.0 +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; filename="image.png" +Content-Description: the description +Content-ID: + +ZGF0YQ== +--===limit1==--""" + + if sys.version_info<(3, 0): + expected_raw=expected_raw.replace('','') + else: + expected_raw=expected_raw.replace('','\n') + + self.assertEqual(raw, expected_raw) + + parts=get_mail_parts(msg) + # [MailPart<*text/plain charset=us-ascii len=9>, MailPart] + + self.assertEqual(len(parts), 2) + + self.assertEqual(parts[0].type, 'text/plain') + self.assertEqual(parts[0].is_body, 'text/plain') # not a error, is_body must be type + self.assertEqual(parts[0].charset, 'us-ascii') + self.assertEqual(parts[0].get_payload().decode(parts[0].charset), 'The text.') + + self.assertEqual(parts[1].type, 'image/png') + self.assertEqual(parts[1].is_body, False) + self.assertEqual(parts[1].charset, None) + self.assertEqual(parts[1].filename, 'image.png') + self.assertEqual(parts[1].description, 'the description') + self.assertEqual(parts[1].content_id, 'this.is.the.normaly.unique.contentid') + self.assertEqual(parts[1].get_payload(), b'data') + + + raw_1='''Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Subject: simple test +From: Me +To: A , B +Cc: C , d@foo.com +User-Agent: pyzmail + +The text. +''' + + def check_message_1(self, msg): + self.assertEqual(msg.get_subject(), 'simple test') + self.assertEqual(msg.get_decoded_header('subject'), 'simple test') + self.assertEqual(msg.get_decoded_header('User-Agent'), 'pyzmail') + self.assertEqual(msg.get('User-Agent'), 'pyzmail') + self.assertEqual(msg.get_address('from'), ('Me', 'me@foo.com')) + self.assertEqual(msg.get_addresses('to'), [('A', 'a@foo.com'), ('B', 'b@foo.com')]) + self.assertEqual(msg.get_addresses('cc'), [('C', 'c@foo.com'), ('d@foo.com', 'd@foo.com')]) + self.assertEqual(len(msg.mailparts), 1) + self.assertEqual(msg.text_part, msg.mailparts[0]) + self.assertEqual(msg.html_part, None) + + # use 8bits encoding and 2 different charsets ! python 3.0 & 3.1 are not eable to parse this sample + raw_2=b"""From: sender@domain.com +To: recipient@domain.com +Date: Tue, 7 Jun 2011 16:32:17 +0200 +Subject: contains 8bits attachments using different encoding +Content-Type: multipart/mixed; boundary=mixed + +--mixed +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +body +--mixed +Content-Type: text/plain; charset="windows-1252" +MIME-Version: 1.0 +Content-Transfer-Encoding: 8bit +Content-Disposition: attachment; filename="file1.txt" + +bo\xeete mail = mailbox +--mixed +Content-Type: text/plain; charset="utf-8" +MIME-Version: 1.0 +Content-Transfer-Encoding: 8bit +Content-Disposition: attachment; filename="file2.txt" + +bo\xc3\xaete mail = mailbox +--mixed-- +""" + + def check_message_2(self, msg): + self.assertEqual(msg.get_subject(), 'contains 8bits attachments using different encoding') + + body, file1, file2=msg.mailparts + + self.assertEqual('file1.txt', file1.filename) + self.assertEqual('file2.txt', file2.filename) + self.assertEqual('windows-1252', file1.charset) + self.assertEqual('utf-8', file2.charset) + content=b'bo\xeete mail = mailbox'.decode("windows-1252") + content1=file1.get_payload().decode(file1.charset) + content2=file2.get_payload().decode(file2.charset) + self.assertEqual(content, content1) + self.assertEqual(content, content2) + + # this one contain non us-ascii chars in the header + # py 2x and py3k return different value here + raw_3=b'Content-Type: text/plain; charset="us-ascii"\n' \ + b'MIME-Version: 1.0\n' \ + b'Content-Transfer-Encoding: 7bit\n' \ + + 'Subject: Beno\xeet & Ant\xf3nio\n'.encode('utf8') +\ + b'From: =?utf-8?q?Beno=C3=AEt?= \n' \ + + 'To: Ant\xf3nio Foo \n'.encode('utf8') \ + + 'Cc: Beno\xeet , d@foo.com\n'.encode('utf8') +\ + b'User-Agent: pyzmail\n' \ + b'\n' \ + b'The text.\n' + + def check_message_3(self, msg): + subject='Beno\ufffd\ufffdt & Ant\ufffd\ufffdnio' # if sys.version_info<(3, 0) else u'Beno??t & Ant??nio' + self.assertEqual(msg.get_subject(), subject) + self.assertEqual(msg.get_decoded_header('subject'), subject) + self.assertEqual(msg.get_decoded_header('User-Agent'), 'pyzmail') + self.assertEqual(msg.get('User-Agent'), 'pyzmail') + self.assertEqual(msg.get_address('from'), ('Beno\xeet', 'benoit@example.com')) + + to=msg.get_addresses('to') + self.assertEqual(to[0][1], 'a.foo@example.com') + self.assertEqual(to[0][0], 'Ant\ufffd\ufffdnio Foo' if sys.version_info<(3, 0) else 'Ant??nio Foo') + + cc=msg.get_addresses('cc') + self.assertEqual(cc[0][1], 'benoit@foo.com') + self.assertEqual(cc[0][0], 'Beno\ufffd\ufffdt' if sys.version_info<(3, 0) else 'Beno??t') + self.assertEqual(cc[1], ('d@foo.com', 'd@foo.com')) + + self.assertEqual(len(msg.mailparts), 1) + self.assertEqual(msg.text_part, msg.mailparts[0]) + self.assertEqual(msg.html_part, None) + + + def check_pyzmessage_factories(self, input, check): + """test PyzMessage from different sources""" + if isinstance(input, bytes) and sys.version_info>=(3, 2): + check(PyzMessage.factory(input)) + check(message_from_bytes(input)) + + import io + check(PyzMessage.factory(io.BytesIO(input))) + check(message_from_binary_file(io.BytesIO(input))) + + if isinstance(input, str): + + check(PyzMessage.factory(input)) + check(message_from_string(input)) + + import io + check(PyzMessage.factory(io.StringIO(input))) + check(message_from_file(io.StringIO(input))) + + def test_pyzmessage_factories(self): + """test PyzMessage class different sources""" + self.check_pyzmessage_factories(self.raw_1, self.check_message_1) + self.check_pyzmessage_factories(self.raw_2, self.check_message_2) + self.check_pyzmessage_factories(self.raw_3, self.check_message_3) + + +# Add doctest +def load_tests(loader, tests, ignore): + # this works with python 2.7 and 3.x + if sys.version_info<(3, 0): + tests.addTests(doctest.DocTestSuite(pyzmail.parse)) + return tests + +def additional_tests(): + # Add doctest for python 2.6 and below + if sys.version_info<(2, 7): + return doctest.DocTestSuite(pyzmail.parse) + else: + return unittest.TestSuite() + diff --git a/pyzmail/tests/test_send.py b/pyzmail/tests/test_send.py new file mode 100644 index 000000000..554f8549a --- /dev/null +++ b/pyzmail/tests/test_send.py @@ -0,0 +1,77 @@ +import threading, smtpd, asyncore, socket, smtplib, time +import unittest +import pyzmail +from pyzmail.generate import * + + +smtpd_addr='127.0.0.1' +smtpd_port=32525 +smtp_bad_port=smtpd_port-1 + +smtp_mode='normal' +smtp_login=None +smtp_password=None + + +class SMTPServer(smtpd.SMTPServer): + def __init__(self, localaddr, remoteaddr, received): + smtpd.SMTPServer.__init__(self, localaddr, remoteaddr) + self.set_reuse_addr() + # put the received mail into received list + self.received=received + + def process_message(self, peer, mail_from, rcpt_to, data): + ret=None + if mail_from.startswith('data_error'): + ret='552 Requested mail action aborted: exceeded storage allocation' + self.received.append((ret, peer, mail_from, rcpt_to, data)) + return ret + +class TestSend(unittest.TestCase): + + def setUp(self): + self.received=[] + self.smtp_server=SMTPServer((smtpd_addr, smtpd_port), None, self.received) + + def asyncloop(): + # check every sec if all channel are close + asyncore.loop(1) + + + self.payload, self.mail_from, self.rcpt_to, self.msg_id=compose_mail(('Me', 'me@foo.com'), [('Him', 'him@bar.com')], 'the subject', 'iso-8859-1', ('Hello world', 'us-ascii')) + + # start the server after having built the payload, to handle failure in + # the code above + self.smtpd_thread=threading.Thread(target=asyncloop) + self.smtpd_thread.daemon=True + self.smtpd_thread.start() + + + def tearDown(self): + self.smtp_server.close() + self.smtpd_thread.join() + + def test_simple_send(self): + """simple send""" + ret=send_mail(self.payload, self.mail_from, self.rcpt_to, smtpd_addr, smtpd_port, smtp_mode=smtp_mode, smtp_login=smtp_login, smtp_password=smtp_password) + self.assertEqual(ret, dict()) + (ret, peer, mail_from, rcpt_to, payload)=self.received[0] + self.assertEqual(self.payload, payload) + self.assertEqual(self.mail_from, mail_from) + self.assertEqual(self.rcpt_to, rcpt_to) + self.assertEqual('127.0.0.1', peer[0]) + + def test_send_to_a_wrong_port(self): + """send to a wrong port""" + self.smtp_server.close() + ret=send_mail(self.payload, self.mail_from, self.rcpt_to, smtpd_addr, smtpd_port, smtp_mode=smtp_mode, smtp_login=smtp_login, smtp_password=smtp_password) + self.assertEqual(type(ret), str) + + def test_send_data_error(self): + """smtp server return error code""" + ret=send_mail(self.payload, 'data_error@foo.com', self.rcpt_to, smtpd_addr, smtp_bad_port, smtp_mode=smtp_mode, smtp_login=smtp_login, smtp_password=smtp_password) + self.assertEqual(type(ret), str) + +if __name__ == '__main__': + unittest.main() + diff --git a/pyzmail/tests/test_utils.py b/pyzmail/tests/test_utils.py new file mode 100644 index 000000000..e03d07d0d --- /dev/null +++ b/pyzmail/tests/test_utils.py @@ -0,0 +1,24 @@ +import unittest, doctest +import pyzmail +from pyzmail.utils import * + +class TestUtils(unittest.TestCase): + + def setUp(self): + pass + + def test_nothing(self): + pass + +# Add doctest +def load_tests(loader, tests, ignore): + # this works with python 2.7 and 3.x + tests.addTests(doctest.DocTestSuite(pyzmail.utils)) + return tests + +def additional_tests(): + # Add doctest for python 2.6 and below + if sys.version_info<(2, 7): + return doctest.DocTestSuite(pyzmail.utils) + else: + return unittest.TestSuite() diff --git a/pyzmail/utils.py b/pyzmail/utils.py new file mode 100644 index 000000000..3f4ef6d5f --- /dev/null +++ b/pyzmail/utils.py @@ -0,0 +1,155 @@ +# +# pyzmail/utils.py +# (c) Alain Spineux +# http://www.magiksys.net/pyzmail +# Released under LGPL + +""" +Various functions used by other modules +@var invalid_chars_in_filename: a mix of characters not permitted in most used filesystems +@var invalid_windows_name: a list of unauthorized filenames under Windows +""" + +import sys + +invalid_chars_in_filename=b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f' \ + b'\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f' \ + b'<>:"/\\|?*\%\'' + +invalid_windows_name=[b'CON', b'PRN', b'AUX', b'NUL', b'COM1', b'COM2', b'COM3', + b'COM4', b'COM5', b'COM6', b'COM7', b'COM8', b'COM9', + b'LPT1', b'LPT2', b'LPT3', b'LPT4', b'LPT5', b'LPT6', b'LPT7', + b'LPT8', b'LPT9' ] + +def sanitize_filename(filename, alt_name, alt_ext): + """ + Convert the given filename into a name that should work on all + platform. Remove non us-ascii characters, and drop invalid filename. + Use the I{alternative} filename if needed. + + @type filename: unicode or None + @param filename: the originale filename or None. Can be unicode. + @type alt_name: str + @param alt_name: the alternative filename if filename is None or useless + @type alt_ext: str + @param alt_ext: the alternative filename extension (including the '.') + + @rtype: str + @returns: a valid filename. + + >>> sanitize_filename('document.txt', 'file', '.txt') + 'document.txt' + >>> sanitize_filename('number1.txt', 'file', '.txt') + 'number1.txt' + >>> sanitize_filename(None, 'file', '.txt') + 'file.txt' + >>> sanitize_filename(u'R\\xe9pertoir.txt', 'file', '.txt') + 'Rpertoir.txt' + >>> # the '\\xe9' has been removed + >>> sanitize_filename(u'\\xe9\\xe6.html', 'file', '.txt') + 'file.html' + >>> # all non us-ascii characters have been removed, the alternative name + >>> # has been used the replace empty string. The originale extention + >>> # is still valid + >>> sanitize_filename(u'COM1.txt', 'file', '.txt') + 'COM1A.txt' + >>> # if name match an invalid name or assimilated then a A is added + """ + + if not filename: + return alt_name+alt_ext + + if ((sys.version_info<(3, 0) and isinstance(filename, str)) or \ + (sys.version_info>=(3, 0) and isinstance(filename, str))): + filename=filename.encode('ascii', 'ignore') + + filename=filename.translate(None, invalid_chars_in_filename) + filename=filename.strip() + + upper=filename.upper() + for name in invalid_windows_name: + if upper==name: + filename=filename+b'A' + break + if upper.startswith(name+b'.'): + filename=filename[:len(name)]+b'A'+filename[len(name):] + break + + if sys.version_info>=(3, 0): + # back to string + filename=filename.decode('us-ascii') + + if filename.rfind('.')==0: + filename=alt_name+filename + + return filename + +def handle_filename_collision(filename, filenames): + """ + Avoid filename collision, add a sequence number to the name when required. + 'file.txt' will be renamed into 'file-01.txt' then 'file-02.txt' ... + until their is no more collision. The file is not added to the list. + + Windows don't make the difference between lower and upper case. To avoid + "case" collision, the function compare C{filename.lower()} to the list. + If you provide a list in lower case only, then any collisions will be avoided. + + @type filename: str + @param filename: the filename + @type filenames: list or set + @param filenames: a list of filenames. + + @rtype: str + @returns: the I{filename} or the appropriately I{indexed} I{filename} + + >>> handle_filename_collision('file.txt', [ ]) + 'file.txt' + >>> handle_filename_collision('file.txt', [ 'file.txt' ]) + 'file-01.txt' + >>> handle_filename_collision('file.txt', [ 'file.txt', 'file-01.txt',]) + 'file-02.txt' + >>> handle_filename_collision('foo', [ 'foo',]) + 'foo-01' + >>> handle_filename_collision('foo', [ 'foo', 'foo-01',]) + 'foo-02' + >>> handle_filename_collision('FOO', [ 'foo', 'foo-01',]) + 'FOO-02' + """ + if filename.lower() in filenames: + try: + basename, ext=filename.rsplit('.', 1) + ext='.'+ext + except ValueError: + basename, ext=filename, '' + + i=1 + while True: + filename='%s-%02d%s' % (basename, i, ext) + if filename.lower() not in filenames: + break + i+=1 + + return filename + +def is_usascii(value): + """" + test if string contains us-ascii characters only + + >>> is_usascii('foo') + True + >>> is_usascii(u'foo') + True + >>> is_usascii(u'Fran\xe7ais') + False + >>> is_usascii('bad\x81') + False + """ + try: + # if value is byte string, it will be decoded first using us-ascii + # and will generate UnicodeEncodeError, this is fine too + value.encode('us-ascii') + except UnicodeError: + return False + + return True + \ No newline at end of file diff --git a/pyzmail/version.py b/pyzmail/version.py new file mode 100644 index 000000000..03e1b0c6d --- /dev/null +++ b/pyzmail/version.py @@ -0,0 +1 @@ +__version__='1.0.3' diff --git a/release-coverage.json.gz b/release-coverage.json.gz index 668933033..fcef9a9e7 100644 Binary files a/release-coverage.json.gz and b/release-coverage.json.gz differ diff --git a/requirements.txt b/requirements.txt index 47c5bd3a0..1042d270f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ # -*- conf-mode -*- setuptools>=18.5 # Require this first, to prevent later errors # -anora>=0.1.2 argon2-cffi>=16.1.0 # For the Argon2 password hasher option beautifulsoup4>=4.5.0 bibtexparser>=0.6.2,<1.0 # Version 1.0 doesn't work under python 2.7. 1.0.1 doesn't recognize month names or abbreviations. @@ -15,6 +14,7 @@ django-bcrypt>=0.9.2 # for the BCrypt password hasher option. Remove when all django-bootstrap3>=8.2.1,<9.0.0 django-csp>=3.5 django-cors-headers>=2.4.0 +django-form-utils>=1.0.3 django-formtools>=1.0 # instead of django.contrib.formtools in 1.8 django-markup>=1.1 django-password-strength>=1.2.1 @@ -32,15 +32,14 @@ httplib2>=0.10.3 jsonfield>=1.0.3 # for SubmissionCheck. This is https://github.com/bradjasper/django-jsonfield/. jwcrypto>=0.4.0 # for signed notifications #lxml>=3.4.0 # from PyQuery; -mimeparse>=0.1.3 # from TastyPie mock>=2.0.0 -MySQL-python>=1.2.5 +mysqlclient>=1.3.13 oauth2client>=4.0.0 # required by google-api-python-client, but not always pulled in patch>=1.16,<2.0 pathlib>=1.0 pathlib2>=2.3.0 Pillow>=3.0 -pip==9.0.1 # Earlier pip has issues, 9.0.2 and 9.0.3, 10.0.0b1 leaves dross when down- and up-grading packages +#pip==9.0.1 # Earlier pip has issues, 9.0.2 and 9.0.3, 10.0.0b1 leaves dross when down- and up-grading packages pyang>=1.7.2,<2.0 pyflakes>=0.9.2 pyopenssl>=17.5.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency @@ -48,16 +47,17 @@ pyquery>=1.2.13,!=1.2.14 # Pyqyery 1.2.14 fails on some selectors or stacked sel python-dateutil>=2.2 python-magic>=0.4.6 python-memcached>=1.48 # for django.core.cache.backends.memcached +python-mimeparse>=1.6 # from TastyPie pytz>=2014.7 -pyzmail>=1.0.3 +#pyzmail>=1.0.3 requests!=2.12.* -rfc2html>=2.0.0 +rfc2html>=2.0.1 selenium>=2.42,<3.8.1 six>=1.9.0 sqlparse>=0.2.2 tblib>=1.3.0 tqdm>=3.7.0 -Trac>=1.0.10,<1.2 +#Trac>=1.0.10,<1.2 Unidecode>=0.4.18 #wsgiref>=0.1.2 xml2rfc>=2.9.3,!=2.6.0 diff --git a/tzparse.py b/tzparse.py index 8ba624730..ad264406b 100755 --- a/tzparse.py +++ b/tzparse.py @@ -1,3 +1,4 @@ +# Copyright The IETF Trust 2012-2019, All Rights Reserved #!/usr/bin/env python """ @@ -8,7 +9,7 @@ SYNOPSIS >>> tzparse("2008-09-08 14:40:35 +0200", "%Y-%m-%d %H:%M:%S %Z") datetime.datetime(2008, 9, 8, 14, 40, 35, tzinfo=pytz.FixedOffset(120)) - >>> print tzparse("14:40:35 CEST, 08 Sep 2008", "%H:%M:%S %Z, %d %b %Y") + >>> print(tzparse("14:40:35 CEST, 08 Sep 2008", "%H:%M:%S %Z, %d %b %Y")) 2008-09-08 14:40:35+02:00 DESCRIPTION @@ -94,31 +95,31 @@ def tzparse(string, format): Given a time specification string and a format, tzparse() returns a localized datetime.datetime. - >>> print tzparse("9 Oct 2009 CEST 13:58", "%d %b %Y %Z %H:%M") + >>> print(tzparse("9 Oct 2009 CEST 13:58", "%d %b %Y %Z %H:%M")) 2009-10-09 13:58:00+02:00 - >>> print tzparse("9 Oct 2009 13:58:00 Europe/Stockholm", "%d %b %Y %H:%M:%S %Z") + >>> print(tzparse("9 Oct 2009 13:58:00 Europe/Stockholm", "%d %b %Y %H:%M:%S %Z")) 2009-10-09 13:58:00+02:00 - >>> print tzparse("9 Oct 2009 13:58:00 +0200", "%d %b %Y %H:%M:%S %Z") + >>> print(tzparse("9 Oct 2009 13:58:00 +0200", "%d %b %Y %H:%M:%S %Z")) 2009-10-09 13:58:00+02:00 - >>> print tzparse("Fri, 9 Oct 2009 13:58:00 +0200", "%a, %d %b %Y %H:%M:%S %Z") + >>> print(tzparse("Fri, 9 Oct 2009 13:58:00 +0200", "%a, %d %b %Y %H:%M:%S %Z")) 2009-10-09 13:58:00+02:00 - >>> print tzparse("2009-10-09 13:58:00 EST", '%Y-%m-%d %H:%M:%S %Z') + >>> print(tzparse("2009-10-09 13:58:00 EST", '%Y-%m-%d %H:%M:%S %Z')) 2009-10-09 13:58:00-05:00 - >>> print tzparse("2009-10-09 13:58:00+02:00", "%Y-%m-%d %H:%M:%S%Z") + >>> print(tzparse("2009-10-09 13:58:00+02:00", "%Y-%m-%d %H:%M:%S%Z")) 2009-10-09 13:58:00+02:00 - >>> print tzparse("1985-04-12T23:20:50Z", "%Y-%m-%dT%H:%M:%S%Z") + >>> print(tzparse("1985-04-12T23:20:50Z", "%Y-%m-%dT%H:%M:%S%Z")) 1985-04-12 23:20:50+00:00 - >>> print tzparse("1996-12-19T16:39:57-08:00", "%Y-%m-%dT%H:%M:%S%Z") + >>> print(tzparse("1996-12-19T16:39:57-08:00", "%Y-%m-%dT%H:%M:%S%Z")) 1996-12-19 16:39:57-08:00 - >>> print tzparse("1996-12-19T16:39:57", "%Y-%m-%dT%H:%M:%S") + >>> print(tzparse("1996-12-19T16:39:57", "%Y-%m-%dT%H:%M:%S")) 1996-12-19 16:39:57+01:00 """ @@ -131,8 +132,8 @@ def tzparse(string, format): # from the string, too. def fmt2pat(s): - s = re.sub("%[dHIjmMSUwWyY]", "\d+", s) - s = re.sub("%[aAbBp]", "\w+", s) + s = re.sub("%[dHIjmMSUwWyY]", r"\\d+", s) + s = re.sub("%[aAbBp]", r"\\w+", s) s = re.sub("%[cxX]", ".+", s) s = s.replace("%%", "%") return s @@ -177,9 +178,9 @@ def tzparse(string, format): if __name__ == "__main__": import sys if len(sys.argv[1:]) == 2: - print tzparse(sys.argv[1], sys.argv[2]) + print(tzparse(sys.argv[1], sys.argv[2])) else: - print "Running module tests:\n" + print("Running module tests:\n") import doctest - print doctest.testmod() + print(doctest.testmod()) \ No newline at end of file