diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 75351c798..1972a3e00 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1,5 +1,6 @@ # Copyright The IETF Trust 2015-2019, All Rights Reserved +import json import os import sys diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py index 026f7c96b..fa679b25b 100644 --- a/ietf/doc/tests_ballot.py +++ b/ietf/doc/tests_ballot.py @@ -19,7 +19,7 @@ from ietf.name.models import BallotPositionName from ietf.iesg.models import TelechatDate from ietf.person.models import Person, PersonalApiKey from ietf.person.factories import PersonFactory -from ietf.utils.test_utils import TestCase, login_testing_unauthorized +from ietf.utils.test_utils import TestCase, unicontent, login_testing_unauthorized from ietf.utils.mail import outbox, empty_outbox from ietf.utils.text import unwrap diff --git a/ietf/doc/tests_charter.py b/ietf/doc/tests_charter.py index 0bad514ea..49d8c5b5e 100644 --- a/ietf/doc/tests_charter.py +++ b/ietf/doc/tests_charter.py @@ -20,7 +20,7 @@ from ietf.group.factories import RoleFactory, GroupFactory from ietf.group.models import Group, GroupMilestone from ietf.iesg.models import TelechatDate from ietf.person.models import Person -from ietf.utils.test_utils import TestCase +from ietf.utils.test_utils import TestCase, unicontent from ietf.utils.mail import outbox, empty_outbox from ietf.utils.test_utils import login_testing_unauthorized diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index fcc7118cd..c728f477b 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -25,7 +25,7 @@ from ietf.person.factories import PersonFactory from ietf.person.models import Person, Email from ietf.meeting.models import Meeting, MeetingTypeName from ietf.iesg.models import TelechatDate -from ietf.utils.test_utils import login_testing_unauthorized +from ietf.utils.test_utils import login_testing_unauthorized, unicontent from ietf.utils.mail import outbox, empty_outbox from ietf.utils.test_utils import TestCase diff --git a/ietf/doc/tests_material.py b/ietf/doc/tests_material.py index cc414d9d7..1640e38db 100644 --- a/ietf/doc/tests_material.py +++ b/ietf/doc/tests_material.py @@ -19,7 +19,7 @@ from ietf.meeting.factories import MeetingFactory from ietf.meeting.models import Meeting, Session, SessionPresentation from ietf.name.models import SessionStatusName from ietf.person.models import Person -from ietf.utils.test_utils import TestCase, login_testing_unauthorized +from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent class GroupMaterialTests(TestCase): diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index 1cf5c0510..bd9fb73ac 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -769,7 +769,7 @@ def search_mail_archive(request, name, assignment_id): try: res["messages"] = mailarch.retrieve_messages(res["query_data_url"])[:MAX_RESULTS] except KeyError as e: - res["error"] = "No results found (%s)" % str(e) + res["error"] = "No results found" except Exception as e: res["error"] = "Retrieval from mail archive failed: %s" % str(e) # raise # useful when debugging diff --git a/ietf/group/tests.py b/ietf/group/tests.py index 6b37b4f90..72fd4552f 100644 --- a/ietf/group/tests.py +++ b/ietf/group/tests.py @@ -19,7 +19,7 @@ from ietf.group.factories import GroupFactory, RoleFactory from ietf.utils.test_runner import set_coverage_checking from ietf.person.factories import EmailFactory from ietf.person.models import Person -from ietf.utils.test_utils import login_testing_unauthorized, TestCase +from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent if getattr(settings,'SKIP_DOT_TO_PDF', False): skip_dot_to_pdf = True diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py index ea8e80e89..71863a22c 100644 --- a/ietf/group/tests_review.py +++ b/ietf/group/tests_review.py @@ -8,7 +8,7 @@ from pyquery import PyQuery from django.urls import reverse as urlreverse -from ietf.utils.test_utils import login_testing_unauthorized, TestCase, reload_db_objects +from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent, reload_db_objects from ietf.doc.models import TelechatDocEvent from ietf.group.models import Role from ietf.iesg.models import TelechatDate diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index ce478b7b1..1fceb0323 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -179,7 +179,7 @@ def agenda_json(request, date=None): s["docs"].append(docinfo) - return HttpResponse(json.dumps(res, indent=2), content_type='application/json') + return HttpResponse(json.dumps(res, indent=2), content_type='text/plain') # def past_agendas(request): # # This is not particularly useful with the current way of constructing diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py index cb96b73e1..a8bd0a93a 100644 --- a/ietf/ietfauth/tests.py +++ b/ietf/ietfauth/tests.py @@ -14,7 +14,7 @@ from django.conf import settings import debug # pyflakes:ignore -from ietf.utils.test_utils import TestCase, login_testing_unauthorized +from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.utils.mail import outbox, empty_outbox from ietf.group.models import Group, Role, RoleName from ietf.group.factories import GroupFactory, RoleFactory diff --git a/ietf/ipr/mail.py b/ietf/ipr/mail.py index 0daa54f2d..809af13e8 100644 --- a/ietf/ipr/mail.py +++ b/ietf/ipr/mail.py @@ -1,5 +1,4 @@ # Copyright The IETF Trust 2014-2019, All Rights Reserved - import base64 import email import datetime @@ -7,11 +6,8 @@ from dateutil.tz import tzoffset import os import pytz import re - from django.template.loader import render_to_string -import debug # pyflakes:ignore - from ietf.ipr.models import IprEvent from ietf.message.models import Message from ietf.person.models import Person diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index 3d56cccf0..3a57ad591 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -1,5 +1,6 @@ # Copyright The IETF Trust 2009-2019, All Rights Reserved import datetime, os, shutil +import json import debug # pyflakes:ignore @@ -10,7 +11,7 @@ from django.db.models import Q from io import StringIO from pyquery import PyQuery -from ietf.utils.test_utils import TestCase, login_testing_unauthorized +from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.utils.mail import outbox from ietf.group.factories import GroupFactory, RoleFactory diff --git a/ietf/mailtrigger/tests.py b/ietf/mailtrigger/tests.py index db5d0281a..6868c5b77 100644 --- a/ietf/mailtrigger/tests.py +++ b/ietf/mailtrigger/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2015-2019, All Rights Reserved from django.urls import reverse as urlreverse -from ietf.utils.test_utils import TestCase +from ietf.utils.test_utils import TestCase, unicontent class EventMailTests(TestCase): diff --git a/ietf/message/tests.py b/ietf/message/tests.py index 13e599568..679404e2a 100644 --- a/ietf/message/tests.py +++ b/ietf/message/tests.py @@ -3,7 +3,7 @@ import datetime from django.urls import reverse as urlreverse -from ietf.utils.test_utils import TestCase +from ietf.utils.test_utils import TestCase, unicontent from ietf.utils.mail import outbox from ietf.message.models import Message, SendQueue diff --git a/ietf/nomcom/models.py b/ietf/nomcom/models.py index 249958cb7..16d229ed7 100644 --- a/ietf/nomcom/models.py +++ b/ietf/nomcom/models.py @@ -22,10 +22,7 @@ from ietf.nomcom.utils import (initialize_templates_for_group, initialize_questionnaire_for_position, initialize_requirements_for_position, initialize_description_for_topic, - delete_nomcom_templates, - EncryptedException, - ) -from ietf.utils.log import log + delete_nomcom_templates) from ietf.utils.models import ForeignKey from ietf.utils.pipe import pipe from ietf.utils.storage import NoLocationMigrationFileSystemStorage diff --git a/ietf/person/models.py b/ietf/person/models.py index 06bb77cad..bfc329d0f 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -338,7 +338,7 @@ class PersonalApiKey(models.Model): import struct, hashlib, base64 try: key = base64.urlsafe_b64decode(s) - except TypeError: + except TypeError as e: return None id, salt, hash = struct.unpack(KEY_STRUCT, key) diff --git a/ietf/person/tests.py b/ietf/person/tests.py index 21ec57091..0da5b98ef 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -3,6 +3,7 @@ import datetime +import json from pyquery import PyQuery from io import StringIO from django.urls import reverse as urlreverse diff --git a/ietf/redirects/tests.py b/ietf/redirects/tests.py index 9dbf8f574..175a1bd43 100644 --- a/ietf/redirects/tests.py +++ b/ietf/redirects/tests.py @@ -50,7 +50,7 @@ REDIRECT_TESTS = { '/public/idindex.cgi?command=do_search_id&filename=draft-mills-sntp-v4-00.txt': '/drafts/?filename=draft-mills-sntp-v4-00.txt', '/public/idindex.cgi?command=do_search_id&filename=draft-ietf-isis-interoperable&search_button=SEARCH': - '/drafts/?filename=draft-ietf-isis-interoperable&search_button=SEARCH', + '/drafts/?search_button=SEARCH&filename=draft-ietf-isis-interoperable', '/public/idindex.cgi?command=do_search_id&filename=rfc0038.txt': '/drafts/?filename=rfc0038.txt', '/public/idindex.cgi?command=id_detail&id=7096': diff --git a/ietf/secr/meetings/blue_sheets.py b/ietf/secr/meetings/blue_sheets.py index 676542bad..327e2d970 100644 --- a/ietf/secr/meetings/blue_sheets.py +++ b/ietf/secr/meetings/blue_sheets.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved from django.conf import settings -r''' +''' RTF quick reference (from Word2007RTFSpec9.doc): \fs24 : sets the font size to 24 half points \header : header on all pages diff --git a/ietf/secr/sreq/tests.py b/ietf/secr/sreq/tests.py index 4108b48f3..18f7750c5 100644 --- a/ietf/secr/sreq/tests.py +++ b/ietf/secr/sreq/tests.py @@ -4,7 +4,7 @@ import datetime import debug # pyflakes:ignore -from ietf.utils.test_utils import TestCase +from ietf.utils.test_utils import TestCase, unicontent from ietf.group.factories import GroupFactory, RoleFactory from ietf.meeting.models import Session, ResourceAssociation from ietf.meeting.factories import MeetingFactory, SessionFactory diff --git a/ietf/secr/utils/test.py b/ietf/secr/utils/test.py index 9fca188b1..d0864d230 100644 --- a/ietf/secr/utils/test.py +++ b/ietf/secr/utils/test.py @@ -18,4 +18,4 @@ def copy_roles(person): me.role_set.all().delete() for role in person.role_set.all(): Role.objects.create(person=me,email_id='rcross@amsl.com',name=role.name,group=role.group) - print(me.role_set.all()) \ No newline at end of file + print me.role_set.all() \ No newline at end of file diff --git a/ietf/utils/management/commands/create_group_wikis.py b/ietf/utils/management/commands/create_group_wikis.py index c9a007ecf..459ccc47b 100644 --- a/ietf/utils/management/commands/create_group_wikis.py +++ b/ietf/utils/management/commands/create_group_wikis.py @@ -1,4 +1,5 @@ # Copyright The IETF Trust 2016-2019, All Rights Reserved +# Copyright 2016 IETF Trust import os import copy @@ -60,8 +61,6 @@ class Command(BaseCommand): self.note("Running %s %s ..." % (os.path.basename(cmd), " ".join(quoted_args))) command = [ cmd, ] + list(args) code, out, err = pipe(command) - out = out.decode() - err = err.decode() msg = None if code != 0: msg = "Error %s: %s when executing '%s'" % (code, err, " ".join(command)) diff --git a/ietf/utils/management/commands/pyflakes.py b/ietf/utils/management/commands/pyflakes.py index a0cefbac5..ab2b5e046 100644 --- a/ietf/utils/management/commands/pyflakes.py +++ b/ietf/utils/management/commands/pyflakes.py @@ -72,7 +72,6 @@ def check(codeString, filename, verbosity=1): sys.stderr.write('F') else: sys.stderr.write('.') - sys.stderr.flush() if verbosity > 1: sys.stderr.write(" %s\n" % filename) return messages @@ -85,7 +84,7 @@ def checkPath(filename, verbosity): @return: the number of warnings printed """ try: - return check(open(filename).read() + '\n', filename, verbosity) + return check(open(filename, 'U').read() + '\n', filename, verbosity) except IOError as msg: return ["%s: %s" % (filename, msg.args[1])] except TypeError: @@ -101,7 +100,7 @@ def checkPaths(filenames, verbosity): try: warnings.extend(checkPath(os.path.join(dirpath, filename), verbosity)) except TypeError as e: - print("Exception while processing dirpath=%s, filename=%s: %s" % (dirpath, filename, e )) + print(("Exception while processing dirpath=%s, filename=%s: %s" % (dirpath, filename,e ))) raise else: warnings.extend(checkPath(arg, verbosity)) diff --git a/ietf/utils/management/commands/send_gdpr_consent_request.py b/ietf/utils/management/commands/send_gdpr_consent_request.py index ee0282f68..b9a37e4a3 100644 --- a/ietf/utils/management/commands/send_gdpr_consent_request.py +++ b/ietf/utils/management/commands/send_gdpr_consent_request.py @@ -97,7 +97,7 @@ class Command(BaseCommand): 'person': person, 'settings': settings, }, ) - PersonEvent.objects.create(person=person, type='gdpr_notice_email', + e = PersonEvent.objects.create(person=person, type='gdpr_notice_email', desc="Sent GDPR notice email to %s with confirmation deadline %s" % (to, date)) time.sleep(delay) diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 4f7a67dbb..252643668 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -239,6 +239,7 @@ def save_test_results(failures, test_labels): tfile.write("%s OK\n" % (timestr, )) tfile.close() + def set_coverage_checking(flag=True): global template_coverage_collection global code_coverage_collection diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 815869ada..855139547 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -11,8 +11,8 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from fnmatch import fnmatch from importlib import import_module -#from .pipe import pipe -#from io import StringIO +from .pipe import pipe +from io import StringIO from textwrap import dedent from unittest import skipIf from tempfile import mkdtemp @@ -20,7 +20,7 @@ from tempfile import mkdtemp from django.apps import apps from django.contrib.auth.models import User from django.conf import settings -#from django.core.management import call_command +from django.core.management import call_command from django.template import Context from django.template.defaulttags import URLNode from django.template.loader import get_template @@ -29,8 +29,8 @@ from django.urls import reverse as urlreverse import debug # pyflakes:ignore -#from ietf.group.factories import GroupFactory -#from ietf.group.models import Group +from ietf.group.factories import GroupFactory +from ietf.group.models import Group from ietf.person.name import name_parts, unidecode_name from ietf.submit.tests import submission_file from ietf.utils.bower_storage import BowerStorageFinder @@ -294,43 +294,43 @@ class TestWikiGlueManagementCommand(TestCase): shutil.rmtree(os.path.dirname(self.wiki_dir_pattern)) shutil.rmtree(os.path.dirname(self.svn_dir_pattern)) -# def test_wiki_create_output(self): -# for type in ['wg','rg','ag','area']: -# GroupFactory(type_id=type) -# groups = Group.objects.filter( -# type__slug__in=['wg','rg','ag','area'], -# state__slug='active' -# ).order_by('acronym') -# out = StringIO() -# err = StringIO() -# call_command('create_group_wikis', stdout=out, stderr=err, verbosity=2, -# wiki_dir_pattern=self.wiki_dir_pattern, -# svn_dir_pattern=self.svn_dir_pattern, -# ) -# command_output = out.getvalue() -# command_errors = err.getvalue() -# self.assertEqual("", command_errors) -# for group in groups: -# self.assertIn("Processing group '%s'" % group.acronym, command_output) -# # Do a bit of verification using trac-admin, too -# admin_code, admin_output, admin_error = pipe( -# 'trac-admin %s permission list' % (self.wiki_dir_pattern % group.acronym)) -# self.assertEqual(admin_code, 0) -# roles = group.role_set.filter(name_id__in=['chair', 'secr', 'ad']) -# for role in roles: -# user = role.email.address.lower() -# self.assertIn("Granting admin permission for %s" % user, command_output) -# self.assertIn(user, admin_output) -# docs = group.document_set.filter(states__slug='active', type_id='draft') -# for doc in docs: -# name = doc.name -# name = name.replace('draft-','') -# name = name.replace(doc.stream_id+'-', '') -# name = name.replace(group.acronym+'-', '') -# self.assertIn("Adding component %s"%name, command_output) -# for page in settings.TRAC_WIKI_PAGES_TEMPLATES: -# self.assertIn("Adding page %s" % os.path.basename(page), command_output) -# self.assertIn("Indexing default repository", command_output) + def test_wiki_create_output(self): + for type in ['wg','rg','ag','area']: + GroupFactory(type_id=type) + groups = Group.objects.filter( + type__slug__in=['wg','rg','ag','area'], + state__slug='active' + ).order_by('acronym') + out = StringIO() + err = StringIO() + call_command('create_group_wikis', stdout=out, stderr=err, verbosity=2, + wiki_dir_pattern=self.wiki_dir_pattern, + svn_dir_pattern=self.svn_dir_pattern, + ) + command_output = out.getvalue() + command_errors = err.getvalue() + self.assertEqual("", command_errors) + for group in groups: + self.assertIn("Processing group '%s'" % group.acronym, command_output) + # Do a bit of verification using trac-admin, too + admin_code, admin_output, admin_error = pipe( + 'trac-admin %s permission list' % (self.wiki_dir_pattern % group.acronym)) + self.assertEqual(admin_code, 0) + roles = group.role_set.filter(name_id__in=['chair', 'secr', 'ad']) + for role in roles: + user = role.email.address.lower() + self.assertIn("Granting admin permission for %s" % user, command_output) + self.assertIn(user, admin_output) + docs = group.document_set.filter(states__slug='active', type_id='draft') + for doc in docs: + name = doc.name + name = name.replace('draft-','') + name = name.replace(doc.stream_id+'-', '') + name = name.replace(group.acronym+'-', '') + self.assertIn("Adding component %s"%name, command_output) + for page in settings.TRAC_WIKI_PAGES_TEMPLATES: + self.assertIn("Adding page %s" % os.path.basename(page), command_output) + self.assertIn("Indexing default repository", command_output) OMITTED_APPS = [ 'ietf.secr.meetings', diff --git a/ietf/utils/text.py b/ietf/utils/text.py index 232ea1489..d82a2fcb1 100644 --- a/ietf/utils/text.py +++ b/ietf/utils/text.py @@ -3,6 +3,7 @@ import re import textwrap +import types import unicodedata from django.utils.functional import keep_lazy diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py index aa627df7b..77e4bbf09 100644 --- a/ietf/utils/validators.py +++ b/ietf/utils/validators.py @@ -77,12 +77,9 @@ def validate_mime_type(file, valid): mime_type, encoding = get_mime_type(raw) # work around mis-identification of text where a line has 'virtual' as # the first word: - if mime_type == 'text/x-c++' and re.search(rb'(?m)^virtual\s', raw): - mod = raw.replace(b'virtual', b' virtual')) + if mime_type == 'text/x-c++' and re.search('(?m)^virtual\s', raw): + mod = raw.replace(str('virtual'), str(' virtual')) mime_type, encoding = get_mime_type(mod) - debug.show('mime_type') - debug.show('encoding') - debug.show('valid') if valid and not mime_type in valid: raise ValidationError('Found content with unexpected mime type: %s. Expected one of %s.' % (mime_type, ', '.join(valid) )) diff --git a/requirements.txt b/requirements.txt index bca7b425d..f034e6b8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,6 +32,7 @@ httplib2>=0.10.3 jsonfield>=1.0.3 # for SubmissionCheck. This is https://github.com/bradjasper/django-jsonfield/. jwcrypto>=0.4.0 # for signed notifications #lxml>=3.4.0 # from PyQuery; +mimeparse>=0.1.3 # from TastyPie mock>=2.0.0 mysqlclient>=1.3.13 oauth2client>=4.0.0 # required by google-api-python-client, but not always pulled in @@ -47,7 +48,6 @@ pyquery>=1.2.13,!=1.2.14 # Pyqyery 1.2.14 fails on some selectors or stacked sel python-dateutil>=2.2 python-magic>=0.4.6 python-memcached>=1.48 # for django.core.cache.backends.memcached -python-mimeparse>=1.6 # from TastyPie pytz>=2014.7 #pyzmail>=1.0.3 requests!=2.12.*