From 6d7c5b52eea2b89cdafd6a3f55689558ce7afe43 Mon Sep 17 00:00:00 2001 From: Henrik Levkowetz Date: Fri, 8 May 2020 13:07:07 +0000 Subject: [PATCH] Django 2.0 conversion. Test suite passes, except for javascript tests. - Legacy-Id: 17762 --- ietf/dbtemplate/template.py | 70 +++++++------ ietf/group/views.py | 1 + ietf/meeting/tests_js.py | 10 ++ ietf/meeting/views.py | 1 + ietf/person/models.py | 2 + ietf/settings.py | 24 +---- ietf/sync/iana.py | 2 +- ietf/sync/rfceditor.py | 2 +- ietf/templates/meeting/agenda_by_room.html | 2 +- ietf/templates/meeting/agenda_by_type.html | 2 +- ietf/templates/meeting/floor-plan.html | 6 +- ietf/templates/meeting/meeting_heading.html | 24 ++--- ietf/templates/meeting/room-view.html | 2 +- .../person/mail/possible_duplicates.txt | 2 +- .../commands/check_referential_integrity.py | 98 +++++++------------ .../purge_request_profiler_records.py | 2 +- ietf/utils/management/commands/sqldumpdump.py | 82 +++++++++------- ietf/utils/test_runner.py | 33 ++++--- requirements.txt | 7 +- 19 files changed, 177 insertions(+), 195 deletions(-) diff --git a/ietf/dbtemplate/template.py b/ietf/dbtemplate/template.py index 02d6a512c..6ada20d08 100644 --- a/ietf/dbtemplate/template.py +++ b/ietf/dbtemplate/template.py @@ -8,10 +8,8 @@ from docutils.core import publish_string from docutils.utils import SystemMessage import debug # pyflakes:ignore +from django.template import Origin, TemplateDoesNotExist, Template as DjangoTemplate from django.template.loaders.base import Loader as BaseLoader -from django.template.base import Template as DjangoTemplate, TemplateEncodingError # type: ignore -from django.template.exceptions import TemplateDoesNotExist -from django.utils.encoding import smart_text from ietf.dbtemplate.models import DBTemplate @@ -20,15 +18,11 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) RST_TEMPLATE = os.path.join(BASE_DIR, 'resources/rst.txt') -class Template(object): +class Template(DjangoTemplate): - def __init__(self, template_string, origin=None, name=''): - try: - template_string = smart_text(template_string) - except UnicodeDecodeError: - raise TemplateEncodingError("Templates can only be constructed from unicode or UTF-8 strings.") + def __init__(self, template_string, origin=None, name='', engine=None): + super(Template, self).__init__(template_string, origin, name, engine) self.template_string = string.Template(template_string) - self.name = name def render(self, context): raise NotImplementedError @@ -70,27 +64,43 @@ class Loader(BaseLoader): super(Loader, self).__init__(engine) self.is_usable = True - def load_template(self, template_name, template_dirs=None): - try: - template = DBTemplate.objects.get(path=template_name) - if template.type.slug == 'rst': - return (RSTTemplate(template.content), template) - elif template.type.slug == 'django': - return (DjangoTemplate(template.content), template) - return (PlainTemplate(template.content), template) - except DBTemplate.DoesNotExist: - raise TemplateDoesNotExist(template_name) + def get_template(self, template_name, skip=None): + """ + Call self.get_template_sources() and return a Template object for + the first template matching template_name. If skip is provided, ignore + template origins in skip. This is used to avoid recursion during + template extending. + """ + tried = [] + for origin in self.get_template_sources(template_name): + if skip is not None and origin in skip: + tried.append((origin, 'Skipped')) + continue -_loader = Loader(engine='django') + try: + template = DBTemplate.objects.get(path=origin) + contents = template.content + except DBTemplate.DoesNotExist: + tried.append((origin, 'Source does not exist')) + continue + else: + if template.type_id == 'rst': + return RSTTemplate(contents, origin, origin.template_name, self.engine) + elif template.type_id == 'plain': + return PlainTemplate(contents, origin, origin.template_name, self.engine) + elif template.type_id == 'django': + return DjangoTemplate(contents, origin, origin.template_name, self.engine) + else: + return Template(contents, origin, origin.template_name, self.engine) + raise TemplateDoesNotExist(template_name, tried=tried) + + def get_template_sources(self, template_name): + for template in DBTemplate.objects.filter(path__endswith=template_name): + yield Origin( + name=template.path, + template_name=template_name, + loader=self, + ) -def load_template_source(template_name, template_dirs=None): - # For backwards compatibility - import warnings - warnings.warn( - "'ietf.dbtemplate.template.load_template_source' is deprecated; use 'ietf.dbtemplate.template.Loader' instead.", - PendingDeprecationWarning - ) - return _loader.load_template_source(template_name, template_dirs) -load_template_source.is_usable = True # type: ignore # https://github.com/python/mypy/issues/2087 diff --git a/ietf/group/views.py b/ietf/group/views.py index 8d9ceaed3..abdde45e5 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -1000,6 +1000,7 @@ def edit(request, group_type=None, acronym=None, action="edit", field=None): changed_personnel.update(set(old)^set(new)) if personnel_change_text!="": + changed_personnel = [ str(p) for p in changed_personnel ] personnel_change_text = "%s has updated %s personnel:\n\n" % (request.user.person.plain_name(), group.acronym.upper() ) + personnel_change_text email_personnel_change(request, group, personnel_change_text, changed_personnel) diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index 5760b396a..422591332 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -234,8 +234,18 @@ class ScheduleEditTests(IetfLiveServerTestCase): self.assertEqual(SchedTimeSessAssignment.objects.filter(session__meeting=meeting, session__group__acronym='mars', schedule__name='test-schedule').count(),1) + + ss = list(SchedTimeSessAssignment.objects.filter(session__meeting__number=72,session__group__acronym='mars',schedule__name='test-schedule')) # pyflakes:ignore + debug.pprint('ss[0].json_dict("http:")') + + self.login() url = self.absreverse('ietf.meeting.views.edit_schedule',kwargs=dict(num='72',name='test-schedule',owner='plain@example.com')) + js = self.absreverse('ietf.meeting.ajax.sessions_json', kwargs=dict(num='72')) + debug.show('js') + r = self.client.get(js) # pyflakes:ignore + from ietf.utils.test_utils import unicontent # pyflakes:ignore + debug.pprint('r.json()') self.driver.get(url) s1 = Session.objects.filter(group__acronym='mars', meeting=meeting).first() diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index d5cb61f78..61b5eafd1 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -2767,6 +2767,7 @@ def floor_plan(request, num=None, floor=None, ): except FileNotFoundError: raise Http404('Missing floorplan image for %s' % floor) return render(request, 'meeting/floor-plan.html', { + "meeting": meeting, "schedule": schedule, "number": num, "floors": floors, diff --git a/ietf/person/models.py b/ietf/person/models.py index 74eb8fbca..a79487b43 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -99,6 +99,8 @@ class Person(models.Model): return name_parts(self.name)[3] def first_name(self): return name_parts(self.name)[1] + def aliases(self): + return [ str(a) for a in self.alias_set.all() ] def role_email(self, role_name, group=None): """Lookup email for role for person, optionally on group which may be an object or the group acronym.""" diff --git a/ietf/settings.py b/ietf/settings.py index 328fc02f1..d97593896 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -13,26 +13,9 @@ import warnings from typing import Any, Dict, List, Tuple # pyflakes:ignore warnings.simplefilter("always", DeprecationWarning) +warnings.filterwarnings("ignore", message="Add the `renderer` argument to the render\(\) method of", module="bootstrap3") +warnings.filterwarnings("ignore", message="The logout\(\) view is superseded by") warnings.filterwarnings("ignore", message="Report.file_reporters will no longer be available in Coverage.py 4.2", module="coverage.report") -warnings.filterwarnings("ignore", message="The popen2 module is deprecated. Use the subprocess module.", module="ietf.utils.pipe") -warnings.filterwarnings("ignore", message="Usage of field.rel has been deprecated. Use field.remote_field instead.", module="tastypie.resources") -warnings.filterwarnings("ignore", message="Importing from django.core.urlresolvers is deprecated in favor of django.urls.", module="tastypie.resources") -warnings.filterwarnings("ignore", message="on_delete will be a required arg for OneToOneField in Django 2.0.", module="tastypie") -warnings.filterwarnings("ignore", message=r"The load_template\(\) method is deprecated. Use get_template\(\) instead.") -warnings.filterwarnings("ignore", message="escape isn't the last filter in") -warnings.filterwarnings("ignore", message="Deprecated allow_tags attribute used on field") -warnings.filterwarnings("ignore", message="You passed a bytestring as `filenames`. This will not work on Python 3.") -warnings.filterwarnings("ignore", message="django.forms.extras is deprecated.", module="bootstrap3") -warnings.filterwarnings("ignore", message="defusedxml.lxml is no longer supported and will be removed in a future release.", module="tastypie") -warnings.filterwarnings("ignore", message="Duplicate index '.*' defined on the table") -# Warnings found under Python 3.7: -warnings.filterwarnings("ignore", message="Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated") -warnings.filterwarnings("ignore", message="'U' mode is deprecated", module="docutils.io") -warnings.filterwarnings("ignore", message="'U' mode is deprecated", module="xml2rfc") -warnings.filterwarnings("ignore", message="'U' mode is deprecated", module="site") -warnings.filterwarnings("ignore", message="Flags not at the start of the expression", module="genshi") -warnings.filterwarnings("ignore", message="Flags not at the start of the expression", module="coverage") -warnings.filterwarnings("ignore", message="encodestring\(\) is a deprecated alias since 3.1, use encodebytes\(\)") try: @@ -367,7 +350,6 @@ MIDDLEWARE = [ 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.http.ConditionalGetMiddleware', 'simple_history.middleware.HistoryRequestMiddleware', @@ -901,7 +883,6 @@ SECR_PROCEEDINGS_DIR = '/a/www/www6s/proceedings/' SECR_PPT2PDF_COMMAND = ['/usr/bin/soffice','--headless','--convert-to','pdf:writer_globaldocument_pdf_Export','--outdir'] STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://ietf.org/registration/attendees/{number}' NEW_PROCEEDINGS_START = 95 -USE_ETAGS=True YOUTUBE_API_KEY = '' YOUTUBE_API_SERVICE_NAME = 'youtube' YOUTUBE_API_VERSION = 'v3' @@ -1061,7 +1042,6 @@ SILENCED_SYSTEM_CHECKS = [ ] CHECKS_LIBRARY_PATCHES_TO_APPLY = [ - 'patch/fix-django-unicode-comparison-bug.patch', 'patch/fix-unidecode-argument-warning.patch', 'patch/fix-request-profiler-streaming-length.patch', ] diff --git a/ietf/sync/iana.py b/ietf/sync/iana.py index 7e6085e93..e997139c5 100644 --- a/ietf/sync/iana.py +++ b/ietf/sync/iana.py @@ -68,7 +68,7 @@ def fetch_changes_json(url, start, end): # HTTP basic auth username = "ietfsync" password = settings.IANA_SYNC_PASSWORD - headers = { "Authorization": "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", "") } + headers = { "Authorization": "Basic %s" % force_str(base64.encodebytes(smart_bytes("%s:%s" % (username, password)))).replace("\n", "") } text = requests.get(url, headers=headers).text return text diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index 258883b68..af886581c 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -532,7 +532,7 @@ def post_approved_draft(url, name): headers = { "Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain", - "Authorization": "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""), + "Authorization": "Basic %s" % force_str(base64.encodebytes(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""), } log("Posting RFC-Editor notifcation of approved draft '%s' to '%s'" % (name, url)) diff --git a/ietf/templates/meeting/agenda_by_room.html b/ietf/templates/meeting/agenda_by_room.html index f17ec7988..5d67c8ec2 100644 --- a/ietf/templates/meeting/agenda_by_room.html +++ b/ietf/templates/meeting/agenda_by_room.html @@ -18,7 +18,7 @@ ul.sessionlist { list-style:none; padding-left:2em; margin-bottom:10px;} {% block content %} - {% include "meeting/meeting_heading.html" with meeting=schedule.meeting updated=meeting.updated selected="by-room" title_extra="by Room" %} + {% include "meeting/meeting_heading.html" with updated=meeting.updated selected="by-room" title_extra="by Room" %}
    {% for day,sessions in ss_by_day.items %} diff --git a/ietf/templates/meeting/agenda_by_type.html b/ietf/templates/meeting/agenda_by_type.html index 616293fdb..afe1eb462 100644 --- a/ietf/templates/meeting/agenda_by_type.html +++ b/ietf/templates/meeting/agenda_by_type.html @@ -27,7 +27,7 @@ li.daylistentry { margin-left:2em; font-weight: 400; } {% block title %}Agenda for {{meeting}} by Session Type{% endblock %} {% block content %} -{% include "meeting/meeting_heading.html" with meeting=schedule.meeting updated=meeting.updated selected="by-type" title_extra="by Session Type" %} +{% include "meeting/meeting_heading.html" with updated=meeting.updated selected="by-type" title_extra="by Session Type" %} {% regroup assignments by session.type.slug as type_list %}
      diff --git a/ietf/templates/meeting/floor-plan.html b/ietf/templates/meeting/floor-plan.html index 7a5da96f3..8708952ec 100644 --- a/ietf/templates/meeting/floor-plan.html +++ b/ietf/templates/meeting/floor-plan.html @@ -7,7 +7,7 @@ {% load staticfiles %} {% block title %} - IETF {{ schedule.meeting.number }} meeting agenda + IETF {{ meeting.number }} meeting agenda {% if "-utc" in request.path %} (UTC) {% endif %} @@ -40,7 +40,7 @@ hr.slim {
      - {% include "meeting/meeting_heading.html" with meeting=schedule.meeting selected="floor-plan" title_extra="Floor Plan" %} + {% include "meeting/meeting_heading.html" with selected="floor-plan" title_extra="Floor Plan" %}
      @@ -85,7 +85,6 @@ hr.slim { {% endblock %} {% block js %} -{% with meeting=schedule.meeting %} -{% endwith %} {% endblock %} diff --git a/ietf/templates/meeting/meeting_heading.html b/ietf/templates/meeting/meeting_heading.html index 133415f12..0a5c371d3 100644 --- a/ietf/templates/meeting/meeting_heading.html +++ b/ietf/templates/meeting/meeting_heading.html @@ -15,7 +15,7 @@ IETF {{ meeting.number }} Meeting Agenda {{ title_extra }}
      - {{ meeting.city }}, {{ meeting.date|date:"F j" }} - + {{ meeting.city|default:"Location TBD" }}, {{ meeting.date|date:"F j" }} - {% if meeting.date.month != meeting.end_date.month %} {{ meeting.end_date|date:"F " }} {% endif %} @@ -30,28 +30,28 @@ diff --git a/ietf/templates/meeting/room-view.html b/ietf/templates/meeting/room-view.html index 7d6dc1766..ea577d5c8 100644 --- a/ietf/templates/meeting/room-view.html +++ b/ietf/templates/meeting/room-view.html @@ -582,7 +582,7 @@ -
      {% include "meeting/meeting_heading.html" with meeting=schedule.meeting updated=schedule.meeting.updated selected="room-view" title_extra="Room Grid" %}
      +
      {% include "meeting/meeting_heading.html" with updated=schedule.meeting.updated selected="room-view" title_extra="Room Grid" %}