altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

Django 2.0 conversion. Test suite passes, except for javascript tests.

Browse source 
- Legacy-Id: 17762
This commit is contained in:
Henrik Levkowetz 2020-05-08 13:07:07 +00:00
parent 01027b977c
commit 6d7c5b52ee
19 changed files with 177 additions and 195 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
ietf/dbtemplate/template.py
View file

@ -8,10 +8,8 @@ from docutils.core import publish_string
from docutils.utils import SystemMessage from docutils.utils import SystemMessage
import debug # pyflakes:ignore import debug # pyflakes:ignore
from django.template import Origin, TemplateDoesNotExist, Template as DjangoTemplate
from django.template.loaders.base import Loader as BaseLoader from django.template.loaders.base import Loader as BaseLoader
from django.template.base import Template as DjangoTemplate, TemplateEncodingError # type: ignore
from django.template.exceptions import TemplateDoesNotExist
from django.utils.encoding import smart_text
from ietf.dbtemplate.models import DBTemplate from ietf.dbtemplate.models import DBTemplate
@ -20,15 +18,11 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__))
RST_TEMPLATE = os.path.join(BASE_DIR, 'resources/rst.txt') RST_TEMPLATE = os.path.join(BASE_DIR, 'resources/rst.txt')
class Template(object): class Template(DjangoTemplate):
def __init__(self, template_string, origin=None, name='<Unknown Template>'): def __init__(self, template_string, origin=None, name='<Unknown Template>', engine=None):
try: super(Template, self).__init__(template_string, origin, name, engine)
template_string = smart_text(template_string)
except UnicodeDecodeError:
raise TemplateEncodingError("Templates can only be constructed from unicode or UTF-8 strings.")
self.template_string = string.Template(template_string) self.template_string = string.Template(template_string)
self.name = name
def render(self, context): def render(self, context):
raise NotImplementedError raise NotImplementedError
@ -70,27 +64,43 @@ class Loader(BaseLoader):
super(Loader, self).__init__(engine) super(Loader, self).__init__(engine)
self.is_usable = True self.is_usable = True
def load_template(self, template_name, template_dirs=None): def get_template(self, template_name, skip=None):
try: """
template = DBTemplate.objects.get(path=template_name) Call self.get_template_sources() and return a Template object for
if template.type.slug == 'rst': the first template matching template_name. If skip is provided, ignore
return (RSTTemplate(template.content), template) template origins in skip. This is used to avoid recursion during
elif template.type.slug == 'django': template extending.
return (DjangoTemplate(template.content), template) """
return (PlainTemplate(template.content), template) tried = []
except DBTemplate.DoesNotExist:
raise TemplateDoesNotExist(template_name)
for origin in self.get_template_sources(template_name):
if skip is not None and origin in skip:
tried.append((origin, 'Skipped'))
continue
_loader = Loader(engine='django') try:
template = DBTemplate.objects.get(path=origin)
contents = template.content
except DBTemplate.DoesNotExist:
tried.append((origin, 'Source does not exist'))
continue
else:
if template.type_id == 'rst':
return RSTTemplate(contents, origin, origin.template_name, self.engine)
elif template.type_id == 'plain':
return PlainTemplate(contents, origin, origin.template_name, self.engine)
elif template.type_id == 'django':
return DjangoTemplate(contents, origin, origin.template_name, self.engine)
else:
return Template(contents, origin, origin.template_name, self.engine)
raise TemplateDoesNotExist(template_name, tried=tried)
def get_template_sources(self, template_name):
for template in DBTemplate.objects.filter(path__endswith=template_name):
yield Origin(
name=template.path,
template_name=template_name,
loader=self,
)
def load_template_source(template_name, template_dirs=None):
# For backwards compatibility
import warnings
warnings.warn(
"'ietf.dbtemplate.template.load_template_source' is deprecated; use 'ietf.dbtemplate.template.Loader' instead.",
PendingDeprecationWarning
)
return _loader.load_template_source(template_name, template_dirs)
load_template_source.is_usable = True # type: ignore # https://github.com/python/mypy/issues/2087

1
ietf/group/views.py
View file

@ -1000,6 +1000,7 @@ def edit(request, group_type=None, acronym=None, action="edit", field=None):
changed_personnel.update(set(old)^set(new)) changed_personnel.update(set(old)^set(new))
if personnel_change_text!="": if personnel_change_text!="":
changed_personnel = [ str(p) for p in changed_personnel ]
personnel_change_text = "%s has updated %s personnel:\n\n" % (request.user.person.plain_name(), group.acronym.upper() ) + personnel_change_text personnel_change_text = "%s has updated %s personnel:\n\n" % (request.user.person.plain_name(), group.acronym.upper() ) + personnel_change_text
email_personnel_change(request, group, personnel_change_text, changed_personnel) email_personnel_change(request, group, personnel_change_text, changed_personnel)

10
ietf/meeting/tests_js.py
View file

@ -234,8 +234,18 @@ class ScheduleEditTests(IetfLiveServerTestCase):
self.assertEqual(SchedTimeSessAssignment.objects.filter(session__meeting=meeting, session__group__acronym='mars', schedule__name='test-schedule').count(),1) self.assertEqual(SchedTimeSessAssignment.objects.filter(session__meeting=meeting, session__group__acronym='mars', schedule__name='test-schedule').count(),1)
ss = list(SchedTimeSessAssignment.objects.filter(session__meeting__number=72,session__group__acronym='mars',schedule__name='test-schedule')) # pyflakes:ignore
debug.pprint('ss[0].json_dict("http:")')
self.login() self.login()
url = self.absreverse('ietf.meeting.views.edit_schedule',kwargs=dict(num='72',name='test-schedule',owner='plain@example.com')) url = self.absreverse('ietf.meeting.views.edit_schedule',kwargs=dict(num='72',name='test-schedule',owner='plain@example.com'))
js = self.absreverse('ietf.meeting.ajax.sessions_json', kwargs=dict(num='72'))
debug.show('js')
r = self.client.get(js) # pyflakes:ignore
from ietf.utils.test_utils import unicontent # pyflakes:ignore
debug.pprint('r.json()')
self.driver.get(url) self.driver.get(url)
s1 = Session.objects.filter(group__acronym='mars', meeting=meeting).first() s1 = Session.objects.filter(group__acronym='mars', meeting=meeting).first()

1
ietf/meeting/views.py
View file

@ -2767,6 +2767,7 @@ def floor_plan(request, num=None, floor=None, ):
except FileNotFoundError: except FileNotFoundError:
raise Http404('Missing floorplan image for %s' % floor) raise Http404('Missing floorplan image for %s' % floor)
return render(request, 'meeting/floor-plan.html', { return render(request, 'meeting/floor-plan.html', {
"meeting": meeting,
"schedule": schedule, "schedule": schedule,
"number": num, "number": num,
"floors": floors, "floors": floors,

2
ietf/person/models.py
View file

@ -99,6 +99,8 @@ class Person(models.Model):
return name_parts(self.name)[3] return name_parts(self.name)[3]
def first_name(self): def first_name(self):
return name_parts(self.name)[1] return name_parts(self.name)[1]
def aliases(self):
return [ str(a) for a in self.alias_set.all() ]
def role_email(self, role_name, group=None): def role_email(self, role_name, group=None):
"""Lookup email for role for person, optionally on group which """Lookup email for role for person, optionally on group which
may be an object or the group acronym.""" may be an object or the group acronym."""

24
ietf/settings.py
View file

@ -13,26 +13,9 @@ import warnings
from typing import Any, Dict, List, Tuple # pyflakes:ignore from typing import Any, Dict, List, Tuple # pyflakes:ignore
warnings.simplefilter("always", DeprecationWarning) warnings.simplefilter("always", DeprecationWarning)
warnings.filterwarnings("ignore", message="Add the `renderer` argument to the render\(\) method of", module="bootstrap3")
warnings.filterwarnings("ignore", message="The logout\(\) view is superseded by")
warnings.filterwarnings("ignore", message="Report.file_reporters will no longer be available in Coverage.py 4.2", module="coverage.report") warnings.filterwarnings("ignore", message="Report.file_reporters will no longer be available in Coverage.py 4.2", module="coverage.report")
warnings.filterwarnings("ignore", message="The popen2 module is deprecated. Use the subprocess module.", module="ietf.utils.pipe")
warnings.filterwarnings("ignore", message="Usage of field.rel has been deprecated. Use field.remote_field instead.", module="tastypie.resources")
warnings.filterwarnings("ignore", message="Importing from django.core.urlresolvers is deprecated in favor of django.urls.", module="tastypie.resources")
warnings.filterwarnings("ignore", message="on_delete will be a required arg for OneToOneField in Django 2.0.", module="tastypie")
warnings.filterwarnings("ignore", message=r"The load_template\(\) method is deprecated. Use get_template\(\) instead.")
warnings.filterwarnings("ignore", message="escape isn't the last filter in")
warnings.filterwarnings("ignore", message="Deprecated allow_tags attribute used on field")
warnings.filterwarnings("ignore", message="You passed a bytestring as `filenames`. This will not work on Python 3.")
warnings.filterwarnings("ignore", message="django.forms.extras is deprecated.", module="bootstrap3")
warnings.filterwarnings("ignore", message="defusedxml.lxml is no longer supported and will be removed in a future release.", module="tastypie")
warnings.filterwarnings("ignore", message="Duplicate index '.*' defined on the table")
# Warnings found under Python 3.7:
warnings.filterwarnings("ignore", message="Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated")
warnings.filterwarnings("ignore", message="'U' mode is deprecated", module="docutils.io")
warnings.filterwarnings("ignore", message="'U' mode is deprecated", module="xml2rfc")
warnings.filterwarnings("ignore", message="'U' mode is deprecated", module="site")
warnings.filterwarnings("ignore", message="Flags not at the start of the expression", module="genshi")
warnings.filterwarnings("ignore", message="Flags not at the start of the expression", module="coverage")
warnings.filterwarnings("ignore", message="encodestring\(\) is a deprecated alias since 3.1, use encodebytes\(\)")
try: try:
@ -367,7 +350,6 @@ MIDDLEWARE = [
'django.middleware.common.CommonMiddleware', 'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware', 'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.http.ConditionalGetMiddleware', 'django.middleware.http.ConditionalGetMiddleware',
'simple_history.middleware.HistoryRequestMiddleware', 'simple_history.middleware.HistoryRequestMiddleware',
@ -901,7 +883,6 @@ SECR_PROCEEDINGS_DIR = '/a/www/www6s/proceedings/'
SECR_PPT2PDF_COMMAND = ['/usr/bin/soffice','--headless','--convert-to','pdf:writer_globaldocument_pdf_Export','--outdir'] SECR_PPT2PDF_COMMAND = ['/usr/bin/soffice','--headless','--convert-to','pdf:writer_globaldocument_pdf_Export','--outdir']
STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://ietf.org/registration/attendees/{number}' STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://ietf.org/registration/attendees/{number}'
NEW_PROCEEDINGS_START = 95 NEW_PROCEEDINGS_START = 95
USE_ETAGS=True
YOUTUBE_API_KEY = '' YOUTUBE_API_KEY = ''
YOUTUBE_API_SERVICE_NAME = 'youtube' YOUTUBE_API_SERVICE_NAME = 'youtube'
YOUTUBE_API_VERSION = 'v3' YOUTUBE_API_VERSION = 'v3'
@ -1061,7 +1042,6 @@ SILENCED_SYSTEM_CHECKS = [
] ]
CHECKS_LIBRARY_PATCHES_TO_APPLY = [ CHECKS_LIBRARY_PATCHES_TO_APPLY = [
'patch/fix-django-unicode-comparison-bug.patch',
'patch/fix-unidecode-argument-warning.patch', 'patch/fix-unidecode-argument-warning.patch',
'patch/fix-request-profiler-streaming-length.patch', 'patch/fix-request-profiler-streaming-length.patch',
] ]

2
ietf/sync/iana.py
View file

@ -68,7 +68,7 @@ def fetch_changes_json(url, start, end):
# HTTP basic auth # HTTP basic auth
username = "ietfsync" username = "ietfsync"
password = settings.IANA_SYNC_PASSWORD password = settings.IANA_SYNC_PASSWORD
headers = { "Authorization": "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", "") } headers = { "Authorization": "Basic %s" % force_str(base64.encodebytes(smart_bytes("%s:%s" % (username, password)))).replace("\n", "") }
text = requests.get(url, headers=headers).text text = requests.get(url, headers=headers).text
return text return text

2
ietf/sync/rfceditor.py
View file

@ -532,7 +532,7 @@ def post_approved_draft(url, name):
headers = { headers = {
"Content-type": "application/x-www-form-urlencoded", "Content-type": "application/x-www-form-urlencoded",
"Accept": "text/plain", "Accept": "text/plain",
"Authorization": "Basic %s" % force_str(base64.encodestring(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""), "Authorization": "Basic %s" % force_str(base64.encodebytes(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""),
} }
log("Posting RFC-Editor notifcation of approved draft '%s' to '%s'" % (name, url)) log("Posting RFC-Editor notifcation of approved draft '%s' to '%s'" % (name, url))

2
ietf/templates/meeting/agenda_by_room.html
View file

@ -18,7 +18,7 @@ ul.sessionlist { list-style:none; padding-left:2em; margin-bottom:10px;}
{% block content %} {% block content %}
{% include "meeting/meeting_heading.html" with meeting=schedule.meeting updated=meeting.updated selected="by-room" title_extra="by Room" %} {% include "meeting/meeting_heading.html" with updated=meeting.updated selected="by-room" title_extra="by Room" %}
<ul class="daylist"> <ul class="daylist">
{% for day,sessions in ss_by_day.items %} {% for day,sessions in ss_by_day.items %}

2
ietf/templates/meeting/agenda_by_type.html
View file

@ -27,7 +27,7 @@ li.daylistentry { margin-left:2em; font-weight: 400; }
{% block title %}Agenda for {{meeting}} by Session Type{% endblock %} {% block title %}Agenda for {{meeting}} by Session Type{% endblock %}
{% block content %} {% block content %}
{% include "meeting/meeting_heading.html" with meeting=schedule.meeting updated=meeting.updated selected="by-type" title_extra="by Session Type" %} {% include "meeting/meeting_heading.html" with updated=meeting.updated selected="by-type" title_extra="by Session Type" %}
{% regroup assignments by session.type.slug as type_list %} {% regroup assignments by session.type.slug as type_list %}
<ul class="typelist"> <ul class="typelist">

6
ietf/templates/meeting/floor-plan.html
View file

@ -7,7 +7,7 @@
{% load staticfiles %} {% load staticfiles %}
{% block title %} {% block title %}
IETF {{ schedule.meeting.number }} meeting agenda IETF {{ meeting.number }} meeting agenda
{% if "-utc" in request.path %} {% if "-utc" in request.path %}
(UTC) (UTC)
{% endif %} {% endif %}
@ -40,7 +40,7 @@ hr.slim {
<div class="row"> <div class="row">
<div class="col-md-12 col-sm-12 col-xs-12" > <div class="col-md-12 col-sm-12 col-xs-12" >
{% include "meeting/meeting_heading.html" with meeting=schedule.meeting selected="floor-plan" title_extra="Floor Plan" %} {% include "meeting/meeting_heading.html" with selected="floor-plan" title_extra="Floor Plan" %}
</div> </div>
</div> </div>
@ -85,7 +85,6 @@ hr.slim {
{% endblock %} {% endblock %}
{% block js %} {% block js %}
{% with meeting=schedule.meeting %}
<script src="{% static 'ietf/js/room_params.js' %}"></script> <script src="{% static 'ietf/js/room_params.js' %}"></script>
<script> <script>
// These must match the 'arrowdiv' divs above // These must match the 'arrowdiv' divs above
@ -118,5 +117,4 @@ hr.slim {
return [left, top, right, bottom, floor, width]; return [left, top, right, bottom, floor, width];
} }
</script> </script>
{% endwith %}
{% endblock %} {% endblock %}

24
ietf/templates/meeting/meeting_heading.html
View file

@ -15,7 +15,7 @@
IETF {{ meeting.number }} Meeting Agenda {{ title_extra }} IETF {{ meeting.number }} Meeting Agenda {{ title_extra }}
<br> <br>
<small> <small>
{{ meeting.city }}, {{ meeting.date|date:"F j" }} - {{ meeting.city|default:"Location TBD" }}, {{ meeting.date|date:"F j" }} -
{% if meeting.date.month != meeting.end_date.month %} {% if meeting.date.month != meeting.end_date.month %}
{{ meeting.end_date|date:"F " }} {{ meeting.end_date|date:"F " }}
{% endif %} {% endif %}
@ -30,28 +30,28 @@
<ul class="nav nav-tabs" role="tablist"> <ul class="nav nav-tabs" role="tablist">
<li {% if selected == "agenda" %}class="active"{% endif %}> <li {% if selected == "agenda" %}class="active"{% endif %}>
<a href="{% url 'ietf.meeting.views.agenda' num=schedule.meeting.number %}">Agenda</a></li> <a href="{% url 'ietf.meeting.views.agenda' num=meeting.number %}">Agenda</a></li>
<li {% if selected == "agenda-utc" %}class="active"{% endif %}> <li {% if selected == "agenda-utc" %}class="active"{% endif %}>
<a href="{% url 'ietf.meeting.views.agenda' num=schedule.meeting.number utc='-utc' %}">UTC Agenda</a></li> <a href="{% url 'ietf.meeting.views.agenda' num=meeting.number utc='-utc' %}">UTC Agenda</a></li>
{% if user|has_role:"Secretariat,Area Director,IAB" %} {% if user|has_role:"Secretariat,Area Director,IAB" %}
{% if schedule != meeting.schedule %} {% if schedule != meeting.schedule %}
<li {% if selected == "by-room" %}class="active"{% endif %}> <li {% if selected == "by-room" %}class="active"{% endif %}>
<a href="{% url 'ietf.meeting.views.agenda_by_room' num=schedule.meeting.number name=schedule.name owner=schedule.owner.email %}">by Room</a></li> <a href="{% url 'ietf.meeting.views.agenda_by_room' num=meeting.number name=schedule.name owner=schedule.owner.email %}">by Room</a></li>
<li {% if selected == "by-type" %}class="active"{% endif %}> <li {% if selected == "by-type" %}class="active"{% endif %}>
<a href="{% url 'ietf.meeting.views.agenda_by_type' num=schedule.meeting.number name=schedule.name owner=schedule.owner.email %}">by Type</a></li> <a href="{% url 'ietf.meeting.views.agenda_by_type' num=meeting.number name=schedule.name owner=schedule.owner.email %}">by Type</a></li>
<li {% if selected == "room-view" %}class="active"{% endif %}> <li {% if selected == "room-view" %}class="active"{% endif %}>
<a href="{% url 'ietf.meeting.views.room_view' num=schedule.meeting.number name=schedule.name owner=schedule.owner.email %}">Room grid</a></li> <a href="{% url 'ietf.meeting.views.room_view' num=meeting.number name=schedule.name owner=schedule.owner.email %}">Room grid</a></li>
{% else %} {% else %}
<li {% if selected == "by-room" %}class="active"{% endif %}> <li {% if selected == "by-room" %}class="active"{% endif %}>
<a href="{% url 'ietf.meeting.views.agenda_by_room' num=schedule.meeting.number %}">by Room</a></li> <a href="{% url 'ietf.meeting.views.agenda_by_room' num=meeting.number %}">by Room</a></li>
<li {% if selected == "by-type" %}class="active"{% endif %}> <li {% if selected == "by-type" %}class="active"{% endif %}>
<a href="{% url 'ietf.meeting.views.agenda_by_type' num=schedule.meeting.number %}">by Type</a></li> <a href="{% url 'ietf.meeting.views.agenda_by_type' num=meeting.number %}">by Type</a></li>
<li {% if selected == "room-view" %}class="active"{% endif %}> <li {% if selected == "room-view" %}class="active"{% endif %}>
<a href="{% url 'ietf.meeting.views.room_view' num=schedule.meeting.number %}">Room grid</a></li> <a href="{% url 'ietf.meeting.views.room_view' num=meeting.number %}">Room grid</a></li>
{% endif %} {% endif %}
{% endif %} {% endif %}
<li {% if selected == "floor-plan" %}class="active"{% endif %}> <li {% if selected == "floor-plan" %}class="active"{% endif %}>
<a href="{% url 'ietf.meeting.views.floor_plan' num=schedule.meeting.number %}">Floor plan</a></li> <a href="{% url 'ietf.meeting.views.floor_plan' num=meeting.number %}">Floor plan</a></li>
<li><a href="{% url 'ietf.meeting.views.agenda' num=schedule.meeting.number ext='.txt' %}">Plaintext</a></li> <li><a href="{% url 'ietf.meeting.views.agenda' num=meeting.number ext='.txt' %}">Plaintext</a></li>
<li><a href="https://tools.ietf.org/agenda/{{schedule.meeting.number}}/">Tools-style &raquo;</a></li> <li><a href="https://tools.ietf.org/agenda/{{meeting.number}}/">Tools-style &raquo;</a></li>
</ul> </ul>

2
ietf/templates/meeting/room-view.html
View file

@ -582,7 +582,7 @@
</script> </script>
</head> </head>
<body onload="draw_calendar()" onresize="draw_calendar()" id="body"> <body onload="draw_calendar()" onresize="draw_calendar()" id="body">
<div id="mtgheader" style="overflow:auto">{% include "meeting/meeting_heading.html" with meeting=schedule.meeting updated=schedule.meeting.updated selected="room-view" title_extra="Room Grid" %}</div> <div id="mtgheader" style="overflow:auto">{% include "meeting/meeting_heading.html" with updated=schedule.meeting.updated selected="room-view" title_extra="Room Grid" %}</div>
<div id="daycontainer" role="tabpanel"> <div id="daycontainer" role="tabpanel">
<ul id="daytabs" class="nav nav-tabs" role="tablist"> <ul id="daytabs" class="nav nav-tabs" role="tablist">
{% for day in days %} {% for day in days %}

2
ietf/templates/person/mail/possible_duplicates.txt
View file

@ -12,7 +12,7 @@ Please check to see if they represent the same actual person, and if so, merge t
time: {{person.time}} time: {{person.time}}
ascii: {{person.ascii}} ascii: {{person.ascii}}
email: {% for email in person.email_set.all %}{{ email.address }} {% endfor %} email: {% for email in person.email_set.all %}{{ email.address }} {% endfor %}
aliases: {{ person.alias_set.all|join:", " }} aliases: {{ person.aliases|join:", " }}
username: {% if person.user %}{{person.user.username}}{% else %}None{% endif %} username: {% if person.user %}{{person.user.username}}{% else %}None{% endif %}
{% endfor %} {% endautoescape %} {% endfor %} {% endautoescape %}

98
ietf/utils/management/commands/check_referential_integrity.py
View file

@ -2,6 +2,8 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
from tqdm import tqdm from tqdm import tqdm
import django import django
@ -9,7 +11,6 @@ django.setup()
from django.apps import apps from django.apps import apps
from django.core.management.base import BaseCommand #, CommandError from django.core.management.base import BaseCommand #, CommandError
from django.core.exceptions import FieldError
from django.db import IntegrityError from django.db import IntegrityError
from django.db.models.fields.related import ForeignKey, OneToOneField, ManyToManyField from django.db.models.fields.related import ForeignKey, OneToOneField, ManyToManyField
@ -33,7 +34,7 @@ class Command(BaseCommand):
self.stdout.ending = None self.stdout.ending = None
self.stderr.ending = None self.stderr.ending = None
def check_field(field): def check_field(field, through_table=False):
try: try:
foreign_model = field.related_model foreign_model = field.related_model
except Exception: except Exception:
@ -46,7 +47,25 @@ class Command(BaseCommand):
used = set(field.model.objects.values_list(field.name, flat=True)) used = set(field.model.objects.values_list(field.name, flat=True))
used.discard(None) used.discard(None)
exists = set(foreign_model.objects.values_list('pk', flat=True)) exists = set(foreign_model.objects.values_list('pk', flat=True))
if through_table:
used = set( int(i) if isinstance(i, str) and i.isdigit() else i for i in used )
exists = set( int(i) if isinstance(i, str) and i.isdigit() else i for i in exists )
dangling = used - exists dangling = used - exists
if dangling:
debug.say('')
debug.show('len(used)')
debug.show('len(exists)')
used_list = list(used)
used_list.sort()
debug.show('used_list[:20]')
exists_list = list(exists)
exists_list.sort()
debug.show('exists_list[:20]')
for d in dangling:
if d in exists:
debug.say("%s exists, it isn't dangling!" % d)
exit()
exit()
if verbosity > 1: if verbosity > 1:
if dangling: if dangling:
self.stdout.write("\r ["+self.style.ERROR("fail")+"]\n ** Bad key values: %s\n" % sorted(list(dangling))) self.stdout.write("\r ["+self.style.ERROR("fail")+"]\n ** Bad key values: %s\n" % sorted(list(dangling)))
@ -63,14 +82,17 @@ class Command(BaseCommand):
kwargs = { field.name: value } kwargs = { field.name: value }
for obj in field.model.objects.filter(**kwargs): for obj in field.model.objects.filter(**kwargs):
try: try:
if isinstance(field, (ForeignKey, OneToOneField)): if through_table:
setattr(obj, field.name, None) obj.delete()
obj.save()
elif isinstance(field, (ManyToManyField, )):
manager = getattr(obj, field.name)
manager.remove(value)
else: else:
self.stderr.write("\nUnexpected field type: %s\n" % type(field)) if isinstance(field, (ForeignKey, OneToOneField)):
setattr(obj, field.name, None)
obj.save()
elif isinstance(field, (ManyToManyField, )):
manager = getattr(obj, field.name)
manager.remove(value)
else:
self.stderr.write("\nUnexpected field type: %s\n" % type(field))
except IntegrityError as e: except IntegrityError as e:
self.stderr.write('\n') self.stderr.write('\n')
self.stderr.write("Tried setting %s[%s].%s to %s, but got:\n" % (model.__name__, obj.pk, field.name, None)) self.stderr.write("Tried setting %s[%s].%s to %s, but got:\n" % (model.__name__, obj.pk, field.name, None))
@ -79,60 +101,11 @@ class Command(BaseCommand):
self.stdout.write('\n') self.stdout.write('\n')
def check_many_to_many_field(field): def check_many_to_many_field(field):
try: model = field.remote_field.through
foreign_model = field.related_model self.stdout.write(" %s.%s (through table)\n" % (model.__module__,model.__name__))
except Exception:
debug.pprint('dir(field)')
raise
if foreign_model == field.model:
return
foreign_field_name = field.remote_field.name
foreign_accessor_name = field.remote_field.get_accessor_name()
if verbosity > 1:
self.stdout.write(" [....] %s <- %s ( -> %s.%s)" %
(field.model.__name__, field.remote_field.through._meta.db_table,
foreign_model.__module__, foreign_model.__name__))
self.stdout.flush()
try:
used = set(foreign_model.objects.values_list(foreign_field_name, flat=True))
accessor_name = foreign_field_name
except FieldError:
try:
used = set(foreign_model.objects.values_list(foreign_accessor_name, flat=True))
accessor_name = foreign_accessor_name
except FieldError:
self.stdout.write("\n ** Warning: could not find foreign field name for %s.%s -> %s.%s\n" %
(field.model.__module__, field.model.__name__,
foreign_model.__name__, foreign_field_name))
used.discard(None)
exists = set(field.model.objects.values_list('pk',flat=True))
dangling = used - exists
if verbosity > 1:
if dangling:
self.stdout.write("\r ["+self.style.ERROR("fail")+"]\n ** Bad key values:\n %s\n" % sorted(list(dangling)))
else:
self.stdout.write("\r [ "+self.style.SUCCESS("ok")+" ]\n")
else:
if dangling:
self.stdout.write("\n%s.%s <- %s (-> %s.%s) ** Bad target key values:\n %s\n" %
(field.model.__module__, field.model.__name__,
field.remote_field.through._meta.db_table,
foreign_model.__module__, foreign_model.__name__,
sorted(list(dangling))))
if dangling and options.get('delete'):
through = field.remote_field.through
if verbosity > 1:
self.stdout.write("Removing dangling entries from %s.%s\n" % (through._meta.app_label, through.__name__))
kwargs = { accessor_name+'_id__in': dangling }
to_delete = field.remote_field.through.objects.filter(**kwargs)
count = to_delete.count()
to_delete.delete()
if verbosity > 1:
self.stdout.write("Removed %s entries from through table %s.%s\n" % (count, through._meta.app_label, through.__name__))
for ff in [f for f in model._meta.fields if isinstance(f, (ForeignKey, OneToOneField)) ]:
check_field(ff, through_table=True)
for conf in tqdm([ c for c in apps.get_app_configs() if c.name.startswith('ietf')], desc='apps ', disable=verbose): for conf in tqdm([ c for c in apps.get_app_configs() if c.name.startswith('ietf')], desc='apps ', disable=verbose):
if verbosity > 1: if verbosity > 1:
@ -145,5 +118,4 @@ class Command(BaseCommand):
for field in [f for f in model._meta.fields if isinstance(f, (ForeignKey, OneToOneField)) ]: for field in [f for f in model._meta.fields if isinstance(f, (ForeignKey, OneToOneField)) ]:
check_field(field) check_field(field)
for field in [f for f in model._meta.many_to_many ]: for field in [f for f in model._meta.many_to_many ]:
check_field(field)
check_many_to_many_field(field) check_many_to_many_field(field)

2
ietf/utils/management/commands/purge_request_profiler_records.py
View file

@ -21,7 +21,7 @@ class Command(BaseCommand):
def add_arguments(self, parser): def add_arguments(self, parser):
parser.add_argument('-d', '--days', dest='days', type=int, default=30, parser.add_argument('-d', '--days', dest='days', type=int, default=3,
help='Purge records older than this (default %(default)s days).') help='Purge records older than this (default %(default)s days).')
def handle(self, *filenames, **options): def handle(self, *filenames, **options):

82
ietf/utils/management/commands/sqldumpdump.py
View file

@ -1,8 +1,9 @@
# Copyright The IETF Trust 2020, All Rights Reserved # Copyright The IETF Trust 2020, All Rights Reserved
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import collections import collections
import gzip
import io import io
import re import re
import sys import sys
@ -44,6 +45,8 @@ class Command(DumpdataCommand):
help="One or more files to process") help="One or more files to process")
parser.add_argument('--pk-name', default='id', type=str, parser.add_argument('--pk-name', default='id', type=str,
help="Use the specified name as the primary key filed name (default: '%(default)s')" ) help="Use the specified name as the primary key filed name (default: '%(default)s')" )
parser.add_argument('--list-tables', action='store_true', default=False,
help="Just list the tables found in the input files, with record counts")
def note(self, msg): def note(self, msg):
if self.verbosity > 1: if self.verbosity > 1:
@ -79,11 +82,11 @@ class Command(DumpdataCommand):
for fn in filenames: for fn in filenames:
prev = '' prev = ''
lc = 0 lc = 0
with io.open(fn) as f: with gzip.open(fn, 'rt') if fn.endswith('.gz') else io.open(fn) as f:
for line in f: for line in f:
lc += 1 lc += 1
line = line.strip() line = line.strip()
if line[0] in ['<', '>']: if line and line[0] in ['<', '>']:
self.err("Input file '%s' looks like a diff file. Please provide just the SQL 'INSERT' statements for the records to be dumped." % (fn, )) self.err("Input file '%s' looks like a diff file. Please provide just the SQL 'INSERT' statements for the records to be dumped." % (fn, ))
if prev: if prev:
line = prev + line line = prev + line
@ -93,14 +96,15 @@ class Command(DumpdataCommand):
continue continue
sql = line sql = line
if not sql.upper().startswith('INSERT '): if not sql.upper().startswith('INSERT '):
self.warn("Skipping sql '%s...'" % sql[:64]) if self.verbosity > 2:
self.warn("Skipping sql '%s...'" % sql[:64])
else: else:
sql = sql.replace("\\'", "\\x27") sql = sql.replace("\\'", "\\x27")
match = re.match(r"INSERT( +(LOW_PRIORITY|DELAYED|HIGH_PRIORITY))*( +IGNORE)?( +INTO)?" match = re.match(r"INSERT( +(LOW_PRIORITY|DELAYED|HIGH_PRIORITY))*( +IGNORE)?( +INTO)?"
r" +(?P<table>\S+)" r" +(?P<table>\S+)"
r" +\((?P<fields>([^ ,]+)(, [^ ,]+)*)\)" r" +\((?P<fields>([^ ,]+)(, [^ ,]+)*)\)"
r" +(VALUES|VALUE)" r" +(VALUES|VALUE)"
r" +\((?P<values>(\d+|'[^']*'|NULL)(,(\d+|'[^']*'|NULL))*)\)" r" +\((?P<values>(\d+|'[^']*'|-1|NULL)(,(\d+|'[^']*'|-1|NULL))*)\)"
r" *;" r" *;"
, sql) , sql)
if not match: if not match:
@ -178,41 +182,45 @@ class Command(DumpdataCommand):
tables = self.get_tables() tables = self.get_tables()
pks, count = self.get_pks(filenames, tables) pks, count = self.get_pks(filenames, tables)
sys.stdout.write("Found %s SQL records.\n" % count) if options.get('list_tables', False):
for key in pks:
self.stdout.write("%-32s %6d\n" % (key, len(pks[key])))
else:
self.stdout.write("Found %s SQL records.\n" % count)
app_list = collections.OrderedDict() app_list = collections.OrderedDict()
for t in tables: for t in tables:
#print("%32s\t%s" % (t, ','.join(pks[t]))) #print("%32s\t%s" % (t, ','.join(pks[t])))
app_config = tables[t]['app_config'] app_config = tables[t]['app_config']
app_list.setdefault(app_config, []) app_list.setdefault(app_config, [])
app_list[app_config].append(tables[t]['model']) app_list[app_config].append(tables[t]['model'])
#debug.pprint('app_list') #debug.pprint('app_list')
try:
self.stdout.ending = None
progress_output = None
object_count = 0
# If dumpdata is outputting to stdout, there is no way to display progress
if (output and self.stdout.isatty() and options['verbosity'] > 0):
progress_output = self.stdout
object_count = sum(self.get_objects(app_list, pks, count_only=True))
stream = open(output, 'w') if output else None
try: try:
serializers.serialize( self.stdout.ending = None
format, self.get_objects(app_list, pks), indent=indent, progress_output = None
use_natural_foreign_keys=use_natural_foreign_keys, object_count = 0
use_natural_primary_keys=use_natural_primary_keys, # If dumpdata is outputting to stdout, there is no way to display progress
stream=stream or self.stdout, progress_output=progress_output, if (output and self.stdout.isatty() and options['verbosity'] > 0):
object_count=object_count, progress_output = self.stdout
) object_count = sum(self.get_objects(app_list, pks, count_only=True))
sys.stdout.write("Dumped %s objects.\n" % object_count) stream = open(output, 'w') if output else None
finally: try:
if stream: serializers.serialize(
stream.close() format, self.get_objects(app_list, pks), indent=indent,
except Exception as e: use_natural_foreign_keys=use_natural_foreign_keys,
if show_traceback: use_natural_primary_keys=use_natural_primary_keys,
raise stream=stream or self.stdout, progress_output=progress_output,
raise CommandError("Unable to serialize database: %s" % e) object_count=object_count,
)
self.stdout.write("Dumped %s objects.\n" % object_count)
finally:
if stream:
stream.close()
except Exception as e:
if show_traceback:
raise
raise CommandError("Unable to serialize database: %s" % e)

33
ietf/utils/test_runner.py
View file

@ -61,10 +61,10 @@ from django.db.migrations.operations.fields import FieldOperation
from django.db.migrations.operations.models import ModelOperation from django.db.migrations.operations.models import ModelOperation
from django.db.migrations.operations.base import Operation from django.db.migrations.operations.base import Operation
from django.template import TemplateDoesNotExist from django.template import TemplateDoesNotExist
from django.template.loaders.base import Loader as BaseLoader from django.template.loaders.filesystem import Loader as BaseLoader
from django.test.runner import DiscoverRunner from django.test.runner import DiscoverRunner
from django.core.management import call_command from django.core.management import call_command
from django.urls import RegexURLResolver # type: ignore from django.urls import URLResolver # type: ignore
import debug # pyflakes:ignore import debug # pyflakes:ignore
debug.debug = True debug.debug = True
@ -88,7 +88,7 @@ url_coverage_collection = None
def load_and_run_fixtures(verbosity): def load_and_run_fixtures(verbosity):
loadable = [f for f in settings.GLOBAL_TEST_FIXTURES if "." not in f] loadable = [f for f in settings.GLOBAL_TEST_FIXTURES if "." not in f]
call_command('loaddata', *loadable, verbosity=int(verbosity)-1, commit=False, database="default") call_command('loaddata', *loadable, verbosity=int(verbosity)-1, database="default")
for f in settings.GLOBAL_TEST_FIXTURES: for f in settings.GLOBAL_TEST_FIXTURES:
if f not in loadable: if f not in loadable:
@ -157,12 +157,11 @@ class MyPyTest(TestCase):
class TemplateCoverageLoader(BaseLoader): class TemplateCoverageLoader(BaseLoader):
is_usable = True is_usable = True
def load_template_source(self, template_name, dirs): def get_template(self, template_name, skip=None):
global template_coverage_collection, loaded_templates global template_coverage_collection, loaded_templates
if template_coverage_collection == True: if template_coverage_collection == True:
loaded_templates.add(str(template_name)) loaded_templates.add(str(template_name))
raise TemplateDoesNotExist(template_name) raise TemplateDoesNotExist(template_name)
load_template_source.is_usable = True # type: ignore # https://github.com/python/mypy/issues/2087
def record_urls_middleware(get_response): def record_urls_middleware(get_response):
def record_urls(request): def record_urls(request):
@ -177,38 +176,40 @@ def get_url_patterns(module, apps=None):
if not apps: if not apps:
return True return True
for app in apps: for app in apps:
if name.startswith(app+'.'): if str(name).startswith(app+'.'):
return True return True
return False return False
def exclude(name): def exclude(name):
for pat in settings.TEST_URL_COVERAGE_EXCLUDE: for pat in settings.TEST_URL_COVERAGE_EXCLUDE:
if re.search(pat, name): if re.search(pat, str(name)):
return True return True
return False return False
def append(res, p0, p1, item): def do_append(res, p0, p1, item):
p0 = str(p0)
p1 = str(p1)
if p1.startswith("^"): if p1.startswith("^"):
res.append((p0 + p1[1:], item)) res.append((p0 + p1[1:], item))
else: else:
res.append((item.p0 + ".*" + p1, item)) res.append((p0 + p1, item))
if not hasattr(module, 'urlpatterns'): if not hasattr(module, 'urlpatterns'):
return [] return []
res = [] res = []
for item in module.urlpatterns: for item in module.urlpatterns:
if isinstance(item, RegexURLResolver): if isinstance(item, URLResolver):
if type(item.urlconf_module) is list: if type(item.urlconf_module) is list:
for subitem in item.urlconf_module: for subitem in item.urlconf_module:
if isinstance(subitem, RegexURLResolver): if isinstance(subitem, URLResolver):
res += get_url_patterns(subitem.urlconf_module) res += get_url_patterns(subitem.urlconf_module)
else: else:
sub = subitem.regex.pattern sub = subitem.pattern
append(res, item.regex.pattern, subitem.regex.pattern, subitem) do_append(res, item.pattern, subitem.pattern, subitem)
else: else:
if include(item.urlconf_module.__name__) and not exclude(item.regex.pattern): if include(item.urlconf_module.__name__) and not exclude(item.pattern):
subpatterns = get_url_patterns(item.urlconf_module) subpatterns = get_url_patterns(item.urlconf_module)
for sub, subitem in subpatterns: for sub, subitem in subpatterns:
append(res, item.regex.pattern, sub, subitem) do_append(res, item.pattern, sub, subitem)
else: else:
res.append((item.regex.pattern, item)) res.append((str(item.pattern), item))
return res return res
_all_templates = None _all_templates = None

7
requirements.txt
View file

@ -5,13 +5,12 @@ argon2-cffi>=16.1.0 # For the Argon2 password hasher option
beautifulsoup4>=4.5.0 beautifulsoup4>=4.5.0
bibtexparser>=0.6.2,<1.0 # Version 1.0 doesn't work under python 2.7. 1.0.1 doesn't recognize month names or abbreviations. bibtexparser>=0.6.2,<1.0 # Version 1.0 doesn't work under python 2.7. 1.0.1 doesn't recognize month names or abbreviations.
bleach>=2.0.0,!=3.0.0,!=3.0.1,!=3.0.2 bleach>=2.0.0,!=3.0.0,!=3.0.1,!=3.0.2
coverage>=4.0.1,!=4.0.2,<5.0 coverage>=4.0.1,!=4.0.2,<5.0 # Coverage 5.x moves from a json database to SQLite. Moving to 5.x will require substantial rewrites in ietf.utils.test_runner and ietf.release.views
#cssselect>=0.6.1 # for PyQuery #cssselect>=0.6.1 # for PyQuery
decorator>=4.0.4 decorator>=4.0.4
defusedxml>=0.4.1 # for TastyPie when ussing xml; not a declared dependency defusedxml>=0.4.1 # for TastyPie when ussing xml; not a declared dependency
Django>=1.11,!=1.11.18,<1.12 # 1.11.18 has problems exporting BinaryField from django.db.models Django>=2.0,<2.1
django-bcrypt>=0.9.2 # for the BCrypt password hasher option. Remove when all bcrypt upgraded to argon2 django-bootstrap3>=9.1.0
django-bootstrap3>=8.2.1,<9.0.0
django-csp>=3.5 django-csp>=3.5
django-cors-headers>=2.4.0 django-cors-headers>=2.4.0
django-form-utils>=1.0.3 django-form-utils>=1.0.3