diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6b112ba62..e2226163d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,10 +91,10 @@ jobs: echo "pkg_version=$NEXT_VERSION" >> $GITHUB_OUTPUT echo "::notice::Release $NEXT_VERSION created using branch $GITHUB_REF_NAME" else - echo "Using TEST mode: 10.0.0-dev.$GITHUB_RUN_NUMBER" + echo "Using TEST mode: 11.0.0-dev.$GITHUB_RUN_NUMBER" echo "should_deploy=false" >> $GITHUB_OUTPUT - echo "pkg_version=10.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT - echo "::notice::Non-production build 10.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME" + echo "pkg_version=11.0.0-dev.$GITHUB_RUN_NUMBER" >> $GITHUB_OUTPUT + echo "::notice::Non-production build 11.0.0-dev.$GITHUB_RUN_NUMBER created using branch $GITHUB_REF_NAME" fi # ----------------------------------------------------------------- diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index e8e9fe324..8a7ee696b 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - 'main' + - 'feat/django4' paths: - 'client/**' - 'ietf/**' diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index d9bf00f22..60981ba56 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -10,7 +10,7 @@ DATABASES = { 'HOST': '__DBHOST__', 'PORT': 5432, 'NAME': 'datatracker', - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', 'USER': 'django', 'PASSWORD': 'RkTkDPFnKpko', }, diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index cd3923d50..593ccadd7 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -10,7 +10,7 @@ DATABASES = { 'HOST': '__DBHOST__', 'PORT': 5432, 'NAME': 'datatracker', - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', 'USER': 'django', 'PASSWORD': 'RkTkDPFnKpko', }, diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index fdc60a849..0cd761c0a 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -10,7 +10,7 @@ DATABASES = { 'HOST': 'db', 'PORT': 5432, 'NAME': 'datatracker', - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', 'USER': 'django', 'PASSWORD': 'RkTkDPFnKpko', }, diff --git a/docker/configs/settings_postgresqldb.py b/docker/configs/settings_postgresqldb.py index fe0c827ff..05d19b9a8 100644 --- a/docker/configs/settings_postgresqldb.py +++ b/docker/configs/settings_postgresqldb.py @@ -3,7 +3,7 @@ DATABASES = { 'HOST': 'db', 'PORT': 5432, 'NAME': 'datatracker', - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', 'USER': 'django', 'PASSWORD': 'RkTkDPFnKpko', }, diff --git a/ietf/__init__.py b/ietf/__init__.py index b0dad6c65..59f9802de 100644 --- a/ietf/__init__.py +++ b/ietf/__init__.py @@ -6,7 +6,7 @@ from . import checks # pyflakes:ignore # Version must stay in single quotes for automatic CI replace # Don't add patch number here: -__version__ = '10.0.0-dev' +__version__ = '11.0.0-dev' # Release hash must stay in single quotes for automatic CI replace __release_hash__ = '' diff --git a/ietf/api/serializer.py b/ietf/api/serializer.py index 9d6cf1ebb..27f194c5b 100644 --- a/ietf/api/serializer.py +++ b/ietf/api/serializer.py @@ -9,11 +9,12 @@ from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist, FieldError from django.core.serializers.json import Serializer from django.http import HttpResponse -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str from django.db.models import Field -from django.db.models.query import QuerySet from django.db.models.signals import post_save, post_delete, m2m_changed +from django_stubs_ext import QuerySetAny + import debug # pyflakes:ignore @@ -121,7 +122,7 @@ class AdminJsonSerializer(Serializer): for name in expansions: try: field = getattr(obj, name) - #self._current["_"+name] = smart_text(field) + #self._current["_"+name] = smart_str(field) if not isinstance(field, Field): options = self.options.copy() options["expand"] = [ v[len(name)+2:] for v in options["expand"] if v.startswith(name+"__") ] @@ -145,7 +146,7 @@ class AdminJsonSerializer(Serializer): field_value = None else: field_value = field - if isinstance(field_value, QuerySet) or isinstance(field_value, list): + if isinstance(field_value, QuerySetAny) or isinstance(field_value, list): self._current[name] = dict([ (rel.pk, self.expand_related(rel, name)) for rel in field_value ]) else: if hasattr(field_value, "_meta"): @@ -188,10 +189,10 @@ class AdminJsonSerializer(Serializer): related = related.natural_key() elif field.remote_field.field_name == related._meta.pk.name: # Related to remote object via primary key - related = smart_text(related._get_pk_val(), strings_only=True) + related = smart_str(related._get_pk_val(), strings_only=True) else: # Related to remote object via other field - related = smart_text(getattr(related, field.remote_field.field_name), strings_only=True) + related = smart_str(getattr(related, field.remote_field.field_name), strings_only=True) self._current[field.name] = related def handle_m2m_field(self, obj, field): @@ -201,7 +202,7 @@ class AdminJsonSerializer(Serializer): elif self.use_natural_keys and hasattr(field.remote_field.to, 'natural_key'): m2m_value = lambda value: value.natural_key() else: - m2m_value = lambda value: smart_text(value._get_pk_val(), strings_only=True) + m2m_value = lambda value: smart_str(value._get_pk_val(), strings_only=True) self._current[field.name] = [m2m_value(related) for related in getattr(obj, field.name).iterator()] @@ -221,7 +222,7 @@ class JsonExportMixin(object): # obj = None # # if obj is None: -# raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_text(self.model._meta.verbose_name), 'key': escape(object_id)}) +# raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_str(self.model._meta.verbose_name), 'key': escape(object_id)}) # # content_type = 'application/json' # return HttpResponse(serialize([ obj ], sort_keys=True, indent=3)[2:-2], content_type=content_type) @@ -264,6 +265,6 @@ class JsonExportMixin(object): qd = dict( ( k, json.loads(v)[0] ) for k,v in items ) except (FieldError, ValueError) as e: return HttpResponse(json.dumps({"error": str(e)}, sort_keys=True, indent=3), content_type=content_type) - text = json.dumps({smart_text(self.model._meta): qd}, sort_keys=True, indent=3) + text = json.dumps({smart_str(self.model._meta): qd}, sort_keys=True, indent=3) return HttpResponse(text, content_type=content_type) diff --git a/ietf/api/tests.py b/ietf/api/tests.py index c910b6f90..2285fa153 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -691,7 +691,7 @@ class CustomApiTests(TestCase): self.assertEqual(set(missing_fields), set(drop_fields)) def test_api_version(self): - DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=timezone.utc), host='testapi.example.com',tz='UTC') + DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.timezone.utc), host='testapi.example.com',tz='UTC') url = urlreverse('ietf.api.views.version') r = self.client.get(url) data = r.json() diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 5185b9f88..7ee55cf70 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2017, All Rights Reserved from django.conf import settings -from django.conf.urls import include +from django.urls import include from django.views.generic import TemplateView from ietf import api diff --git a/ietf/community/views.py b/ietf/community/views.py index b0646424a..054bed302 100644 --- a/ietf/community/views.py +++ b/ietf/community/views.py @@ -22,6 +22,7 @@ from ietf.community.utils import docs_tracked_by_community_list, docs_matching_c from ietf.community.utils import states_of_significant_change, reset_name_contains_index_for_rule from ietf.doc.models import DocEvent, Document from ietf.doc.utils_search import prepare_document_table +from ietf.utils.http import is_ajax from ietf.utils.response import permission_denied def view_list(request, username=None): @@ -142,7 +143,7 @@ def track_document(request, name, username=None, acronym=None): if not doc in clist.added_docs.all(): clist.added_docs.add(doc) - if request.is_ajax(): + if is_ajax(request): return HttpResponse(json.dumps({ 'success': True }), content_type='application/json') else: return HttpResponseRedirect(clist.get_absolute_url()) @@ -162,7 +163,7 @@ def untrack_document(request, name, username=None, acronym=None): if clist.pk is not None: clist.added_docs.remove(doc) - if request.is_ajax(): + if is_ajax(request): return HttpResponse(json.dumps({ 'success': True }), content_type='application/json') else: return HttpResponseRedirect(clist.get_absolute_url()) diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py index ddb2843cc..8f5d0eb67 100644 --- a/ietf/doc/mails.py +++ b/ietf/doc/mails.py @@ -11,7 +11,7 @@ from django.utils.html import strip_tags from django.conf import settings from django.urls import reverse as urlreverse from django.utils import timezone -from django.utils.encoding import force_text +from django.utils.encoding import force_str import debug # pyflakes:ignore from ietf.doc.templatetags.mail_filters import std_level_prompt @@ -175,7 +175,7 @@ def generate_ballot_writeup(request, doc): e.doc = doc e.rev = doc.rev e.desc = "Ballot writeup was generated" - e.text = force_text(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana, 'doc': doc })) + e.text = force_str(render_to_string("doc/mail/ballot_writeup.txt", {'iana': iana, 'doc': doc })) # caller is responsible for saving, if necessary return e @@ -187,7 +187,7 @@ def generate_ballot_rfceditornote(request, doc): e.doc = doc e.rev = doc.rev e.desc = "RFC Editor Note for ballot was generated" - e.text = force_text(render_to_string("doc/mail/ballot_rfceditornote.txt")) + e.text = force_str(render_to_string("doc/mail/ballot_rfceditornote.txt")) e.save() return e @@ -232,7 +232,7 @@ def generate_last_call_announcement(request, doc): e.doc = doc e.rev = doc.rev e.desc = "Last call announcement was generated" - e.text = force_text(mail) + e.text = force_str(mail) # caller is responsible for saving, if necessary return e @@ -252,7 +252,7 @@ def generate_approval_mail(request, doc): e.doc = doc e.rev = doc.rev e.desc = "Ballot approval text was generated" - e.text = force_text(mail) + e.text = force_str(mail) # caller is responsible for saving, if necessary return e diff --git a/ietf/doc/migrations/0004_alter_dochistory_ad_alter_dochistory_shepherd_and_more.py b/ietf/doc/migrations/0004_alter_dochistory_ad_alter_dochistory_shepherd_and_more.py new file mode 100644 index 000000000..adc0e6962 --- /dev/null +++ b/ietf/doc/migrations/0004_alter_dochistory_ad_alter_dochistory_shepherd_and_more.py @@ -0,0 +1,36 @@ +# Generated by Django 4.0.10 on 2023-05-16 20:36 + +from django.db import migrations +import django.db.models.deletion +import ietf.utils.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('person', '0001_initial'), + ('doc', '0003_remove_document_info_order'), + ] + + operations = [ + migrations.AlterField( + model_name='dochistory', + name='ad', + field=ietf.utils.models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='ad_%(class)s_set', to='person.person', verbose_name='area director'), + ), + migrations.AlterField( + model_name='dochistory', + name='shepherd', + field=ietf.utils.models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='shepherd_%(class)s_set', to='person.email'), + ), + migrations.AlterField( + model_name='document', + name='ad', + field=ietf.utils.models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='ad_%(class)s_set', to='person.person', verbose_name='area director'), + ), + migrations.AlterField( + model_name='document', + name='shepherd', + field=ietf.utils.models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='shepherd_%(class)s_set', to='person.email'), + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 5d0e22683..85b01c3d0 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -24,7 +24,7 @@ from django.urls import reverse as urlreverse from django.contrib.contenttypes.models import ContentType from django.conf import settings from django.utils import timezone -from django.utils.encoding import force_text +from django.utils.encoding import force_str from django.utils.html import mark_safe # type:ignore from django.contrib.staticfiles import finders @@ -1134,7 +1134,7 @@ class DocHistory(DocumentInfo): name = models.CharField(max_length=255) def __str__(self): - return force_text(self.doc.name) + return force_str(self.doc.name) def get_related_session(self): return self.doc.get_related_session() @@ -1196,7 +1196,7 @@ class DocAlias(models.Model): return self.docs.first() def __str__(self): - return u"%s-->%s" % (self.name, ','.join([force_text(d.name) for d in self.docs.all() if isinstance(d, Document) ])) + return u"%s-->%s" % (self.name, ','.join([force_str(d.name) for d in self.docs.all() if isinstance(d, Document) ])) document_link = admin_link("document") class Meta: verbose_name = "document alias" diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 332e5ca15..1c5836328 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -13,8 +13,7 @@ from django.utils.html import escape from django.template.defaultfilters import truncatewords_html, linebreaksbr, stringfilter, striptags from django.utils.safestring import mark_safe, SafeData from django.utils.html import strip_tags -from django.utils.encoding import force_text -from django.utils.encoding import force_str # pyflakes:ignore force_str is used in the doctests +from django.utils.encoding import force_str from django.urls import reverse as urlreverse from django.core.cache import cache from django.core.exceptions import ValidationError @@ -132,7 +131,7 @@ register.filter('fill', fill) @register.filter def prettystdname(string, space=" "): from ietf.doc.utils import prettify_std_name - return prettify_std_name(force_text(string or ""), space) + return prettify_std_name(force_str(string or ""), space) @register.filter def rfceditor_info_url(rfcnum : str): diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 65859bf4b..47c4e146c 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -1779,7 +1779,7 @@ class DocTestCase(TestCase): self.client.login(username='ad', password='ad+password') r = self.client.post(urlreverse('ietf.doc.views_status_change.change_state',kwargs=dict(name=doc.name)),dict(new_state=iesgeval_pk)) self.assertEqual(r.status_code, 302) - r = self.client.get(r._headers["location"][1]) + r = self.client.get(r.headers["location"]) self.assertContains(r, ">IESG Evaluation<") self.assertEqual(len(outbox), 2) self.assertIn('iesg-secretary',outbox[0]['To']) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index 0ebfd6b8f..5b6dd63b9 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -573,7 +573,7 @@ class ResurrectTests(DraftFileMixin, TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('form [type=submit]')), 1) + self.assertEqual(len(q('#content form [type=submit]')), 1) # request resurrect @@ -609,7 +609,7 @@ class ResurrectTests(DraftFileMixin, TestCase): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('form [type=submit]')), 1) + self.assertEqual(len(q('#content form [type=submit]')), 1) # complete resurrect events_before = draft.docevent_set.count() diff --git a/ietf/doc/tests_js.py b/ietf/doc/tests_js.py index 02daaae90..ac63c0995 100644 --- a/ietf/doc/tests_js.py +++ b/ietf/doc/tests_js.py @@ -118,7 +118,7 @@ class EditAuthorsTests(IetfSeleniumTestCase): # Must provide a "basis" (change reason) self.driver.find_element(By.ID, 'id_basis').send_keys('change testing') # Now click the 'submit' button and check that the update was accepted. - submit_button = self.driver.find_element(By.CSS_SELECTOR, 'button[type="submit"]') + submit_button = self.driver.find_element(By.CSS_SELECTOR, '#content button[type="submit"]') self.driver.execute_script("arguments[0].click();", submit_button) # FIXME: no idea why this fails: # self.scroll_to_element(submit_button) # submit_button.click() @@ -132,4 +132,4 @@ class EditAuthorsTests(IetfSeleniumTestCase): self.assertEqual( list(draft.documentauthor_set.values_list('person', flat=True)), [first_auth.person.pk] + [auth.pk for auth in authors] - ) \ No newline at end of file + ) diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py index edfb89f38..f4b672804 100644 --- a/ietf/doc/urls.py +++ b/ietf/doc/urls.py @@ -33,9 +33,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from django.conf.urls import include -from django.views.generic import RedirectView from django.conf import settings +from django.urls import include +from django.views.generic import RedirectView from ietf.doc import views_search, views_draft, views_ballot, views_status_change, views_doc, views_downref, views_stats, views_help, views_bofreq from ietf.utils.urls import url diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index b2e65066a..6c9409475 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -17,7 +17,6 @@ from zoneinfo import ZoneInfo from django.conf import settings from django.contrib import messages -from django.db.models import QuerySet from django.forms import ValidationError from django.http import Http404 from django.template.loader import render_to_string @@ -25,6 +24,7 @@ from django.utils import timezone from django.utils.html import escape from django.urls import reverse as urlreverse +from django_stubs_ext import QuerySetAny import debug # pyflakes:ignore from ietf.community.models import CommunityList @@ -345,7 +345,7 @@ def augment_events_with_revision(doc, events): """Take a set of events for doc and add a .rev attribute with the revision they refer to by checking NewRevisionDocEvents.""" - if isinstance(events, QuerySet): + if isinstance(events, QuerySetAny): qs = events.filter(newrevisiondocevent__isnull=False) else: qs = NewRevisionDocEvent.objects.filter(doc=doc) @@ -353,7 +353,7 @@ def augment_events_with_revision(doc, events): if doc.type_id == "draft" and doc.get_state_slug() == "rfc": # add fake "RFC" revision - if isinstance(events, QuerySet): + if isinstance(events, QuerySetAny): e = events.filter(type="published_rfc").order_by('time').last() else: e = doc.latest_event(type="published_rfc") diff --git a/ietf/doc/utils_charter.py b/ietf/doc/utils_charter.py index d14684d42..2e85b3cc1 100644 --- a/ietf/doc/utils_charter.py +++ b/ietf/doc/utils_charter.py @@ -12,7 +12,7 @@ from django.conf import settings from django.urls import reverse as urlreverse from django.template.loader import render_to_string from django.utils import timezone -from django.utils.encoding import smart_text, force_text +from django.utils.encoding import smart_str, force_str import debug # pyflakes:ignore @@ -153,7 +153,7 @@ def generate_ballot_writeup(request, doc): e.doc = doc e.rev = doc.rev, e.desc = "Ballot writeup was generated" - e.text = force_text(render_to_string("doc/charter/ballot_writeup.txt")) + e.text = force_str(render_to_string("doc/charter/ballot_writeup.txt")) # caller is responsible for saving, if necessary return e @@ -197,7 +197,7 @@ def derive_new_work_text(review_text,group): 'Reply_to':''}) if not addrs.cc: del m['Cc'] - return smart_text(m.as_string()) + return smart_str(m.as_string()) def default_review_text(group, charter, by): now = timezone.now() diff --git a/ietf/doc/views_charter.py b/ietf/doc/views_charter.py index c49710895..d3173291d 100644 --- a/ietf/doc/views_charter.py +++ b/ietf/doc/views_charter.py @@ -17,7 +17,7 @@ from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required from django.utils import timezone -from django.utils.encoding import force_text +from django.utils.encoding import force_str from django.utils.html import escape import debug # pyflakes:ignore @@ -821,7 +821,7 @@ def charter_with_milestones_txt(request, name, rev): try: with io.open(os.path.join(settings.CHARTER_PATH, filename), 'r') as f: - charter_text = force_text(f.read(), errors='ignore') + charter_text = force_str(f.read(), errors='ignore') except IOError: charter_text = "Error reading charter text %s" % filename diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index a13e9eb08..fa6e3a7ff 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -11,7 +11,7 @@ import requests import email.utils from django.utils import timezone -from django.utils.http import is_safe_url +from django.utils.http import url_has_allowed_host_and_scheme from simple_history.utils import update_change_reason @@ -53,6 +53,7 @@ from ietf.utils.textupload import get_cleaned_text_file_content from ietf.utils.mail import send_mail_message from ietf.mailtrigger.utils import gather_address_lists from ietf.utils.fields import MultiEmailField +from ietf.utils.http import is_ajax from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO @@ -1087,11 +1088,16 @@ def review_wishes_remove(request, name): def _generate_ajax_or_redirect_response(request, doc): - redirect_url = request.GET.get('next') - url_is_safe = is_safe_url(url=redirect_url, allowed_hosts=request.get_host(), - require_https=request.is_secure()) - if request.is_ajax(): - return HttpResponse(json.dumps({'success': True}), content_type='application/json') + redirect_url = request.GET.get("next") + url_is_safe = url_has_allowed_host_and_scheme( + url=redirect_url, + allowed_hosts=request.get_host(), + require_https=request.is_secure(), + ) + if is_ajax(request): + return HttpResponse( + json.dumps({"success": True}), content_type="application/json" + ) elif url_is_safe: return HttpResponseRedirect(redirect_url) else: diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 0eca0007e..5bd1c8517 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -765,7 +765,9 @@ def drafts_in_iesg_process(request): if s.slug == "lc": for d in docs: e = d.latest_event(LastCallDocEvent, type="sent_last_call") - d.lc_expires = e.expires if e else datetime.datetime.min + # If we don't have an event, use an arbitrary date in the past (but not datetime.datetime.min, + # which causes problems with timezone conversions) + d.lc_expires = e.expires if e else datetime.datetime(1950, 1, 1) docs = list(docs) docs.sort(key=lambda d: d.lc_expires) diff --git a/ietf/doc/views_stats.py b/ietf/doc/views_stats.py index 7d56e8569..34f670d5c 100644 --- a/ietf/doc/views_stats.py +++ b/ietf/doc/views_stats.py @@ -7,6 +7,7 @@ from django.conf import settings from django.core.cache import cache from django.urls import reverse as urlreverse from django.db.models.aggregates import Count +from django.db.models.functions import TruncDate from django.http import JsonResponse, HttpResponseBadRequest from django.shortcuts import render from django.views.decorators.cache import cache_page @@ -24,7 +25,6 @@ from ietf.utils.timezone import date_today epochday = datetime.datetime.utcfromtimestamp(0).date().toordinal() -column_chart_conf = settings.CHART_TYPE_COLUMN_OPTIONS def dt(s): "Convert the date string returned by sqlite's date() to a datetime.date" @@ -40,15 +40,12 @@ def model_to_timeline_data(model, field='time', **kwargs): assert field in [ f.name for f in model._meta.get_fields() ] objects = ( model.objects.filter(**kwargs) + .annotate(date=TruncDate(field)) .order_by('date') - .extra(select={'date': 'date(%s.%s)'% (model._meta.db_table, field) }) .values('date') .annotate(count=Count('id'))) if objects.exists(): obj_list = list(objects) - # This is needed for sqlite, when we're running tests: - if type(obj_list[0]['date']) != datetime.date: - obj_list = [ {'date': dt(e['date']), 'count': e['count']} for e in obj_list ] today = date_today(datetime.timezone.utc) if not obj_list[-1]['date'] == today: obj_list += [ {'date': today, 'count': 0} ] diff --git a/ietf/doc/views_status_change.py b/ietf/doc/views_status_change.py index 99f82d435..ec914eebe 100644 --- a/ietf/doc/views_status_change.py +++ b/ietf/doc/views_status_change.py @@ -15,7 +15,7 @@ from django.http import Http404, HttpResponseRedirect from django.urls import reverse from django.template.loader import render_to_string from django.conf import settings -from django.utils.encoding import force_text +from django.utils.encoding import force_str from django.utils.html import escape import debug # pyflakes:ignore @@ -531,7 +531,7 @@ def rfc_status_changes(request): ) @role_required("Area Director","Secretariat") -def start_rfc_status_change(request,name): +def start_rfc_status_change(request, name=None): """Start the RFC status change review process, setting the initial shepherding AD, and possibly putting the review on a telechat.""" if name: @@ -665,7 +665,7 @@ def generate_last_call_text(request, doc): e.doc = doc e.rev = doc.rev e.desc = 'Last call announcement was generated' - e.text = force_text(new_text) + e.text = force_str(new_text) e.save() return e diff --git a/ietf/group/admin.py b/ietf/group/admin.py index 0773a4ce2..afaa87c0b 100644 --- a/ietf/group/admin.py +++ b/ietf/group/admin.py @@ -14,9 +14,9 @@ from django.contrib.admin.utils import unquote from django.core.management import load_command_class from django.http import Http404 from django.shortcuts import render -from django.utils.encoding import force_text +from django.utils.encoding import force_str from django.utils.html import escape -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from ietf.group.models import (Group, GroupFeatures, GroupHistory, GroupEvent, GroupURL, GroupMilestone, GroupMilestoneHistory, GroupStateTransitions, Role, RoleHistory, ChangeStateGroupEvent, @@ -152,7 +152,7 @@ class GroupAdmin(admin.ModelAdmin): permission_denied(request, "You don't have edit permissions for this change.") if obj is None: - raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_text(opts.verbose_name), 'key': escape(object_id)}) + raise Http404(_('%(name)s object with primary key %(key)r does not exist.') % {'name': force_str(opts.verbose_name), 'key': escape(object_id)}) return self.send_reminder(request, sdo=obj) diff --git a/ietf/group/urls.py b/ietf/group/urls.py index 713a0b7ee..0e4f7ef2f 100644 --- a/ietf/group/urls.py +++ b/ietf/group/urls.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2013-2020, All Rights Reserved from django.conf import settings -from django.conf.urls import include +from django.urls import include from django.views.generic import RedirectView from ietf.community import views as community_views diff --git a/ietf/group/views.py b/ietf/group/views.py index 4d83a6b4c..95bf4c5e9 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -300,8 +300,27 @@ def active_groups(request, group_type=None): raise Http404 def active_group_types(request): - grouptypes = GroupTypeName.objects.filter(slug__in=['wg','rg','ag','rag','team','dir','review','area','program','iabasg','adm']).filter(group__state='active').annotate(group_count=Count('group')) - return render(request, 'group/active_groups.html', {'grouptypes':grouptypes}) + grouptypes = ( + GroupTypeName.objects.filter( + slug__in=[ + "wg", + "rg", + "ag", + "rag", + "team", + "dir", + "review", + "area", + "program", + "iabasg", + "adm", + ] + ) + .filter(group__state="active") + .order_by('order', 'name') # default ordering ignored for "GROUP BY" queries, make it explicit + .annotate(group_count=Count("group")) + ) + return render(request, "group/active_groups.html", {"grouptypes": grouptypes}) def active_dirs(request): dirs = Group.objects.filter(type__in=['dir', 'review'], state="active").order_by("name") diff --git a/ietf/ietfauth/forms.py b/ietf/ietfauth/forms.py index ce9f58f49..9b8ee22e0 100644 --- a/ietf/ietfauth/forms.py +++ b/ietf/ietfauth/forms.py @@ -11,14 +11,14 @@ from django.core.exceptions import ValidationError from django.db import models from django.contrib.auth.models import User -from django_password_strength.widgets import PasswordStrengthInput, PasswordConfirmationInput - import debug # pyflakes:ignore from ietf.person.models import Person, Email from ietf.mailinglists.models import Allowlisted from ietf.utils.text import isascii +from .widgets import PasswordStrengthInput, PasswordConfirmationInput + class RegistrationForm(forms.Form): email = forms.EmailField(label="Your email (lowercase)") diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py index 631da9870..0e5fcb3c4 100644 --- a/ietf/ietfauth/tests.py +++ b/ietf/ietfauth/tests.py @@ -98,7 +98,7 @@ class IetfAuthTests(TestCase): self.assertEqual(urlsplit(r["Location"])[2], urlreverse(ietf.ietfauth.views.profile)) # try logging out - r = self.client.get(urlreverse('django.contrib.auth.views.logout')) + r = self.client.post(urlreverse('django.contrib.auth.views.logout'), {}) self.assertEqual(r.status_code, 200) self.assertNotContains(r, "accounts/logout") @@ -215,7 +215,7 @@ class IetfAuthTests(TestCase): self.assertContains(r, "Allowlist entry creation successful") # log out - r = self.client.get(urlreverse('django.contrib.auth.views.logout')) + r = self.client.post(urlreverse('django.contrib.auth.views.logout'), {}) self.assertEqual(r.status_code, 200) # register and verify allowlisted email @@ -664,7 +664,7 @@ class IetfAuthTests(TestCase): "new_password_confirmation": "foobar", }) self.assertEqual(r.status_code, 200) - self.assertFormError(r, 'form', 'current_password', 'Invalid password') + self.assertFormError(r.context["form"], 'current_password', 'Invalid password') # mismatching new passwords r = self.client.post(chpw_url, {"current_password": "password", @@ -672,7 +672,7 @@ class IetfAuthTests(TestCase): "new_password_confirmation": "barfoo", }) self.assertEqual(r.status_code, 200) - self.assertFormError(r, 'form', None, "The password confirmation is different than the new password") + self.assertFormError(r.context["form"], None, "The password confirmation is different than the new password") # correct password change r = self.client.post(chpw_url, {"current_password": "password", @@ -711,7 +711,7 @@ class IetfAuthTests(TestCase): "password": "password", }) self.assertEqual(r.status_code, 200) - self.assertFormError(r, 'form', 'username', + self.assertFormError(r.context["form"], 'username', "Select a valid choice. fiddlesticks is not one of the available choices.") # wrong password @@ -719,7 +719,7 @@ class IetfAuthTests(TestCase): "password": "foobar", }) self.assertEqual(r.status_code, 200) - self.assertFormError(r, 'form', 'password', 'Invalid password') + self.assertFormError(r.context["form"], 'password', 'Invalid password') # correct username change r = self.client.post(chun_url, {"username": "othername@example.org", diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index 029342e7f..52f582ca8 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -7,7 +7,8 @@ import oidc_provider.lib.claims -from functools import wraps +from functools import wraps, WRAPPER_ASSIGNMENTS +from urllib.parse import quote as urlquote from django.conf import settings from django.contrib.auth import REDIRECT_FIELD_NAME @@ -15,8 +16,6 @@ from django.core.exceptions import PermissionDenied from django.db.models import Q from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404 -from django.utils.decorators import available_attrs -from django.utils.http import urlquote import debug # pyflakes:ignore @@ -113,7 +112,7 @@ def passes_test_decorator(test_func, message): error. The test function should be on the form fn(user) -> true/false.""" def decorate(view_func): - @wraps(view_func, assigned=available_attrs(view_func)) + @wraps(view_func, assigned=WRAPPER_ASSIGNMENTS) def inner(request, *args, **kwargs): if not request.user.is_authenticated: return HttpResponseRedirect('%s?%s=%s' % (settings.LOGIN_URL, REDIRECT_FIELD_NAME, urlquote(request.get_full_path()))) diff --git a/ietf/ietfauth/widgets.py b/ietf/ietfauth/widgets.py new file mode 100644 index 000000000..6b01a67bd --- /dev/null +++ b/ietf/ietfauth/widgets.py @@ -0,0 +1,114 @@ +from django.forms import PasswordInput +from django.utils.safestring import mark_safe +from django.utils.translation import gettext as _ + +# The PasswordStrengthInput and PasswordConfirmationInput widgets come from the +# django-password-strength project, https://pypi.org/project/django-password-strength/ +# +# Original license: +# +# Copyright © 2015 A.J. May and individual contributors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +# following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +# following disclaimer in the documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +# products derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +class PasswordStrengthInput(PasswordInput): + """ + Form widget to show the user how strong his/her password is. + """ + + def render(self, name, value, attrs=None, renderer=None): + strength_markup = """ +
+
+
+
+ +
+ """ % ( + _("Warning"), + _( + 'This password would take to crack.' + ), + ) + + try: + self.attrs["class"] = "%s password_strength".strip() % self.attrs["class"] + except KeyError: + self.attrs["class"] = "password_strength" + + return mark_safe( + super(PasswordInput, self).render(name, value, attrs, renderer) + + strength_markup + ) + + class Media: + js = ( + "ietf/js/zxcvbn.js", + "ietf/js/password_strength.js", + ) + + +class PasswordConfirmationInput(PasswordInput): + """ + Form widget to confirm the users password by letting him/her type it again. + """ + + def __init__(self, confirm_with=None, attrs=None, render_value=False): + super(PasswordConfirmationInput, self).__init__(attrs, render_value) + self.confirm_with = confirm_with + + def render(self, name, value, attrs=None, renderer=None): + if self.confirm_with: + self.attrs["data-confirm-with"] = "id_%s" % self.confirm_with + + confirmation_markup = """ + + """ % ( + _("Warning"), + _("Your passwords don't match."), + ) + + try: + self.attrs["class"] = ( + "%s password_confirmation".strip() % self.attrs["class"] + ) + except KeyError: + self.attrs["class"] = "password_confirmation" + + return mark_safe( + super(PasswordInput, self).render(name, value, attrs, renderer) + + confirmation_markup + ) diff --git a/ietf/ipr/feeds.py b/ietf/ipr/feeds.py index b5f4c4e6a..4979c649b 100644 --- a/ietf/ipr/feeds.py +++ b/ietf/ipr/feeds.py @@ -6,7 +6,7 @@ from django.contrib.syndication.views import Feed from django.utils.feedgenerator import Atom1Feed from django.urls import reverse_lazy from django.utils.safestring import mark_safe -from django.utils.encoding import force_text +from django.utils.encoding import force_str from ietf.ipr.models import IprDisclosureBase @@ -25,7 +25,7 @@ class LatestIprDisclosuresFeed(Feed): return mark_safe(item.title) def item_description(self, item): - return force_text(item.title) + return force_str(item.title) def item_pubdate(self, item): return item.time diff --git a/ietf/ipr/mail.py b/ietf/ipr/mail.py index f1d8039db..842426d82 100644 --- a/ietf/ipr/mail.py +++ b/ietf/ipr/mail.py @@ -12,7 +12,7 @@ from email import message_from_bytes from email.utils import parsedate_tz from django.template.loader import render_to_string -from django.utils.encoding import force_text, force_bytes +from django.utils.encoding import force_str, force_bytes import debug # pyflakes:ignore @@ -102,7 +102,7 @@ def get_reply_to(): address with "plus addressing" using a random string. Guaranteed to be unique""" local,domain = get_base_ipr_request_address().split('@') while True: - rand = force_text(base64.urlsafe_b64encode(os.urandom(12))) + rand = force_str(base64.urlsafe_b64encode(os.urandom(12))) address = "{}+{}@{}".format(local,rand,domain) q = Message.objects.filter(reply_to=address) if not q: diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index fa1f550d0..b41351b94 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -9,16 +9,15 @@ import operator from typing import Union # pyflakes:ignore from email.utils import parseaddr -from form_utils.forms import BetterModelForm from django import forms from django.conf import settings from django.core.exceptions import ObjectDoesNotExist, ValidationError -from django.db.models.query import QuerySet from django.forms.utils import ErrorList from django.db.models import Q #from django.forms.widgets import RadioFieldRenderer from django.core.validators import validate_email +from django_stubs_ext import QuerySetAny import debug # pyflakes:ignore @@ -132,7 +131,7 @@ class AddCommentForm(forms.Form): # def render(self): # output = [] # for widget in self: -# output.append(format_html(force_text(widget))) +# output.append(format_html(force_str(widget))) # return mark_safe('\n'.join(output)) @@ -204,7 +203,7 @@ class SearchLiaisonForm(forms.Form): class CustomModelMultipleChoiceField(forms.ModelMultipleChoiceField): '''If value is a QuerySet, return it as is (for use in widget.render)''' def prepare_value(self, value): - if isinstance(value, QuerySet): + if isinstance(value, QuerySetAny): return value if (hasattr(value, '__iter__') and not isinstance(value, str) and @@ -213,7 +212,7 @@ class CustomModelMultipleChoiceField(forms.ModelMultipleChoiceField): return super(CustomModelMultipleChoiceField, self).prepare_value(value) -class LiaisonModelForm(BetterModelForm): +class LiaisonModelForm(forms.ModelForm): '''Specify fields which require a custom widget or that are not part of the model. ''' from_groups = forms.ModelMultipleChoiceField(queryset=Group.objects.all(),label='Groups',required=False) @@ -238,13 +237,6 @@ class LiaisonModelForm(BetterModelForm): class Meta: model = LiaisonStatement exclude = ('attachments','state','from_name','to_name') - fieldsets = [('From', {'fields': ['from_groups','from_contact', 'response_contacts'], 'legend': ''}), - ('To', {'fields': ['to_groups','to_contacts'], 'legend': ''}), - ('Other email addresses', {'fields': ['technical_contacts','action_holder_contacts','cc_contacts'], 'legend': ''}), - ('Purpose', {'fields':['purpose', 'deadline'], 'legend': ''}), - ('Reference', {'fields': ['other_identifiers','related_to'], 'legend': ''}), - ('Liaison Statement', {'fields': ['title', 'submitted_date', 'body', 'attachments'], 'legend': ''}), - ('Add attachment', {'fields': ['attach_title', 'attach_file', 'attach_button'], 'legend': ''})] def __init__(self, user, *args, **kwargs): super(LiaisonModelForm, self).__init__(*args, **kwargs) @@ -476,14 +468,6 @@ class OutgoingLiaisonForm(LiaisonModelForm): class Meta: model = LiaisonStatement exclude = ('attachments','state','from_name','to_name','action_holder_contacts') - # add approved field, no action_holder_contacts - fieldsets = [('From', {'fields': ['from_groups','from_contact','response_contacts','approved'], 'legend': ''}), - ('To', {'fields': ['to_groups','to_contacts'], 'legend': ''}), - ('Other email addresses', {'fields': ['technical_contacts','cc_contacts'], 'legend': ''}), - ('Purpose', {'fields':['purpose', 'deadline'], 'legend': ''}), - ('Reference', {'fields': ['other_identifiers','related_to'], 'legend': ''}), - ('Liaison Statement', {'fields': ['title', 'submitted_date', 'body', 'attachments'], 'legend': ''}), - ('Add attachment', {'fields': ['attach_title', 'attach_file', 'attach_button'], 'legend': ''})] def is_approved(self): return self.cleaned_data['approved'] diff --git a/ietf/liaisons/widgets.py b/ietf/liaisons/widgets.py index d6e2fe936..74368e83f 100644 --- a/ietf/liaisons/widgets.py +++ b/ietf/liaisons/widgets.py @@ -3,11 +3,12 @@ from django.urls import reverse as urlreverse -from django.db.models.query import QuerySet from django.forms.widgets import Widget from django.utils.safestring import mark_safe from django.utils.html import conditional_escape +from django_stubs_ext import QuerySetAny + class ButtonWidget(Widget): def __init__(self, *args, **kwargs): @@ -34,7 +35,7 @@ class ShowAttachmentsWidget(Widget): html = '
' % name html += 'No files attached' html += '
' - if value and isinstance(value, QuerySet): + if value and isinstance(value, QuerySetAny): for attachment in value: html += '%s ' % (conditional_escape(attachment.document.get_href()), conditional_escape(attachment.document.title)) html += 'Edit '.format(urlreverse("ietf.liaisons.views.liaison_edit_attachment", kwargs={'object_id':attachment.statement.pk,'doc_id':attachment.document.pk})) @@ -43,4 +44,4 @@ class ShowAttachmentsWidget(Widget): else: html += 'No files attached' html += '
' - return mark_safe(html) \ No newline at end of file + return mark_safe(html) diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index 1590f12e6..e69afe5ca 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -7,9 +7,7 @@ import datetime import shutil import os import re -from unittest import skipIf -import django from django.utils import timezone from django.utils.text import slugify from django.db.models import F @@ -880,42 +878,6 @@ class EditMeetingScheduleTests(IetfSeleniumTestCase): self.assertNotIn('would-violate-hint', session_elements[4].get_attribute('class'), 'Constraint violation should not be indicated on non-conflicting session') -@ifSeleniumEnabled -@skipIf(django.VERSION[0]==2, "Skipping test with race conditions under Django 2") -class ScheduleEditTests(IetfSeleniumTestCase): - def testUnschedule(self): - - meeting = make_meeting_test_data() - - self.assertEqual(SchedTimeSessAssignment.objects.filter(session__meeting=meeting, session__group__acronym='mars', schedule__name='test-schedule').count(),1) - - - ss = list(SchedTimeSessAssignment.objects.filter(session__meeting__number=72,session__group__acronym='mars',schedule__name='test-schedule')) # pyflakes:ignore - - self.login() - url = self.absreverse('ietf.meeting.views.edit_meeting_schedule',kwargs=dict(num='72',name='test-schedule',owner='plain@example.com')) - self.driver.get(url) - - # driver.get() will wait for scripts to finish, but not ajax - # requests. Wait for completion of the permissions check: - read_only_note = self.driver.find_element(By.ID, 'read_only') - WebDriverWait(self.driver, 10).until(expected_conditions.invisibility_of_element(read_only_note), "Read-only schedule") - - s1 = Session.objects.filter(group__acronym='mars', meeting=meeting).first() - selector = "#session_{}".format(s1.pk) - WebDriverWait(self.driver, 30).until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, selector)), "Did not find %s"%selector) - - self.assertEqual(self.driver.find_elements(By.CSS_SELECTOR, "#sortable-list #session_{}".format(s1.pk)), []) - - element = self.driver.find_element(By.ID, 'session_{}'.format(s1.pk)) - target = self.driver.find_element(By.ID, 'sortable-list') - ActionChains(self.driver).drag_and_drop(element,target).perform() - - self.assertTrue(self.driver.find_elements(By.CSS_SELECTOR, "#sortable-list #session_{}".format(s1.pk))) - - time.sleep(0.1) # The API that modifies the database runs async - - self.assertEqual(SchedTimeSessAssignment.objects.filter(session__meeting__number=72,session__group__acronym='mars',schedule__name='test-schedule').count(),0) @ifSeleniumEnabled class SlideReorderTests(IetfSeleniumTestCase): @@ -1531,7 +1493,7 @@ class EditTimeslotsTests(IetfSeleniumTestCase): """Test the timeslot editor""" def setUp(self): super().setUp() - self.meeting: Meeting = MeetingFactory( + self.meeting: Meeting = MeetingFactory( # type: ignore[annotation-unchecked] type_id='ietf', number=120, date=date_today() + datetime.timedelta(days=10), @@ -1608,13 +1570,13 @@ class EditTimeslotsTests(IetfSeleniumTestCase): delete_time = delete_time_local.astimezone(datetime.timezone.utc) duration = datetime.timedelta(minutes=60) - delete: [TimeSlot] = TimeSlotFactory.create_batch( + delete: [TimeSlot] = TimeSlotFactory.create_batch( # type: ignore[annotation-unchecked] 2, meeting=self.meeting, time=delete_time_local, duration=duration, ) - keep: [TimeSlot] = [ + keep: [TimeSlot] = [ # type: ignore[annotation-unchecked] TimeSlotFactory( meeting=self.meeting, time=keep_time, @@ -1651,14 +1613,14 @@ class EditTimeslotsTests(IetfSeleniumTestCase): hours = [10, 12] other_days = [self.meeting.get_meeting_date(d) for d in range(1, 3)] - delete: [TimeSlot] = [ + delete: [TimeSlot] = [ # type: ignore[annotation-unchecked] TimeSlotFactory( meeting=self.meeting, time=datetime_from_date(delete_day, self.meeting.tz()).replace(hour=hour), ) for hour in hours ] - keep: [TimeSlot] = [ + keep: [TimeSlot] = [ # type: ignore[annotation-unchecked] TimeSlotFactory( meeting=self.meeting, time=datetime_from_date(day, self.meeting.tz()).replace(hour=hour), diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index e65adaf0c..618fbd842 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -556,7 +556,7 @@ class MeetingTests(BaseMeetingTestCase): self.assertContains(r, "1. More work items underway") - cont_disp = r._headers.get('content-disposition', ('Content-Disposition', ''))[1] + cont_disp = r.headers.get('content-disposition', ('Content-Disposition', ''))[1] cont_disp = re.split('; ?', cont_disp) cont_disp_settings = dict( e.split('=', 1) for e in cont_disp if '=' in e ) filename = cont_disp_settings.get('filename', '').strip('"') @@ -2357,7 +2357,7 @@ class EditTimeslotsTests(TestCase): def test_invalid_edit_timeslot(self): meeting = self.create_bare_meeting() - ts: TimeSlot = TimeSlotFactory(meeting=meeting, name='slot') # n.b., colon indicates type hinting + ts: TimeSlot = TimeSlotFactory(meeting=meeting, name='slot') # type: ignore[annotation-unchecked] self.login() r = self.client.post( self.edit_timeslot_url(ts), @@ -3931,7 +3931,7 @@ class EditTests(TestCase): 'base': meeting.schedule.base_id, }) self.assertEqual(r.status_code, 200) - self.assertFormError(r, 'form', 'name', 'Enter a valid value.') + self.assertFormError(r.context["form"], 'name', 'Enter a valid value.') self.assertEqual(meeting.schedule_set.count(), orig_schedule_count, 'Schedule should not be created') r = self.client.post(url, { @@ -3941,7 +3941,7 @@ class EditTests(TestCase): 'base': meeting.schedule.base_id, }) self.assertEqual(r.status_code, 200) - self.assertFormError(r, 'form', 'name', 'Enter a valid value.') + self.assertFormError(r.context["form"], 'name', 'Enter a valid value.') self.assertEqual(meeting.schedule_set.count(), orig_schedule_count, 'Schedule should not be created') # Non-ASCII alphanumeric characters @@ -3952,7 +3952,7 @@ class EditTests(TestCase): 'base': meeting.schedule.base_id, }) self.assertEqual(r.status_code, 200) - self.assertFormError(r, 'form', 'name', 'Enter a valid value.') + self.assertFormError(r.context["form"], 'name', 'Enter a valid value.') self.assertEqual(meeting.schedule_set.count(), orig_schedule_count, 'Schedule should not be created') def test_edit_session(self): @@ -4039,9 +4039,9 @@ class EditTests(TestCase): self.assertIn(return_url_unofficial, r.content.decode()) r = self.client.post(url, {}) - self.assertFormError(r, 'form', 'confirmed', 'This field is required.') + self.assertFormError(r.context["form"], 'confirmed', 'This field is required.') r = self.client.post(url_unofficial, {}) - self.assertFormError(r, 'form', 'confirmed', 'This field is required.') + self.assertFormError(r.context["form"], 'confirmed', 'This field is required.') r = self.client.post(url, {'confirmed': 'on'}) self.assertRedirects(r, return_url) @@ -6520,8 +6520,8 @@ class ImportNotesTests(TestCase): r = self.client.get(url) # try to import the same text self.assertContains(r, "This document is identical", status_code=200) q = PyQuery(r.content) - self.assertEqual(len(q('button:disabled[type="submit"]')), 1) - self.assertEqual(len(q('button:enabled[type="submit"]')), 0) + self.assertEqual(len(q('#content button:disabled[type="submit"]')), 1) + self.assertEqual(len(q('#content button:enabled[type="submit"]')), 0) def test_allows_import_on_existing_bad_unicode(self): """Should not be able to import text identical to the current revision""" @@ -6545,8 +6545,8 @@ class ImportNotesTests(TestCase): r = self.client.get(url) # try to import the same text self.assertNotContains(r, "This document is identical", status_code=200) q = PyQuery(r.content) - self.assertEqual(len(q('button:enabled[type="submit"]')), 1) - self.assertEqual(len(q('button:disabled[type="submit"]')), 0) + self.assertEqual(len(q('#content button:enabled[type="submit"]')), 1) + self.assertEqual(len(q('#content button:disabled[type="submit"]')), 0) def test_handles_missing_previous_revision_file(self): """Should still allow import if the file for the previous revision is missing""" @@ -7975,7 +7975,7 @@ class ProceedingsTests(BaseMeetingTestCase): invalid_file.seek(0) # read the file contents again r = self.client.post(url, {'file': invalid_file, 'external_url': ''}) self.assertEqual(r.status_code, 200) - self.assertFormError(r, 'form', 'file', 'Found an unexpected extension: .png. Expected one of .pdf') + self.assertFormError(r.context["form"], 'file', 'Found an unexpected extension: .png. Expected one of .pdf') def test_add_proceedings_material_doc_empty(self): """Upload proceedings materials document without specifying a file""" @@ -7988,7 +7988,7 @@ class ProceedingsTests(BaseMeetingTestCase): ) r = self.client.post(url, {'external_url': ''}) self.assertEqual(r.status_code, 200) - self.assertFormError(r, 'form', 'file', 'This field is required') + self.assertFormError(r.context["form"], 'file', 'This field is required') def test_add_proceedings_material_url(self): """Add a URL as proceedings material""" @@ -8012,7 +8012,7 @@ class ProceedingsTests(BaseMeetingTestCase): ) r = self.client.post(url, {'use_url': 'on', 'external_url': "Ceci n'est pas une URL"}) self.assertEqual(r.status_code, 200) - self.assertFormError(r, 'form', 'external_url', 'Enter a valid URL.') + self.assertFormError(r.context["form"], 'external_url', 'Enter a valid URL.') def test_add_proceedings_material_url_empty(self): """Add proceedings materials URL without specifying the URL""" @@ -8025,7 +8025,7 @@ class ProceedingsTests(BaseMeetingTestCase): ) r = self.client.post(url, {'use_url': 'on', 'external_url': ''}) self.assertEqual(r.status_code, 200) - self.assertFormError(r, 'form', 'external_url', 'This field is required') + self.assertFormError(r.context["form"], 'external_url', 'This field is required') @override_settings(MEETING_DOC_HREFS={'procmaterials': '{doc.name}:{doc.rev}'}) def test_replace_proceedings_material(self): diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py index 8284f8fcf..d7a623899 100644 --- a/ietf/meeting/urls.py +++ b/ietf/meeting/urls.py @@ -1,8 +1,8 @@ # Copyright The IETF Trust 2007-2020, All Rights Reserved -from django.conf.urls import include -from django.views.generic import RedirectView from django.conf import settings +from django.urls import include +from django.views.generic import RedirectView from ietf.meeting import views, views_proceedings from ietf.utils.urls import url diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 1f4896c88..a99f29463 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -15,7 +15,7 @@ from django.conf import settings from django.contrib import messages from django.template.loader import render_to_string from django.utils import timezone -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str import debug # pyflakes:ignore @@ -699,7 +699,7 @@ def handle_upload_file(file, filename, meeting, subdir, request=None, encoding=N ) else: try: - text = smart_text(text) + text = smart_str(text) except UnicodeDecodeError as e: return "Failure trying to save '%s'. Hint: Try to upload as UTF-8: %s..." % (filename, str(e)[:120]) # Whole file sanitization; add back what's missing from a complete diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 28beaa113..873bdad75 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -17,6 +17,7 @@ import tempfile from calendar import timegm from collections import OrderedDict, Counter, deque, defaultdict, namedtuple +from functools import partialmethod from urllib.parse import parse_qs, unquote, urlencode, urlsplit, urlunsplit from tempfile import mkstemp from wsgiref.handlers import format_date_time @@ -38,7 +39,6 @@ from django.template import TemplateDoesNotExist from django.template.loader import render_to_string from django.utils import timezone from django.utils.encoding import force_str -from django.utils.functional import curry from django.utils.text import slugify from django.views.decorators.cache import cache_page from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt @@ -2730,7 +2730,7 @@ def upload_session_agenda(request, session_id, num): }) -def upload_session_slides(request, session_id, num, name): +def upload_session_slides(request, session_id, num, name=None): # num is redundant, but we're dragging it along an artifact of where we are in the current URL structure session = get_object_or_404(Session,pk=session_id) if not session.can_manage_materials(request.user): @@ -3210,8 +3210,8 @@ def interim_request(request): if meeting_type in ('single', 'multi-day'): meeting = form.save(date=get_earliest_session_date(formset)) - # need to use curry here to pass custom variable to form init - SessionFormset.form.__init__ = curry( + # need to use partialmethod here to pass custom variable to form init + SessionFormset.form.__init__ = partialmethod( InterimSessionModelForm.__init__, user=request.user, group=group, @@ -3233,7 +3233,7 @@ def interim_request(request): # subsequently dealt with individually elif meeting_type == 'series': series = [] - SessionFormset.form.__init__ = curry( + SessionFormset.form.__init__ = partialmethod( InterimSessionModelForm.__init__, user=request.user, group=group, @@ -3453,7 +3453,7 @@ def interim_request_edit(request, number): group = Group.objects.get(pk=form.data['group']) is_approved = is_interim_meeting_approved(meeting) - SessionFormset.form.__init__ = curry( + SessionFormset.form.__init__ = partialmethod( InterimSessionModelForm.__init__, user=request.user, group=group, diff --git a/ietf/name/management/commands/generate_name_fixture.py b/ietf/name/management/commands/generate_name_fixture.py index 02dc08faf..bbf33e600 100644 --- a/ietf/name/management/commands/generate_name_fixture.py +++ b/ietf/name/management/commands/generate_name_fixture.py @@ -67,7 +67,7 @@ class Command(BaseCommand): pprint(connection.queries) raise - objects = [] # type: List[object] + objects: List[object] = [] # type: ignore[annotation-unchecked] model_objects = {} import ietf.name.models diff --git a/ietf/nomcom/decorators.py b/ietf/nomcom/decorators.py index a002f7c7e..43250bd30 100644 --- a/ietf/nomcom/decorators.py +++ b/ietf/nomcom/decorators.py @@ -3,10 +3,11 @@ import functools +from urllib.parse import quote as urlquote from django.urls import reverse from django.http import HttpResponseRedirect -from django.utils.http import urlquote + def nomcom_private_key_required(view_func): diff --git a/ietf/nomcom/factories.py b/ietf/nomcom/factories.py index 8ef4e07fa..6fd6819b0 100644 --- a/ietf/nomcom/factories.py +++ b/ietf/nomcom/factories.py @@ -66,7 +66,7 @@ Tdb0MiLc+r/zvx8oXtgDjDUa def provide_private_key_to_test_client(testcase): session = testcase.client.session - session['NOMCOM_PRIVATE_KEY_%s'%testcase.nc.year()] = key + session['NOMCOM_PRIVATE_KEY_%s'%testcase.nc.year()] = key.decode("utf8") session.save() def nomcom_kwargs_for_year(year=None, *args, **kwargs): diff --git a/ietf/nomcom/models.py b/ietf/nomcom/models.py index f2f9c7b31..6c1281f9b 100644 --- a/ietf/nomcom/models.py +++ b/ietf/nomcom/models.py @@ -188,8 +188,9 @@ class NomineePosition(models.Model): def save(self, **kwargs): if not self.pk and not self.state_id: + # Don't need to set update_fields because the self.pk test means this is a new instance self.state = NomineePositionStateName.objects.get(slug='pending') - super(NomineePosition, self).save(**kwargs) + super().save(**kwargs) def __str__(self): return "%s - %s - %s" % (self.nominee, self.state, self.position) diff --git a/ietf/nomcom/templatetags/nomcom_tags.py b/ietf/nomcom/templatetags/nomcom_tags.py index 05a2c2e8b..38dfc61b9 100644 --- a/ietf/nomcom/templatetags/nomcom_tags.py +++ b/ietf/nomcom/templatetags/nomcom_tags.py @@ -6,7 +6,7 @@ import re from django import template from django.conf import settings from django.template.defaultfilters import linebreaksbr, force_escape -from django.utils.encoding import force_text, DjangoUnicodeDecodeError +from django.utils.encoding import force_str, DjangoUnicodeDecodeError from django.utils.safestring import mark_safe import debug # pyflakes:ignore @@ -55,8 +55,10 @@ def formatted_email(address): @register.simple_tag def decrypt(string, request, year, plain=False): - key = retrieve_nomcom_private_key(request, year) - + try: + key = retrieve_nomcom_private_key(request, year) + except UnicodeError: + return f"-*- Encrypted text [Error retrieving private key, contact the secretariat ({settings.SECRETARIAT_SUPPORT_EMAIL})]" if not key: return '-*- Encrypted text [No private key provided] -*-' @@ -68,7 +70,7 @@ def decrypt(string, request, year, plain=False): code, out, error = pipe(command % (settings.OPENSSL_COMMAND, encrypted_file.name), key) try: - out = force_text(out) + out = force_str(out) except DjangoUnicodeDecodeError: pass if code != 0: diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index 805f1ab74..07c893fdc 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -4,6 +4,7 @@ import datetime import io +import mock import random import shutil @@ -1577,6 +1578,16 @@ class NewActiveNomComTests(TestCase): login_testing_unauthorized(self,self.chair.user.username,url) response = self.client.get(url) self.assertEqual(response.status_code,200) + # Check that we get an error if there's an encoding problem talking to openssl + # "\xc3\x28" is an invalid utf8 string + with mock.patch("ietf.nomcom.utils.pipe", return_value=(0, b"\xc3\x28", None)): + response = self.client.post(url, {'key': force_str(key)}) + self.assertFormError( + response.context["form"], + None, + "An internal error occurred while adding your private key to your session." + f"Please contact the secretariat for assistance ({settings.SECRETARIAT_SUPPORT_EMAIL})", + ) response = self.client.post(url,{'key': force_str(key)}) self.assertEqual(response.status_code,302) @@ -1993,7 +2004,7 @@ class NoPublicKeyTests(TestCase): text_bits = [x.xpath('.//text()') for x in q('.alert-warning')] flat_text_bits = [item for sublist in text_bits for item in sublist] self.assertTrue(any(['not yet' in y for y in flat_text_bits])) - self.assertEqual(bool(q('form:not(.navbar-form)')),expected_form) + self.assertEqual(bool(q('#content form:not(.navbar-form)')),expected_form) self.client.logout() def test_not_yet(self): diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index d666ed816..2fdbe1382 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -172,6 +172,12 @@ def command_line_safe_secret(secret): return base64.encodebytes(secret).decode('utf-8').rstrip() def retrieve_nomcom_private_key(request, year): + """Retrieve decrypted nomcom private key from the session store + + Retrieves encrypted, ascii-armored private key from the session store, encodes + as utf8 bytes, then decrypts. Raises UnicodeError if the value in the session + store cannot be encoded as utf8. + """ private_key = request.session.get('NOMCOM_PRIVATE_KEY_%s' % year, None) if not private_key: @@ -183,7 +189,8 @@ def retrieve_nomcom_private_key(request, year): settings.OPENSSL_COMMAND, command_line_safe_secret(settings.NOMCOM_APP_SECRET) ), - private_key + # The openssl command expects ascii-armored input, so utf8 encoding should be valid + private_key.encode("utf8") ) if code != 0: log("openssl error: %s:\n Error %s: %s" %(command, code, error)) @@ -191,6 +198,12 @@ def retrieve_nomcom_private_key(request, year): def store_nomcom_private_key(request, year, private_key): + """Put encrypted nomcom private key in the session store + + Encrypts the private key using openssl, then decodes the ascii-armored output + as utf8 and adds to the session store. Raises UnicodeError if the openssl's + output cannot be decoded as utf8. + """ if not private_key: request.session['NOMCOM_PRIVATE_KEY_%s' % year] = '' else: @@ -205,8 +218,9 @@ def store_nomcom_private_key(request, year, private_key): if code != 0: log("openssl error: %s:\n Error %s: %s" %(command, code, error)) if error and error!=b"*** WARNING : deprecated key derivation used.\nUsing -iter or -pbkdf2 would be better.\n": - out = '' - request.session['NOMCOM_PRIVATE_KEY_%s' % year] = out + out = b'' + # The openssl command output in 'out' is an ascii-armored value, so should be utf8-decodable + request.session['NOMCOM_PRIVATE_KEY_%s' % year] = out.decode("utf8") def validate_private_key(key): diff --git a/ietf/nomcom/views.py b/ietf/nomcom/views.py index d862b3a40..77a3c3b76 100644 --- a/ietf/nomcom/views.py +++ b/ietf/nomcom/views.py @@ -18,7 +18,7 @@ from django.http import Http404, HttpResponseRedirect, HttpResponse from django.shortcuts import render, get_object_or_404, redirect from django.template.loader import render_to_string from django.urls import reverse -from django.utils.encoding import force_bytes, force_text +from django.utils.encoding import force_bytes, force_str from ietf.dbtemplate.models import DBTemplate @@ -158,8 +158,16 @@ def private_key(request, year): if request.method == 'POST': form = PrivateKeyForm(data=request.POST) if form.is_valid(): - store_nomcom_private_key(request, year, force_bytes(form.cleaned_data.get('key', ''))) - return HttpResponseRedirect(back_url) + try: + store_nomcom_private_key(request, year, force_bytes(form.cleaned_data.get('key', ''))) + except UnicodeError: + form.add_error( + None, + "An internal error occurred while adding your private key to your session." + f"Please contact the secretariat for assistance ({settings.SECRETARIAT_SUPPORT_EMAIL})" + ) + else: + return HttpResponseRedirect(back_url) else: form = PrivateKeyForm() @@ -684,7 +692,7 @@ def private_questionnaire(request, year): if form.is_valid(): form.save() messages.success(request, 'The questionnaire response has been registered.') - questionnaire_response = force_text(form.cleaned_data['comment_text']) + questionnaire_response = force_str(form.cleaned_data['comment_text']) form = QuestionnaireForm(nomcom=nomcom, user=request.user) else: form = QuestionnaireForm(nomcom=nomcom, user=request.user) diff --git a/ietf/person/factories.py b/ietf/person/factories.py index 8e80932c9..4761a3f4e 100644 --- a/ietf/person/factories.py +++ b/ietf/person/factories.py @@ -16,7 +16,7 @@ from unicodedata import normalize from django.conf import settings from django.contrib.auth.models import User from django.utils.text import slugify -from django.utils.encoding import force_text +from django.utils.encoding import force_str import debug # pyflakes:ignore @@ -68,7 +68,7 @@ class PersonFactory(factory.django.DjangoModelFactory): # Some i18n names, e.g., "शिला के.सी." have a dot at the end that is also part of the ASCII, e.g., "Shilaa Kesii." # That trailing dot breaks extract_authors(). Avoid this issue by stripping the dot from the ASCII. # Some others have a trailing semicolon (e.g., "உயிரோவியம் தங்கராஐ;") - strip those, too. - ascii = factory.LazyAttribute(lambda p: force_text(unidecode_name(p.name)).rstrip(".;")) + ascii = factory.LazyAttribute(lambda p: force_str(unidecode_name(p.name)).rstrip(".;")) class Params: with_bio = factory.Trait(biography = "\n\n".join(fake.paragraphs())) # type: ignore diff --git a/ietf/review/policies.py b/ietf/review/policies.py index e834891f7..fe6519a5e 100644 --- a/ietf/review/policies.py +++ b/ietf/review/policies.py @@ -183,9 +183,12 @@ class AbstractReviewerQueuePolicy: role__group=review_req.team ).exclude( person_id__in=rejecting_reviewer_ids ) - one_assignment = (review_req.reviewassignment_set - .exclude(state__slug__in=('rejected', 'no-response')) - .first()) + one_assignment = None + if review_req.pk is not None: + # cannot use reviewassignment_set relation until review_req has been created + one_assignment = (review_req.reviewassignment_set + .exclude(state__slug__in=('rejected', 'no-response')) + .first()) if one_assignment: field.initial = one_assignment.reviewer_id diff --git a/ietf/review/utils.py b/ietf/review/utils.py index 1e9a237b5..31b6b401f 100644 --- a/ietf/review/utils.py +++ b/ietf/review/utils.py @@ -382,7 +382,8 @@ def assign_review_request_to_reviewer(request, review_req, reviewer, add_skip=Fa # with a different view on a ReviewAssignment. log.assertion('reviewer is not None') - if review_req.reviewassignment_set.filter(reviewer=reviewer).exists(): + # cannot reference reviewassignment_set relation until pk exists + if review_req.pk is not None and review_req.reviewassignment_set.filter(reviewer=reviewer).exists(): return # Note that assigning a review no longer unassigns other reviews @@ -598,7 +599,9 @@ def suggested_review_requests_for_team(team): res.sort(key=lambda r: (r.deadline, r.doc_id), reverse=True) return res -def extract_revision_ordered_review_assignments_for_documents_and_replaced(review_assignment_queryset, names): +def extract_revision_ordered_review_assignments_for_documents_and_replaced( + review_assignment_queryset, names +): """Extracts all review assignments for document names (including replaced ancestors), return them neatly sorted.""" names = set(names) @@ -607,8 +610,13 @@ def extract_revision_ordered_review_assignments_for_documents_and_replaced(revie assignments_for_each_doc = defaultdict(list) replacement_name_set = set(e for l in replaces.values() for e in l) | names - for r in ( review_assignment_queryset.filter(review_request__doc__name__in=replacement_name_set) - .order_by("-reviewed_rev","-assigned_on", "-id").iterator()): + for r in ( + review_assignment_queryset.filter( + review_request__doc__name__in=replacement_name_set + ) + .order_by("-reviewed_rev", "-assigned_on", "-id") + .iterator(chunk_size=2000) # chunk_size not tested, using pre-Django 5 default value + ): assignments_for_each_doc[r.review_request.doc.name].append(r) # now collect in breadth-first order to keep the revision order intact @@ -646,7 +654,10 @@ def extract_revision_ordered_review_assignments_for_documents_and_replaced(revie return res -def extract_revision_ordered_review_requests_for_documents_and_replaced(review_request_queryset, names): + +def extract_revision_ordered_review_requests_for_documents_and_replaced( + review_request_queryset, names +): """Extracts all review requests for document names (including replaced ancestors), return them neatly sorted.""" names = set(names) @@ -654,7 +665,13 @@ def extract_revision_ordered_review_requests_for_documents_and_replaced(review_r replaces = extract_complete_replaces_ancestor_mapping_for_docs(names) requests_for_each_doc = defaultdict(list) - for r in review_request_queryset.filter(doc__name__in=set(e for l in replaces.values() for e in l) | names).order_by("-time", "-id").iterator(): + for r in ( + review_request_queryset.filter( + doc__name__in=set(e for l in replaces.values() for e in l) | names + ) + .order_by("-time", "-id") + .iterator(chunk_size=2000) # chunk_size not tested, using pre-Django 5 default value + ): requests_for_each_doc[r.doc.name].append(r) # now collect in breadth-first order to keep the revision order intact diff --git a/ietf/secr/telechat/views.py b/ietf/secr/telechat/views.py index 4dd4fba4e..f13a082f2 100644 --- a/ietf/secr/telechat/views.py +++ b/ietf/secr/telechat/views.py @@ -4,10 +4,11 @@ import datetime +from functools import partialmethod + from django.contrib import messages from django.forms.formsets import formset_factory from django.shortcuts import render, get_object_or_404, redirect -from django.utils.functional import curry import debug # pyflakes:ignore @@ -215,7 +216,7 @@ def doc_detail(request, date, name): initial_state = {'state':doc.get_state(state_type).pk, 'substate':tag} - # need to use curry here to pass custom variable to form init + # need to use partialmethod here to pass custom variable to form init if doc.active_ballot(): ballot_type = doc.active_ballot().ballot_type elif doc.type.slug == 'draft': @@ -223,7 +224,7 @@ def doc_detail(request, date, name): else: ballot_type = BallotType.objects.get(doc_type=doc.type) BallotFormset = formset_factory(BallotForm, extra=0) - BallotFormset.form.__init__ = curry(BallotForm.__init__, ballot_type=ballot_type) + BallotFormset.form.__init__ = partialmethod(BallotForm.__init__, ballot_type=ballot_type) agenda = agenda_data(date=date) header = get_section_header(doc, agenda) diff --git a/ietf/secr/templates/announcement/confirm.html b/ietf/secr/templates/announcement/confirm.html index 7ad745a09..ddf2a6de6 100644 --- a/ietf/secr/templates/announcement/confirm.html +++ b/ietf/secr/templates/announcement/confirm.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Announcement{% endblock %} diff --git a/ietf/secr/templates/areas/list.html b/ietf/secr/templates/areas/list.html index a0ed1ae4a..0d9946efc 100644 --- a/ietf/secr/templates/areas/list.html +++ b/ietf/secr/templates/areas/list.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Areas{% endblock %} {% block extrahead %}{{ block.super }} diff --git a/ietf/secr/templates/areas/people.html b/ietf/secr/templates/areas/people.html index e84dc1a79..168089f52 100644 --- a/ietf/secr/templates/areas/people.html +++ b/ietf/secr/templates/areas/people.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Areas - People{% endblock %} {% block extrahead %}{{ block.super }} diff --git a/ietf/secr/templates/areas/view.html b/ietf/secr/templates/areas/view.html index e3ecac70a..2bcb9619c 100644 --- a/ietf/secr/templates/areas/view.html +++ b/ietf/secr/templates/areas/view.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Areas - View{% endblock %} {% block extrahead %}{{ block.super }} diff --git a/ietf/secr/templates/base_secr.html b/ietf/secr/templates/base_secr.html index 47b893f04..18d77e47b 100644 --- a/ietf/secr/templates/base_secr.html +++ b/ietf/secr/templates/base_secr.html @@ -1,5 +1,5 @@ -{% load staticfiles %} +{% load static %} diff --git a/ietf/secr/templates/base_secr_bootstrap.html b/ietf/secr/templates/base_secr_bootstrap.html index 2eee566a1..a32634684 100644 --- a/ietf/secr/templates/base_secr_bootstrap.html +++ b/ietf/secr/templates/base_secr_bootstrap.html @@ -1,5 +1,5 @@ -{% load staticfiles %} +{% load static %} diff --git a/ietf/secr/templates/base_site.html b/ietf/secr/templates/base_site.html index d369a40ec..5e3ddc62d 100644 --- a/ietf/secr/templates/base_site.html +++ b/ietf/secr/templates/base_site.html @@ -1,7 +1,7 @@ {% extends "base_secr.html" %} {% load i18n %} {% load ietf_filters %} -{% load staticfiles %} +{% load static %} {% block title %}{{ title }}{% if user|has_role:"Secretariat" %} Secretariat Dashboard {% else %} IETF Dashboard {% endif %}{% endblock %} diff --git a/ietf/secr/templates/base_site_bootstrap.html b/ietf/secr/templates/base_site_bootstrap.html index c1c2fdac6..1653b26b8 100644 --- a/ietf/secr/templates/base_site_bootstrap.html +++ b/ietf/secr/templates/base_site_bootstrap.html @@ -1,7 +1,7 @@ {% extends "base_secr_bootstrap.html" %} {% load i18n %} {% load ietf_filters %} -{% load staticfiles %} +{% load static %} {% block title %}{{ title }}{% if user|has_role:"Secretariat" %} Secretariat Dashboard {% else %} WG Chair Dashboard {% endif %}{% endblock %} diff --git a/ietf/secr/templates/confirm_cancel.html b/ietf/secr/templates/confirm_cancel.html index 6bae631a7..541c82863 100644 --- a/ietf/secr/templates/confirm_cancel.html +++ b/ietf/secr/templates/confirm_cancel.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Confirm Cancel{% endblock %} diff --git a/ietf/secr/templates/confirm_delete.html b/ietf/secr/templates/confirm_delete.html index ccfc7b1c2..3f8fd19c8 100644 --- a/ietf/secr/templates/confirm_delete.html +++ b/ietf/secr/templates/confirm_delete.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Confirm Delete{% endblock %} diff --git a/ietf/secr/templates/meetings/add.html b/ietf/secr/templates/meetings/add.html index 5a7825526..b2cc2617d 100644 --- a/ietf/secr/templates/meetings/add.html +++ b/ietf/secr/templates/meetings/add.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Meetings - Add{% endblock %} diff --git a/ietf/secr/templates/meetings/base_rooms_times.html b/ietf/secr/templates/meetings/base_rooms_times.html index f856b1f62..263418fab 100644 --- a/ietf/secr/templates/meetings/base_rooms_times.html +++ b/ietf/secr/templates/meetings/base_rooms_times.html @@ -1,5 +1,5 @@ {% extends "base_site_bootstrap.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Meetings{% endblock %} diff --git a/ietf/secr/templates/meetings/blue_sheet.html b/ietf/secr/templates/meetings/blue_sheet.html index d67efd9f6..9bda80f2e 100644 --- a/ietf/secr/templates/meetings/blue_sheet.html +++ b/ietf/secr/templates/meetings/blue_sheet.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Meetings - Blue Sheet{% endblock %} diff --git a/ietf/secr/templates/meetings/edit_meeting.html b/ietf/secr/templates/meetings/edit_meeting.html index 773536e65..474373dbe 100644 --- a/ietf/secr/templates/meetings/edit_meeting.html +++ b/ietf/secr/templates/meetings/edit_meeting.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Meetings - Edit{% endblock %} diff --git a/ietf/secr/templates/meetings/main.html b/ietf/secr/templates/meetings/main.html index 90c380289..ff110dd97 100755 --- a/ietf/secr/templates/meetings/main.html +++ b/ietf/secr/templates/meetings/main.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Meetings{% endblock %} diff --git a/ietf/secr/templates/meetings/notifications.html b/ietf/secr/templates/meetings/notifications.html index bf7099577..dbe66ff28 100644 --- a/ietf/secr/templates/meetings/notifications.html +++ b/ietf/secr/templates/meetings/notifications.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Meetings{% endblock %} diff --git a/ietf/secr/templates/meetings/regular_session_edit.html b/ietf/secr/templates/meetings/regular_session_edit.html index fbfba4f96..9993858be 100644 --- a/ietf/secr/templates/meetings/regular_session_edit.html +++ b/ietf/secr/templates/meetings/regular_session_edit.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles tz %} +{% load static tz %} {% block title %}Meetings{% endblock %} diff --git a/ietf/secr/templates/meetings/view.html b/ietf/secr/templates/meetings/view.html index d552d38dc..d54346dae 100644 --- a/ietf/secr/templates/meetings/view.html +++ b/ietf/secr/templates/meetings/view.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Meetings{% endblock %} diff --git a/ietf/secr/templates/rolodex/add.html b/ietf/secr/templates/rolodex/add.html index 272b844fa..5adb738f2 100644 --- a/ietf/secr/templates/rolodex/add.html +++ b/ietf/secr/templates/rolodex/add.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Rolodex - Add{% endblock %} diff --git a/ietf/secr/templates/rolodex/edit.html b/ietf/secr/templates/rolodex/edit.html index 28a125f10..ed4c0f97e 100644 --- a/ietf/secr/templates/rolodex/edit.html +++ b/ietf/secr/templates/rolodex/edit.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Rolodex - Edit{% endblock %} diff --git a/ietf/secr/templates/rolodex/search.html b/ietf/secr/templates/rolodex/search.html index 8994cfabd..065b0463f 100644 --- a/ietf/secr/templates/rolodex/search.html +++ b/ietf/secr/templates/rolodex/search.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Rolodex - Search{% endblock %} diff --git a/ietf/secr/templates/sreq/confirm.html b/ietf/secr/templates/sreq/confirm.html index 4215c89c3..025375af3 100755 --- a/ietf/secr/templates/sreq/confirm.html +++ b/ietf/secr/templates/sreq/confirm.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Sessions - Confirm{% endblock %} diff --git a/ietf/secr/templates/sreq/edit.html b/ietf/secr/templates/sreq/edit.html index b0bfbc1e0..f6e62104b 100755 --- a/ietf/secr/templates/sreq/edit.html +++ b/ietf/secr/templates/sreq/edit.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Sessions - Edit{% endblock %} {% block extrahead %}{{ block.super }} diff --git a/ietf/secr/templates/sreq/locked.html b/ietf/secr/templates/sreq/locked.html index 5f619f37c..c27cf578e 100755 --- a/ietf/secr/templates/sreq/locked.html +++ b/ietf/secr/templates/sreq/locked.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Sessions{% endblock %} diff --git a/ietf/secr/templates/sreq/main.html b/ietf/secr/templates/sreq/main.html index bdb33bb77..a6695cd4f 100755 --- a/ietf/secr/templates/sreq/main.html +++ b/ietf/secr/templates/sreq/main.html @@ -1,6 +1,6 @@ {% extends "base_site.html" %} {% load ietf_filters %} -{% load staticfiles %} +{% load static %} {% block title %}Sessions{% endblock %} diff --git a/ietf/secr/templates/sreq/new.html b/ietf/secr/templates/sreq/new.html index 2c6afb557..3f46e6f89 100755 --- a/ietf/secr/templates/sreq/new.html +++ b/ietf/secr/templates/sreq/new.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Sessions- New{% endblock %} diff --git a/ietf/secr/templates/sreq/tool_status.html b/ietf/secr/templates/sreq/tool_status.html index cf5131c22..b91e73a12 100755 --- a/ietf/secr/templates/sreq/tool_status.html +++ b/ietf/secr/templates/sreq/tool_status.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Sessions{% endblock %} diff --git a/ietf/secr/templates/sreq/view.html b/ietf/secr/templates/sreq/view.html index c7ae2d27b..9a0a3b01c 100644 --- a/ietf/secr/templates/sreq/view.html +++ b/ietf/secr/templates/sreq/view.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Sessions - View{% endblock %} diff --git a/ietf/secr/templates/telechat/base_telechat.html b/ietf/secr/templates/telechat/base_telechat.html index 73d42ea71..1c8feaff6 100644 --- a/ietf/secr/templates/telechat/base_telechat.html +++ b/ietf/secr/templates/telechat/base_telechat.html @@ -1,5 +1,5 @@ {% extends "base_site.html" %} -{% load staticfiles %} +{% load static %} {% block title %}Telechat{% endblock %} diff --git a/ietf/secr/urls.py b/ietf/secr/urls.py index 5a3df23d0..e5d3c19c3 100644 --- a/ietf/secr/urls.py +++ b/ietf/secr/urls.py @@ -1,13 +1,13 @@ -from django.conf.urls import url, include +from django.urls import re_path, include from django.views.generic import TemplateView urlpatterns = [ - url(r'^$', TemplateView.as_view(template_name='main.html')), - url(r'^announcement/', include('ietf.secr.announcement.urls')), - url(r'^areas/', include('ietf.secr.areas.urls')), - url(r'^console/', include('ietf.secr.console.urls')), - url(r'^meetings/', include('ietf.secr.meetings.urls')), - url(r'^rolodex/', include('ietf.secr.rolodex.urls')), - url(r'^sreq/', include('ietf.secr.sreq.urls')), - url(r'^telechat/', include('ietf.secr.telechat.urls')), + re_path(r'^$', TemplateView.as_view(template_name='main.html')), + re_path(r'^announcement/', include('ietf.secr.announcement.urls')), + re_path(r'^areas/', include('ietf.secr.areas.urls')), + re_path(r'^console/', include('ietf.secr.console.urls')), + re_path(r'^meetings/', include('ietf.secr.meetings.urls')), + re_path(r'^rolodex/', include('ietf.secr.rolodex.urls')), + re_path(r'^sreq/', include('ietf.secr.sreq.urls')), + re_path(r'^telechat/', include('ietf.secr.telechat.urls')), ] diff --git a/ietf/secr/utils/decorators.py b/ietf/secr/utils/decorators.py index f635bc7ec..5887c3c9c 100644 --- a/ietf/secr/utils/decorators.py +++ b/ietf/secr/utils/decorators.py @@ -1,12 +1,12 @@ # Copyright The IETF Trust 2013-2020, All Rights Reserved from functools import wraps +from urllib.parse import quote as urlquote from django.conf import settings from django.contrib.auth import REDIRECT_FIELD_NAME from django.core.exceptions import ObjectDoesNotExist from django.http import HttpResponseRedirect from django.shortcuts import render, get_object_or_404 -from django.utils.http import urlquote from ietf.ietfauth.utils import has_role from ietf.doc.models import Document diff --git a/ietf/settings.py b/ietf/settings.py index db3acab1e..7d07f1220 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -13,12 +13,18 @@ import warnings from typing import Any, Dict, List, Tuple # pyflakes:ignore warnings.simplefilter("always", DeprecationWarning) -warnings.filterwarnings("ignore", message="'urllib3\[secure\]' extra is deprecated") -warnings.filterwarnings("ignore", message="The logout\(\) view is superseded by") +warnings.filterwarnings("ignore", message="pkg_resources is deprecated as an API") +warnings.filterwarnings("ignore", "Log out via GET requests is deprecated") # happens in oidc_provider +warnings.filterwarnings("ignore", module="tastypie", message="The django.utils.datetime_safe module is deprecated.") +warnings.filterwarnings("ignore", module="oidc_provider", message="The django.utils.timezone.utc alias is deprecated.") +warnings.filterwarnings("ignore", message="The USE_DEPRECATED_PYTZ setting,") # https://github.com/ietf-tools/datatracker/issues/5635 +warnings.filterwarnings("ignore", message="The USE_L10N setting is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5648 +warnings.filterwarnings("ignore", message="django.contrib.auth.hashers.CryptPasswordHasher is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5663 +warnings.filterwarnings("ignore", message="'urllib3\\[secure\\]' extra is deprecated") +warnings.filterwarnings("ignore", message="The logout\\(\\) view is superseded by") warnings.filterwarnings("ignore", message="Report.file_reporters will no longer be available in Coverage.py 4.2", module="coverage.report") -warnings.filterwarnings("ignore", message="{% load staticfiles %} is deprecated") warnings.filterwarnings("ignore", message="Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated", module="bleach") -warnings.filterwarnings("ignore", message="HTTPResponse.getheader\(\) is deprecated", module='selenium.webdriver') +warnings.filterwarnings("ignore", message="HTTPResponse.getheader\\(\\) is deprecated", module='selenium.webdriver') try: import syslog syslog.openlog(str("datatracker"), syslog.LOG_PID, syslog.LOG_USER) @@ -101,7 +107,23 @@ SITE_ID = 1 # to load the internationalization machinery. USE_I18N = False +# Django 4.0 changed the default setting of USE_L10N to True. The setting +# is deprecated and will be removed in Django 5.0. +USE_L10N = False + USE_TZ = True +USE_DEPRECATED_PYTZ = True # supported until Django 5 + +# The DjangoDivFormRenderer is a transitional class that opts in to defaulting to the div.html +# template for formsets. This will become the default behavior in Django 5.0. This configuration +# can be removed at that point. +# See https://docs.djangoproject.com/en/4.2/releases/4.1/#forms +FORM_RENDERER = "django.forms.renderers.DjangoDivFormRenderer" + +# Default primary key field type to use for models that don’t have a field with primary_key=True. +# In the future (relative to 4.2), the default will become 'django.db.models.BigAutoField.' +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + if SERVER_MODE == 'production': MEDIA_ROOT = '/a/www/www6s/lib/dt/media/' @@ -317,10 +339,10 @@ UTILS_LOGGER_LEVELS: Dict[str, str] = { X_FRAME_OPTIONS = 'SAMEORIGIN' CSRF_TRUSTED_ORIGINS = [ - 'ietf.org', - '*.ietf.org', - 'meetecho.com', - '*.meetecho.com', + "https://ietf.org", + "https://*.ietf.org", + 'https://meetecho.com', + 'https://*.meetecho.com', ] CSRF_COOKIE_SAMESITE = 'None' CSRF_COOKIE_SECURE = True @@ -331,11 +353,7 @@ SESSION_COOKIE_SAMESITE = 'None' SESSION_COOKIE_SECURE = True SESSION_EXPIRE_AT_BROWSER_CLOSE = False -# We want to use the JSON serialisation, as it's safer -- but there is /secr/ -# code which stashes objects in the session that can't be JSON serialized. -# Switch when that code is rewritten. -#SESSION_SERIALIZER = "django.contrib.sessions.serializers.JSONSerializer" -SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' +SESSION_SERIALIZER = "django.contrib.sessions.serializers.JSONSerializer" SESSION_ENGINE = "django.contrib.sessions.backends.cache" SESSION_SAVE_EVERY_REQUEST = True SESSION_CACHE_ALIAS = 'sessions' @@ -435,8 +453,6 @@ INSTALLED_APPS = [ 'django_celery_beat', 'corsheaders', 'django_markup', - 'django_password_strength', - 'form_utils', 'oidc_provider', 'simple_history', 'tastypie', @@ -523,6 +539,8 @@ SECURE_HSTS_SECONDS = 3600 #SECURE_REDIRECT_EXEMPT #SECURE_SSL_HOST #SECURE_SSL_REDIRECT = True +# Relax the COOP policy to allow Meetecho authentication pop-up +SECURE_CROSS_ORIGIN_OPENER_POLICY = "unsafe-none" # Override this in your settings_local with the IP addresses relevant for you: INTERNAL_IPS = ( @@ -708,13 +726,13 @@ CACHE_MIDDLEWARE_KEY_PREFIX = '' # This setting is possibly overridden further down, after the import of settings_local CACHES = { 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', 'LOCATION': '127.0.0.1:11211', 'VERSION': __version__, 'KEY_PREFIX': 'ietf:dt', }, 'sessions': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', 'LOCATION': '127.0.0.1:11211', # No release-specific VERSION setting. 'KEY_PREFIX': 'ietf:dt', @@ -1123,14 +1141,13 @@ ACCOUNT_REQUEST_EMAIL = 'account-request@ietf.org' SILENCED_SYSTEM_CHECKS = [ "fields.W342", # Setting unique=True on a ForeignKey has the same effect as using a OneToOneField. + "fields.W905", # django.contrib.postgres.fields.CICharField is deprecated. (see https://github.com/ietf-tools/datatracker/issues/5660) ] CHECKS_LIBRARY_PATCHES_TO_APPLY = [ 'patch/change-oidc-provider-field-sizes-228.patch', 'patch/fix-oidc-access-token-post.patch', 'patch/fix-jwkest-jwt-logging.patch', - 'patch/fix-django-password-strength-kwargs.patch', - 'patch/add-django-http-cookie-value-none.patch', 'patch/django-cookie-delete-with-all-settings.patch', 'patch/tastypie-django22-fielderror-response.patch', ] diff --git a/ietf/settings_postgrestest.py b/ietf/settings_postgrestest.py index 450fd9180..13bbc9239 100755 --- a/ietf/settings_postgrestest.py +++ b/ietf/settings_postgrestest.py @@ -39,7 +39,7 @@ DATABASES = { 'HOST': 'db', 'PORT': '5432', 'NAME': 'test.db', - 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.postgresql', 'USER': 'django', 'PASSWORD': 'RkTkDPFnKpko', }, diff --git a/ietf/stats/models.py b/ietf/stats/models.py index 422c5b78a..0871804b0 100644 --- a/ietf/stats/models.py +++ b/ietf/stats/models.py @@ -24,7 +24,8 @@ class AffiliationAlias(models.Model): def save(self, *args, **kwargs): self.alias = self.alias.lower() - super(AffiliationAlias, self).save(*args, **kwargs) + update_fields = {"alias"}.union(kwargs.pop("update_fields", set())) + super(AffiliationAlias, self).save(update_fields=update_fields, *args, **kwargs) class Meta: verbose_name_plural = "affiliation aliases" diff --git a/ietf/submit/mail.py b/ietf/submit/mail.py index 93f97026c..1953ad81c 100644 --- a/ietf/submit/mail.py +++ b/ietf/submit/mail.py @@ -13,7 +13,7 @@ from django.urls import reverse as urlreverse from django.core.exceptions import ValidationError from django.contrib.sites.models import Site from django.template.loader import render_to_string -from django.utils.encoding import force_text, force_str +from django.utils.encoding import force_str import debug # pyflakes:ignore @@ -202,7 +202,7 @@ def get_reply_to(): address with "plus addressing" using a random string. Guaranteed to be unique""" local,domain = get_base_submission_message_address().split('@') while True: - rand = force_text(base64.urlsafe_b64encode(os.urandom(12))) + rand = force_str(base64.urlsafe_b64encode(os.urandom(12))) address = "{}+{}@{}".format(local,rand,domain) q = Message.objects.filter(reply_to=address) if not q: diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 2e9f2da71..3f57f7cf9 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -24,7 +24,7 @@ from django.test import override_settings from django.test.client import RequestFactory from django.urls import reverse as urlreverse from django.utils import timezone -from django.utils.encoding import force_str, force_text +from django.utils.encoding import force_str import debug # pyflakes:ignore from ietf.submit.utils import (expirable_submissions, expire_submission, find_submission_filenames, @@ -750,10 +750,10 @@ class SubmitTests(BaseSubmitTestCase): self.assertTrue("New Version Notification" in outbox[-2]["Subject"]) self.assertTrue(name in get_payload_text(outbox[-2])) interesting_address = {'ietf':'mars', 'irtf':'irtf-chair', 'iab':'iab-chair', 'ise':'rfc-ise'}[draft.stream_id] - self.assertTrue(interesting_address in force_text(outbox[-2].as_string())) + self.assertTrue(interesting_address in force_str(outbox[-2].as_string())) if draft.stream_id == 'ietf': - self.assertTrue(draft.ad.role_email("ad").address in force_text(outbox[-2].as_string())) - self.assertTrue(ballot_position.balloter.role_email("ad").address in force_text(outbox[-2].as_string())) + self.assertTrue(draft.ad.role_email("ad").address in force_str(outbox[-2].as_string())) + self.assertTrue(ballot_position.balloter.role_email("ad").address in force_str(outbox[-2].as_string())) self.assertTrue("New Version Notification" in outbox[-1]["Subject"]) self.assertTrue(name in get_payload_text(outbox[-1])) r = self.client.get(urlreverse('ietf.doc.views_search.recent_drafts')) @@ -1204,15 +1204,15 @@ class SubmitTests(BaseSubmitTestCase): Unlike some other tests in this module, does not confirm draft if this would be required. """ - orig_draft = DocumentFactory( + orig_draft: Document = DocumentFactory( # type: ignore[annotation-unchecked] type_id='draft', group=GroupFactory(type_id=group_type) if group_type else None, stream_id=stream_type, - ) # type: Document + ) name = orig_draft.name group = orig_draft.group new_rev = '%02d' % (int(orig_draft.rev) + 1) - author = PersonFactory() # type: Person + author: Person = PersonFactory() # type: ignore[annotation-unchecked] DocumentAuthor.objects.create(person=author, document=orig_draft) orig_draft.docextresource_set.create(name_id='faq', value='https://faq.example.com/') orig_draft.docextresource_set.create(name_id='wiki', value='https://wiki.example.com', display_name='Test Wiki') @@ -1982,7 +1982,7 @@ class SubmitTests(BaseSubmitTestCase): group = GroupFactory() # someone to be notified of resource suggestion when permission not granted RoleFactory(group=group, person=PersonFactory(), name_id='chair') - submission = SubmissionFactory(state_id='grp-appr', group=group) # type: Submission + submission: Submission = SubmissionFactory(state_id='grp-appr', group=group) # type: ignore[annotation-unchecked] SubmissionExtResourceFactory(submission=submission) # use secretary user to ensure we have permission to approve @@ -2000,7 +2000,7 @@ class SubmitTests(BaseSubmitTestCase): group = GroupFactory() # someone to be notified of resource suggestion when permission not granted RoleFactory(group=group, person=PersonFactory(), name_id='chair') - submission = SubmissionFactory(state_id=state, group=group) # type: Submission + submission: Submission = SubmissionFactory(state_id=state, group=group) # type: ignore[annotation-unchecked] SubmissionExtResourceFactory(submission=submission) url = urlreverse( @@ -2052,7 +2052,7 @@ class SubmitTests(BaseSubmitTestCase): def test_forcepost_with_extresources(self): # state needs to be one that has 'posted' as a next state - submission = SubmissionFactory(state_id='grp-appr') # type: Submission + submission: Submission = SubmissionFactory(state_id='grp-appr') # type: ignore[annotation-unchecked] SubmissionExtResourceFactory(submission=submission) url = urlreverse( diff --git a/ietf/sync/iana.py b/ietf/sync/iana.py index 9993d492a..dc61f9159 100644 --- a/ietf/sync/iana.py +++ b/ietf/sync/iana.py @@ -10,11 +10,11 @@ import re import requests from email.utils import parsedate_to_datetime +from urllib.parse import quote as urlquote from django.conf import settings from django.utils import timezone from django.utils.encoding import smart_bytes, force_str -from django.utils.http import urlquote import debug # pyflakes:ignore diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index 59356dd48..784e7a2f0 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -12,7 +12,7 @@ from xml.dom import pulldom, Node from django.conf import settings from django.utils import timezone -from django.utils.encoding import smart_bytes, force_str, force_text +from django.utils.encoding import smart_bytes, force_str import debug # pyflakes:ignore @@ -583,7 +583,7 @@ def post_approved_draft(url, name): if r.status_code != 200: raise RuntimeError("Status code is not 200 OK (it's %s)." % r.status_code) - if force_text(r.text) != "OK": + if force_str(r.text) != "OK": raise RuntimeError('Response is not "OK" (it\'s "%s").' % r.text) except Exception as e: diff --git a/ietf/templates/base/menu_user.html b/ietf/templates/base/menu_user.html index bb68855b6..9e6bbde56 100644 --- a/ietf/templates/base/menu_user.html +++ b/ietf/templates/base/menu_user.html @@ -32,11 +32,13 @@ {% else %} {% if user.is_authenticated %}
  • - - Sign out - +
    + {% csrf_token %} + +
  • - {% ifequal ref.source.get_state.slug 'rfc' %} + {% if ref.source.get_state.slug == 'rfc' %} {% with ref.source.std_level as lvl %} {% if lvl %}{{ lvl }}{% endif %} {% endwith %} @@ -72,7 +72,7 @@ {% with ref.source.intended_std_level as lvl %} {% if lvl %}{{ lvl }}{% endif %} {% endwith %} - {% endifequal %} + {% endif %} {{ ref.relationship.name }} {{ ref.is_downref|default:'' }} diff --git a/ietf/templates/doc/document_references.html b/ietf/templates/doc/document_references.html index 4578d6b8c..d9134be6f 100644 --- a/ietf/templates/doc/document_references.html +++ b/ietf/templates/doc/document_references.html @@ -51,7 +51,7 @@ - {% ifequal ref.target.document.get_state.slug 'rfc' %} + {% if ref.target.document.get_state.slug == 'rfc' %} {% with ref.target.document.std_level as lvl %} {% if lvl %}{{ lvl }}{% endif %} {% endwith %} @@ -59,7 +59,7 @@ {% with ref.target.document.intended_std_level as lvl %} {% if lvl %}{{ lvl }}{% endif %} {% endwith %} - {% endifequal %} + {% endif %} {{ ref.relationship.name }} {{ ref.is_downref|default:'' }} diff --git a/ietf/templates/liaisons/edit.html b/ietf/templates/liaisons/edit.html index 2c7b4b65b..c8023e50c 100644 --- a/ietf/templates/liaisons/edit.html +++ b/ietf/templates/liaisons/edit.html @@ -47,21 +47,43 @@ method="post" enctype="multipart/form-data" data-edit-form="{{ form.edit }}" - data-ajax-info-url="{% url "ietf.liaisons.views.ajax_get_liaison_info" %}"> + data-ajax-info-url="{% url 'ietf.liaisons.views.ajax_get_liaison_info' %}"> {% csrf_token %} - {% for fieldset in form.fieldsets %} -

    {{ fieldset.name }}

    - {% for field in fieldset %} - {% if field.id_for_label != "id_attachments" %} - {% bootstrap_field field layout="horizontal" %} - {% else %} -
    -

    {{ field.label }}

    -
    {{ field }}
    -
    - {% endif %} - {% endfor %} - {% endfor %} +

    From

    + {% bootstrap_field form.from_groups layout="horizontal" %} + {% bootstrap_field form.from_contact layout="horizontal" %} + {% bootstrap_field form.response_contacts layout="horizontal" %} + {% if form.approved %} + {% bootstrap_field form.approved layout="horizontal" %} + {% endif %} +

    To

    + {% bootstrap_field form.to_groups layout="horizontal" %} + {% bootstrap_field form.to_contacts layout="horizontal" %} +

    Other email addresses

    + {% bootstrap_field form.technical_contacts layout="horizontal" %} + {% if form.action_holder_contacts %} + {% bootstrap_field form.action_holder_contacts layout="horizontal" %} + {% endif %} + {% bootstrap_field form.cc_contacts layout="horizontal" %} +

    Purpose

    + {% bootstrap_field form.purpose layout="horizontal" %} + {% bootstrap_field form.deadline layout="horizontal" %} +

    Reference

    + {% bootstrap_field form.other_identifiers layout="horizontal" %} + {% bootstrap_field form.related_to layout="horizontal" %} +

    Liaison Statement

    + {% bootstrap_field form.title layout="horizontal" %} + {% bootstrap_field form.submitted_date layout="horizontal" %} + {% bootstrap_field form.body layout="horizontal" %} +
    +

    {{ form.attachments.label }}

    +
    {{ form.attachments }}
    +
    +

    Add attachment

    + {% bootstrap_field form.attach_title layout="horizontal" %} + {% bootstrap_field form.attach_file layout="horizontal" %} + {% bootstrap_field form.attach_button layout="horizontal" %} + Cancel diff --git a/ietf/templates/meeting/edit_meeting_timeslots_and_misc_sessions.html b/ietf/templates/meeting/edit_meeting_timeslots_and_misc_sessions.html index 214845035..5e51ab5a8 100644 --- a/ietf/templates/meeting/edit_meeting_timeslots_and_misc_sessions.html +++ b/ietf/templates/meeting/edit_meeting_timeslots_and_misc_sessions.html @@ -1,7 +1,7 @@ {% extends "base.html" %} {# Copyright The IETF Trust 2015-2020, All Rights Reserved #} {% load origin %} -{% load staticfiles %} +{% load static %} {% load ietf_filters %} {% load django_bootstrap5 %} {% block title %}{{ schedule.name }}: IETF {{ meeting.number }} meeting agenda{% endblock %} diff --git a/ietf/templates/meeting/new_meeting_schedule.html b/ietf/templates/meeting/new_meeting_schedule.html index c93d72b2a..d1b5263c5 100644 --- a/ietf/templates/meeting/new_meeting_schedule.html +++ b/ietf/templates/meeting/new_meeting_schedule.html @@ -1,7 +1,7 @@ {% extends "base.html" %} {# Copyright The IETF Trust 2015-2020, All Rights Reserved #} {% load origin %} -{% load staticfiles %} +{% load static %} {% load ietf_filters %} {% load django_bootstrap5 %} {% block content %} diff --git a/ietf/templates/meeting/previously_approved_slides.html b/ietf/templates/meeting/previously_approved_slides.html index 060be438e..25a4c9786 100644 --- a/ietf/templates/meeting/previously_approved_slides.html +++ b/ietf/templates/meeting/previously_approved_slides.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {# Copyright The IETF Trust 2020, All Rights Reserved #} -{% load origin staticfiles django_bootstrap5 %} +{% load origin static django_bootstrap5 %} {% block title %} Approved Slides for {{ submission.session.meeting }} : {{ submission.session.group.acronym }} {% endblock %} diff --git a/ietf/templates/registration/change_password.html b/ietf/templates/registration/change_password.html index 1df189031..21c102bd0 100644 --- a/ietf/templates/registration/change_password.html +++ b/ietf/templates/registration/change_password.html @@ -6,8 +6,7 @@ {% block title %}Change password{% endblock %} {% block js %} {{ block.super }} - - + {{ form.media.js }} {% endblock %} {% block content %} {% origin %} diff --git a/ietf/templates/registration/confirm_account.html b/ietf/templates/registration/confirm_account.html index b419b5f34..d6639d8e7 100644 --- a/ietf/templates/registration/confirm_account.html +++ b/ietf/templates/registration/confirm_account.html @@ -6,8 +6,7 @@ {% block title %}Complete account creation{% endblock %} {% block js %} {{ block.super }} - - + {{ form.media.js }} {% endblock %} {% block content %} {% origin %} diff --git a/ietf/urls.py b/ietf/urls.py index 972e06cb3..8eb08b8d0 100644 --- a/ietf/urls.py +++ b/ietf/urls.py @@ -1,15 +1,14 @@ # Copyright The IETF Trust 2007-2022, All Rights Reserved from django.conf import settings -from django.conf.urls import include from django.conf.urls.static import static as static_url from django.contrib import admin from django.contrib.sitemaps import views as sitemap_views from django.contrib.staticfiles.urls import staticfiles_urlpatterns +from django.urls import include, path from django.views import static as static_view from django.views.generic import TemplateView from django.views.defaults import server_error -from django.urls import path import debug # pyflakes:ignore diff --git a/ietf/utils/accesstoken.py b/ietf/utils/accesstoken.py index b2a93f77d..243d3f24d 100644 --- a/ietf/utils/accesstoken.py +++ b/ietf/utils/accesstoken.py @@ -5,7 +5,7 @@ import time, random, hashlib from django.conf import settings -from django.utils.encoding import force_bytes, force_text +from django.utils.encoding import force_bytes, force_str def generate_random_key(max_length=32): @@ -18,4 +18,4 @@ def generate_access_token(key, max_length=32): # we hash it with the private key to make sure only we can # generate and use the final token - so storing the key in the # database is safe - return force_text(hashlib.sha256(force_bytes(settings.SECRET_KEY) + force_bytes(key)).hexdigest()[:max_length]) + return force_str(hashlib.sha256(force_bytes(settings.SECRET_KEY) + force_bytes(key)).hexdigest()[:max_length]) diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index 3e562c2bc..fa1ebb708 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -3,7 +3,7 @@ from django.contrib import admin -from django.utils.encoding import force_text +from django.utils.encoding import force_str from ietf.utils.models import VersionInfo @@ -14,7 +14,7 @@ def name(obj): if callable(obj.name): name = obj.name() else: - name = force_text(obj.name) + name = force_str(obj.name) if name: return name return str(obj) diff --git a/ietf/utils/decorators.py b/ietf/utils/decorators.py index d37d255a3..db919bd4b 100644 --- a/ietf/utils/decorators.py +++ b/ietf/utils/decorators.py @@ -5,6 +5,7 @@ import datetime from decorator import decorator, decorate +from functools import wraps from django.conf import settings from django.contrib.auth import login @@ -39,52 +40,54 @@ def person_required(f, request, *args, **kwargs): return render(request, 'registration/missing_person.html') return f(request, *args, **kwargs) -@decorator -def require_api_key(f, request, *args, **kwargs): - - def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') - # Check method and get hash - if request.method == 'POST': - hash = request.POST.get('apikey') - elif request.method == 'GET': - hash = request.GET.get('apikey') - else: - return err(405, "Method not allowed") - if not hash: - return err(400, "Missing apikey parameter") - # Check hash - key = PersonalApiKey.validate_key(force_bytes(hash)) - if not key: - return err(403, "Invalid apikey") - # Check endpoint - urlpath = request.META.get('PATH_INFO') - if not (urlpath and urlpath == key.endpoint): - return err(400, "Apikey endpoint mismatch") - # Check time since regular login - person = key.person - last_login = person.user.last_login - if not person.user.is_staff: - time_limit = (timezone.now() - datetime.timedelta(days=settings.UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS)) - if last_login == None or last_login < time_limit: - return err(400, "Too long since last regular login") - # Log in - login(request, person.user) - # restore the user.last_login field, so it reflects only gui logins - person.user.last_login = last_login - person.user.save() - # Update stats - key.count += 1 - key.latest = timezone.now() - key.save() - PersonApiKeyEvent.objects.create(person=person, type='apikey_login', key=key, desc="Logged in with key ID %s, endpoint %s" % (key.id, key.endpoint)) - # Execute decorated function - try: - ret = f(request, *args, **kwargs) - except AttributeError as e: - log.log("Bad API call: args: %s, kwargs: %s, exception: %s" % (args, kwargs, e)) - return err(400, "Bad or missing parameters") - return ret + +def require_api_key(f): + @wraps(f) + def _wrapper(request, *args, **kwargs): + def err(code, text): + return HttpResponse(text, status=code, content_type='text/plain') + # Check method and get hash + if request.method == 'POST': + hash = request.POST.get('apikey') + elif request.method == 'GET': + hash = request.GET.get('apikey') + else: + return err(405, "Method not allowed") + if not hash: + return err(400, "Missing apikey parameter") + # Check hash + key = PersonalApiKey.validate_key(force_bytes(hash)) + if not key: + return err(403, "Invalid apikey") + # Check endpoint + urlpath = request.META.get('PATH_INFO') + if not (urlpath and urlpath == key.endpoint): + return err(400, "Apikey endpoint mismatch") + # Check time since regular login + person = key.person + last_login = person.user.last_login + if not person.user.is_staff: + time_limit = (timezone.now() - datetime.timedelta(days=settings.UTILS_APIKEY_GUI_LOGIN_LIMIT_DAYS)) + if last_login == None or last_login < time_limit: + return err(400, "Too long since last regular login") + # Log in + login(request, person.user) + # restore the user.last_login field, so it reflects only gui logins + person.user.last_login = last_login + person.user.save() + # Update stats + key.count += 1 + key.latest = timezone.now() + key.save() + PersonApiKeyEvent.objects.create(person=person, type='apikey_login', key=key, desc="Logged in with key ID %s, endpoint %s" % (key.id, key.endpoint)) + # Execute decorated function + try: + ret = f(request, *args, **kwargs) + except AttributeError as e: + log.log("Bad API call: args: %s, kwargs: %s, exception: %s" % (args, kwargs, e)) + return err(400, "Bad or missing parameters") + return ret + return _wrapper def _memoize(func, self, *args, **kwargs): diff --git a/ietf/utils/http.py b/ietf/utils/http.py new file mode 100644 index 000000000..6e6409e31 --- /dev/null +++ b/ietf/utils/http.py @@ -0,0 +1,10 @@ +# Copyright The IETF Trust 2023, All Rights Reserved +# -*- coding: utf-8 -*- + +def is_ajax(request): + """Checks whether a request was an AJAX call + + See https://docs.djangoproject.com/en/3.1/releases/3.1/#id2 - this implements the + exact reproduction of the deprecated method suggested there. + """ + return request.headers.get("x-requested-with") == "XMLHttpRequest" diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py index 722b40581..a901df66f 100644 --- a/ietf/utils/jstest.py +++ b/ietf/utils/jstest.py @@ -81,7 +81,7 @@ class IetfSeleniumTestCase(IetfLiveServerTestCase): self.driver.get(url) self.driver.find_element(By.NAME, 'username').send_keys(username) self.driver.find_element(By.NAME, 'password').send_keys(password) - self.driver.find_element(By.XPATH, '//button[@type="submit"]').click() + self.driver.find_element(By.XPATH, '//*[@id="content"]//button[@type="submit"]').click() def scroll_to_element(self, element): """Scroll an element into view""" @@ -108,4 +108,4 @@ class presence_of_element_child_by_css_selector: def __call__(self, driver): child = self.element.find_element(By.CSS_SELECTOR, self.child_selector) - return child if child is not None else False \ No newline at end of file + return child if child is not None else False diff --git a/ietf/utils/mail.py b/ietf/utils/mail.py index b2b9f0b9d..e747c7477 100644 --- a/ietf/utils/mail.py +++ b/ietf/utils/mail.py @@ -27,7 +27,7 @@ from django.core.validators import validate_email from django.template.loader import render_to_string from django.template import Context,RequestContext from django.utils import timezone -from django.utils.encoding import force_text, force_str, force_bytes +from django.utils.encoding import force_str, force_bytes import debug # pyflakes:ignore @@ -137,7 +137,7 @@ def send_smtp(msg, bcc=None): server.quit() except smtplib.SMTPServerDisconnected: pass - subj = force_text(msg.get('Subject', '[no subject]')) + subj = force_str(msg.get('Subject', '[no subject]')) tau = time.time() - mark log("sent email (%.3fs) from '%s' to %s id %s subject '%s'" % (tau, frm, to, msg.get('Message-ID', ''), subj)) @@ -166,7 +166,7 @@ def copy_email(msg, to, toUser=False, originalBcc=None): # Overwrite the From: header, so that the copy from a development or # test server doesn't look like spam. new['From'] = settings.DEFAULT_FROM_EMAIL - new['Subject'] = '[Django %s] %s' % (settings.SERVER_MODE, force_text(msg.get('Subject', '[no subject]'))) + new['Subject'] = '[Django %s] %s' % (settings.SERVER_MODE, force_str(msg.get('Subject', '[no subject]'))) new['To'] = to send_smtp(new) @@ -325,7 +325,7 @@ def show_that_mail_was_sent(request,leadline,msg,bcc): from ietf.ietfauth.utils import has_role if has_role(request.user,['Area Director','Secretariat','IANA','RFC Editor','ISE','IAD','IRTF Chair','WG Chair','RG Chair','WG Secretary','RG Secretary']): info = "%s at %s %s\n" % (leadline,timezone.now().strftime("%Y-%m-%d %H:%M:%S"),settings.TIME_ZONE) - info += "Subject: %s\n" % force_text(msg.get('Subject','[no subject]')) + info += "Subject: %s\n" % force_str(msg.get('Subject','[no subject]')) info += "To: %s\n" % msg.get('To','[no to]') if msg.get('Cc'): info += "Cc: %s\n" % msg.get('Cc') @@ -336,7 +336,7 @@ def show_that_mail_was_sent(request,leadline,msg,bcc): def save_as_message(request, msg, bcc): by = ((request and request.user and not request.user.is_anonymous and request.user.person) or ietf.person.models.Person.objects.get(name="(System)")) - headers, body = force_text(str(msg)).split('\n\n', 1) + headers, body = force_str(str(msg)).split('\n\n', 1) kwargs = {'by': by, 'body': body, 'content_type': msg.get_content_type(), 'bcc': bcc or '' } for (arg, field) in [ ('cc', 'Cc'), diff --git a/ietf/utils/management/commands/loadrelated.py b/ietf/utils/management/commands/loadrelated.py index e3d84990c..da9d00d5d 100644 --- a/ietf/utils/management/commands/loadrelated.py +++ b/ietf/utils/management/commands/loadrelated.py @@ -18,7 +18,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.core import serializers from django.db import DEFAULT_DB_ALIAS, DatabaseError, IntegrityError, connections from django.db.models.signals import post_save -from django.utils.encoding import force_text +from django.utils.encoding import force_str import django.core.management.commands.loaddata as loaddata import debug # pyflakes:ignore @@ -91,7 +91,7 @@ class Command(loaddata.Command): obj.save(using=self.using) self.loaded_object_count += 1 except (DatabaseError, IntegrityError, ObjectDoesNotExist, AttributeError) as e: - error_msg = force_text(e) + error_msg = force_str(e) if "Duplicate entry" in error_msg: pass else: diff --git a/ietf/utils/management/commands/mergedata.py b/ietf/utils/management/commands/mergedata.py index 861973482..e73014c78 100644 --- a/ietf/utils/management/commands/mergedata.py +++ b/ietf/utils/management/commands/mergedata.py @@ -15,7 +15,7 @@ from django.core.management.base import CommandError from django.core.management.commands.loaddata import Command as LoadCommand, humanize from django.db import DatabaseError, IntegrityError, router, transaction from django.db.models import ManyToManyField -from django.utils.encoding import force_text +from django.utils.encoding import force_str from ietf.utils.models import ForeignKey @@ -234,7 +234,7 @@ class Command(LoadCommand): 'object_name': obj.object._meta.object_name, 'pk': obj.object.pk, 'data': obj_to_dict(obj.object), - 'error_msg': force_text(e) + 'error_msg': force_str(e) },) raise if objects and show_progress: diff --git a/ietf/utils/test_data.py b/ietf/utils/test_data.py index 3e3324211..1e7097625 100644 --- a/ietf/utils/test_data.py +++ b/ietf/utils/test_data.py @@ -7,7 +7,7 @@ import datetime from django.conf import settings from django.contrib.auth.models import User from django.utils import timezone -from django.utils.encoding import smart_text +from django.utils.encoding import smart_str import debug # pyflakes:ignore @@ -38,7 +38,7 @@ def create_person(group, role_name, name=None, username=None, email_address=None user = User.objects.create(username=username,is_staff=is_staff,is_superuser=is_superuser) user.set_password(password) user.save() - person = Person.objects.create(name=name, ascii=unidecode_name(smart_text(name)), user=user) + person = Person.objects.create(name=name, ascii=unidecode_name(smart_str(name)), user=user) email = Email.objects.create(address=email_address, person=person, origin=user.username) Role.objects.create(group=group, name_id=role_name, person=person, email=email) return person diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index 7453b40a4..ea07a4c2a 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -74,7 +74,7 @@ from django.urls import URLResolver # type: ignore from django.template.backends.django import DjangoTemplates from django.template.backends.django import Template # type: ignore[attr-defined] from django.utils import timezone -# from django.utils.safestring import mark_safe +from django.views.generic import RedirectView, TemplateView import debug # pyflakes:ignore debug.debug = True @@ -95,7 +95,7 @@ old_create = None template_coverage_collection = None code_coverage_collection = None url_coverage_collection = None - +validation_settings = {"validate_html": None, "validate_html_harder": None, "show_logging": False} def start_vnu_server(port=8888): "Start a vnu validation server on the indicated port" @@ -282,7 +282,7 @@ class ValidatingTemplates(DjangoTemplates): def __init__(self, params): super().__init__(params) - if not settings.validate_html: + if not validation_settings["validate_html"]: return self.validation_cache = set() self.cwd = str(pathlib.Path.cwd()) @@ -298,7 +298,7 @@ class ValidatingTemplate(Template): def render(self, context=None, request=None): content = super().render(context, request) - if not settings.validate_html: + if not validation_settings["validate_html"]: return content if not self.origin.name.endswith("html"): @@ -310,7 +310,7 @@ class ValidatingTemplate(Template): return content fingerprint = hash(content) + sys.maxsize + 1 # make hash positive - if not settings.validate_html_harder and fingerprint in self.backend.validation_cache: + if not validation_settings["validate_html_harder"] and fingerprint in self.backend.validation_cache: # already validated this HTML fragment, skip it # as an optimization, make page a bit smaller by not returning HTML for the menus # FIXME: figure out why this still includes base/menu.html @@ -326,7 +326,7 @@ class ValidatingTemplate(Template): # don't validate each template by itself, causes too much overhead # instead, save a batch of them and then validate them all in one go # this delays error detection a bit, but is MUCH faster - settings.validate_html.batches[kind].append( + validation_settings["validate_html"].batches[kind].append( (self.origin.name, content, fingerprint) ) return content @@ -550,8 +550,10 @@ class CoverageTest(unittest.TestCase): return (regex in ("^_test500/$", "^accounts/testemail/$") or regex.startswith("^admin/") or re.search('^api/v1/[^/]+/[^/]+/', regex) - or getattr(pattern.callback, "__name__", "") == "RedirectView" - or getattr(pattern.callback, "__name__", "") == "TemplateView" + or ( + hasattr(pattern.callback, "view_class") + and issubclass(pattern.callback.view_class, (RedirectView, TemplateView)) + ) or pattern.callback == django.views.static.serve) patterns = [(regex, re.compile(regex, re.U), obj) for regex, obj in url_patterns @@ -726,9 +728,10 @@ class IetfTestRunner(DiscoverRunner): self.html_report = html_report self.permit_mixed_migrations = permit_mixed_migrations self.show_logging = show_logging - settings.validate_html = self if validate_html else None - settings.validate_html_harder = self if validate_html and validate_html_harder else None - settings.show_logging = show_logging + global validation_settings + validation_settings["validate_html"] = self if validate_html else None + validation_settings["validate_html_harder"] = self if validate_html and validate_html_harder else None + validation_settings["show_logging"] = show_logging # self.root_dir = os.path.dirname(settings.BASE_DIR) self.coverage_file = os.path.join(self.root_dir, settings.TEST_COVERAGE_MAIN_FILE) @@ -739,6 +742,11 @@ class IetfTestRunner(DiscoverRunner): "as the collection of test coverage data isn't currently threadsafe.") sys.exit(1) self.check_coverage = False + from ietf.doc.tests import TemplateTagTest # import here to prevent circular imports + # Ensure that the coverage tests come last. Specifically list TemplateTagTest before CoverageTest. If this list + # contains parent classes to later subclasses, the parent classes will determine the ordering, so use the most + # specific classes necessary to get the right ordering: + self.reorder_by = (PyFlakesTestCase, MyPyTest,) + self.reorder_by + (StaticLiveServerTestCase, TemplateTagTest, CoverageTest,) def setup_test_environment(self, **kwargs): global template_coverage_collection @@ -843,7 +851,7 @@ class IetfTestRunner(DiscoverRunner): s[1] = tuple(s[1]) # random.setstate() won't accept a list in lieu of a tuple factory.random.set_random_state(s) - if not settings.validate_html: + if not validation_settings["validate_html"]: print(" Not validating any generated HTML; " "please do so at least once before committing changes") else: @@ -912,7 +920,7 @@ class IetfTestRunner(DiscoverRunner): self.config_file[kind].flush() pathlib.Path(self.config_file[kind].name).chmod(0o644) - if not settings.validate_html_harder: + if not validation_settings["validate_html_harder"]: print("") self.vnu = None else: @@ -941,7 +949,7 @@ class IetfTestRunner(DiscoverRunner): with open(self.coverage_file, "w") as file: json.dump(self.coverage_master, file, indent=2, sort_keys=True) - if settings.validate_html: + if validation_settings["validate_html"]: for kind in self.batches: if len(self.batches[kind]): print(f" WARNING: not all templates of kind '{kind}' were validated") @@ -1007,7 +1015,7 @@ class IetfTestRunner(DiscoverRunner): + "\n" ) - if settings.validate_html_harder and kind != "frag": + if validation_settings["validate_html_harder"] and kind != "frag": files = [ os.path.join(d, f) for d, dirs, files in os.walk(tmppath) @@ -1058,20 +1066,57 @@ class IetfTestRunner(DiscoverRunner): test_paths = [ os.path.join(*app.split('.')) for app in test_apps ] return test_apps, test_paths + # Django 5 will drop the extra_tests mechanism for the test runner. Work around + # by adding a special label to the test suite, then injecting our extra tests + # in load_tests_for_label() + def build_suite(self, test_labels=None, extra_tests=None, **kwargs): + if test_labels is None: + # Base class sets test_labels to ["."] if it was None. The label we're + # adding will interfere with that, so replicate that behavior here. + test_labels = ["."] + test_labels = ("_ietf_extra_tests",) + tuple(test_labels) + return super().build_suite(test_labels, extra_tests, **kwargs) + + def load_tests_for_label(self, label, discover_kwargs): + if label == "_ietf_extra_tests": + return self._extra_tests() or None + return super().load_tests_for_label(label, discover_kwargs) + + def _extra_tests(self): + """Get extra tests that should be added to the test suite""" + tests = [] + if validation_settings["validate_html"]: + tests += [ + TemplateValidationTests( + test_runner=self, + validate_html=self, + methodName='run_template_validation', + ), + ] + if self.check_coverage: + global template_coverage_collection, code_coverage_collection, url_coverage_collection + template_coverage_collection = True + code_coverage_collection = True + url_coverage_collection = True + tests += [ + PyFlakesTestCase(test_runner=self, methodName='pyflakes_test'), + MyPyTest(test_runner=self, methodName='mypy_test'), + #CoverageTest(test_runner=self, methodName='interleaved_migrations_test'), + CoverageTest(test_runner=self, methodName='url_coverage_test'), + CoverageTest(test_runner=self, methodName='template_coverage_test'), + CoverageTest(test_runner=self, methodName='code_coverage_test'), + ] + return tests + def run_tests(self, test_labels, extra_tests=None, **kwargs): - global old_destroy, old_create, test_database_name, template_coverage_collection, code_coverage_collection, url_coverage_collection - from django.db import connection - from ietf.doc.tests import TemplateTagTest - - if extra_tests is None: - extra_tests=[] - # Tests that involve switching back and forth between the real # database and the test database are way too dangerous to run # against the production database if socket.gethostname().split('.')[0] in ['core3', 'ietfa', 'ietfb', 'ietfc', ]: raise EnvironmentError("Refusing to run tests on production server") + from django.db import connection + global old_destroy, old_create old_create = connection.creation.__class__.create_test_db connection.creation.__class__.create_test_db = safe_create_test_db old_destroy = connection.creation.__class__.destroy_test_db @@ -1084,35 +1129,6 @@ class IetfTestRunner(DiscoverRunner): self.test_apps, self.test_paths = self.get_test_paths(test_labels) - if settings.validate_html: - extra_tests += [ - TemplateValidationTests( - test_runner=self, - validate_html=self, - methodName='run_template_validation', - ), - ] - - if self.check_coverage: - template_coverage_collection = True - code_coverage_collection = True - url_coverage_collection = True - extra_tests += [ - PyFlakesTestCase(test_runner=self, methodName='pyflakes_test'), - MyPyTest(test_runner=self, methodName='mypy_test'), - #CoverageTest(test_runner=self, methodName='interleaved_migrations_test'), - CoverageTest(test_runner=self, methodName='url_coverage_test'), - CoverageTest(test_runner=self, methodName='template_coverage_test'), - CoverageTest(test_runner=self, methodName='code_coverage_test'), - ] - - # ensure that the coverage tests come last. Specifically list - # TemplateTagTest before CoverageTest. If this list contains - # parent classes to later subclasses, the parent classes will - # determine the ordering, so use the most specific classes - # necessary to get the right ordering: - self.reorder_by = (PyFlakesTestCase, MyPyTest, ) + self.reorder_by + (StaticLiveServerTestCase, TemplateTagTest, CoverageTest, ) - failures = super(IetfTestRunner, self).run_tests(test_labels, extra_tests=extra_tests, **kwargs) if self.check_coverage: @@ -1136,10 +1152,10 @@ class IetfTestRunner(DiscoverRunner): if self.run_full_test_suite: print((" %8s coverage: %6.2f%% (%s: %6.2f%%)" % - (test.capitalize(), test_coverage*100, latest_coverage_version, master_coverage*100, ))) + (test.capitalize(), test_coverage*100, latest_coverage_version, master_coverage*100, ))) else: print((" %8s coverage: %6.2f%%" % - (test.capitalize(), test_coverage*100, ))) + (test.capitalize(), test_coverage*100, ))) print((""" Per-file code and template coverage and per-url-pattern url coverage data diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 69c16be6d..fe715b14d 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -209,7 +209,7 @@ class TemplateChecksTestCase(TestCase): errors = [] for path, template in self.templates.items(): origin = str(template.origin).replace(settings.BASE_DIR, '') - for node in template: + for node in template.nodelist: for child in node.get_nodes_by_type(node_type): errors += func(child, origin, *args, **kwargs) if errors: diff --git a/ietf/utils/urls.py b/ietf/utils/urls.py index 7f46d9c43..6abda9b97 100644 --- a/ietf/utils/urls.py +++ b/ietf/utils/urls.py @@ -6,12 +6,14 @@ import debug # pyflakes:ignore from inspect import isclass -from django.conf.urls import url as django_url -from django.views.generic import View +from django.urls import re_path from django.utils.encoding import force_str +from django.views.generic import View def url(regex, view, kwargs=None, name=None): - if callable(view) and hasattr(view, '__name__'): + if hasattr(view, "view_class"): + view_name = "%s.%s" % (view.__module__, view.view_class.__name__) + elif callable(view) and hasattr(view, '__name__'): view_name = "%s.%s" % (view.__module__, view.__name__) else: view_name = regex @@ -42,5 +44,5 @@ def url(regex, view, kwargs=None, name=None): #debug.show('branch') #debug.show('name') pass - return django_url(regex, view, kwargs=kwargs, name=name) + return re_path(regex, view, kwargs=kwargs, name=name) diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py index ec7366fd1..9642a2877 100644 --- a/ietf/utils/validators.py +++ b/ietf/utils/validators.py @@ -11,10 +11,11 @@ from urllib.parse import urlparse, urlsplit, urlunsplit from django.apps import apps from django.conf import settings from django.core.exceptions import ObjectDoesNotExist, ValidationError -from django.core.validators import RegexValidator, URLValidator, EmailValidator, _lazy_re_compile, BaseValidator +from django.core.validators import RegexValidator, URLValidator, EmailValidator, BaseValidator from django.template.defaultfilters import filesizeformat from django.utils.deconstruct import deconstructible from django.utils.ipv6 import is_valid_ipv6_address +from django.utils.regex_helper import _lazy_re_compile # type: ignore from django.utils.translation import gettext_lazy as _ import debug # pyflakes:ignore diff --git a/mypy.ini b/mypy.ini index 825bf0316..19df7ec9b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,7 +1,5 @@ [mypy] -#mypy_path = ./stubs/ - ignore_missing_imports = True plugins = @@ -9,12 +7,3 @@ plugins = [mypy.plugins.django-stubs] django_settings_module = ietf.settings - -[mypy-ietf.group.migrations.0004_add_group_feature_fields] -ignore_errors = True - -[mypy-ietf.group.migrations.0002_groupfeatures_historicalgroupfeatures] -ignore_errors = True - -[mypy-ietf.doc.migrations.0001_initial] -ignore_errors = True diff --git a/patch/add-django-http-cookie-value-none.patch b/patch/add-django-http-cookie-value-none.patch deleted file mode 100644 index ab75235f2..000000000 --- a/patch/add-django-http-cookie-value-none.patch +++ /dev/null @@ -1,13 +0,0 @@ ---- django/http/response.py.orig 2020-07-08 14:34:42.776562458 +0200 -+++ django/http/response.py 2020-07-08 14:35:56.454687322 +0200 -@@ -197,8 +197,8 @@ - if httponly: - self.cookies[key]['httponly'] = True - if samesite: -- if samesite.lower() not in ('lax', 'strict'): -- raise ValueError('samesite must be "lax" or "strict".') -+ if samesite.lower() not in ('lax', 'strict', 'none'): -+ raise ValueError('samesite must be "lax", "strict", or "none", not "%s".' % samesite) - self.cookies[key]['samesite'] = samesite - - def setdefault(self, key, value): diff --git a/patch/change-oidc-provider-field-sizes-228.patch b/patch/change-oidc-provider-field-sizes-228.patch index 8203f163b..849e2960a 100644 --- a/patch/change-oidc-provider-field-sizes-228.patch +++ b/patch/change-oidc-provider-field-sizes-228.patch @@ -281,7 +281,7 @@ diff -ur oidc_provider.orig/migrations/0021_refresh_token_not_unique.py oidc_pro diff -ur oidc_provider.orig/models.py oidc_provider/models.py --- oidc_provider.orig/models.py 2018-09-14 21:34:52.000000000 +0200 +++ oidc_provider/models.py 2020-06-07 13:34:26.830716635 +0200 -@@ -67,8 +67,8 @@ +@@ -66,8 +66,8 @@ verbose_name=_(u'Client Type'), help_text=_(u'Confidential clients are capable of maintaining the confidentiality' u' of their credentials. Public clients are incapable.')) @@ -292,7 +292,7 @@ diff -ur oidc_provider.orig/models.py oidc_provider/models.py response_types = models.ManyToManyField(ResponseType) jwt_alg = models.CharField( max_length=10, -@@ -78,15 +78,15 @@ +@@ -77,15 +77,15 @@ help_text=_(u'Algorithm used to encode ID Tokens.')) date_created = models.DateField(auto_now_add=True, verbose_name=_(u'Date Created')) website_url = models.CharField( @@ -311,7 +311,7 @@ diff -ur oidc_provider.orig/models.py oidc_provider/models.py logo = models.FileField( blank=True, default='', upload_to='oidc_provider/clients', verbose_name=_(u'Logo Image')) reuse_consent = models.BooleanField( -@@ -186,12 +186,12 @@ +@@ -185,12 +185,12 @@ user = models.ForeignKey( settings.AUTH_USER_MODEL, verbose_name=_(u'User'), on_delete=models.CASCADE) @@ -328,7 +328,7 @@ diff -ur oidc_provider.orig/models.py oidc_provider/models.py class Meta: verbose_name = _(u'Authorization Code') -@@ -205,8 +205,8 @@ +@@ -204,8 +204,8 @@ user = models.ForeignKey( settings.AUTH_USER_MODEL, null=True, verbose_name=_(u'User'), on_delete=models.CASCADE) diff --git a/patch/django-cookie-delete-with-all-settings.patch b/patch/django-cookie-delete-with-all-settings.patch index 830f031d7..fb8bbbe4f 100644 --- a/patch/django-cookie-delete-with-all-settings.patch +++ b/patch/django-cookie-delete-with-all-settings.patch @@ -1,6 +1,6 @@ --- django/contrib/messages/storage/cookie.py.orig 2020-08-13 11:10:36.719177122 +0200 +++ django/contrib/messages/storage/cookie.py 2020-08-13 11:45:23.503463150 +0200 -@@ -92,6 +92,8 @@ +@@ -109,6 +109,8 @@ response.delete_cookie( self.cookie_name, domain=settings.SESSION_COOKIE_DOMAIN, @@ -11,36 +11,47 @@ --- django/http/response.py.orig 2020-08-13 11:16:04.060627793 +0200 +++ django/http/response.py 2020-08-13 11:54:03.482476973 +0200 -@@ -210,12 +210,18 @@ +@@ -282,20 +282,28 @@ value = signing.get_cookie_signer(salt=key + salt).sign(value) return self.set_cookie(key, value, **kwargs) - -- def delete_cookie(self, key, path='/', domain=None, samesite=None): -+ def delete_cookie(self, key, path='/', domain=None, secure=False, httponly=False, samesite=None): - # Most browsers ignore the Set-Cookie header if the cookie name starts - # with __Host- or __Secure- and the cookie doesn't use the secure flag. -- secure = key.startswith(('__Secure-', '__Host-')) + +- def delete_cookie(self, key, path="/", domain=None, samesite=None): ++ def delete_cookie(self, key, path="/", domain=None, secure=False, httponly=False, samesite=None): + # Browsers can ignore the Set-Cookie header if the cookie doesn't use + # the secure flag and: + # - the cookie name starts with "__Host-" or "__Secure-", or + # - the samesite is "none". +- secure = key.startswith(("__Secure-", "__Host-")) or ( +- samesite and samesite.lower() == "none" +- ) + if key in self.cookies: -+ domain = self.cookies[key].get('domain', domain) -+ secure = self.cookies[key].get('secure', secure) -+ httponly = self.cookies[key].get('httponly', httponly) -+ samesite = self.cookies[key].get('samesite', samesite) ++ domain = self.cookies[key].get("domain", domain) ++ secure = self.cookies[key].get("secure", secure) ++ httponly = self.cookies[key].get("httponly", httponly) ++ samesite = self.cookies[key].get("samesite", samesite) + else: -+ secure = secure or key.startswith(('__Secure-', '__Host-')) ++ secure = secure or ( ++ key.startswith(("__Secure-", "__Host-")) or ++ (samesite and samesite.lower() == "none") ++ ) self.set_cookie( -- key, max_age=0, path=path, domain=domain, secure=secure, -+ key, max_age=0, path=path, domain=domain, secure=secure, httponly=httponly, - expires='Thu, 01 Jan 1970 00:00:00 GMT', samesite=samesite, + key, + max_age=0, + path=path, + domain=domain, + secure=secure, ++ httponly=httponly, + expires="Thu, 01 Jan 1970 00:00:00 GMT", + samesite=samesite, ) - --- django/contrib/sessions/middleware.py.orig 2020-08-13 12:12:12.401898114 +0200 +++ django/contrib/sessions/middleware.py 2020-08-13 12:14:52.690520659 +0200 -@@ -39,6 +39,8 @@ - settings.SESSION_COOKIE_NAME, - path=settings.SESSION_COOKIE_PATH, - domain=settings.SESSION_COOKIE_DOMAIN, -+ secure=settings.SESSION_COOKIE_SECURE or None, -+ httponly=settings.SESSION_COOKIE_HTTPONLY or None, - samesite=settings.SESSION_COOKIE_SAMESITE, - ) - else: +@@ -38,6 +38,8 @@ + settings.SESSION_COOKIE_NAME, + path=settings.SESSION_COOKIE_PATH, + domain=settings.SESSION_COOKIE_DOMAIN, ++ secure=settings.SESSION_COOKIE_SECURE or None, ++ httponly=settings.SESSION_COOKIE_HTTPONLY or None, + samesite=settings.SESSION_COOKIE_SAMESITE, + ) + patch_vary_headers(response, ("Cookie",)) diff --git a/patch/fix-django-password-strength-kwargs.patch b/patch/fix-django-password-strength-kwargs.patch deleted file mode 100644 index 9f24ce932..000000000 --- a/patch/fix-django-password-strength-kwargs.patch +++ /dev/null @@ -1,36 +0,0 @@ ---- django_password_strength/widgets.py.orig 2020-06-24 16:07:28.479533134 +0200 -+++ django_password_strength/widgets.py 2020-06-24 16:08:09.540714290 +0200 -@@ -8,7 +8,7 @@ - Form widget to show the user how strong his/her password is. - """ - -- def render(self, name, value, attrs=None): -+ def render(self, name, value, **kwargs): - strength_markup = """ -
    -
    -@@ -30,7 +30,7 @@ - except KeyError: - self.attrs['class'] = 'password_strength' - -- return mark_safe( super(PasswordInput, self).render(name, value, attrs) + strength_markup ) -+ return mark_safe( super(PasswordInput, self).render(name, value, **kwargs) + strength_markup ) - - class Media: - js = ( -@@ -48,7 +48,7 @@ - super(PasswordConfirmationInput, self).__init__(attrs, render_value) - self.confirm_with=confirm_with - -- def render(self, name, value, attrs=None): -+ def render(self, name, value, **kwargs): - if self.confirm_with: - self.attrs['data-confirm-with'] = 'id_%s' % self.confirm_with - -@@ -68,4 +68,4 @@ - except KeyError: - self.attrs['class'] = 'password_confirmation' - -- return mark_safe( super(PasswordInput, self).render(name, value, attrs) + confirmation_markup ) -+ return mark_safe( super(PasswordInput, self).render(name, value, **kwargs) + confirmation_markup ) - diff --git a/patch/fix-oidc-access-token-post.patch b/patch/fix-oidc-access-token-post.patch index 4234fdf61..00271633e 100644 --- a/patch/fix-oidc-access-token-post.patch +++ b/patch/fix-oidc-access-token-post.patch @@ -11,12 +11,10 @@ diff -ur oidc_provider.orig/lib/utils/common.py oidc_provider/lib/utils/common.p --- oidc_provider.orig/lib/utils/oauth2.py 2020-05-22 15:09:21.009044320 +0200 +++ oidc_provider/lib/utils/oauth2.py 2020-06-05 17:05:23.271285858 +0200 -@@ -21,10 +21,14 @@ - """ +@@ -22,9 +22,13 @@ auth_header = request.META.get('HTTP_AUTHORIZATION', '') -- if re.compile('^[Bb]earer\s{1}.+$').match(auth_header): -+ if re.compile(r'^[Bb]earer\s{1}.+$').match(auth_header): + if re.compile(r'^[Bb]earer\s{1}.+$').match(auth_header): access_token = auth_header.split()[1] - else: + elif request.method == 'GET': @@ -27,13 +25,4 @@ diff -ur oidc_provider.orig/lib/utils/common.py oidc_provider/lib/utils/common.p + access_token = '' return access_token - -@@ -39,7 +43,7 @@ - """ - auth_header = request.META.get('HTTP_AUTHORIZATION', '') - -- if re.compile('^Basic\s{1}.+$').match(auth_header): -+ if re.compile(r'^Basic\s{1}.+$').match(auth_header): - b64_user_pass = auth_header.split()[1] - try: - user_pass = b64decode(b64_user_pass).decode('utf-8').split(':') + diff --git a/patch/tastypie-django22-fielderror-response.patch b/patch/tastypie-django22-fielderror-response.patch index bb4decd75..ffb152d31 100644 --- a/patch/tastypie-django22-fielderror-response.patch +++ b/patch/tastypie-django22-fielderror-response.patch @@ -1,15 +1,15 @@ --- tastypie/resources.py.orig 2020-08-24 13:14:25.463166100 +0200 +++ tastypie/resources.py 2020-08-24 13:15:55.133759224 +0200 -@@ -15,7 +15,7 @@ +@@ -12,7 +12,7 @@ ObjectDoesNotExist, MultipleObjectsReturned, ValidationError, FieldDoesNotExist ) from django.core.signals import got_request_exception -from django.core.exceptions import ImproperlyConfigured +from django.core.exceptions import ImproperlyConfigured, FieldError from django.db.models.fields.related import ForeignKey - try: - from django.contrib.gis.db.models.fields import GeometryField -@@ -2207,6 +2207,8 @@ + from django.urls.conf import re_path + from tastypie.utils.timezone import make_naive_utc +@@ -2198,6 +2198,8 @@ return self.authorized_read_list(objects, bundle) except ValueError: raise BadRequest("Invalid resource lookup data provided (mismatched type).") @@ -20,7 +20,7 @@ """ --- tastypie/paginator.py.orig 2020-08-25 15:24:46.391588425 +0200 +++ tastypie/paginator.py 2020-08-25 15:24:53.591797122 +0200 -@@ -128,6 +128,8 @@ +@@ -124,6 +124,8 @@ except (AttributeError, TypeError): # If it's not a QuerySet (or it's ilk), fallback to ``len``. return len(self.objects) diff --git a/requirements.txt b/requirements.txt index 44dcb66a7..81f72ce60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,30 +5,31 @@ argon2-cffi>=21.3.0 # For the Argon2 password hasher option beautifulsoup4>=4.11.1 # Only used in tests bibtexparser>=1.2.0 # Only used in tests bleach>=6 +types-bleach>=6 celery>=5.2.6 coverage>=4.5.4,<5.0 # Coverage 5.x moves from a json database to SQLite. Moving to 5.x will require substantial rewrites in ietf.utils.test_runner and ietf.release.views decorator>=5.1.1 +types-decorator>=5.1.1 defusedxml>=0.7.1 # for TastyPie when using xml; not a declared dependency -Django>=2.2.28,<3.0 +Django>4.2,<5 django-analytical>=3.1.0 django-bootstrap5>=21.3 django-celery-beat>=2.3.0 django-csp>=3.7 django-cors-headers>=3.11.0 django-debug-toolbar>=3.2.4 -django-form-utils>=1.0.3 # Only one use, in the liaisons app. Last release was in 2015. django-markup>=1.5 # Limited use - need to reconcile against direct use of markdown -django-oidc-provider>=0.7,<0.8 # 0.8 dropped Django 2 support -django-password-strength>=1.2.1 +django-oidc-provider>=0.8 # 0.8 dropped Django 2 support django-referrer-policy>=1.0 django-simple-history>=3.0.0 -django-stubs==1.6.0 # The django-stubs version used determines the the mypy version indicated below -django-tastypie==0.14.3 # Version must be locked in sync with version of Django +django-stubs>=4.2.0 # The django-stubs version used determines the the mypy version indicated below +django-tastypie>=0.14.5 # Version must be locked in sync with version of Django django-vite>=2.0.2 django-webtest>=1.9.10 # Only used in tests django-widget-tweaks>=1.4.12 djlint>=1.0.0 # To auto-indent templates via "djlint --profile django --reformat" docutils>=0.18.1 # Used only by dbtemplates for RestructuredText +types-docutils>=0.18.1 factory-boy>=3.2.1 github3.py>=3.2.0 gunicorn>=20.1.0 @@ -41,21 +42,25 @@ jwcrypto>=1.2 # for signed notifications - this is aspirational, and is not r logging_tree>=1.9 # Used only by the showloggers management command lxml>=4.8.0,<5 markdown>=3.3.6 +types-markdown>=3.3.6 mock>=4.0.3 # Used only by tests, of course -mypy>=0.782,<0.790 # Version requirements determined by django-stubs. +types-mock>=4.0.3 +mypy~=1.2.0 # Version requirements determined by django-stubs. oic>=1.3 # Used only by tests Pillow>=9.1.0 -psycopg2<2.9 +psycopg2>=2.9.6 pyang>=2.5.3 pyflakes>=2.4.0 pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency pyquery>=1.4.3 python-dateutil>=2.8.2 +types-python-dateutil>=2.8.2 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures -python-memcached>=1.59 # for django.core.cache.backends.memcached +pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=1.6 # from TastyPie pytz==2022.2.1 # Pinned as changes need to be vetted for their effect on Meeting fields requests>=2.27.1 +types-requests>=2.27.1 requests-mock>=1.9.3 rfc2html>=2.0.3 scout-apm>=2.24.2