altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

ci: merge main to release (#7848)

Browse source 
ci: merge main to release
This commit is contained in:
Robert Sparks 2024-08-22 13:59:02 -05:00 committed by GitHub
parent 162aa54ffe d3fe1c0008
commit 2bf7374716
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 222 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
ietf/doc/migrations/0022_remove_dochistory_internal_comments_and_more.py Normal file
View file

@ -0,0 +1,21 @@
# Generated by Django 4.2.15 on 2024-08-16 16:43
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("doc", "0021_narrativeminutes"),
]
operations = [
migrations.RemoveField(
model_name="dochistory",
name="internal_comments",
),
migrations.RemoveField(
model_name="document",
name="internal_comments",
),
]

1
ietf/doc/models.py
View file

@ -122,7 +122,6 @@ class DocumentInfo(models.Model):
external_url = models.URLField(blank=True)
uploaded_filename = models.TextField(blank=True)
note = models.TextField(blank=True)
internal_comments = models.TextField(blank=True)
rfc_number = models.PositiveIntegerField(blank=True, null=True) # only valid for type="rfc"
def file_extension(self):

2
ietf/doc/resources.py
View file

@ -130,7 +130,6 @@ class DocumentResource(ModelResource):
"external_url": ALL,
"uploaded_filename": ALL,
"note": ALL,
"internal_comments": ALL,
"name": ALL,
"type": ALL_WITH_RELATIONS,
"stream": ALL_WITH_RELATIONS,
@ -247,7 +246,6 @@ class DocHistoryResource(ModelResource):
"external_url": ALL,
"uploaded_filename": ALL,
"note": ALL,
"internal_comments": ALL,
"name": ALL,
"type": ALL_WITH_RELATIONS,
"stream": ALL_WITH_RELATIONS,

4
ietf/doc/templatetags/ietf_filters.py
View file

@ -856,10 +856,10 @@ def badgeify(blob):
Add an appropriate bootstrap badge around "text", based on its contents.
"""
config = [
(r"rejected|not ready", "danger", "x-lg"),
(r"rejected|not ready|serious issues", "danger", "x-lg"),
(r"complete|accepted|ready", "success", ""),
(r"has nits|almost ready", "info", "info-lg"),
(r"has issues", "warning", "exclamation-lg"),
(r"has issues|on the right track", "warning", "exclamation-lg"),
(r"assigned", "info", "person-plus-fill"),
(r"will not review|overtaken by events|withdrawn", "secondary", "dash-lg"),
(r"no response", "warning", "question-lg"),

30
ietf/doc/tests_ballot.py
View file

@ -20,6 +20,7 @@ from ietf.doc.factories import (DocumentFactory, IndividualDraftFactory, Individ
BallotPositionDocEventFactory, BallotDocEventFactory, IRSGBallotDocEventFactory)
from ietf.doc.templatetags.ietf_filters import can_defer
from ietf.doc.utils import create_ballot_if_not_open
from ietf.doc.views_ballot import parse_ballot_edit_return_point
from ietf.doc.views_doc import document_ballot_content
from ietf.group.models import Group, Role
from ietf.group.factories import GroupFactory, RoleFactory, ReviewTeamFactory
@ -1451,3 +1452,32 @@ class BallotContentTests(TestCase):
self._assertBallotMessage(q, balloters[0], 'No discuss send log available')
self._assertBallotMessage(q, balloters[1], 'No comment send log available')
self._assertBallotMessage(q, old_balloter, 'No ballot position send log available')
class ReturnToUrlTests(TestCase):
def test_invalid_return_to_url(self):
self.assertRaises(
Exception,
lambda: parse_ballot_edit_return_point('/doc/', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718'),
)
self.assertRaises(
Exception,
lambda: parse_ballot_edit_return_point('/a-route-that-does-not-exist/', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718'),
)
self.assertRaises(
Exception,
lambda: parse_ballot_edit_return_point('https://example.com/phishing', 'draft-ietf-opsawg-ipfix-tcpo-v6eh', '998718'),
)
def test_valid_default_return_to_url(self):
self.assertEqual(parse_ballot_edit_return_point(
None,
'draft-ietf-opsawg-ipfix-tcpo-v6eh',
'998718'
), '/doc/draft-ietf-opsawg-ipfix-tcpo-v6eh/ballot/998718/')
def test_valid_return_to_url(self):
self.assertEqual(parse_ballot_edit_return_point(
'/doc/draft-ietf-opsawg-ipfix-tcpo-v6eh/ballot/998718/',
'draft-ietf-opsawg-ipfix-tcpo-v6eh',
'998718'
), '/doc/draft-ietf-opsawg-ipfix-tcpo-v6eh/ballot/998718/')

45
ietf/doc/views_ballot.py
View file

@ -8,13 +8,14 @@ import datetime, json
from django import forms
from django.conf import settings
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseBadRequest
from django.shortcuts import render, get_object_or_404, redirect
from django.template.defaultfilters import striptags
from django.template.loader import render_to_string
from django.urls import reverse as urlreverse
from django.views.decorators.csrf import csrf_exempt
from django.utils.html import escape
from urllib.parse import urlencode as urllib_urlencode
import debug # pyflakes:ignore
@ -39,6 +40,7 @@ from ietf.message.utils import infer_message
from ietf.name.models import BallotPositionName, DocTypeName
from ietf.person.models import Person
from ietf.utils.fields import ModelMultipleChoiceField
from ietf.utils.http import validate_return_to_path
from ietf.utils.mail import send_mail_text, send_mail_preformatted
from ietf.utils.decorators import require_api_key
from ietf.utils.response import permission_denied
@ -185,11 +187,11 @@ def edit_position(request, name, ballot_id):
balloter = login = request.user.person
if 'ballot_edit_return_point' in request.session:
return_to_url = request.session['ballot_edit_return_point']
else:
return_to_url = urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name, ballot_id=ballot_id))
try:
return_to_url = parse_ballot_edit_return_point(request.GET.get('ballot_edit_return_point'), doc.name, ballot_id)
except ValueError:
return HttpResponseBadRequest('ballot_edit_return_point is invalid')
# if we're in the Secretariat, we can select a balloter to act as stand-in for
if has_role(request.user, "Secretariat"):
balloter_id = request.GET.get('balloter')
@ -209,9 +211,14 @@ def edit_position(request, name, ballot_id):
save_position(form, doc, ballot, balloter, login, send_mail)
if send_mail:
qstr=""
query = {}
if request.GET.get('balloter'):
qstr += "?balloter=%s" % request.GET.get('balloter')
query['balloter'] = request.GET.get('balloter')
if request.GET.get('ballot_edit_return_point'):
query['ballot_edit_return_point'] = request.GET.get('ballot_edit_return_point')
qstr = ""
if len(query) > 0:
qstr = "?" + urllib_urlencode(query, safe='/')
return HttpResponseRedirect(urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=doc.name, ballot_id=ballot_id)) + qstr)
elif request.POST.get("Defer") and doc.stream.slug != "irtf":
return redirect('ietf.doc.views_ballot.defer_ballot', name=doc)
@ -337,11 +344,11 @@ def send_ballot_comment(request, name, ballot_id):
balloter = request.user.person
if 'ballot_edit_return_point' in request.session:
return_to_url = request.session['ballot_edit_return_point']
else:
return_to_url = urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name, ballot_id=ballot_id))
try:
return_to_url = parse_ballot_edit_return_point(request.GET.get('ballot_edit_return_point'), doc.name, ballot_id)
except ValueError:
return HttpResponseBadRequest('ballot_edit_return_point is invalid')
if 'HTTP_REFERER' in request.META:
back_url = request.META['HTTP_REFERER']
else:
@ -1302,3 +1309,15 @@ def rsab_ballot_status(request):
# Possible TODO: add a menu item to show this? Maybe only if you're in rsab or an rswg chair?
# There will be so few of these that the general community would follow them from the rswg docs page.
# Maybe the view isn't actually needed at all...
def parse_ballot_edit_return_point(path, doc_name, ballot_id):
get_default_path = lambda: urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc_name, ballot_id=ballot_id))
allowed_path_handlers = {
"ietf.doc.views_doc.document_ballot",
"ietf.doc.views_doc.document_irsg_ballot",
"ietf.doc.views_doc.document_rsab_ballot",
"ietf.iesg.views.agenda",
"ietf.iesg.views.agenda_documents",
}
return validate_return_to_path(path, get_default_path, allowed_path_handlers)

5
ietf/doc/views_doc.py
View file

@ -1538,7 +1538,6 @@ def document_ballot(request, name, ballot_id=None):
top = render_document_top(request, doc, ballot_tab, name)
c = document_ballot_content(request, doc, ballot.id, editable=True)
request.session['ballot_edit_return_point'] = request.path_info
return render(request, "doc/document_ballot.html",
dict(doc=doc,
@ -1556,8 +1555,6 @@ def document_irsg_ballot(request, name, ballot_id=None):
c = document_ballot_content(request, doc, ballot_id, editable=True)
request.session['ballot_edit_return_point'] = request.path_info
return render(request, "doc/document_ballot.html",
dict(doc=doc,
top=top,
@ -1575,8 +1572,6 @@ def document_rsab_ballot(request, name, ballot_id=None):
c = document_ballot_content(request, doc, ballot_id, editable=True)
request.session['ballot_edit_return_point'] = request.path_info
return render(
request,
"doc/document_ballot.html",

3
ietf/iesg/views.py
View file

@ -209,7 +209,6 @@ def agenda(request, date=None):
urlreverse("ietf.iesg.views.telechat_agenda_content_view", kwargs={"section": "minutes"})
))
request.session['ballot_edit_return_point'] = request.path_info
return render(request, "iesg/agenda.html", {
"date": data["date"],
"sections": sorted(data["sections"].items(), key=lambda x:[int(p) for p in x[0].split('.')]),
@ -398,7 +397,7 @@ def agenda_documents(request):
"sections": sorted((num, section) for num, section in sections.items()
if "2" <= num < "5")
})
request.session['ballot_edit_return_point'] = request.path_info
return render(request, 'iesg/agenda_documents.html', { 'telechats': telechats })
def past_documents(request):

2
ietf/ipr/forms.py
View file

@ -112,7 +112,7 @@ class DraftForm(forms.ModelForm):
if not document:
self.add_error("document", "Identifying the Internet-Draft or RFC for this disclosure is required.")
elif not document.name.startswith("rfc"):
if revisions.strip() == "":
if revisions is None or revisions.strip() == "":
self.add_error("revisions", "Revisions of this Internet-Draft for which this disclosure is relevant must be specified.")
return cleaned_data

60
ietf/ipr/tests.py
View file

@ -28,9 +28,11 @@ from ietf.group.factories import RoleFactory
from ietf.ipr.factories import (
HolderIprDisclosureFactory,
GenericIprDisclosureFactory,
IprDisclosureBaseFactory,
IprDocRelFactory,
IprEventFactory
)
from ietf.ipr.forms import DraftForm
from ietf.ipr.mail import (process_response_email, get_reply_to, get_update_submitter_emails,
get_pseudo_submitter, get_holders, get_update_cc_addrs)
from ietf.ipr.models import (IprDisclosureBase,GenericIprDisclosure,HolderIprDisclosure,
@ -935,3 +937,61 @@ Subject: test
no_revisions_message(iprdocrel),
"No revisions for this Internet-Draft were specified in this disclosure. However, there is only one revision of this Internet-Draft."
)
class DraftFormTests(TestCase):
def setUp(self):
super().setUp()
self.disclosure = IprDisclosureBaseFactory()
self.draft = WgDraftFactory.create_batch(10)[-1]
self.rfc = RfcFactory()
def test_revisions_valid(self):
post_data = {
# n.b., "document" is a SearchableDocumentField, which is a multiple choice field limited
# to a single choice. Its value must be an array of pks with one element.
"document": [str(self.draft.pk)],
"disclosure": str(self.disclosure.pk),
}
# The revisions field is just a char field that allows descriptions of the applicable
# document revisions. It's usually just a rev or "00-02", but the form allows anything
# not empty. The secretariat will review the value before the disclosure is posted so
# minimal validation is ok here.
self.assertTrue(DraftForm(post_data | {"revisions": "00"}).is_valid())
self.assertTrue(DraftForm(post_data | {"revisions": "00-02"}).is_valid())
self.assertTrue(DraftForm(post_data | {"revisions": "01,03, 05"}).is_valid())
self.assertTrue(DraftForm(post_data | {"revisions": "all but 01"}).is_valid())
# RFC instead of draft - allow empty / missing revisions
post_data["document"] = [str(self.rfc.pk)]
self.assertTrue(DraftForm(post_data).is_valid())
self.assertTrue(DraftForm(post_data | {"revisions": ""}).is_valid())
def test_revisions_invalid(self):
missing_rev_error_msg = (
"Revisions of this Internet-Draft for which this disclosure is relevant must be specified."
)
null_char_error_msg = "Null characters are not allowed."
post_data = {
# n.b., "document" is a SearchableDocumentField, which is a multiple choice field limited
# to a single choice. Its value must be an array of pks with one element.
"document": [str(self.draft.pk)],
"disclosure": str(self.disclosure.pk),
}
self.assertFormError(
DraftForm(post_data), "revisions", missing_rev_error_msg
)
self.assertFormError(
DraftForm(post_data | {"revisions": ""}), "revisions", missing_rev_error_msg
)
self.assertFormError(
DraftForm(post_data | {"revisions": "1\x00"}),
"revisions",
[null_char_error_msg, missing_rev_error_msg],
)
# RFC instead of draft still validates the revisions field
self.assertFormError(
DraftForm(post_data | {"document": [str(self.rfc.pk)], "revisions": "1\x00"}),
"revisions",
null_char_error_msg,
)

17
ietf/meeting/migrations/0008_remove_schedtimesessassignment_notes.py Normal file
View file

@ -0,0 +1,17 @@
# Generated by Django 4.2.15 on 2024-08-16 13:49
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("meeting", "0007_attended_origin_attended_time"),
]
operations = [
migrations.RemoveField(
model_name="schedtimesessassignment",
name="notes",
),
]

3
ietf/meeting/models.py
View file

@ -788,7 +788,6 @@ class SchedTimeSessAssignment(models.Model):
schedule = ForeignKey('Schedule', null=False, blank=False, related_name='assignments')
extendedfrom = ForeignKey('self', null=True, default=None, help_text="Timeslot this session is an extension of.")
modified = models.DateTimeField(auto_now=True)
notes = models.TextField(blank=True)
badness = models.IntegerField(default=0, blank=True, null=True)
pinned = models.BooleanField(default=False, help_text="Do not move session during automatic placement.")
@ -1423,7 +1422,7 @@ class MeetingHost(models.Model):
validate_file_extension,
settings.MEETING_VALID_UPLOAD_EXTENSIONS['meetinghostlogo'],
),
WrappedValidator(
WrappedValidator(
validate_mime_type,
settings.MEETING_VALID_UPLOAD_MIME_TYPES['meetinghostlogo'],
True,

1
ietf/meeting/resources.py
View file

@ -269,7 +269,6 @@ class SchedTimeSessAssignmentResource(ModelResource):
filtering = {
"id": ALL,
"modified": ALL,
"notes": ALL,
"badness": ALL,
"pinned": ALL,
"timeslot": ALL_WITH_RELATIONS,

29
ietf/meeting/utils.py
View file

@ -12,7 +12,8 @@ from pathlib import Path
from django.conf import settings
from django.contrib import messages
from django.db.models import Q
from django.db.models import OuterRef, Subquery, TextField, Q, Value
from django.db.models.functions import Coalesce
from django.template.loader import render_to_string
from django.utils import timezone
from django.utils.encoding import smart_str
@ -149,19 +150,27 @@ def create_proceedings_templates(meeting):
def bluesheet_data(session):
def affiliation(meeting, person):
# from OidcExtraScopeClaims.scope_registration()
email_list = person.email_set.values_list("address")
q = Q(person=person, meeting=meeting) | Q(email__in=email_list, meeting=meeting)
reg = MeetingRegistration.objects.filter(q).exclude(affiliation="").first()
return reg.affiliation if reg else ""
attendance = (
Attended.objects.filter(session=session)
.annotate(
affiliation=Coalesce(
Subquery(
MeetingRegistration.objects.filter(
Q(meeting=session.meeting),
Q(person=OuterRef("person")) | Q(email=OuterRef("person__email")),
).values("affiliation")[:1]
),
Value(""),
output_field=TextField(),
)
).distinct()
.order_by("time")
)
attendance = Attended.objects.filter(session=session).order_by("time")
meeting = session.meeting
return [
{
"name": attended.person.plain_name(),
"affiliation": affiliation(meeting, attended.person),
"affiliation": attended.affiliation,
}
for attended in attendance
]

7
ietf/secr/static/css/custom.css
View file

@ -319,11 +319,6 @@ input.draft-file-input {
width: 4em;
}
.draft-container #id_internal_comments {
height: 4em;
width: 40em;
}
.draft-container #id_abstract {
height: 15em;
width: 40em;
@ -842,4 +837,4 @@ td, th, li, h2 {
thead th {
font-size: 12px;
}
}

2
ietf/templates/doc/ballot_popup.html
View file

@ -27,7 +27,7 @@
{% if editable and user|has_role:"Area Director,Secretariat,IRSG Member,RSAB Member" %}
{% if user|can_ballot:doc %}
<a class="btn btn-primary"
href="{% url "ietf.doc.views_ballot.edit_position" name=doc.name ballot_id=ballot_id %}">
href="{% url "ietf.doc.views_ballot.edit_position" name=doc.name ballot_id=ballot_id %}?ballot_edit_return_point={{ request.path|urlencode }}">
Edit position
</a>
{% endif %}

2
ietf/templates/doc/document_ballot_content.html
View file

@ -60,7 +60,7 @@
</a>
{% if user|can_ballot:doc %}
<a class="btn btn-primary"
href="{% url "ietf.doc.views_ballot.edit_position" name=doc.name ballot_id=ballot.pk %}">
href="{% url "ietf.doc.views_ballot.edit_position" name=doc.name ballot_id=ballot.pk %}?ballot_edit_return_point={{ request.path|urlencode }}">
Edit position
</a>
{% endif %}

26
ietf/utils/http.py
View file

@ -1,6 +1,8 @@
# Copyright The IETF Trust 2023, All Rights Reserved
# Copyright The IETF Trust 2023-2024, All Rights Reserved
# -*- coding: utf-8 -*-
from django.urls import resolve as urlresolve, Resolver404
def is_ajax(request):
"""Checks whether a request was an AJAX call
@ -8,3 +10,25 @@ def is_ajax(request):
exact reproduction of the deprecated method suggested there.
"""
return request.headers.get("x-requested-with") == "XMLHttpRequest"
def validate_return_to_path(path, get_default_path, allowed_path_handlers):
if path is None:
path = get_default_path()
# we need to ensure the path isn't used for attacks (eg phishing).
# `path` can be used in HttpResponseRedirect() which could redirect to Datatracker or offsite.
# Eg http://datatracker.ietf.org/...?ballot_edit_return_point=https://example.com/phish
# offsite links could be phishing attempts so let's reject them all, and require valid Datatracker
# routes
try:
# urlresolve will throw if the url doesn't match a route known to Django
match = urlresolve(path)
# further restrict by whether it's in the list of valid routes to prevent
# (eg) redirecting to logout
if match.url_name not in allowed_path_handlers:
raise ValueError("Invalid return to path not among valid matches")
pass
except Resolver404:
raise ValueError("Invalid return to path doesn't match a route")
return path

4
k8s/nginx-auth.conf
View file

@ -34,5 +34,9 @@ server {
proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $${keepempty}remote_addr;
proxy_pass http://localhost:8000;
# Set timeouts longer than Cloudflare proxy limits
proxy_connect_timeout 60; # nginx default (Cf = 15)
proxy_read_timeout 120; # nginx default = 60 (Cf = 100)
proxy_send_timeout 60; # nginx default = 60 (Cf = 30)
}
}

4
k8s/nginx-datatracker.conf
View file

@ -23,6 +23,10 @@ server {
proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $${keepempty}remote_addr;
proxy_pass http://localhost:8000;
# Set timeouts longer than Cloudflare proxy limits
proxy_connect_timeout 60; # nginx default (Cf = 15)
proxy_read_timeout 120; # nginx default = 60 (Cf = 100)
proxy_send_timeout 60; # nginx default = 60 (Cf = 30)
client_max_body_size 0; # disable size check
}
}

1
playwright/package.json
View file

@ -3,6 +3,7 @@
"install-deps": "playwright install --with-deps",
"test": "playwright test",
"test:legacy": "playwright test -c playwright-legacy.config.js",
"test:legacy:visual": "playwright test -c playwright-legacy.config.js --headed --workers=1",
"test:visual": "playwright test --headed --workers=1",
"test:debug": "playwright test --debug"
},

2
requirements.txt
View file

@ -71,5 +71,5 @@ tqdm>=4.64.0
Unidecode>=1.3.4
urllib3>=2
weasyprint>=59
xml2rfc>=3.12.4
xml2rfc[pdf]>=3.23.0
xym>=0.6,<1.0