ci: merge main to release (#7777)

ci: merge main to release
This commit is contained in:
Robert Sparks 2024-08-01 15:32:26 -05:00 committed by GitHub
commit 9704eed16a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 172 additions and 42 deletions

View file

@ -47,6 +47,10 @@ on:
required: true
type: boolean
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
# -----------------------------------------------------------------
# PREPARE

View file

@ -8,7 +8,7 @@
[![Python Version](https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white)](#prerequisites)
[![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites)
[![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites)
[![MariaDB Version](https://img.shields.io/badge/postgres-14-blue?logo=postgresql&logoColor=white)](#prerequisites)
[![MariaDB Version](https://img.shields.io/badge/postgres-16-blue?logo=postgresql&logoColor=white)](#prerequisites)
##### The day-to-day front-end to the IETF database for people who work on IETF standards.

View file

@ -429,6 +429,7 @@ def directauth(request):
data = None
if raw_data is None or data is None:
log.log("Request body is either missing or invalid")
return HttpResponse(json.dumps(dict(result="failure",reason="invalid post")), content_type='application/json')
authtoken = data.get('authtoken', None)
@ -436,9 +437,11 @@ def directauth(request):
password = data.get('password', None)
if any([item is None for item in (authtoken, username, password)]):
log.log("One or more mandatory fields are missing: authtoken, username, password")
return HttpResponse(json.dumps(dict(result="failure",reason="invalid post")), content_type='application/json')
if not is_valid_token("ietf.api.views.directauth", authtoken):
log.log("Auth token provided is invalid")
return HttpResponse(json.dumps(dict(result="failure",reason="invalid authtoken")), content_type='application/json')
user_query = User.objects.filter(username__iexact=username)
@ -449,18 +452,20 @@ def directauth(request):
# Note well that we are using user.username, not what was passed to the API.
if user_query.count() == 1 and authenticate(username = user_query.first().username, password = password):
user_count = user_query.count()
if user_count == 1 and authenticate(username = user_query.first().username, password = password):
user = user_query.get()
if user_query.filter(person__isnull=True).count() == 1: # Can't inspect user.person direclty here
log.log(f"Direct auth of personless user {user.pk}:{user.username}")
log.log(f"Direct auth success (personless user): {user.pk}:{user.username}")
else:
log.log(f"Direct auth: {user.pk}:{user.person.plain_name()}")
log.log(f"Direct auth success: {user.pk}:{user.person.plain_name()}")
return HttpResponse(json.dumps(dict(result="success")), content_type='application/json')
log.log(f"Direct auth failure: {username}")
log.log(f"Direct auth failure: {username} ({user_count} user(s) found)")
return HttpResponse(json.dumps(dict(result="failure", reason="authentication failed")), content_type='application/json')
else:
log.log(f"Request must be POST: {request.method} received")
return HttpResponse(status=405)

View file

@ -142,6 +142,13 @@ admin.site.register(DocumentActionHolder, DocumentActionHolderAdmin)
# events
class DeletedEventAdmin(admin.ModelAdmin):
list_display = ['id', 'content_type', 'json', 'by', 'time']
list_filter = ['time']
raw_id_fields = ['content_type', 'by']
admin.site.register(DeletedEvent, DeletedEventAdmin)
class DocEventAdmin(admin.ModelAdmin):
def event_type(self, obj):
return str(obj.type)
@ -159,39 +166,42 @@ admin.site.register(NewRevisionDocEvent, DocEventAdmin)
admin.site.register(StateDocEvent, DocEventAdmin)
admin.site.register(ConsensusDocEvent, DocEventAdmin)
admin.site.register(BallotDocEvent, DocEventAdmin)
admin.site.register(IRSGBallotDocEvent, DocEventAdmin)
admin.site.register(WriteupDocEvent, DocEventAdmin)
admin.site.register(LastCallDocEvent, DocEventAdmin)
admin.site.register(TelechatDocEvent, DocEventAdmin)
admin.site.register(ReviewRequestDocEvent, DocEventAdmin)
admin.site.register(ReviewAssignmentDocEvent, DocEventAdmin)
admin.site.register(InitialReviewDocEvent, DocEventAdmin)
admin.site.register(AddedMessageEvent, DocEventAdmin)
admin.site.register(SubmissionDocEvent, DocEventAdmin)
admin.site.register(EditedAuthorsDocEvent, DocEventAdmin)
admin.site.register(IanaExpertDocEvent, DocEventAdmin)
class DeletedEventAdmin(admin.ModelAdmin):
list_display = ['id', 'content_type', 'json', 'by', 'time']
list_filter = ['time']
raw_id_fields = ['content_type', 'by']
admin.site.register(DeletedEvent, DeletedEventAdmin)
class BallotPositionDocEventAdmin(DocEventAdmin):
raw_id_fields = ["doc", "by", "balloter", "ballot"]
raw_id_fields = DocEventAdmin.raw_id_fields + ["balloter", "ballot"]
admin.site.register(BallotPositionDocEvent, BallotPositionDocEventAdmin)
class IRSGBallotDocEventAdmin(DocEventAdmin):
raw_id_fields = ["doc", "by"]
admin.site.register(IRSGBallotDocEvent, IRSGBallotDocEventAdmin)
class BofreqEditorDocEventAdmin(DocEventAdmin):
raw_id_fields = ["doc", "by", "editors" ]
raw_id_fields = DocEventAdmin.raw_id_fields + ["editors"]
admin.site.register(BofreqEditorDocEvent, BofreqEditorDocEventAdmin)
class BofreqResponsibleDocEventAdmin(DocEventAdmin):
raw_id_fields = ["doc", "by", "responsible" ]
raw_id_fields = DocEventAdmin.raw_id_fields + ["responsible"]
admin.site.register(BofreqResponsibleDocEvent, BofreqResponsibleDocEventAdmin)
class ReviewRequestDocEventAdmin(DocEventAdmin):
raw_id_fields = DocEventAdmin.raw_id_fields + ["review_request"]
admin.site.register(ReviewRequestDocEvent, ReviewRequestDocEventAdmin)
class ReviewAssignmentDocEventAdmin(DocEventAdmin):
raw_id_fields = DocEventAdmin.raw_id_fields + ["review_assignment"]
admin.site.register(ReviewAssignmentDocEvent, ReviewAssignmentDocEventAdmin)
class AddedMessageEventAdmin(DocEventAdmin):
raw_id_fields = DocEventAdmin.raw_id_fields + ["message"]
admin.site.register(AddedMessageEvent, AddedMessageEventAdmin)
class SubmissionDocEventAdmin(DocEventAdmin):
raw_id_fields = DocEventAdmin.raw_id_fields + ["submission"]
admin.site.register(SubmissionDocEvent, SubmissionDocEventAdmin)
class DocumentUrlAdmin(admin.ModelAdmin):
list_display = ['id', 'doc', 'tag', 'url', 'desc', ]
search_fields = ['doc__name', 'url', ]

View file

@ -265,6 +265,8 @@ def document_main(request, name, rev=None, document_html=False):
can_change_stream = bool(can_edit or roles)
file_urls, found_types = build_file_urls(doc)
if not request.user.is_authenticated:
file_urls = [fu for fu in file_urls if fu[0] != "pdfized"]
content = doc.text_or_error() # pyflakes:ignore
content = markup_txt.markup(maybe_split(content, split=split_content))
@ -406,6 +408,8 @@ def document_main(request, name, rev=None, document_html=False):
latest_revision = None
file_urls, found_types = build_file_urls(doc)
if not request.user.is_authenticated:
file_urls = [fu for fu in file_urls if fu[0] != "pdfized"]
content = doc.text_or_error() # pyflakes:ignore
content = markup_txt.markup(maybe_split(content, split=split_content))
@ -1039,6 +1043,8 @@ def document_html(request, name, rev=None):
document_html=True,
)
@login_required
def document_pdfized(request, name, rev=None, ext=None):
found = fuzzy_find_documents(name, rev)

View file

@ -52,6 +52,15 @@ class IESGTests(TestCase):
self.assertContains(r, draft.name)
self.assertContains(r, escape(pos.balloter.plain_name()))
# Mark draft as replaced
draft.set_state(State.objects.get(type="draft", slug="repl"))
r = self.client.get(urlreverse("ietf.iesg.views.discusses"))
self.assertEqual(r.status_code, 200)
self.assertNotContains(r, draft.name)
self.assertNotContains(r, escape(pos.balloter.plain_name()))
def test_milestones_needing_review(self):
draft = WgDraftFactory()
RoleFactory(name_id='ad',group=draft.group,person=Person.objects.get(user__username='ad'))

View file

@ -483,6 +483,7 @@ def discusses(request):
models.Q(states__type__in=("statchg", "conflrev"),
states__slug__in=("iesgeval", "defer")),
docevent__ballotpositiondocevent__pos__blocking=True)
possible_docs = possible_docs.exclude(states__in=State.objects.filter(type="draft", slug="repl"))
possible_docs = possible_docs.select_related("stream", "group", "ad").distinct()
docs = []

View file

@ -6246,6 +6246,12 @@ class MaterialsTests(TestCase):
q = PyQuery(r.content)
self.assertTrue(q('form input[type="checkbox"]'))
# test not submitting a file
r = self.client.post(url, dict(submission_method="upload"))
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertTrue(q("form .is-invalid"))
test_file = BytesIO(b'this is some text for a test')
test_file.name = "not_really.json"
r = self.client.post(url,dict(submission_method="upload",file=test_file))

View file

@ -2790,7 +2790,8 @@ class UploadOrEnterAgendaForm(UploadAgendaForm):
def clean_file(self):
submission_method = self.cleaned_data.get("submission_method")
if submission_method == "upload":
return super().clean_file()
if self.cleaned_data.get("file", None) is not None:
return super().clean_file()
return None
def clean(self):

View file

@ -3464,7 +3464,7 @@
"parent_types": [],
"req_subm_approval": true,
"role_order": "[\n \"chair\",\n \"delegate\"\n]",
"session_purposes": "[\n \"officehours\"\n]",
"session_purposes": "[\n \"officehours\",\n \"regular\"\n]",
"show_on_agenda": true
},
"model": "group.groupfeatures",

View file

@ -2,7 +2,7 @@ from ietf.person import views, ajax
from ietf.utils.urls import url
urlpatterns = [
url(r'^merge/$', views.merge),
url(r'^merge/?$', views.merge),
url(r'^search/(?P<model_name>(person|email))/$', views.ajax_select2_search),
url(r'^(?P<personid>[0-9]+)/email.json$', ajax.person_email_json),
url(r'^(?P<email_or_name>[^/]+)$', views.profile),

View file

@ -42,8 +42,11 @@ def get_from_choices(user):
nomcom_choices = get_nomcom_choices(user)
if nomcom_choices:
addresses = list(addresses) + nomcom_choices
return list(zip(addresses, addresses))
choices = list(zip(addresses, addresses))
if len(choices) > 1:
choices.insert(0, ('', '(Choose an option)'))
return choices
def get_nomcom_choices(user):

View file

@ -48,7 +48,7 @@ class SecrAnnouncementTestCase(TestCase):
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
q = PyQuery(r.content)
self.assertEqual(len(q('#id_frm option')),3)
self.assertEqual(len(q('#id_frm option')),4)
# IAB Chair
self.client.login(username="iab-chair", password="iab-chair+password")

View file

@ -1188,3 +1188,20 @@ blockquote {
padding-left: 1rem;
border-left: solid 1px var(--bs-body-color);
}
.overflow-shadows {
transition: box-shadow 0.5s;
}
.overflow-shadows--both {
box-shadow: inset 0px 21px 18px -20px var(--bs-body-color),
inset 0px -21px 18px -20px var(--bs-body-color);
}
.overflow-shadows--top-only {
box-shadow: inset 0px 21px 18px -20px var(--bs-body-color);
}
.overflow-shadows--bottom-only {
box-shadow: inset 0px -21px 18px -20px var(--bs-body-color);
}

View file

@ -91,6 +91,27 @@ $(document)
// });
});
function overflowShadows(el) {
function handleScroll(){
const canScrollUp = el.scrollTop > 0
const canScrollDown = el.offsetHeight + el.scrollTop < el.scrollHeight
el.classList.toggle("overflow-shadows--both", canScrollUp && canScrollDown)
el.classList.toggle("overflow-shadows--top-only", canScrollUp && !canScrollDown)
el.classList.toggle("overflow-shadows--bottom-only", !canScrollUp && canScrollDown)
}
el.addEventListener("scroll", handleScroll, {passive: true})
handleScroll()
const observer = new IntersectionObserver(handleScroll)
observer.observe(el) // el won't have scrollTop etc when hidden, so we need to recalculate when it's revealed
return () => {
el.removeEventListener("scroll", handleScroll)
observer.unobserve(el)
}
}
$(document)
.ready(function () {
// load data for the menu
@ -108,7 +129,7 @@ $(document)
}
attachTo.find(".dropdown-menu")
.remove();
var menu = ['<ul class="dropdown-menu ms-n1 mt-n1">'];
var menu = ['<ul class="dropdown-menu ms-n1 mt-n1 overflow-shadows">'];
var groups = data[parentId];
var gtype = "";
for (var i = 0; i < groups.length; ++i) {
@ -127,6 +148,8 @@ $(document)
attachTo.closest(".dropdown-menu");
}
attachTo.append(menu.join(""));
attachTo.find(".overflow-shadows").each(function(){ overflowShadows(this)})
}
}
});

View file

@ -5,8 +5,10 @@ import {
function text_sort(a, b, options) {
function prep(e, options) {
return $($.parseHTML(e.values()[options.valueName]))
.text()
const el = $($.parseHTML(e.values()[options.valueName]));
const cell_el = e.elm.querySelector(`.${options.valueName}`)
const sort_by_number = cell_el?.getAttribute('data-sort-number')
return sort_by_number ?? el.text()
.trim()
.replaceAll(/\s+/g, ' ');
}

View file

@ -66,7 +66,7 @@
</p>
<p>
If you enter the review below, the review will be sent
to {% for addr in to %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% if review_cc %}, with a CC to {% for addr in cc %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% endif %}.
to {% for addr in review_to %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% if review_cc %}, with a CC to {% for addr in review_cc %}{{ addr|linkify }}{% if not forloop.last %}, {% endif %}{% endfor %}{% endif %}.
</p>
{% elif assignment %}
<p>

View file

@ -40,8 +40,8 @@
<tr>
<th scope="col" data-sort="group">Group</th>
<th scope="col" data-sort="name">Name</th>
<th scope="col" data-sort="date">Start</th>
<th scope="col" data-sort="date">Concluded</th>
<th scope="col" data-sort="date-start">Start</th>
<th scope="col" data-sort="date-concluded">Concluded</th>
</tr>
</thead>
<tbody>
@ -51,8 +51,8 @@
<a href="{{ g.about_url }}">{{ g.acronym }}</a>
</td>
<td>{{ g.name }}</td>
<td>{{ g.start_date|date:"Y-m" }}</td>
<td>{{ g.conclude_date|date:"Y-m" }}</td>
<td data-sort-number="{{ g.start_date|date:"U" }}">{{ g.start_date|date:"Y-m" }}</td>
<td data-sort-number="{{ g.conclude_date|date:"U" }}">{{ g.conclude_date|date:"Y-m" }}</td>
</tr>
{% endfor %}
</tbody>

View file

@ -49,7 +49,7 @@
<td>{{ milestone.desc|urlize_ietf_docs }}</td>
<td>
{% for d in milestone.docs.all %}
<a href="{% url "ietf.doc.views_doc.document_main" name=d.name %}">{{ d.name }}</a>
<a href="{% url "ietf.doc.views_doc.document_main" name=d.name %}">{% if d.became_rfc %}{{ d.became_rfc }} (was {% endif %}{{ d.name }}{% if d.became_rfc %}){% endif %}</a>
<br>
{% endfor %}
</td>

View file

@ -106,7 +106,7 @@
<td>
<a class="text-nowrap" href="{{ doc.get_absolute_url }}">RFC {{ doc.rfc_number }}</a>
</td>
<td>{{ doc.pub_date|date:"b Y"|title }}</td>
<td data-sort-number="{{ doc.pub_date|date:"U" }}">{{ doc.pub_date|date:"b Y"|title }}</td>
<td>{{ doc.title|urlize_ietf_docs }}</td>
<td class="text-end">
{% with doc.referenced_by_rfcs_as_rfc_or_draft.count as refbycount %}

View file

@ -24,3 +24,11 @@ class GunicornRequestJsonFormatter(DatatrackerJsonFormatter):
log_record.setdefault("user_agent", record.args["a"])
log_record.setdefault("len_bytes", record.args["B"])
log_record.setdefault("duration_ms", record.args["M"])
log_record.setdefault("host", record.args["{host}i"])
log_record.setdefault("x_request_start", record.args["{x-request-start}i"])
log_record.setdefault("x_real_ip", record.args["{x-real-ip}i"])
log_record.setdefault("x_forwarded_for", record.args["{x-forwarded-for}i"])
log_record.setdefault("x_forwarded_proto", record.args["{x-forwarded-proto}i"])
log_record.setdefault("cf_connecting_ip", record.args["{cf-connecting-ip}i"])
log_record.setdefault("cf_connecting_ipv6", record.args["{cf-connecting-ipv6}i"])
log_record.setdefault("cf_ray", record.args["{cf-ray}i"])

View file

@ -30,7 +30,7 @@ if skip_selenium:
print(" "+skip_message)
def start_web_driver():
service = Service(log_output=f"{executable_name}.log", service_args=['--log-no-truncate'])
service = Service(executable_path=f"/usr/bin/{executable_name}", log_output=f"{executable_name}.log", service_args=['--log-no-truncate'])
options = Options()
options.add_argument("--headless")
os.environ["MOZ_REMOTE_SETTINGS_DEVTOOLS"] = "1"

View file

@ -80,6 +80,9 @@ spec:
volumeMounts:
- name: nginx-tmp
mountPath: /tmp
- name: dt-cfg
mountPath: /etc/nginx/conf.d/00logging.conf
subPath: nginx-logging.conf
- name: dt-cfg
mountPath: /etc/nginx/conf.d/auth.conf
subPath: nginx-auth.conf

View file

@ -80,6 +80,9 @@ spec:
volumeMounts:
- name: nginx-tmp
mountPath: /tmp
- name: dt-cfg
mountPath: /etc/nginx/conf.d/00logging.conf
subPath: nginx-logging.conf
- name: dt-cfg
mountPath: /etc/nginx/conf.d/datatracker.conf
subPath: nginx-datatracker.conf

View file

@ -3,6 +3,7 @@ namePrefix: dt-
configMapGenerator:
- name: files-cfgmap
files:
- nginx-logging.conf
- nginx-auth.conf
- nginx-datatracker.conf
- settings_local.py

View file

@ -2,9 +2,13 @@ server {
listen 8080 default_server;
server_name _;
# Replace default "main" formatter with the ietfjson formatter from nginx-logging.conf
access_log /var/log/nginx/access.log ietfjson;
# Note that regex location matches take priority over non-regex "prefix" matches. Use regexes so that
# our deny all rule does not squelch the other locations.
location ~ ^/health/nginx$ {
access_log off;
return 200;
}
@ -19,14 +23,14 @@ server {
# n.b. (?!...) is a negative lookahead group
location ~ ^(/(?!(api/openid/|accounts/login/|accounts/logout/|accounts/reset/|person/.*/photo|group/groupmenu.json)).*) {
deny all;
return 302 https://datatracker.ietf.org$${keepempty}request_uri;
}
location / {
add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' data: https://datatracker.ietf.org/ https://www.ietf.org/ http://ietf.org/ https://analytics.ietf.org https://static.ietf.org; frame-ancestors 'self' ietf.org *.ietf.org meetecho.com *.meetecho.com gather.town *.gather.town";
proxy_set_header Host $${keepempty}host;
proxy_set_header Connection close;
proxy_set_header X-Request-Start "t=${msec}";
proxy_set_header X-Request-Start "t=$${keepempty}msec";
proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $${keepempty}remote_addr;
proxy_pass http://localhost:8000;

View file

@ -2,7 +2,11 @@ server {
listen 8080 default_server;
server_name _;
# Replace default "main" formatter with the ietfjson formatter from nginx-logging.conf
access_log /var/log/nginx/access.log ietfjson;
location /health/nginx {
access_log off;
return 200;
}
@ -15,7 +19,7 @@ server {
add_header Content-Security-Policy "default-src 'self' 'unsafe-inline' data: https://datatracker.ietf.org/ https://www.ietf.org/ http://ietf.org/ https://analytics.ietf.org https://static.ietf.org; frame-ancestors 'self' ietf.org *.ietf.org meetecho.com *.meetecho.com";
proxy_set_header Host $${keepempty}host;
proxy_set_header Connection close;
proxy_set_header X-Request-Start "t=${msec}";
proxy_set_header X-Request-Start "t=$${keepempty}msec";
proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $${keepempty}remote_addr;
proxy_pass http://localhost:8000;

20
k8s/nginx-logging.conf Normal file
View file

@ -0,0 +1,20 @@
# Define JSON log format - must be loaded before config that references it
log_format ietfjson escape=json
'{'
'"time":"$${keepempty}time_iso8601",'
'"remote_ip":"$${keepempty}remote_addr",'
'"request":"$${keepempty}request",'
'"host":"$${keepempty}host",'
'"path":"$${keepempty}request_uri",'
'"method":"$${keepempty}request_method",'
'"status":"$${keepempty}status",'
'"len_bytes":"$${keepempty}body_bytes_sent",'
'"duration_ms":"$${keepempty}request_time",'
'"referer":"$${keepempty}http_referer",'
'"user_agent":"$${keepempty}http_user_agent",'
'"x_forwarded_for":"$${keepempty}http_x_forwarded_for",'
'"x_forwarded_proto":"$${keepempty}http_x_forwarded_proto",'
'"cf_connecting_ip":"$${keepempty}http_cf_connecting_ip",'
'"cf_connecting_ipv6":"$${keepempty}http_cf_connecting_ipv6",'
'"cf_ray":"$${keepempty}http_cf_ray"'
'}';