altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

ci: merge main to release (#7551)

Browse source 
ci: merge main to release
This commit is contained in:
Robert Sparks 2024-06-17 08:53:45 -05:00 committed by GitHub
parent 3509d75822 345c2659b0
commit 99b3f628bf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 456 additions and 288 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
bin/monthly
View file

@ -1,17 +0,0 @@
#!/bin/bash
# Weekly datatracker jobs.
#
# This script is expected to be triggered by cron from
# /etc/cron.d/datatracker
export LANG=en_US.UTF-8
export PYTHONIOENCODING=utf-8
DTDIR=/a/www/ietf-datatracker/web
cd $DTDIR/
# Set up the virtual environment
source $DTDIR/env/bin/activate
logger -p user.info -t cron "Running $DTDIR/bin/monthly"

8
dev/build/celery-start.sh
View file

@ -5,6 +5,14 @@
echo "Running Datatracker checks..."
./ietf/manage.py check
if ! ietf/manage.py migrate --skip-checks --check ; then
echo "Unapplied migrations found, waiting to start..."
sleep 5
while ! ietf/manage.py migrate --skip-checks --check ; do
sleep 5
done
fi
cleanup () {
# Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit.
if [[ -n "${celery_pid}" ]]; then

21
dev/build/datatracker-start.sh
View file

@ -4,14 +4,31 @@ echo "Running Datatracker checks..."
./ietf/manage.py check
echo "Running Datatracker migrations..."
./ietf/manage.py migrate --settings=settings_local
./ietf/manage.py migrate --skip-checks --settings=settings_local
echo "Starting Datatracker..."
# trap TERM and shut down gunicorn
cleanup () {
if [[ -n "${gunicorn_pid}" ]]; then
echo "Terminating gunicorn..."
kill -TERM "${gunicorn_pid}"
wait "${gunicorn_pid}"
fi
}
trap 'trap "" TERM; cleanup' TERM
# start gunicorn in the background so we can trap the TERM signal
gunicorn \
--workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \
--max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \
--timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \
--bind :8000 \
--log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \
ietf.wsgi:application
--capture-output \
--access-logfile -\
${DATATRACKER_GUNICORN_EXTRA_ARGS} \
ietf.wsgi:application &
gunicorn_pid=$!
wait "${gunicorn_pid}"

61
ietf/api/tests.py
View file

@ -10,6 +10,7 @@ import sys
from importlib import import_module
from pathlib import Path
from random import randrange
from django.apps import apps
from django.conf import settings
@ -1072,8 +1073,20 @@ class CustomApiTests(TestCase):
self.assertEqual(r.status_code, 400)
self.assertFalse(any(m.called for m in mocks))
# test that valid requests call handlers appropriately
# bad destination
message_b64 = base64.b64encode(b"This is a message").decode()
r = self.client.post(
url,
{"dest": "not-a-destination", "message": message_b64},
content_type="application/json",
headers={"X-Api-Key": "valid-token"},
)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers["Content-Type"], "application/json")
self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
self.assertFalse(any(m.called for m in mocks))
# test that valid requests call handlers appropriately
r = self.client.post(
url,
{"dest": "iana-review", "message": message_b64},
@ -1081,6 +1094,8 @@ class CustomApiTests(TestCase):
headers={"X-Api-Key": "valid-token"},
)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers["Content-Type"], "application/json")
self.assertEqual(json.loads(r.content), {"result": "ok"})
self.assertTrue(mock_iana_ingest.called)
self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
@ -1093,20 +1108,44 @@ class CustomApiTests(TestCase):
headers={"X-Api-Key": "valid-token"},
)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers["Content-Type"], "application/json")
self.assertEqual(json.loads(r.content), {"result": "ok"})
self.assertTrue(mock_ipr_ingest.called)
self.assertEqual(mock_ipr_ingest.call_args, mock.call(b"This is a message"))
self.assertFalse(any(m.called for m in (mocks - {mock_ipr_ingest})))
mock_ipr_ingest.reset_mock()
# bad nomcom-feedback dest
for bad_nomcom_dest in [
"nomcom-feedback", # no suffix
"nomcom-feedback-", # no year
"nomcom-feedback-squid", # not a year,
"nomcom-feedback-2024-2025", # also not a year
]:
r = self.client.post(
url,
{"dest": bad_nomcom_dest, "message": message_b64},
content_type="application/json",
headers={"X-Api-Key": "valid-token"},
)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers["Content-Type"], "application/json")
self.assertEqual(json.loads(r.content), {"result": "bad_dest"})
self.assertFalse(any(m.called for m in mocks))
# good nomcom-feedback dest
random_year = randrange(100000)
r = self.client.post(
url,
{"dest": "nomcom-feedback", "message": message_b64, "year": 2024}, # arbitrary year
{"dest": f"nomcom-feedback-{random_year}", "message": message_b64},
content_type="application/json",
headers={"X-Api-Key": "valid-token"},
)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers["Content-Type"], "application/json")
self.assertEqual(json.loads(r.content), {"result": "ok"})
self.assertTrue(mock_nomcom_ingest.called)
self.assertEqual(mock_nomcom_ingest.call_args, mock.call(b"This is a message", 2024))
self.assertEqual(mock_nomcom_ingest.call_args, mock.call(b"This is a message", random_year))
self.assertFalse(any(m.called for m in (mocks - {mock_nomcom_ingest})))
mock_nomcom_ingest.reset_mock()
@ -1118,7 +1157,9 @@ class CustomApiTests(TestCase):
content_type="application/json",
headers={"X-Api-Key": "valid-token"},
)
self.assertEqual(r.status_code, 400)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers["Content-Type"], "application/json")
self.assertEqual(json.loads(r.content), {"result": "bad_msg"})
self.assertTrue(mock_iana_ingest.called)
self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
@ -1138,7 +1179,9 @@ class CustomApiTests(TestCase):
content_type="application/json",
headers={"X-Api-Key": "valid-token"},
)
self.assertEqual(r.status_code, 400)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers["Content-Type"], "application/json")
self.assertEqual(json.loads(r.content), {"result": "bad_msg"})
self.assertTrue(mock_iana_ingest.called)
self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
@ -1167,7 +1210,9 @@ class CustomApiTests(TestCase):
content_type="application/json",
headers={"X-Api-Key": "valid-token"},
)
self.assertEqual(r.status_code, 400)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers["Content-Type"], "application/json")
self.assertEqual(json.loads(r.content), {"result": "bad_msg"})
self.assertTrue(mock_iana_ingest.called)
self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))
@ -1192,7 +1237,9 @@ class CustomApiTests(TestCase):
content_type="application/json",
headers={"X-Api-Key": "valid-token"},
)
self.assertEqual(r.status_code, 400)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.headers["Content-Type"], "application/json")
self.assertEqual(json.loads(r.content), {"result": "bad_msg"})
self.assertTrue(mock_iana_ingest.called)
self.assertEqual(mock_iana_ingest.call_args, mock.call(b"This is a message"))
self.assertFalse(any(m.called for m in (mocks - {mock_iana_ingest})))

70
ietf/api/views.py
View file

@ -8,6 +8,7 @@ import jsonschema
import pytz
import re
from contextlib import suppress
from django.conf import settings
from django.contrib.auth import authenticate
from django.contrib.auth.decorators import login_required
@ -533,32 +534,13 @@ _response_email_json_validator = jsonschema.Draft202012Validator(
"type": "object",
"properties": {
"dest": {
"enum": [
"iana-review",
"ipr-response",
"nomcom-feedback",
]
"type": "string",
},
"message": {
"type": "string", # base64-encoded mail message
},
},
"required": ["dest", "message"],
"if": {
# If dest == "nomcom-feedback"...
"properties": {
"dest": {"const": "nomcom-feedback"},
}
},
"then": {
# ... then also require year, an integer, be present
"properties": {
"year": {
"type": "integer",
},
},
"required": ["year"],
},
}
)
@ -630,49 +612,63 @@ class EmailIngestionError(Exception):
@requires_api_token
@csrf_exempt
def ingest_email(request):
"""Ingest incoming email
Returns a 4xx or 5xx status code if the HTTP request was invalid or something went
wrong while processing it. If the request was valid, returns a 200. This may or may
not indicate that the message was accepted.
"""
def _err(code, text):
def _http_err(code, text):
return HttpResponse(text, status=code, content_type="text/plain")
def _api_response(result):
return JsonResponse(data={"result": result})
if request.method != "POST":
return _err(405, "Method not allowed")
return _http_err(405, "Method not allowed")
if request.content_type != "application/json":
return _err(415, "Content-Type must be application/json")
return _http_err(415, "Content-Type must be application/json")
# Validate
try:
payload = json.loads(request.body)
_response_email_json_validator.validate(payload)
except json.decoder.JSONDecodeError as err:
return _err(400, f"JSON parse error at line {err.lineno} col {err.colno}: {err.msg}")
return _http_err(400, f"JSON parse error at line {err.lineno} col {err.colno}: {err.msg}")
except jsonschema.exceptions.ValidationError as err:
return _err(400, f"JSON schema error at {err.json_path}: {err.message}")
return _http_err(400, f"JSON schema error at {err.json_path}: {err.message}")
except Exception:
return _err(400, "Invalid request format")
return _http_err(400, "Invalid request format")
try:
message = base64.b64decode(payload["message"], validate=True)
except binascii.Error:
return _err(400, "Invalid message: bad base64 encoding")
return _http_err(400, "Invalid message: bad base64 encoding")
dest = payload["dest"]
valid_dest = False
try:
if dest == "iana-review":
valid_dest = True
iana_ingest_review_email(message)
elif dest == "ipr-response":
valid_dest = True
ipr_ingest_response_email(message)
elif dest == "nomcom-feedback":
year = payload["year"]
nomcom_ingest_feedback_email(message, year)
else:
# Should never get here - json schema validation should enforce the enum
log.unreachable(date="2024-04-04")
return _err(400, "Invalid dest") # return something reasonable if we got here unexpectedly
elif dest.startswith("nomcom-feedback-"):
maybe_year = dest[len("nomcom-feedback-"):]
if maybe_year.isdecimal():
valid_dest = True
nomcom_ingest_feedback_email(message, int(maybe_year))
except EmailIngestionError as err:
error_email = err.as_emailmessage()
if error_email is not None:
send_smtp(error_email)
return _err(400, err.msg)
with suppress(Exception): # send_smtp logs its own exceptions, ignore them here
send_smtp(error_email)
return _api_response("bad_msg")
return HttpResponse(status=200)
if not valid_dest:
return _api_response("bad_dest")
return _api_response("ok")

38
ietf/group/tasks.py
View file

@ -14,6 +14,7 @@ from ietf.utils import log
from .models import Group
from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles
from .views import extract_last_name, roles
@shared_task
@ -59,3 +60,40 @@ def generate_wg_charters_files_task():
log.log(
f"Error copying {charters_by_acronym_file} to {charter_copy_dest}: {err}"
)
@shared_task
def generate_wg_summary_files_task():
# Active WGs (all should have a parent, but filter to be sure)
groups = (
Group.objects.filter(type="wg", state="active")
.exclude(parent=None)
.order_by("acronym")
)
# Augment groups with chairs list
for group in groups:
group.chairs = sorted(roles(group, "chair"), key=extract_last_name)
# Active areas with one or more active groups in them
areas = Group.objects.filter(
type="area",
state="active",
group__in=groups,
).distinct().order_by("name")
# Augment areas with their groups
for area in areas:
area.groups = [g for g in groups if g.parent_id == area.pk]
summary_path = Path(settings.GROUP_SUMMARY_PATH)
summary_file = summary_path / "1wg-summary.txt"
summary_file.write_text(
render_to_string("group/1wg-summary.txt", {"areas": areas}),
encoding="utf8",
)
summary_by_acronym_file = summary_path / "1wg-summary-by-acronym.txt"
summary_by_acronym_file.write_text(
render_to_string(
"group/1wg-summary-by-acronym.txt",
{"areas": areas, "groups": groups},
),
encoding="utf8",
)

163
ietf/group/tests_info.py
View file

@ -8,7 +8,7 @@ import datetime
import io
import bleach
from unittest.mock import patch
from unittest.mock import call, patch
from pathlib import Path
from pyquery import PyQuery
from tempfile import NamedTemporaryFile
@ -16,6 +16,7 @@ from tempfile import NamedTemporaryFile
import debug # pyflakes:ignore
from django.conf import settings
from django.http import Http404, HttpResponse
from django.test import RequestFactory
from django.test.utils import override_settings
from django.urls import reverse as urlreverse
@ -35,7 +36,8 @@ from ietf.group.factories import (GroupFactory, RoleFactory, GroupEventFactory,
DatedGroupMilestoneFactory, DatelessGroupMilestoneFactory)
from ietf.group.forms import GroupForm
from ietf.group.models import Group, GroupEvent, GroupMilestone, GroupStateTransitions, Role
from ietf.group.tasks import generate_wg_charters_files_task
from ietf.group.tasks import generate_wg_charters_files_task, generate_wg_summary_files_task
from ietf.group.views import response_from_file
from ietf.group.utils import save_group_in_history, setup_default_community_list_for_group
from ietf.meeting.factories import SessionFactory
from ietf.name.models import DocTagName, GroupStateName, GroupTypeName, ExtResourceName, RoleName
@ -58,7 +60,11 @@ def pklist(docs):
return [ str(doc.pk) for doc in docs.all() ]
class GroupPagesTests(TestCase):
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CHARTER_PATH', 'CHARTER_COPY_PATH']
settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [
"CHARTER_PATH",
"CHARTER_COPY_PATH",
"GROUP_SUMMARY_PATH",
]
def test_active_groups(self):
area = GroupFactory.create(type_id='area')
@ -112,63 +118,90 @@ class GroupPagesTests(TestCase):
self.assertContains(r, draft.name)
self.assertContains(r, draft.title)
def test_wg_summaries(self):
group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area')).group
RoleFactory(group=group,name_id='chair',person=PersonFactory())
RoleFactory(group=group,name_id='ad',person=PersonFactory())
chair = Email.objects.filter(role__group=group, role__name="chair")[0]
url = urlreverse('ietf.group.views.wg_summary_area', kwargs=dict(group_type="wg"))
r = self.client.get(url)
def test_response_from_file(self):
# n.b., GROUP_SUMMARY_PATH is a temp dir that will be cleaned up automatically
fp = Path(settings.GROUP_SUMMARY_PATH) / "some-file.txt"
fp.write_text("This is a charters file with an é")
r = response_from_file(fp)
self.assertEqual(r.status_code, 200)
self.assertContains(r, group.parent.name)
self.assertContains(r, group.acronym)
self.assertContains(r, group.name)
self.assertContains(r, chair.address)
url = urlreverse('ietf.group.views.wg_summary_acronym', kwargs=dict(group_type="wg"))
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
self.assertContains(r, group.acronym)
self.assertContains(r, group.name)
self.assertContains(r, chair.address)
def test_wg_charters(self):
# file does not exist = 404
url = urlreverse("ietf.group.views.wg_charters", kwargs=dict(group_type="wg"))
r = self.client.get(url)
self.assertEqual(r.status_code, 404)
# should return expected file with expected encoding
wg_path = Path(settings.CHARTER_PATH) / "1wg-charters.txt"
wg_path.write_text("This is a charters file with an é")
r = self.client.get(url)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.charset, "UTF-8")
self.assertEqual(r.headers["Content-Type"], "text/plain; charset=utf-8")
self.assertEqual(r.content.decode("utf8"), "This is a charters file with an é")
# now try with a nonexistent file
fp.unlink()
with self.assertRaises(Http404):
response_from_file(fp)
# non-wg request = 404 even if the file exists
url = urlreverse("ietf.group.views.wg_charters", kwargs=dict(group_type="rg"))
r = self.client.get(url)
@patch("ietf.group.views.response_from_file")
def test_wg_summary_area(self, mock):
r = self.client.get(
urlreverse("ietf.group.views.wg_summary_area", kwargs={"group_type": "rg"})
) # not wg
self.assertEqual(r.status_code, 404)
def test_wg_charters_by_acronym(self):
url = urlreverse("ietf.group.views.wg_charters_by_acronym", kwargs=dict(group_type="wg"))
r = self.client.get(url)
self.assertEqual(r.status_code, 404)
wg_path = Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt"
wg_path.write_text("This is a charters file with an é")
r = self.client.get(url)
self.assertFalse(mock.called)
mock.return_value = HttpResponse("yay")
r = self.client.get(
urlreverse("ietf.group.views.wg_summary_area", kwargs={"group_type": "wg"})
)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.charset, "UTF-8")
self.assertEqual(r.content.decode("utf8"), "This is a charters file with an é")
self.assertEqual(r.content.decode(), "yay")
self.assertEqual(mock.call_args, call(Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary.txt"))
# non-wg request = 404 even if the file exists
url = urlreverse("ietf.group.views.wg_charters_by_acronym", kwargs=dict(group_type="rg"))
r = self.client.get(url)
@patch("ietf.group.views.response_from_file")
def test_wg_summary_acronym(self, mock):
r = self.client.get(
urlreverse(
"ietf.group.views.wg_summary_acronym", kwargs={"group_type": "rg"}
)
) # not wg
self.assertEqual(r.status_code, 404)
self.assertFalse(mock.called)
mock.return_value = HttpResponse("yay")
r = self.client.get(
urlreverse(
"ietf.group.views.wg_summary_acronym", kwargs={"group_type": "wg"}
)
)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.content.decode(), "yay")
self.assertEqual(
mock.call_args, call(Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary-by-acronym.txt")
)
@patch("ietf.group.views.response_from_file")
def test_wg_charters(self, mock):
r = self.client.get(
urlreverse("ietf.group.views.wg_charters", kwargs={"group_type": "rg"})
) # not wg
self.assertEqual(r.status_code, 404)
self.assertFalse(mock.called)
mock.return_value = HttpResponse("yay")
r = self.client.get(
urlreverse("ietf.group.views.wg_charters", kwargs={"group_type": "wg"})
)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.content.decode(), "yay")
self.assertEqual(mock.call_args, call(Path(settings.CHARTER_PATH) / "1wg-charters.txt"))
@patch("ietf.group.views.response_from_file")
def test_wg_charters_by_acronym(self, mock):
r = self.client.get(
urlreverse(
"ietf.group.views.wg_charters_by_acronym", kwargs={"group_type": "rg"}
)
) # not wg
self.assertEqual(r.status_code, 404)
self.assertFalse(mock.called)
mock.return_value = HttpResponse("yay")
r = self.client.get(
urlreverse(
"ietf.group.views.wg_charters_by_acronym", kwargs={"group_type": "wg"}
)
)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.content.decode(), "yay")
self.assertEqual(
mock.call_args, call(Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt")
)
def test_generate_wg_charters_files_task(self):
group = CharterFactory(
@ -254,6 +287,30 @@ class GroupPagesTests(TestCase):
)
self.assertEqual(not_a_dir.read_text(), "Not a dir")
def test_generate_wg_summary_files_task(self):
group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area')).group
RoleFactory(group=group,name_id='chair',person=PersonFactory())
RoleFactory(group=group,name_id='ad',person=PersonFactory())
chair = Email.objects.filter(role__group=group, role__name="chair")[0]
generate_wg_summary_files_task()
summary_by_area_contents = (
Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary.txt"
).read_text(encoding="utf8")
self.assertIn(group.parent.name, summary_by_area_contents)
self.assertIn(group.acronym, summary_by_area_contents)
self.assertIn(group.name, summary_by_area_contents)
self.assertIn(chair.address, summary_by_area_contents)
summary_by_acronym_contents = (
Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary-by-acronym.txt"
).read_text(encoding="utf8")
self.assertIn(group.acronym, summary_by_acronym_contents)
self.assertIn(group.name, summary_by_acronym_contents)
self.assertIn(chair.address, summary_by_acronym_contents)
def test_chartering_groups(self):
group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area'),states=[('charter','intrev')]).group

43
ietf/group/views.py
View file

@ -152,56 +152,39 @@ def check_group_email_aliases():
return False
def response_from_file(fpath: Path) -> HttpResponse:
"""Helper to shovel a file back in an HttpResponse"""
try:
content = fpath.read_bytes()
except IOError:
raise Http404
return HttpResponse(content, content_type="text/plain; charset=utf-8")
# --- View functions ---------------------------------------------------
def wg_summary_area(request, group_type):
if group_type != "wg":
raise Http404
areas = Group.objects.filter(type="area", state="active").order_by("name")
for area in areas:
area.groups = Group.objects.filter(parent=area, type="wg", state="active").order_by("acronym")
for group in area.groups:
group.chairs = sorted(roles(group, "chair"), key=extract_last_name)
return response_from_file(Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary.txt")
areas = [a for a in areas if a.groups]
return render(request, 'group/1wg-summary.txt',
{ 'areas': areas },
content_type='text/plain; charset=UTF-8')
def wg_summary_acronym(request, group_type):
if group_type != "wg":
raise Http404
areas = Group.objects.filter(type="area", state="active").order_by("name")
groups = Group.objects.filter(type="wg", state="active").order_by("acronym").select_related("parent")
for group in groups:
group.chairs = sorted(roles(group, "chair"), key=extract_last_name)
return render(request, 'group/1wg-summary-by-acronym.txt',
{ 'areas': areas,
'groups': groups },
content_type='text/plain; charset=UTF-8')
return response_from_file(Path(settings.GROUP_SUMMARY_PATH) / "1wg-summary-by-acronym.txt")
def wg_charters(request, group_type):
if group_type != "wg":
raise Http404
fpath = Path(settings.CHARTER_PATH) / "1wg-charters.txt"
try:
content = fpath.read_bytes()
except IOError:
raise Http404
return HttpResponse(content, content_type="text/plain; charset=UTF-8")
return response_from_file(Path(settings.CHARTER_PATH) / "1wg-charters.txt")
def wg_charters_by_acronym(request, group_type):
if group_type != "wg":
raise Http404
fpath = Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt"
try:
content = fpath.read_bytes()
except IOError:
raise Http404
return HttpResponse(content, content_type="text/plain; charset=UTF-8")
return response_from_file(Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt")
def active_groups(request, group_type=None):

1
ietf/settings.py
View file

@ -671,6 +671,7 @@ INTERNET_DRAFT_PDF_PATH = '/a/www/ietf-datatracker/pdf/'
RFC_PATH = '/a/www/ietf-ftp/rfc/'
CHARTER_PATH = '/a/ietfdata/doc/charter/'
CHARTER_COPY_PATH = '/a/www/ietf-ftp/ietf' # copy 1wg-charters files here if set
GROUP_SUMMARY_PATH = '/a/www/ietf-ftp/ietf'
BOFREQ_PATH = '/a/ietfdata/doc/bofreq/'
CONFLICT_REVIEW_PATH = '/a/ietfdata/doc/conflict-review'
STATUS_CHANGE_PATH = '/a/ietfdata/doc/status-change'

10
ietf/utils/management/commands/periodic_tasks.py
View file

@ -231,6 +231,16 @@ class Command(BaseCommand):
),
)
PeriodicTask.objects.get_or_create(
name="Generate WG summary files",
task="ietf.group.tasks.generate_wg_summary_files_task",
defaults=dict(
enabled=False,
crontab=self.crontabs["hourly"],
description="Update 1wg-summary.txt and 1wg-summary-by-acronym.txt",
),
)
PeriodicTask.objects.get_or_create(
name="Generate I-D bibxml files",
task="ietf.doc.tasks.generate_draft_bibxml_files_task",

124
k8s/beat.yaml
View file

@ -1,61 +1,63 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: beat
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: beat
strategy:
type: Recreate
template:
metadata:
labels:
app: beat
spec:
securityContext:
runAsNonRoot: true
containers:
- name: beat
image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
imagePullPolicy: Always
ports:
- containerPort: 8000
name: http
protocol: TCP
volumeMounts:
- name: dt-vol
mountPath: /a
- name: dt-tmp
mountPath: /tmp
- name: dt-cfg
mountPath: /workspace/ietf/settings_local.py
subPath: settings_local.py
env:
- name: "CONTAINER_ROLE"
value: "beat"
envFrom:
- configMapRef:
name: django-config
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsUser: 1000
runAsGroup: 1000
volumes:
# To be overriden with the actual shared volume
- name: dt-vol
- name: dt-tmp
emptyDir:
sizeLimit: "2Gi"
- name: dt-cfg
configMap:
name: files-cfgmap
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
apiVersion: apps/v1
kind: Deployment
metadata:
name: beat
labels:
deleteBeforeUpgrade: yes
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: beat
strategy:
type: Recreate
template:
metadata:
labels:
app: beat
spec:
securityContext:
runAsNonRoot: true
containers:
- name: beat
image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
imagePullPolicy: Always
ports:
- containerPort: 8000
name: http
protocol: TCP
volumeMounts:
- name: dt-vol
mountPath: /a
- name: dt-tmp
mountPath: /tmp
- name: dt-cfg
mountPath: /workspace/ietf/settings_local.py
subPath: settings_local.py
env:
- name: "CONTAINER_ROLE"
value: "beat"
envFrom:
- configMapRef:
name: django-config
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsUser: 1000
runAsGroup: 1000
volumes:
# To be overriden with the actual shared volume
- name: dt-vol
- name: dt-tmp
emptyDir:
sizeLimit: "2Gi"
- name: dt-cfg
configMap:
name: files-cfgmap
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 600

171
k8s/celery.yaml
View file

@ -1,80 +1,91 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: celery
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: celery
strategy:
type: Recreate
template:
metadata:
labels:
app: celery
spec:
securityContext:
runAsNonRoot: true
containers:
# -----------------------------------------------------
# ScoutAPM Container
# -----------------------------------------------------
- name: scoutapm
image: "scoutapp/scoutapm:version-1.4.0"
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- "sh"
- "-c"
- "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
securityContext:
readOnlyRootFilesystem: true
runAsUser: 65534 # "nobody" user by default
runAsGroup: 65534 # "nogroup" group by default
# -----------------------------------------------------
# Celery Container
# -----------------------------------------------------
- name: celery
image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
imagePullPolicy: Always
ports:
- containerPort: 8000
name: http
protocol: TCP
volumeMounts:
- name: dt-vol
mountPath: /a
- name: dt-tmp
mountPath: /tmp
- name: dt-cfg
mountPath: /workspace/ietf/settings_local.py
subPath: settings_local.py
env:
- name: "CONTAINER_ROLE"
value: "celery"
envFrom:
- configMapRef:
name: django-config
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsUser: 1000
runAsGroup: 1000
volumes:
# To be overriden with the actual shared volume
- name: dt-vol
- name: dt-tmp
emptyDir:
sizeLimit: "2Gi"
- name: dt-cfg
configMap:
name: files-cfgmap
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
apiVersion: apps/v1
kind: Deployment
metadata:
name: celery
labels:
deleteBeforeUpgrade: yes
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: celery
strategy:
type: Recreate
template:
metadata:
labels:
app: celery
spec:
securityContext:
runAsNonRoot: true
containers:
# -----------------------------------------------------
# ScoutAPM Container
# -----------------------------------------------------
- name: scoutapm
image: "scoutapp/scoutapm:version-1.4.0"
imagePullPolicy: IfNotPresent
# Replace command with one that will shut down on a TERM signal
# The ./core-agent start command line is from the scoutapm docker image
command:
- "sh"
- "-c"
- >-
trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
wait $!
livenessProbe:
exec:
command:
- "sh"
- "-c"
- "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'"
securityContext:
readOnlyRootFilesystem: true
runAsUser: 65534 # "nobody" user by default
runAsGroup: 65534 # "nogroup" group by default
# -----------------------------------------------------
# Celery Container
# -----------------------------------------------------
- name: celery
image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG"
imagePullPolicy: Always
ports:
- containerPort: 8000
name: http
protocol: TCP
volumeMounts:
- name: dt-vol
mountPath: /a
- name: dt-tmp
mountPath: /tmp
- name: dt-cfg
mountPath: /workspace/ietf/settings_local.py
subPath: settings_local.py
env:
- name: "CONTAINER_ROLE"
value: "celery"
envFrom:
- configMapRef:
name: django-config
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsUser: 1000
runAsGroup: 1000
volumes:
# To be overriden with the actual shared volume
- name: dt-vol
- name: dt-tmp
emptyDir:
sizeLimit: "2Gi"
- name: dt-cfg
configMap:
name: files-cfgmap
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 600

11
k8s/datatracker.yaml
View file

@ -24,6 +24,15 @@ spec:
- name: scoutapm
image: "scoutapp/scoutapm:version-1.4.0"
imagePullPolicy: IfNotPresent
# Replace command with one that will shut down on a TERM signal
# The ./core-agent start command line is from the scoutapm docker image
command:
- "sh"
- "-c"
- >-
trap './core-agent shutdown --tcp 0.0.0.0:6590' TERM;
./core-agent start --daemonize false --log-level debug --tcp 0.0.0.0:6590 &
wait $!
livenessProbe:
exec:
command:
@ -77,7 +86,7 @@ spec:
name: files-cfgmap
dnsPolicy: ClusterFirst
restartPolicy: Always
terminationGracePeriodSeconds: 30
terminationGracePeriodSeconds: 60
---
apiVersion: v1
kind: Service

6
k8s/settings_local.py
View file

@ -262,3 +262,9 @@ CACHES = {
_csrf_trusted_origins_str = os.environ.get("DATATRACKER_CSRF_TRUSTED_ORIGINS")
if _csrf_trusted_origins_str is not None:
CSRF_TRUSTED_ORIGINS = _multiline_to_list(_csrf_trusted_origins_str)
# Send logs to console instead of debug_console when running in kubernetes
LOGGING["loggers"]["django"]["handlers"] = ["console", "mail_admins"]
LOGGING["loggers"]["django.security"]["handlers"] = ["console"]
LOGGING["loggers"]["datatracker"]["handlers"] = ["console"]
LOGGING["loggers"]["celery"]["handlers"] = ["console"]