ci: merge main to release

.devcontainer/devcontainer.json

@@ -7,6 +7,7 @@
     "workspaceFolder": "/workspace",
     "shutdownAction": "stopCompose",
     "postCreateCommand": "/docker-init.sh",
+    "postStartCommand": "/docker-start.sh",
     "containerEnv": {
         "EDITOR_VSCODE": "true"
     },
@@ -66,7 +67,7 @@
     },
     
     // Use 'forwardPorts' to make a list of ports inside the container available locally.
-    "forwardPorts": [3000, 5432, 5433, 8000, 8001],
+    "forwardPorts": [3000, 5432, 5433, 8000],
     
     "portsAttributes": {
         "3000": {
@@ -82,12 +83,12 @@
             "onAutoForward": "silent"
         },
         "8000": {
-            "label": "Datatracker",
+            "label": "NGINX",
             "onAutoForward": "notify"
         },
         "8001": {
-            "label": "Static",
-            "onAutoForward": "silent"
+            "label": "Datatracker",
+            "onAutoForward": "ignore"
         }
     },
     

.devcontainer/docker-compose.extend.yml

@@ -18,5 +18,8 @@ services:
     pgadmin:
         network_mode: service:db
 
+    static:
+        network_mode: service:db
+
 volumes:
     datatracker-vscode-ext:

.vscode/launch.json

@@ -12,7 +12,7 @@
             "program": "${workspaceFolder}/ietf/manage.py",
             "args": [
                 "runserver",
-                "0.0.0.0:8000",
+                "0.0.0.0:8001",
                 "--settings=settings_local"
             ],
             "django": true,
@@ -30,7 +30,7 @@
             "program": "${workspaceFolder}/ietf/manage.py",
             "args": [
                 "runserver",
-                "0.0.0.0:8000",
+                "0.0.0.0:8001",
                 "--settings=settings_local_vite"
             ],
             "django": true,
@@ -48,7 +48,7 @@
             "program": "${workspaceFolder}/ietf/manage.py",
             "args": [
                 "runserver",
-                "0.0.0.0:8000",
+                "0.0.0.0:8001",
                 "--settings=settings_local_debug"
             ],
             "django": true,

README.md

@@ -18,6 +18,7 @@
 - [Changelog](https://github.com/ietf-tools/datatracker/releases)
 - [Contributing](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md)
 - [Getting Started](#getting-started) - *[ tl;dr ](#the-tldr-to-get-going)*
+    - [Creating a Fork](#creating-a-fork)
     - [Git Cloning Tips](#git-cloning-tips)
     - [Docker Dev Environment](docker/README.md)
 - [Database & Assets](#database--assets)
@@ -47,6 +48,12 @@ This project is following the standard **Git Feature Workflow** development mode
 
 You can submit bug reports, enhancement and new feature requests in the [discussions](https://github.com/ietf-tools/datatracker/discussions) area. Accepted tickets will be converted to issues.
 
+#### Creating a Fork
+
+Click the <kbd>Fork</kbd> button in the top-right corner of the repository to create a personal copy that you can work on.
+
+> Note that some GitHub Actions might be enabled by default in your fork. You should disable them by going to **Settings** > **Actions** > **General** and selecting **Disable actions** (then Save).
+
 #### Git Cloning Tips
 
 As outlined in the [Contributing](https://github.com/ietf-tools/.github/blob/main/CONTRIBUTING.md) guide, you will first want to create a fork of the datatracker project in your personal GitHub account before cloning it.

docker-compose.yml

@@ -62,8 +62,6 @@ services:
     static:
         image: ghcr.io/ietf-tools/static:latest
         restart: unless-stopped
-        ports:
-            - 8001:80
 
     mq:
         image: rabbitmq:3-alpine

docker/app.Dockerfile

@@ -30,12 +30,21 @@ COPY docker/scripts/app-setup-python.sh /tmp/library-scripts/docker-setup-python
 RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-python.sh && chmod +x /tmp/library-scripts/docker-setup-python.sh
 RUN bash /tmp/library-scripts/docker-setup-python.sh "none" "/usr/local" "${PIPX_HOME}" "${USERNAME}"
 
+# Setup nginx
+COPY docker/scripts/app-setup-nginx.sh /tmp/library-scripts/docker-setup-nginx.sh
+RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-nginx.sh && chmod +x /tmp/library-scripts/docker-setup-nginx.sh
+RUN bash /tmp/library-scripts/docker-setup-nginx.sh
+COPY docker/configs/nginx-proxy.conf /etc/nginx/sites-available/default
+COPY docker/configs/nginx-502.html /var/www/html/502.html
+
 # Remove library scripts for final image
 RUN rm -rf /tmp/library-scripts
 
 # Copy the startup file
 COPY docker/scripts/app-init.sh /docker-init.sh
+COPY docker/scripts/app-start.sh /docker-start.sh
 RUN sed -i 's/\r$//' /docker-init.sh && chmod +x /docker-init.sh
+RUN sed -i 's/\r$//' /docker-start.sh && chmod +x /docker-start.sh
 
 # Fix user UID / GID to match host
 RUN groupmod --gid $USER_GID $USERNAME \

docker/configs/nginx-502.html

@@ -0,0 +1,59 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>Datatracker DEV</title>
+        <style>
+            body {
+                background-color: #111;
+                color: #FFF;
+                font-family: Arial, Helvetica, sans-serif;
+                text-align: center;
+                padding: 50px;
+                font-size: 16px;
+            }
+            img {
+                height: 80px;
+            }
+            h1 {
+                color: #CCC;
+            }
+            div {
+                background-color: #222;
+                border-radius: 10px;
+                padding: 10px 50px;
+                display: inline-block;
+            }
+            i {
+                font-size: 64px;
+                line-height: 16px;
+                vertical-align: text-bottom;
+                margin-right: 5px;
+            }
+            code {
+                background-color: #444;
+                padding: 3px 6px;
+                border-radius: 5px;
+                margin: 0 5px;
+            }
+            .mt {
+                margin-top: 15px;
+            }
+            a {
+                color: #31CCEC;
+            }
+        </style>
+    </head>
+    <body>
+        <img src="/_static/logos/ietf-inverted.svg" alt="IETF" />
+        <h1>Datatracker</h1>
+        <h2>Could not connect to dev server.</h2>
+        <div>
+            <p>Is the datatracker server running?</p>
+            <p class="mt">Using <strong>VS Code</strong>, open the <strong>Run and Debug</strong> tab on the left and click the <i>&#x2023;</i> symbol (Run Server) to start the server.</p>
+            <p>Otherwise, run the command <code>ietf/manage.py runserver 0.0.0.0:8001</code> from the terminal.</p>
+        </div>
+        <p class="mt">For more information, check out the <a href="https://github.com/ietf-tools/datatracker/blob/main/docker/README.md" target="_blank">Datatracker Development in Docker</a> guide.</p>
+    </body>
+</html>

docker/configs/nginx-proxy.conf

@@ -0,0 +1,24 @@
+server {
+    listen 8000 default_server;
+    listen [::]:8000 default_server;
+    
+    root /var/www/html;
+    index index.html index.htm index.nginx-debian.html;
+
+    server_name _;
+
+    location /_static/ {
+        proxy_pass http://localhost:80/;
+    }
+
+    location / {
+        error_page 502 /502.html;
+        proxy_pass http://localhost:8001/;
+        proxy_set_header Host localhost:8000;
+    }
+
+    location /502.html {
+        root /var/www/html;
+        internal;
+    }
+}

docker/configs/settings_local.py

@@ -56,4 +56,5 @@ SLIDE_STAGING_PATH = 'test/staging/'
 
 DE_GFM_BINARY = '/usr/local/bin/de-gfm'
 
-STATIC_IETF_ORG = "http://localhost:8001"
+STATIC_IETF_ORG = "/_static"
+STATIC_IETF_ORG_INTERNAL = "http://localhost:80"

docker/scripts/app-init.sh

@@ -21,6 +21,11 @@ sudo chown dev:dev "/assets"
 echo "Fix chromedriver /dev/shm permissions..."
 sudo chmod 1777 /dev/shm
 
+# Run nginx
+
+echo "Starting nginx..."
+sudo nginx
+
 # Build node packages that requrie native compilation
 echo "Compiling native node packages..."
 yarn rebuild
@@ -35,37 +40,27 @@ cp $WORKSPACEDIR/docker/configs/settings_postgresqldb.py $WORKSPACEDIR/ietf/sett
 
 if [ ! -f "$WORKSPACEDIR/ietf/settings_local.py" ]; then
     echo "Setting up a default settings_local.py ..."
-    cp $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
-
 else
-    echo "Using existing ietf/settings_local.py file"
-    if ! cmp -s $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py; then
-        echo "NOTE: Differences detected compared to docker/configs/settings_local.py!"
-        echo "We'll assume you made these deliberately."
-    fi
+    echo "Renaming existing ietf/settings_local.py to ietf/settings_local.py.bak"
+    mv -f $WORKSPACEDIR/ietf/settings_local.py $WORKSPACEDIR/ietf/settings_local.py.bak
 fi
+cp $WORKSPACEDIR/docker/configs/settings_local.py $WORKSPACEDIR/ietf/settings_local.py
 
 if [ ! -f "$WORKSPACEDIR/ietf/settings_local_debug.py" ]; then
     echo "Setting up a default settings_local_debug.py ..."
-    cp $WORKSPACEDIR/docker/configs/settings_local_debug.py $WORKSPACEDIR/ietf/settings_local_debug.py
 else
-    echo "Using existing ietf/settings_local_debug.py file"
-    if ! cmp -s $WORKSPACEDIR/docker/configs/settings_local_debug.py $WORKSPACEDIR/ietf/settings_local_debug.py; then
-        echo "NOTE: Differences detected compared to docker/configs/settings_local_debug.py!"
-        echo "We'll assume you made these deliberately."
-    fi
+    echo "Renaming existing ietf/settings_local_debug.py to ietf/settings_local_debug.py.bak"
+    mv -f $WORKSPACEDIR/ietf/settings_local_debug.py $WORKSPACEDIR/ietf/settings_local_debug.py.bak
 fi
+cp $WORKSPACEDIR/docker/configs/settings_local_debug.py $WORKSPACEDIR/ietf/settings_local_debug.py
 
 if [ ! -f "$WORKSPACEDIR/ietf/settings_local_vite.py" ]; then
     echo "Setting up a default settings_local_vite.py ..."
-    cp $WORKSPACEDIR/docker/configs/settings_local_vite.py $WORKSPACEDIR/ietf/settings_local_vite.py
 else
-    echo "Using existing ietf/settings_local_vite.py file"
-    if ! cmp -s $WORKSPACEDIR/docker/configs/settings_local_vite.py $WORKSPACEDIR/ietf/settings_local_vite.py; then
-        echo "NOTE: Differences detected compared to docker/configs/settings_local_vite.py!"
-        echo "We'll assume you made these deliberately."
-    fi
+    echo "Renaming existing ietf/settings_local_vite.py to ietf/settings_local_vite.py.bak"
+    mv -f $WORKSPACEDIR/ietf/settings_local_vite.py $WORKSPACEDIR/ietf/settings_local_vite.py.bak
 fi
+cp $WORKSPACEDIR/docker/configs/settings_local_vite.py $WORKSPACEDIR/ietf/settings_local_vite.py
 
 # Create data directories
 
@@ -99,14 +94,13 @@ echo "Running initial checks..."
 
 /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --fake-initial --settings=settings_local
 
-echo "-----------------------------------------------------------------"
-echo "Done!"
-echo "-----------------------------------------------------------------"
-
 if [ -z "$EDITOR_VSCODE" ]; then
     CODE=0
     python -m smtpd -n -c DebuggingServer localhost:2025 &
     if [ -z "$*" ]; then
+        echo "-----------------------------------------------------------------"
+        echo "Ready!"
+        echo "-----------------------------------------------------------------"
         echo
         echo "You can execute arbitrary commands now, e.g.,"
         echo

docker/scripts/app-setup-nginx.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+apt-get update -y
+apt-get install -y nginx

docker/scripts/app-start.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+sudo service rsyslog start &>/dev/null
+
+# Run nginx
+
+echo "Starting nginx..."
+pidof nginx >/dev/null && echo "nginx is already running [ OK ]" || sudo nginx
+
+# Run memcached
+
+echo "Starting memcached..."
+pidof memcached >/dev/null && echo "memcached is already running [ OK ]" || /usr/bin/memcached -u dev -d
+
+echo "-----------------------------------------------------------------"
+echo "Ready!"
+echo "-----------------------------------------------------------------"

ietf/.gitignore

@@ -1,8 +1,8 @@
 /*.pyc
 /settings_local.py
-/settings_local.py.bak
 /settings_local_debug.py
 /settings_local_vite.py
 /settings_mysqldb.py
 /settings_postgresqldb.py
+/settings_*.py.bak
 /ietfdb.sql.gz

ietf/doc/models.py

@@ -627,7 +627,7 @@ class DocumentInfo(models.Model):
             stylesheets.append(finders.find("ietf/css/document_html_txt.css"))
         else:
             text = self.htmlized()
-        stylesheets.append(f'{settings.STATIC_IETF_ORG}/fonts/noto-sans-mono/import.css')
+        stylesheets.append(f'{settings.STATIC_IETF_ORG_INTERNAL}/fonts/noto-sans-mono/import.css')
 
         cache = caches["pdfized"]
         cache_key = name.split(".")[0]

ietf/doc/views_search.py

@@ -65,6 +65,7 @@ from ietf.name.models import DocTagName, DocTypeName, StreamName
 from ietf.person.models import Person
 from ietf.person.utils import get_active_ads
 from ietf.utils.draft_search import normalize_draftname
+from ietf.utils.log import log
 from ietf.doc.utils_search import prepare_document_table
 
 
@@ -222,12 +223,14 @@ def search(request):
             return HttpResponseBadRequest("form not valid: %s" % form.errors)
 
         cache_key = get_search_cache_key(get_params)
-        results = cache.get(cache_key)
-        if not results:
+        cached_val = cache.get(cache_key)
+        if cached_val:
+            [results, meta] = cached_val
+        else:
             results = retrieve_search_results(form)
-            cache.set(cache_key, results)
-
-        results, meta = prepare_document_table(request, results, get_params)
+            results, meta = prepare_document_table(request, results, get_params)
+            cache.set(cache_key, [results, meta]) # for settings.CACHE_MIDDLEWARE_SECONDS
+            log(f"Search results computed for {get_params}")
         meta['searching'] = True
     else:
         form = SearchForm()
@@ -846,11 +849,12 @@ def index_all_drafts(request):
     return render(request, 'doc/index_all_drafts.html', { "categories": categories })
 
 def index_active_drafts(request):
+    slowcache = caches['slowpages']
     cache_key = 'doc:index_active_drafts'
-    groups = cache.get(cache_key)
+    groups = slowcache.get(cache_key)
     if not groups:
         groups = active_drafts_index_by_group()
-        cache.set(cache_key, groups, 15*60)
+        slowcache.set(cache_key, groups, 15*60)
     return render(request, "doc/index_active_drafts.html", { 'groups': groups })
 
 def ajax_select2_search_docs(request, model_name, doc_type):

ietf/settings.py

@@ -178,7 +178,10 @@ STATICFILES_FINDERS = (
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
 )
 
+# Client-side static.ietf.org URL
 STATIC_IETF_ORG = "https://static.ietf.org"
+# Server-side static.ietf.org URL (used in pdfized)
+STATIC_IETF_ORG_INTERNAL = STATIC_IETF_ORG
 
 WSGI_APPLICATION = "ietf.wsgi.application"
 
@@ -726,13 +729,13 @@ CACHE_MIDDLEWARE_KEY_PREFIX = ''
 # This setting is possibly overridden further down, after the import of settings_local
 CACHES = {
     'default': {
-        'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache',
+        'BACKEND': 'ietf.utils.cache.LenientMemcacheCache',
         'LOCATION': '127.0.0.1:11211',
         'VERSION': __version__,
         'KEY_PREFIX': 'ietf:dt',
     },
     'sessions': {
-        'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache',
+        'BACKEND': 'ietf.utils.cache.LenientMemcacheCache',
         'LOCATION': '127.0.0.1:11211',
         # No release-specific VERSION setting.
         'KEY_PREFIX': 'ietf:dt',
@@ -1239,7 +1242,7 @@ if SERVER_MODE != 'production':
     CACHES = {
         'default': {
             'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
-            #'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+            #'BACKEND': 'ietf.utils.cache.LenientMemcacheCache',
             #'LOCATION': '127.0.0.1:11211',
             #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
             'VERSION': __version__,
@@ -1292,6 +1295,6 @@ if SERVER_MODE != 'production':
     # Cannot have this set to True if we're using http: from the dev-server:
     CSRF_COOKIE_SECURE = False
     CSRF_COOKIE_SAMESITE = 'Lax'
+    CSRF_TRUSTED_ORIGINS = ['http://localhost:8000']
     SESSION_COOKIE_SECURE = False
     SESSION_COOKIE_SAMESITE = 'Lax'
-    

ietf/templates/doc/document_info.html

@@ -100,7 +100,7 @@
     {% endif %}
     <tr>
         <td></td>
-        <th scope="row">Author{{ doc.authors|pluralize }}</th>
+        <th scope="row">Author{% if doc.pk %}{{ doc.authors|pluralize }}{% endif %}</th>
         <td class="edit">
             {% if can_edit_authors %}
                 <a class="btn btn-primary btn-sm"
@@ -109,9 +109,9 @@
         </td>
         <td>
             {# Implementation that uses the current primary email for each author #}
-            {% for author in doc.authors %}
+            {% if doc.pk %}{% for author in doc.authors %}
                 {% person_link author %}{% if not forloop.last %},{% endif %}
-            {% endfor %}
+            {% endfor %}{% endif %}
             {% if document_html and not snapshot or document_html and doc.rev == latest_rev%}
                 <br>
                 <a class="btn btn-primary btn-sm mt-1" href="mailto:{{ doc.name }}@ietf.org?subject={{ doc.name}}" title="Send email to the document authors">Email authors</a>

ietf/templates/doc/index_active_drafts.html

@@ -5,7 +5,7 @@
 {% load cache %}
 {% block title %}Active Internet-Drafts{% endblock %}
 {% block content %}
-    {% cache 900 ietf_doc_index_active_drafts %}
+    {% cache 900 ietf_doc_index_active_drafts using="slowpages" %}
     {% origin %}
     <h1>Active Internet-Drafts</h1>
     <p>

ietf/templates/doc/opengraph.html

@@ -36,7 +36,7 @@
 {% else %}{# TODO: We need a card image for individual I-Ds. #}
 <meta property="article:section" content="Individual Internet-Draft">
 {% endif %}
-{% for author in doc.documentauthor_set.all %}<meta property="article:author" content="{{ author.person.name}}">
-{% endfor %}
+{% if doc.pk %}{% for author in doc.documentauthor_set.all %}<meta property="article:author" content="{{ author.person.name}}">
+{% endfor %}{% endif %}
 {% if published %}<meta property="article:published_time" content="{{ published.time|date:'Y-m-d' }}">{% endif %}
 {% if expires %}<meta property="article:expiration_time" content="{{ expires.time|date:'Y-m-d' }}">{% endif %}

ietf/templates/release/about.html

@@ -23,6 +23,7 @@
         moments.  For the nitty-gritty week-to-week code changes, please check the release
         notes or the commit log.
     </p>
+    <h2>Version 11.0.0: Django 4</h2>
     <h2>Version 10.0.0: Migration to PostgreSQL as the backend database engine</h2>
     <h2>Version 9.0.0: Timezone Aware Data</h2>
     <p>All timestamps in the database are now stored as UTC. Values reported through the API for several 

ietf/utils/cache.py

@@ -0,0 +1,19 @@
+# Copyright The IETF Trust 2023, All Rights Reserved
+# -*- coding: utf-8 -*-
+
+from django.core.cache.backends.memcached import PyMemcacheCache
+from pymemcache.exceptions import MemcacheServerError
+
+from .log import log
+
+
+class LenientMemcacheCache(PyMemcacheCache):
+    """PyMemcacheCache backend that tolerates failed inserts due to object size"""
+    def set(self, key, value, timeout=None, version=None):
+        try:
+            super().set(key, value, timeout, version)
+        except MemcacheServerError as err:
+            if "object too large for cache" in str(err):
+                log(f"Memcache failed to cache large object for {key}")
+            else:
+                raise

requirements.txt

@@ -32,7 +32,6 @@ docutils>=0.18.1    # Used only by dbtemplates for RestructuredText
 types-docutils>=0.18.1
 factory-boy>=3.2.1
 github3.py>=3.2.0
-google-i18n-address<3  # for xml2rfc
 gunicorn>=20.1.0
 hashids>=1.3.1
 html2text>=2020.1.16    # Used only to clean comment field of secr/sreq