Store Auth48 URL as DocumentURL and display in RFC-Editor state. Migrates old data when possible. Alternative to 17563. Fixes #2722. Commit ready for merge.

- Legacy-Id: 18157
2020-07-13 15:34:01 +00:00 · 2020-07-13 15:34:01 +00:00 · fff927b085
parent 15bf49d4b3
commit fff927b085
9 changed files with 308 additions and 53 deletions
--- a/ietf/doc/migrations/0032_populate_auth48_urls.py
+++ b/ietf/doc/migrations/0032_populate_auth48_urls.py
@ -0,0 +1,67 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+from django.db.models import OuterRef, Subquery
+
+from re import match
+
+
+def forward(apps, schema_editor):
+    """Add DocumentURLs for docs in the Auth48 state
+
+    Checks the latest StateDocEvent; if it is in the auth48 state and the
+    event desc has an AUTH48 link, creates an auth48 DocumentURL for that doc.
+    """
+    Document = apps.get_model('doc', 'Document')
+    StateDocEvent = apps.get_model('doc', 'StateDocEvent')
+    DocumentURL = apps.get_model('doc', 'DocumentURL')
+
+    # Regex - extracts auth48 URL as first match group
+    pattern = r'RFC Editor state changed to <a href="(.*)"><b>AUTH48.*</b></a>.*'
+
+    # To avoid 100k queries, set up a subquery to find the latest StateDocEvent for each doc...
+    latest_events = StateDocEvent.objects.filter(doc=OuterRef('pk')).order_by('-time', '-id')
+    # ... then annotate the doc list with that and select only those in the auth48 state...
+    auth48_docs = Document.objects.annotate(
+        current_state_slug=Subquery(latest_events.values('state__slug')[:1])
+    ).filter(current_state_slug='auth48')
+    # ... and add an auth48 DocumentURL if one is found.
+    for doc in auth48_docs:
+        # Retrieve the full StateDocEvent. Results in a query per doc, but
+        # only for the few few in the auth48 state.
+        sde = StateDocEvent.objects.filter(doc=doc).order_by('-time', '-id').first()
+        urlmatch = match(pattern, sde.desc)  # Auth48 URL is usually in the event desc
+        if urlmatch is not None:
+            DocumentURL.objects.create(doc=doc, tag_id='auth48', url=urlmatch[1])
+
+    # Validate the migration using a different approach to find auth48 docs.
+    # This is slower than above, but still avoids querying for every Document.
+    auth48_events = StateDocEvent.objects.filter(state__slug='auth48')
+    for a48_event in auth48_events:
+        doc = a48_event.doc
+        latest_sde = StateDocEvent.objects.filter(doc=doc).order_by('-time', '-id').first()
+        if latest_sde.state and latest_sde.state.slug == 'auth48' and match(pattern, latest_sde.desc) is not None:
+            # Currently in the auth48 state with a URL
+            assert doc.documenturl_set.filter(tag_id='auth48').count() == 1
+        else:
+            # Either no longer in auth48 state or had no URL
+            assert doc.documenturl_set.filter(tag_id='auth48').count() == 0
+
+
+def reverse(apps, schema_editor):
+    """Remove any auth48 DocumentURLs - these did not exist before"""
+    DocumentURL = apps.get_model('doc', 'DocumentURL')
+    DocumentURL.objects.filter(tag_id='auth48').delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('doc', '0031_set_state_for_charters_of_replaced_groups'),
+        ('name', '0012_add_auth48_docurltagname'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]
--- a/ietf/doc/models.py
+++ b/ietf/doc/models.py
@ -916,6 +916,7 @@ class DocHistory(DocumentInfo):
    def related_ipr(self):
        return self.doc.related_ipr()

+    @property
    def documenturl_set(self):
        return self.doc.documenturl_set

--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@ -780,6 +780,44 @@ Man                    Expires September 22, 2015               [Page 3]
            self.assertEqual(r.status_code, 200)
            self.assertContains(r, "%s-00"%docname)

+    def test_rfcqueue_auth48_views(self):
+        """Test view handling of RFC editor queue auth48 state"""
+        def _change_state(doc, state):
+            event = StateDocEventFactory(doc=doc, state=state)
+            doc.set_state(event.state)
+            doc.save_with_history([event])
+
+        draft = IndividualDraftFactory()
+
+        # Put in an rfceditor state other than auth48
+        for state in [('draft-iesg', 'rfcqueue'), ('draft-rfceditor', 'rfc-edit')]:
+            _change_state(draft, state)
+        r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
+        self.assertEqual(r.status_code, 200)
+        self.assertNotContains(r, 'Auth48 status')
+
+        # Put in auth48 state without a URL
+        _change_state(draft, ('draft-rfceditor', 'auth48'))
+        r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
+        self.assertEqual(r.status_code, 200)
+        self.assertNotContains(r, 'Auth48 status')
+
+        # Now add a URL
+        documenturl = draft.documenturl_set.create(tag_id='auth48', 
+                                                   url='http://rfceditor.example.com/auth48-url')
+        r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r, 'Auth48 status')
+        self.assertContains(r, documenturl.url)
+
+        # Put in auth48-done state and delete auth48 DocumentURL
+        draft.documenturl_set.filter(tag_id='auth48').delete()
+        _change_state(draft, ('draft-rfceditor', 'auth48-done'))
+        r = self.client.get(urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=draft.name)))
+        self.assertEqual(r.status_code, 200)
+        self.assertNotContains(r, 'Auth48 status')
+
+
 class DocTestCase(TestCase):
    def test_document_charter(self):
        CharterFactory(name='charter-ietf-mars')
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@ -420,6 +420,15 @@ def document_main(request, name, rev=None):
        exp_comment = doc.latest_event(IanaExpertDocEvent,type="comment")
        iana_experts_comment = exp_comment and exp_comment.desc

+        # See if we should show an Auth48 URL
+        auth48_url = None  # stays None unless we are in the auth48 state
+        if doc.get_state_slug('draft-rfceditor') == 'auth48':
+            document_url = doc.documenturl_set.filter(tag_id='auth48').first()
+            auth48_url = document_url.url if document_url else ''
+
+        # Do not show the Auth48 URL in the "Additional URLs" section
+        additional_urls = doc.documenturl_set.exclude(tag_id='auth48')
+
        return render(request, "doc/document_draft.html",
                                  dict(doc=doc,
                                       group=group,
@ -469,6 +478,7 @@ def document_main(request, name, rev=None):
                                       has_errata=doc.tags.filter(slug="errata"),
                                       published=published,
                                       file_urls=file_urls,
+                                       additional_urls=additional_urls,
                                       stream_state_type_slug=stream_state_type_slug,
                                       stream_state=stream_state,
                                       stream_tags=stream_tags,
@ -477,6 +487,7 @@ def document_main(request, name, rev=None):
                                       iesg_state=iesg_state,
                                       iesg_state_summary=iesg_state_summary,
                                       rfc_editor_state=doc.get_state("draft-rfceditor"),
+                                       rfc_editor_auth48_url=auth48_url,
                                       iana_review_state=doc.get_state("draft-iana-review"),
                                       iana_action_state=doc.get_state("draft-iana-action"),
                                       iana_experts_state=doc.get_state("draft-iana-experts"),
--- a/ietf/name/fixtures/names.json
+++ b/ietf/name/fixtures/names.json
@ -9417,6 +9417,16 @@
    "model": "name.doctypename",
    "pk": "statchg"
  },
+  {
+    "fields": {
+      "desc": "",
+      "name": "RFC Editor Auth48 status",
+      "order": 0,
+      "used": true
+    },
+    "model": "name.docurltagname",
+    "pk": "auth48"
+  },
  {
    "fields": {
      "desc": "",
@ -14481,7 +14491,7 @@
    "fields": {
      "command": "xym",
      "switch": "--version",
-      "time": "2020-05-10T00:12:51.809",
+      "time": "2020-05-29T00:13:35.959",
      "used": true,
      "version": "xym 0.4.8"
    },
@ -14492,7 +14502,7 @@
    "fields": {
      "command": "pyang",
      "switch": "--version",
-      "time": "2020-05-10T00:12:53.489",
+      "time": "2020-05-29T00:13:38.724",
      "used": true,
      "version": "pyang 2.2.1"
    },
@ -14503,7 +14513,7 @@
    "fields": {
      "command": "yanglint",
      "switch": "--version",
-      "time": "2020-05-10T00:12:53.919",
+      "time": "2020-05-29T00:13:39.026",
      "used": true,
      "version": "yanglint SO 1.6.7"
    },
@ -14514,7 +14524,7 @@
    "fields": {
      "command": "xml2rfc",
      "switch": "--version",
-      "time": "2020-05-10T00:12:56.462",
+      "time": "2020-05-29T00:13:40.790",
      "used": true,
      "version": "xml2rfc 2.44.0"
    },
--- a/ietf/name/migrations/0012_add_auth48_docurltagname.py
+++ b/ietf/name/migrations/0012_add_auth48_docurltagname.py
@ -0,0 +1,28 @@
+# Copyright The IETF Trust 2020, All Rights Reserved
+# -*- coding: utf-8 -*-
+# Generated by Django 2.0.13 on 2020-06-02 10:13
+
+from django.db import migrations
+
+def forward(apps, schema_editor):
+    DocUrlTagName = apps.get_model('name', 'DocUrlTagName')
+    DocUrlTagName.objects.create(
+        slug='auth48',
+        name='RFC Editor Auth48 status',
+        used=True,
+    )
+
+def reverse(apps, schema_editor):
+    DocUrlTagName = apps.get_model('name', 'DocUrlTagName')
+    auth48_tag = DocUrlTagName.objects.get(slug='auth48')
+    auth48_tag.delete()
+
+class Migration(migrations.Migration):
+    """Add DocUrlTagName entry for RFC Ed Auth48 URL"""
+    dependencies = [
+        ('name', '0011_constraintname_editor_label'),
+    ]
+
+    operations = [
+        migrations.RunPython(forward, reverse),
+    ]
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@ -199,7 +199,14 @@ def update_drafts_from_queue(drafts):
            if auth48:
                e.desc = re.sub(r"(<b>.*</b>)", "<a href=\"%s\">\\1</a>" % auth48, e.desc)
                e.save()
-
+                # Create or update the auth48 URL whether or not this is a state expected to have one.
+                d.documenturl_set.update_or_create(
+                    tag_id='auth48',  # look up existing based on this field
+                    defaults=dict(url=auth48)  # create or update with this field
+                )
+            else:
+                # Remove any existing auth48 URL when an update does not have one.
+                d.documenturl_set.filter(tag_id='auth48').delete()
            if e:
                events.append(e)

--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@ -381,17 +381,15 @@ class RFCSyncTests(TestCase):
        changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)))
        self.assertEqual(len(changed), 0)

-
-    def test_rfc_queue(self):
-        draft = WgDraftFactory(states=[('draft-iesg','ann')])
-
+    def _generate_rfc_queue_xml(self, draft, state, auth48_url=None):
+        """Generate an RFC queue xml string for a draft"""
        t = '''<rfc-editor-queue xmlns="http://www.rfc-editor.org/rfc-editor-queue">
 <section name="IETF STREAM: WORKING GROUP STANDARDS TRACK">
 <entry xml:id="%(name)s">
 <draft>%(name)s-%(rev)s.txt</draft>
 <date-received>2010-09-08</date-received>
-<state>EDIT*R*A(1G)</state>
-<auth48-url>http://www.rfc-editor.org/auth48/rfc1234</auth48-url>
+<state>%(state)s</state>
+<auth48-url>%(auth48_url)s</auth48-url>
 <normRef>
 <ref-name>%(ref)s</ref-name>
 <ref-state>IN-QUEUE</ref-state>
@ -408,26 +406,24 @@ class RFCSyncTests(TestCase):
                              rev=draft.rev,
                              title=draft.title,
                              group=draft.group.name,
-                              ref="draft-ietf-test")
+                              ref="draft-ietf-test",
+                              state=state,
+                              auth48_url=(auth48_url or ''))
+        t = t.replace('<auth48-url></auth48-url>\n', '')  # strip empty auth48-url tags
+        return t
+
+    def test_rfc_queue(self):
+        draft = WgDraftFactory(states=[('draft-iesg','ann')])
+        expected_auth48_url = "http://www.rfc-editor.org/auth48/rfc1234"
+        t = self._generate_rfc_queue_xml(draft,
+                                         state='EDIT*R*A(1G)',
+                                         auth48_url=expected_auth48_url)

        drafts, warnings = rfceditor.parse_queue(io.StringIO(t))
+        # rfceditor.parse_queue() is tested independently; just sanity check here
        self.assertEqual(len(drafts), 1)
        self.assertEqual(len(warnings), 0)

-        # Test with TI state introduced 11 Sep 2019
-        t = t.replace("<state>EDIT*R*A(1G)</state>", "<state>TI</state>")
-        __, warnings = rfceditor.parse_queue(io.StringIO(t))
-        self.assertEqual(len(warnings), 0)
-
-        draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs = drafts[0]
-
-        # currently, we only check what we actually use
-        self.assertEqual(draft_name, draft.name)
-        self.assertEqual(state, "EDIT")
-        self.assertEqual(set(tags), set(["iana", "ref"]))
-        self.assertEqual(auth48, "http://www.rfc-editor.org/auth48/rfc1234")
-
-
        mailbox_before = len(outbox)

        changed, warnings = rfceditor.update_drafts_from_queue(drafts)
@ -450,6 +446,83 @@ class RFCSyncTests(TestCase):
        self.assertEqual(len(changed), 0)
        self.assertEqual(len(warnings), 0)

+    def test_rfceditor_parse_queue(self):
+        """Test that rfceditor.parse_queue() behaves as expected.
+
+        Currently does a limited test - old comment was 
+        "currently, we only check what we actually use".
+        """
+        draft = WgDraftFactory(states=[('draft-iesg','ann')])
+        t = self._generate_rfc_queue_xml(draft,
+                                         state='EDIT*R*A(1G)',
+                                         auth48_url="http://www.rfc-editor.org/auth48/rfc1234")
+
+        drafts, warnings = rfceditor.parse_queue(io.StringIO(t))
+        self.assertEqual(len(drafts), 1)
+        self.assertEqual(len(warnings), 0)
+
+        draft_name, date_received, state, tags, missref_generation, stream, auth48, cluster, refs = drafts[0]
+        self.assertEqual(draft_name, draft.name)
+        self.assertEqual(state, "EDIT")
+        self.assertEqual(set(tags), set(["iana", "ref"]))
+        self.assertEqual(auth48, "http://www.rfc-editor.org/auth48/rfc1234")
+
+    def test_rfceditor_parse_queue_TI_state(self):
+        # Test with TI state introduced 11 Sep 2019
+        draft = WgDraftFactory(states=[('draft-iesg','ann')])
+        t = self._generate_rfc_queue_xml(draft,
+                                         state='TI',
+                                         auth48_url="http://www.rfc-editor.org/auth48/rfc1234")
+        __, warnings = rfceditor.parse_queue(io.StringIO(t))
+        self.assertEqual(len(warnings), 0)
+
+    def _generate_rfceditor_update(self, draft, state, tags=None, auth48_url=None):
+        """Helper to generate fake output from rfceditor.parse_queue()"""
+        return [[
+            draft.name, # draft_name
+            '2020-06-03',  # date_received
+            state,
+            tags or [],
+            '1',  # missref_generation
+            'ietf',  # stream
+            auth48_url or '',
+            '',  # cluster
+            ['draft-ietf-test'],  # refs
+        ]]
+
+    def test_update_draft_auth48_url(self):
+        """Test that auth48 URLs are handled correctly."""
+        draft = WgDraftFactory(states=[('draft-iesg','ann')])
+
+        # Step 1 setup: update to a state with no auth48 URL
+        changed, warnings = rfceditor.update_drafts_from_queue(
+            self._generate_rfceditor_update(draft, state='EDIT')
+        )
+        self.assertEqual(len(changed), 1)
+        self.assertEqual(len(warnings), 0)
+        auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first()
+        self.assertIsNone(auth48_docurl)
+
+        # Step 2: update to auth48 state with auth48 URL
+        changed, warnings = rfceditor.update_drafts_from_queue(
+            self._generate_rfceditor_update(draft, state='AUTH48', auth48_url='http://www.rfc-editor.org/rfc1234')
+        )
+        self.assertEqual(len(changed), 1)
+        self.assertEqual(len(warnings), 0)
+        auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first()
+        self.assertIsNotNone(auth48_docurl)
+        self.assertEqual(auth48_docurl.url, 'http://www.rfc-editor.org/rfc1234')
+
+        # Step 3: update to auth48-done state without auth48 URL
+        changed, warnings = rfceditor.update_drafts_from_queue(
+            self._generate_rfceditor_update(draft, state='AUTH48-DONE')
+        )
+        self.assertEqual(len(changed), 1)
+        self.assertEqual(len(warnings), 0)
+        auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first()
+        self.assertIsNone(auth48_docurl)
+
+
 class DiscrepanciesTests(TestCase):
    def test_discrepancies(self):

--- a/ietf/templates/doc/document_draft.html
+++ b/ietf/templates/doc/document_draft.html
@ -245,33 +245,31 @@
        </tr>
      {% endif %}

-      {% with doc.documenturl_set.all as urls %}
-        {% if urls or can_edit_stream_info or can_edit_individual %}
-        <tr>
-          <td></td>
-          <th>Additional URLs</th>
-          <td class="edit">
-            {% if can_edit_stream_info or can_edit_individual %}
-              <a class="btn btn-default btn-xs" href="{% url 'ietf.doc.views_draft.edit_document_urls' name=doc.name %}">Edit</a>
-            {% endif %}
-          </td>
-          <td>
-            {% if urls or doc.group and doc.group.list_archive %}
-            <table class="col-md-12 col-sm-12 col-xs-12">
-              <tbody>
-                {% for url in urls|dictsort:"desc" %}
-                  <tr><td> - <a href="{{ url.url }}">{% firstof url.desc url.tag.name %}</a></td></tr>
-                {% endfor %}
-                {% if doc.group and doc.group.list_archive %}
-                  <tr><td> - <a href="{{doc.group.list_archive}}?q={{doc.name}}">Mailing list discussion</a><td></tr>
-                {% endif %}
-              </tbody>
-            </table>
-            {% endif %}
-          </td>
-        </tr>
+    {% if additional_urls or can_edit_stream_info or can_edit_individual %}
+    <tr>
+      <td></td>
+      <th>Additional URLs</th>
+      <td class="edit">
+        {% if can_edit_stream_info or can_edit_individual %}
+          <a class="btn btn-default btn-xs" href="{% url 'ietf.doc.views_draft.edit_document_urls' name=doc.name %}">Edit</a>
        {% endif %}
-      {% endwith %}
+      </td>
+      <td>
+        {% if additional_urls or doc.group and doc.group.list_archive %}
+        <table class="col-md-12 col-sm-12 col-xs-12">
+          <tbody>
+            {% for url in additional_urls|dictsort:"desc" %}
+              <tr><td> - <a href="{{ url.url }}">{% firstof url.desc url.tag.name %}</a></td></tr>
+            {% endfor %}
+            {% if doc.group and doc.group.list_archive %}
+              <tr><td> - <a href="{{doc.group.list_archive}}?q={{doc.name}}">Mailing list discussion</a><td></tr>
+            {% endif %}
+          </tbody>
+        </table>
+        {% endif %}
+      </td>
+    </tr>
+    {% endif %}


    </tbody>
@ -592,7 +590,29 @@
          <th>RFC Editor</th>
          <th><a href="{% url "ietf.help.views.state" doc=doc.type.slug type="rfceditor" %}">RFC Editor state</a></th>
          <td class="edit"></td>
-          <td><a href="https://www.rfc-editor.org/queue2.html#{{ doc.name }}">{{ rfc_editor_state }}</a></td>
+          <td>
+            {{ rfc_editor_state }}
+          </td>
+        </tr>
+        <tr>
+          <th></th>
+          <th>Details</th>
+          <td class="edit"></td>
+          <td>
+            <div>
+              <a href="https://www.rfc-editor.org/queue2.html#{{ doc.name }}">
+                Publication queue entry
+              </a>
+            </div>
+            {% if rfc_editor_auth48_url %}
+              <div>
+                <a href="{{ rfc_editor_auth48_url }}">
+                  Auth48 status
+                </a>
+              </div>
+            {% endif %}
+          </td>
+          <td></td>
        </tr>
      {% endif %}