Provide a per-document status file in the format expected by idnits2. Rework the rfc status blob and obsoleted file to manage the long generation times. Commit ready for merge.

- Legacy-Id: 19265

bin/hourly

@@ -85,6 +85,8 @@ mv $TMPFILE9 $DERIVED/1id-index.txt
 mv $TMPFILEA $DERIVED/1id-abstracts.txt
 mv $TMPFILEB $DERIVED/all_id2.txt
 
+$DTDIR/ietf/manage.py generate_idnits2_rfc_status
+$DTDIR/ietf/manage.py generate_idnits2_rfcs_obsoleted
 
 CHARTER=/a/www/ietf-ftp/charter
 wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt

ietf/doc/management/commands/generate_idnits2_rfc_status.py

@@ -0,0 +1,23 @@
+# Copyright The IETF Trust 2021 All Rights Reserved
+
+import os
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ietf.doc.utils import generate_idnits2_rfc_status
+from ietf.utils.log import log
+
+class Command(BaseCommand):
+    help = ('Generate the rfc_status blob used by idnits2')
+
+    def handle(self, *args, **options):
+        filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfc-status')
+        blob = generate_idnits2_rfc_status()
+        try:
+            bytes = blob.encode('utf-8')
+            with open(filename,'wb') as f:
+                f.write(bytes)
+        except Exception as e:
+            log('failed to write idnits2-rfc-status: '+str(e))
+            raise e

ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py

@@ -0,0 +1,23 @@
+# Copyright The IETF Trust 2021 All Rights Reserved
+
+import os
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from ietf.doc.utils import generate_idnits2_rfcs_obsoleted
+from ietf.utils.log import log
+
+class Command(BaseCommand):
+    help = ('Generate the rfcs-obsoleted file used by idnits2')
+
+    def handle(self, *args, **options):
+        filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfcs-obsoleted')
+        blob = generate_idnits2_rfcs_obsoleted()
+        try:
+            bytes = blob.encode('utf-8')
+            with open(filename,'wb') as f:
+                f.write(bytes)
+        except Exception as e:
+            log('failed to write idnits2-rfcs-obsoleted: '+str(e))
+            raise e

ietf/doc/tests.py

@@ -2380,20 +2380,58 @@ class MaterialsTests(TestCase):
 
 class Idnits2SupportTests(TestCase):
 
+    def setUp(self):
+        self.derived_dir = self.tempdir('derived')
+        self.saved_derived_dir = settings.DERIVED_DIR
+        settings.DERIVED_DIR = self.derived_dir
+
+    def tearDown(self):
+        settings.DERIVED_DIR = self.saved_derived_dir
+        shutil.rmtree(self.derived_dir)
+
     def test_obsoleted(self):
         rfc = WgRfcFactory(alias2__name='rfc1001')
         WgRfcFactory(alias2__name='rfc1003',relations=[('obs',rfc)])
         rfc = WgRfcFactory(alias2__name='rfc1005')
         WgRfcFactory(alias2__name='rfc1007',relations=[('obs',rfc)])
 
+        url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted')
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 404)
+        call_command('generate_idnits2_rfcs_obsoleted')
         url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted')
         r = self.client.get(url)
         self.assertEqual(r.status_code, 200)
         self.assertEqual(r.content, b'1001 1003\n1005 1007\n')
 
     def test_rfc_status(self):
+        for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'):
+            WgRfcFactory(std_level_id=slug)
         url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status')
         r = self.client.get(url)
+        self.assertEqual(r.status_code,404)
+        call_command('generate_idnits2_rfc_status')
+        r = self.client.get(url)
         self.assertEqual(r.status_code,200)
         blob = unicontent(r).replace('\n','')
         self.assertEqual(blob[6312-1],'O')
+
+    def test_idnits2_state(self):
+        rfc = WgRfcFactory()
+        url = urlreverse('ietf.doc.views_doc.idnits2_state', kwargs=dict(name=rfc.canonical_name()))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r,'rfcnum')
+
+        draft = WgDraftFactory()
+        url = urlreverse('ietf.doc.views_doc.idnits2_state', kwargs=dict(name=draft.canonical_name()))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertNotContains(r,'rfcnum')
+        self.assertContains(r,'Unknown')
+
+        draft = WgDraftFactory(intended_std_level_id='ps')
+        url = urlreverse('ietf.doc.views_doc.idnits2_state', kwargs=dict(name=draft.canonical_name()))
+        r = self.client.get(url)
+        self.assertEqual(r.status_code, 200)
+        self.assertContains(r,'Proposed')

ietf/doc/urls.py

@@ -80,6 +80,7 @@ urlpatterns = [
 
     url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main),
     url(r'^%(name)s(?:/%(rev)s)?/bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex),
+    url(r'^%(name)s(?:/%(rev)s)?/idnits2-state/$' % settings.URL_REGEXPS, views_doc.idnits2_state),
     url(r'^bibxml3/reference.I-D.%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml_ref),
     url(r'^bibxml3/%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml),
     url(r'^%(name)s/history/$' % settings.URL_REGEXPS, views_doc.document_history),

ietf/doc/utils.py

@@ -9,6 +9,7 @@ import json
 import math
 import os
 import re
+import textwrap
 
 from collections import defaultdict
 from urllib.parse import quote
@@ -16,9 +17,11 @@ from urllib.parse import quote
 from django.conf import settings
 from django.contrib import messages
 from django.forms import ValidationError
+from django.template.loader import render_to_string
 from django.utils.html import escape
 from django.urls import reverse as urlreverse
 
+
 import debug                            # pyflakes:ignore
 from ietf.community.models import CommunityList
 from ietf.community.utils import docs_tracked_by_community_list
@@ -1232,3 +1235,54 @@ def update_doc_extresources(doc, new_resources, by):
     e.save()
     doc.save_with_history([e])
     return True
+
+def generate_idnits2_rfc_status():
+
+    blob=['N']*10000
+
+    symbols={
+        'ps': 'P',
+        'inf': 'I',
+        'exp': 'E',
+        'ds': 'D',
+        'hist': 'H',
+        'std': 'S',
+        'bcp': 'B',
+        'unkn': 'U',
+    }
+
+    rfcs = Document.objects.filter(type_id='draft',states__slug='rfc',states__type='draft')
+    for rfc in rfcs:
+        offset = int(rfc.rfcnum)-1
+        blob[offset] = symbols[rfc.std_level_id]
+        if rfc.related_that('obs'):
+            blob[offset] = 'O'
+
+    # Workarounds for unusual states in the datatracker
+
+    # Document.get(docalias='rfc6312').rfcnum == 6342 
+    # 6312 was published with the wrong rfc number in it
+    # weird workaround in the datatracker - there are two 
+    # DocAliases starting with rfc - the canonical name code
+    # searches for the lexically highest alias starting with rfc
+    # which is getting lucky.
+    blob[6312 - 1] = 'O'
+
+    # RFC200 is an old RFC List by Number
+    blob[200 -1] = 'O' 
+
+    # End Workarounds
+
+    blob = re.sub('N*$','',''.join(blob))
+    blob = textwrap.fill(blob, width=64)
+
+    return blob
+
+def generate_idnits2_rfcs_obsoleted():
+    obsdict = defaultdict(list)
+    for r in RelatedDocument.objects.filter(relationship_id='obs'):
+        obsdict[int(r.target.document.rfc_number())].append(int(r.source.rfc_number()))
+    for k in obsdict:
+        obsdict[k] = sorted(obsdict[k])
+    return render_to_string('doc/idnits2-rfcs-obsoleted.txt', context={'obsitems':sorted(obsdict.items())})
+

ietf/doc/views_doc.py

@@ -41,9 +41,7 @@ import json
 import os
 import re
 import markdown
-import textwrap
 
-from collections import defaultdict
 from urllib.parse import quote
 
 from django.http import HttpResponse, Http404
@@ -52,14 +50,13 @@ from django.template.loader import render_to_string
 from django.urls import reverse as urlreverse
 from django.conf import settings
 from django import forms
-from django.views.decorators.cache import cache_page
 
 
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import ( Document, DocAlias, DocHistory, DocEvent, BallotDocEvent, BallotType,
     ConsensusDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent,
-    IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS, DocumentActionHolder, DocumentAuthor, RelatedDocument)
+    IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS, DocumentActionHolder, DocumentAuthor)
 from ietf.doc.utils import (add_links_in_new_revision_events, augment_events_with_revision,
     can_adopt_draft, can_unadopt_draft, get_chartering_type, get_tags_for_stream_id,
     needed_ballot_positions, nice_consensus, prettify_std_name, update_telechat, has_same_ballot,
@@ -83,6 +80,7 @@ from ietf.review.models import ReviewAssignment
 from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
 from ietf.utils import markup_txt, log
+from ietf.utils.draft import Draft
 from ietf.utils.response import permission_denied
 from ietf.utils.text import maybe_split
 
@@ -1719,57 +1717,53 @@ def all_presentations(request, name):
         'past' : past+recent,
         })
 
-@cache_page ( 60 * 60, cache="slowpages" )
+
 def idnits2_rfcs_obsoleted(request):
+    filename = os.path.join(settings.DERIVED_DIR,'idnits2-rfcs-obsoleted')
+    try:
+        with open(filename,'rb') as f:
+            blob = f.read()
+            return HttpResponse(blob,content_type='text/plain;charset=utf-8')
+    except Exception as e:
+        log.log('Failed to read idnits2-rfcs-obsoleted:'+str(e))
+        raise Http404
 
-    obsdict = defaultdict(list)
-    for r in RelatedDocument.objects.filter(relationship_id='obs'):
-        obsdict[int(r.target.document.rfc_number())].append(int(r.source.rfc_number()))
 
-    for k in obsdict:
-        obsdict[k] = sorted(obsdict[k])
-
-    return render(request, 'doc/idnits2-rfcs-obsoleted.txt', context={'obsitems':sorted(obsdict.items())},content_type='text/plain;charset=utf-8')
-
-@cache_page ( 60 * 60, cache="slowpages" )
 def idnits2_rfc_status(request):
+    filename = os.path.join(settings.DERIVED_DIR,'idnits2-rfc-status')
+    try:
+        with open(filename,'rb') as f:
+            blob = f.read()
+            return HttpResponse(blob,content_type='text/plain;charset=utf-8')
+    except Exception as e:
+        log.log('Failed to read idnits2-rfc-status:'+str(e))
+        raise Http404
 
-    blob=['N']*10000
 
-    symbols={
+def idnits2_state(request, name, rev=None):
-        'ps': 'P',
+    doc = get_object_or_404(Document, docalias__name=name)
-        'inf': 'I',
+    if doc.type_id!='draft':
-        'exp': 'E',
+        raise Http404
-        'ds': 'D',
+    zero_revision = NewRevisionDocEvent.objects.filter(doc=doc,rev='00').first()
-        'hist': 'H',
+    if zero_revision:
-        'std': 'S',
+        doc.created = zero_revision.time
-        'bcp': 'B',
+    else:
-        'unkn': 'U',
+        doc.created = doc.docevent_set.order_by('-time').first().time
-    }
+    if doc.std_level:
+        doc.deststatus = doc.std_level.name
+    elif doc.intended_std_level:
+        doc.deststatus = doc.intended_std_level.name
+    else:
+        text = doc.text()
+        if text:
+            parsed_draft = Draft(text=doc.text(), source=name, name_from_source=False)
+            doc.deststatus = parsed_draft.get_status()
+        else:
+            doc.deststatus="Unknown"
+    return render(request, 'doc/idnits2-state.txt', context={'doc':doc}, content_type='text/plain;charset=utf-8')    
 
-    rfcs = Document.objects.filter(type_id='draft',states__slug='rfc',states__type='draft')
-    for rfc in rfcs:
-        offset = int(rfc.rfcnum)-1
-        blob[offset] = symbols[rfc.std_level_id]
-        if rfc.related_that('obs'):
-            blob[offset] = 'O'
 
-    # Workarounds for unusual states in the datatracker
 
-    # Document.get(docalias='rfc6312').rfcnum == 6342 
-    # 6312 was published with the wrong rfc number in it
-    # weird workaround in the datatracker - there are two 
-    # DocAliases starting with rfc - the canonical name code
-    # searches for the lexically highest alias starting with rfc
-    # which is getting lucky.
-    blob[6312 - 1] = 'O'
 
-    # RFC200 is an old RFC List by Number
-    blob[200 -1] = 'O' 
 
-    # End Workarounds
-
-    blob = re.sub('N*$','',''.join(blob))
-
-    return HttpResponse(textwrap.fill(blob, width=64),content_type='text/plain;charset=utf-8')
 

ietf/settings.py

@@ -682,6 +682,7 @@ INTERNET_DRAFT_ARCHIVE_DIR = '/a/ietfdata/doc/draft/collection/draft-archive/'
 # write anything to this directory -- its content is maintained by ghostlinkd:
 INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/a/ietfdata/doc/draft/archive'
 MEETING_RECORDINGS_DIR = '/a/www/audio'
+DERIVED_DIR = '/a/ietfdata/derived'
 
 DOCUMENT_FORMAT_WHITELIST = ["txt", "ps", "pdf", "xml", "html", ]
 

ietf/templates/doc/idnits2-state.txt

@@ -0,0 +1,6 @@
+{% load ietf_filters %}{% filter linebreaks_lf %}{% comment %}
+{% endcomment %}Doc-created: {{doc.created|date:"Y-m-d"}};datatracker
+Doc-deststatus: {{doc.deststatus}};datatracker
+Doc-rev: {{doc.rev}};datatracker{% if doc.rfcnum %}
+Doc-rfcnum: {{doc.rfcnum}};datatracker{%endif%}
+{% endfilter %}