diff --git a/bin/hourly b/bin/hourly index 2e4355260..ddda01bec 100755 --- a/bin/hourly +++ b/bin/hourly @@ -85,6 +85,8 @@ mv $TMPFILE9 $DERIVED/1id-index.txt mv $TMPFILEA $DERIVED/1id-abstracts.txt mv $TMPFILEB $DERIVED/all_id2.txt +$DTDIR/ietf/manage.py generate_idnits2_rfc_status +$DTDIR/ietf/manage.py generate_idnits2_rfcs_obsoleted CHARTER=/a/www/ietf-ftp/charter wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt diff --git a/ietf/doc/management/commands/generate_idnits2_rfc_status.py b/ietf/doc/management/commands/generate_idnits2_rfc_status.py new file mode 100644 index 000000000..45be18801 --- /dev/null +++ b/ietf/doc/management/commands/generate_idnits2_rfc_status.py @@ -0,0 +1,23 @@ +# Copyright The IETF Trust 2021 All Rights Reserved + +import os + +from django.conf import settings +from django.core.management.base import BaseCommand + +from ietf.doc.utils import generate_idnits2_rfc_status +from ietf.utils.log import log + +class Command(BaseCommand): + help = ('Generate the rfc_status blob used by idnits2') + + def handle(self, *args, **options): + filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfc-status') + blob = generate_idnits2_rfc_status() + try: + bytes = blob.encode('utf-8') + with open(filename,'wb') as f: + f.write(bytes) + except Exception as e: + log('failed to write idnits2-rfc-status: '+str(e)) + raise e diff --git a/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py b/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py new file mode 100644 index 000000000..8bd122e87 --- /dev/null +++ b/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py @@ -0,0 +1,23 @@ +# Copyright The IETF Trust 2021 All Rights Reserved + +import os + +from django.conf import settings +from django.core.management.base import BaseCommand + +from ietf.doc.utils import generate_idnits2_rfcs_obsoleted +from ietf.utils.log import log + +class Command(BaseCommand): + help = ('Generate the rfcs-obsoleted file used by idnits2') + + def handle(self, *args, **options): + filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfcs-obsoleted') + blob = generate_idnits2_rfcs_obsoleted() + try: + bytes = blob.encode('utf-8') + with open(filename,'wb') as f: + f.write(bytes) + except Exception as e: + log('failed to write idnits2-rfcs-obsoleted: '+str(e)) + raise e diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 8bb521f71..1d21dac67 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -2380,20 +2380,58 @@ class MaterialsTests(TestCase): class Idnits2SupportTests(TestCase): + def setUp(self): + self.derived_dir = self.tempdir('derived') + self.saved_derived_dir = settings.DERIVED_DIR + settings.DERIVED_DIR = self.derived_dir + + def tearDown(self): + settings.DERIVED_DIR = self.saved_derived_dir + shutil.rmtree(self.derived_dir) + def test_obsoleted(self): rfc = WgRfcFactory(alias2__name='rfc1001') WgRfcFactory(alias2__name='rfc1003',relations=[('obs',rfc)]) rfc = WgRfcFactory(alias2__name='rfc1005') WgRfcFactory(alias2__name='rfc1007',relations=[('obs',rfc)]) + url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted') + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + call_command('generate_idnits2_rfcs_obsoleted') url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted') r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertEqual(r.content, b'1001 1003\n1005 1007\n') def test_rfc_status(self): + for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'): + WgRfcFactory(std_level_id=slug) url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status') r = self.client.get(url) + self.assertEqual(r.status_code,404) + call_command('generate_idnits2_rfc_status') + r = self.client.get(url) self.assertEqual(r.status_code,200) blob = unicontent(r).replace('\n','') self.assertEqual(blob[6312-1],'O') + + def test_idnits2_state(self): + rfc = WgRfcFactory() + url = urlreverse('ietf.doc.views_doc.idnits2_state', kwargs=dict(name=rfc.canonical_name())) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r,'rfcnum') + + draft = WgDraftFactory() + url = urlreverse('ietf.doc.views_doc.idnits2_state', kwargs=dict(name=draft.canonical_name())) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertNotContains(r,'rfcnum') + self.assertContains(r,'Unknown') + + draft = WgDraftFactory(intended_std_level_id='ps') + url = urlreverse('ietf.doc.views_doc.idnits2_state', kwargs=dict(name=draft.canonical_name())) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r,'Proposed') diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py index 887f1c95f..1bf8c1752 100644 --- a/ietf/doc/urls.py +++ b/ietf/doc/urls.py @@ -80,6 +80,7 @@ urlpatterns = [ url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main), url(r'^%(name)s(?:/%(rev)s)?/bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex), + url(r'^%(name)s(?:/%(rev)s)?/idnits2-state/$' % settings.URL_REGEXPS, views_doc.idnits2_state), url(r'^bibxml3/reference.I-D.%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml_ref), url(r'^bibxml3/%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml), url(r'^%(name)s/history/$' % settings.URL_REGEXPS, views_doc.document_history), diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 32a15d2cc..5ebcc8124 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -9,6 +9,7 @@ import json import math import os import re +import textwrap from collections import defaultdict from urllib.parse import quote @@ -16,9 +17,11 @@ from urllib.parse import quote from django.conf import settings from django.contrib import messages from django.forms import ValidationError +from django.template.loader import render_to_string from django.utils.html import escape from django.urls import reverse as urlreverse + import debug # pyflakes:ignore from ietf.community.models import CommunityList from ietf.community.utils import docs_tracked_by_community_list @@ -1232,3 +1235,54 @@ def update_doc_extresources(doc, new_resources, by): e.save() doc.save_with_history([e]) return True + +def generate_idnits2_rfc_status(): + + blob=['N']*10000 + + symbols={ + 'ps': 'P', + 'inf': 'I', + 'exp': 'E', + 'ds': 'D', + 'hist': 'H', + 'std': 'S', + 'bcp': 'B', + 'unkn': 'U', + } + + rfcs = Document.objects.filter(type_id='draft',states__slug='rfc',states__type='draft') + for rfc in rfcs: + offset = int(rfc.rfcnum)-1 + blob[offset] = symbols[rfc.std_level_id] + if rfc.related_that('obs'): + blob[offset] = 'O' + + # Workarounds for unusual states in the datatracker + + # Document.get(docalias='rfc6312').rfcnum == 6342 + # 6312 was published with the wrong rfc number in it + # weird workaround in the datatracker - there are two + # DocAliases starting with rfc - the canonical name code + # searches for the lexically highest alias starting with rfc + # which is getting lucky. + blob[6312 - 1] = 'O' + + # RFC200 is an old RFC List by Number + blob[200 -1] = 'O' + + # End Workarounds + + blob = re.sub('N*$','',''.join(blob)) + blob = textwrap.fill(blob, width=64) + + return blob + +def generate_idnits2_rfcs_obsoleted(): + obsdict = defaultdict(list) + for r in RelatedDocument.objects.filter(relationship_id='obs'): + obsdict[int(r.target.document.rfc_number())].append(int(r.source.rfc_number())) + for k in obsdict: + obsdict[k] = sorted(obsdict[k]) + return render_to_string('doc/idnits2-rfcs-obsoleted.txt', context={'obsitems':sorted(obsdict.items())}) + diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 53711ee81..5f443814b 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -41,9 +41,7 @@ import json import os import re import markdown -import textwrap -from collections import defaultdict from urllib.parse import quote from django.http import HttpResponse, Http404 @@ -52,14 +50,13 @@ from django.template.loader import render_to_string from django.urls import reverse as urlreverse from django.conf import settings from django import forms -from django.views.decorators.cache import cache_page import debug # pyflakes:ignore from ietf.doc.models import ( Document, DocAlias, DocHistory, DocEvent, BallotDocEvent, BallotType, ConsensusDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent, - IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS, DocumentActionHolder, DocumentAuthor, RelatedDocument) + IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS, DocumentActionHolder, DocumentAuthor) from ietf.doc.utils import (add_links_in_new_revision_events, augment_events_with_revision, can_adopt_draft, can_unadopt_draft, get_chartering_type, get_tags_for_stream_id, needed_ballot_positions, nice_consensus, prettify_std_name, update_telechat, has_same_ballot, @@ -83,6 +80,7 @@ from ietf.review.models import ReviewAssignment from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs from ietf.review.utils import no_review_from_teams_on_doc from ietf.utils import markup_txt, log +from ietf.utils.draft import Draft from ietf.utils.response import permission_denied from ietf.utils.text import maybe_split @@ -1719,57 +1717,53 @@ def all_presentations(request, name): 'past' : past+recent, }) -@cache_page ( 60 * 60, cache="slowpages" ) + def idnits2_rfcs_obsoleted(request): + filename = os.path.join(settings.DERIVED_DIR,'idnits2-rfcs-obsoleted') + try: + with open(filename,'rb') as f: + blob = f.read() + return HttpResponse(blob,content_type='text/plain;charset=utf-8') + except Exception as e: + log.log('Failed to read idnits2-rfcs-obsoleted:'+str(e)) + raise Http404 - obsdict = defaultdict(list) - for r in RelatedDocument.objects.filter(relationship_id='obs'): - obsdict[int(r.target.document.rfc_number())].append(int(r.source.rfc_number())) - for k in obsdict: - obsdict[k] = sorted(obsdict[k]) - - return render(request, 'doc/idnits2-rfcs-obsoleted.txt', context={'obsitems':sorted(obsdict.items())},content_type='text/plain;charset=utf-8') - -@cache_page ( 60 * 60, cache="slowpages" ) def idnits2_rfc_status(request): + filename = os.path.join(settings.DERIVED_DIR,'idnits2-rfc-status') + try: + with open(filename,'rb') as f: + blob = f.read() + return HttpResponse(blob,content_type='text/plain;charset=utf-8') + except Exception as e: + log.log('Failed to read idnits2-rfc-status:'+str(e)) + raise Http404 - blob=['N']*10000 - symbols={ - 'ps': 'P', - 'inf': 'I', - 'exp': 'E', - 'ds': 'D', - 'hist': 'H', - 'std': 'S', - 'bcp': 'B', - 'unkn': 'U', - } +def idnits2_state(request, name, rev=None): + doc = get_object_or_404(Document, docalias__name=name) + if doc.type_id!='draft': + raise Http404 + zero_revision = NewRevisionDocEvent.objects.filter(doc=doc,rev='00').first() + if zero_revision: + doc.created = zero_revision.time + else: + doc.created = doc.docevent_set.order_by('-time').first().time + if doc.std_level: + doc.deststatus = doc.std_level.name + elif doc.intended_std_level: + doc.deststatus = doc.intended_std_level.name + else: + text = doc.text() + if text: + parsed_draft = Draft(text=doc.text(), source=name, name_from_source=False) + doc.deststatus = parsed_draft.get_status() + else: + doc.deststatus="Unknown" + return render(request, 'doc/idnits2-state.txt', context={'doc':doc}, content_type='text/plain;charset=utf-8') - rfcs = Document.objects.filter(type_id='draft',states__slug='rfc',states__type='draft') - for rfc in rfcs: - offset = int(rfc.rfcnum)-1 - blob[offset] = symbols[rfc.std_level_id] - if rfc.related_that('obs'): - blob[offset] = 'O' - # Workarounds for unusual states in the datatracker - # Document.get(docalias='rfc6312').rfcnum == 6342 - # 6312 was published with the wrong rfc number in it - # weird workaround in the datatracker - there are two - # DocAliases starting with rfc - the canonical name code - # searches for the lexically highest alias starting with rfc - # which is getting lucky. - blob[6312 - 1] = 'O' - # RFC200 is an old RFC List by Number - blob[200 -1] = 'O' - # End Workarounds - - blob = re.sub('N*$','',''.join(blob)) - - return HttpResponse(textwrap.fill(blob, width=64),content_type='text/plain;charset=utf-8') diff --git a/ietf/settings.py b/ietf/settings.py index 4ca18e8f5..0727768aa 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -682,6 +682,7 @@ INTERNET_DRAFT_ARCHIVE_DIR = '/a/ietfdata/doc/draft/collection/draft-archive/' # write anything to this directory -- its content is maintained by ghostlinkd: INTERNET_ALL_DRAFTS_ARCHIVE_DIR = '/a/ietfdata/doc/draft/archive' MEETING_RECORDINGS_DIR = '/a/www/audio' +DERIVED_DIR = '/a/ietfdata/derived' DOCUMENT_FORMAT_WHITELIST = ["txt", "ps", "pdf", "xml", "html", ] diff --git a/ietf/templates/doc/idnits2-state.txt b/ietf/templates/doc/idnits2-state.txt new file mode 100644 index 000000000..da9417891 --- /dev/null +++ b/ietf/templates/doc/idnits2-state.txt @@ -0,0 +1,6 @@ +{% load ietf_filters %}{% filter linebreaks_lf %}{% comment %} +{% endcomment %}Doc-created: {{doc.created|date:"Y-m-d"}};datatracker +Doc-deststatus: {{doc.deststatus}};datatracker +Doc-rev: {{doc.rev}};datatracker{% if doc.rfcnum %} +Doc-rfcnum: {{doc.rfcnum}};datatracker{%endif%} +{% endfilter %} \ No newline at end of file