From 1df545079026162506009ecb2dfe71baf494aa9a Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 9 Aug 2023 11:40:00 -0300
Subject: [PATCH 01/25] style: Run Black on update_docs_from_rfc_index()
---
ietf/sync/rfceditor.py | 136 ++++++++++++++++++++++++++++++-----------
1 file changed, 101 insertions(+), 35 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index dab5cb17c..8dd511459 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -342,7 +342,7 @@ def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=Non
errata = {}
for item in errata_data:
- name = item['doc-id']
+ name = item["doc-id"]
if not name in errata:
errata[name] = []
errata[name].append(item)
@@ -357,7 +357,7 @@ def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=Non
"Best Current Practice": StdLevelName.objects.get(slug="bcp"),
"Historic": StdLevelName.objects.get(slug="hist"),
"Unknown": StdLevelName.objects.get(slug="unkn"),
- }
+ }
stream_mapping = {
"IETF": StreamName.objects.get(slug="ietf"),
@@ -367,15 +367,32 @@ def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=Non
"Legacy": StreamName.objects.get(slug="legacy"),
}
- tag_has_errata = DocTagName.objects.get(slug='errata')
- tag_has_verified_errata = DocTagName.objects.get(slug='verified-errata')
+ tag_has_errata = DocTagName.objects.get(slug="errata")
+ tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata")
relationship_obsoletes = DocRelationshipName.objects.get(slug="obs")
relationship_updates = DocRelationshipName.objects.get(slug="updates")
system = Person.objects.get(name="(System)")
- for rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract in index_data:
-
+ for (
+ rfc_number,
+ title,
+ authors,
+ rfc_published_date,
+ current_status,
+ updates,
+ updated_by,
+ obsoletes,
+ obsoleted_by,
+ also,
+ draft,
+ has_errata,
+ stream,
+ wg,
+ file_formats,
+ pages,
+ abstract,
+ ) in index_data:
if skip_older_than_date and rfc_published_date < skip_older_than_date:
# speed up the process by skipping old entries
continue
@@ -402,7 +419,9 @@ def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=Non
if not doc:
changes.append("created document %s" % prettify_std_name(name))
- doc = Document.objects.create(name=name, type=DocTypeName.objects.get(slug="draft"))
+ doc = Document.objects.create(
+ name=name, type=DocTypeName.objects.get(slug="draft")
+ )
# add alias
alias, __ = DocAlias.objects.get_or_create(name=name)
@@ -435,12 +454,16 @@ def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=Non
doc.stream = stream_mapping[stream]
changes.append("changed stream to %s" % doc.stream)
- if not doc.group: # if we have no group assigned, check if RFC Editor has a suggestion
+ if (
+ not doc.group
+ ): # if we have no group assigned, check if RFC Editor has a suggestion
if wg:
doc.group = Group.objects.get(acronym=wg)
changes.append("set group to %s" % doc.group)
else:
- doc.group = Group.objects.get(type="individ") # fallback for newly created doc
+ doc.group = Group.objects.get(
+ type="individ"
+ ) # fallback for newly created doc
if not doc.latest_event(type="published_rfc"):
e = DocEvent(doc=doc, rev=doc.rev, type="published_rfc")
@@ -469,33 +492,44 @@ def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=Non
e.save()
events.append(e)
- changes.append("added RFC published event at %s" % e.time.strftime("%Y-%m-%d"))
+ changes.append(
+ "added RFC published event at %s" % e.time.strftime("%Y-%m-%d")
+ )
rfc_published = True
- for t in ("draft-iesg", "draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"):
+ for t in (
+ "draft-iesg",
+ "draft-stream-iab",
+ "draft-stream-irtf",
+ "draft-stream-ise",
+ ):
prev_state = doc.get_state(t)
if prev_state is not None:
if prev_state.slug not in ("pub", "idexists"):
- new_state = State.objects.select_related("type").get(used=True, type=t, slug="pub")
+ new_state = State.objects.select_related("type").get(
+ used=True, type=t, slug="pub"
+ )
doc.set_state(new_state)
- changes.append("changed %s to %s" % (new_state.type.label, new_state))
+ changes.append(
+ "changed %s to %s" % (new_state.type.label, new_state)
+ )
e = update_action_holders(doc, prev_state, new_state)
if e:
events.append(e)
- elif t == 'draft-iesg':
- doc.set_state(State.objects.get(type_id='draft-iesg', slug='idexists'))
+ elif t == "draft-iesg":
+ doc.set_state(State.objects.get(type_id="draft-iesg", slug="idexists"))
def parse_relation_list(l):
res = []
for x in l:
- # This lookup wasn't finding anything but maybe some STD and we know
+ # This lookup wasn't finding anything but maybe some STD and we know
# if the STD had more than one RFC the wrong thing happens
#
- #if x[:3] in ("NIC", "IEN", "STD", "RTR"):
+ # if x[:3] in ("NIC", "IEN", "STD", "RTR"):
# # try translating this to RFCs that we can handle
# # sensibly; otherwise we'll have to ignore them
# l = DocAlias.objects.filter(name__startswith="rfc", docs__docalias__name=x.lower())
- #else:
+ # else:
l = Document.objects.filter(name=x.lower())
for a in l:
@@ -504,14 +538,36 @@ def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=Non
return res
for x in parse_relation_list(obsoletes):
- if not RelatedDocument.objects.filter(source=doc, target=x, relationship=relationship_obsoletes):
- r = RelatedDocument.objects.create(source=doc, target=x, relationship=relationship_obsoletes)
- changes.append("created %s relation between %s and %s" % (r.relationship.name.lower(), prettify_std_name(r.source.name), prettify_std_name(r.target.name)))
+ if not RelatedDocument.objects.filter(
+ source=doc, target=x, relationship=relationship_obsoletes
+ ):
+ r = RelatedDocument.objects.create(
+ source=doc, target=x, relationship=relationship_obsoletes
+ )
+ changes.append(
+ "created %s relation between %s and %s"
+ % (
+ r.relationship.name.lower(),
+ prettify_std_name(r.source.name),
+ prettify_std_name(r.target.name),
+ )
+ )
for x in parse_relation_list(updates):
- if not RelatedDocument.objects.filter(source=doc, target=x, relationship=relationship_updates):
- r = RelatedDocument.objects.create(source=doc, target=x, relationship=relationship_updates)
- changes.append("created %s relation between %s and %s" % (r.relationship.name.lower(), prettify_std_name(r.source.name), prettify_std_name(r.target.name)))
+ if not RelatedDocument.objects.filter(
+ source=doc, target=x, relationship=relationship_updates
+ ):
+ r = RelatedDocument.objects.create(
+ source=doc, target=x, relationship=relationship_updates
+ )
+ changes.append(
+ "created %s relation between %s and %s"
+ % (
+ r.relationship.name.lower(),
+ prettify_std_name(r.source.name),
+ prettify_std_name(r.target.name),
+ )
+ )
if also:
for a in also:
@@ -520,14 +576,21 @@ def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=Non
DocAlias.objects.create(name=a).docs.add(doc)
changes.append("created alias %s" % prettify_std_name(a))
- doc_errata = errata.get('RFC%04d'%rfc_number, [])
- all_rejected = doc_errata and all( er['errata_status_code']=='Rejected' for er in doc_errata )
+ doc_errata = errata.get("RFC%04d" % rfc_number, [])
+ all_rejected = doc_errata and all(
+ er["errata_status_code"] == "Rejected" for er in doc_errata
+ )
if has_errata and not all_rejected:
if not doc.tags.filter(pk=tag_has_errata.pk).exists():
doc.tags.add(tag_has_errata)
changes.append("added Errata tag")
- has_verified_errata = any([ er['errata_status_code']=='Verified' for er in doc_errata ])
- if has_verified_errata and not doc.tags.filter(pk=tag_has_verified_errata.pk).exists():
+ has_verified_errata = any(
+ [er["errata_status_code"] == "Verified" for er in doc_errata]
+ )
+ if (
+ has_verified_errata
+ and not doc.tags.filter(pk=tag_has_verified_errata.pk).exists()
+ ):
doc.tags.add(tag_has_verified_errata)
changes.append("added Verified Errata tag")
else:
@@ -542,13 +605,16 @@ def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=Non
changes.append("removed Verified Errata tag")
if changes:
- events.append(DocEvent.objects.create(
- doc=doc,
- rev=doc.rev,
- by=system,
- type="sync_from_rfc_editor",
- desc="Received changes through RFC Editor sync (%s)" % ", ".join(changes),
- ))
+ events.append(
+ DocEvent.objects.create(
+ doc=doc,
+ rev=doc.rev,
+ by=system,
+ type="sync_from_rfc_editor",
+ desc="Received changes through RFC Editor sync (%s)"
+ % ", ".join(changes),
+ )
+ )
doc.save_with_history(events)
From f464f3172d441b16f61e51fc21085d1dc5a029ec Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 9 Aug 2023 11:41:18 -0300
Subject: [PATCH 02/25] chore: Add type annotation to complex return value
---
ietf/sync/rfceditor.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 8dd511459..0e811c731 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -7,6 +7,7 @@ import datetime
import re
import requests
+from typing import Iterator
from urllib.parse import urlencode
from xml.dom import pulldom, Node
@@ -332,7 +333,9 @@ def parse_index(response):
return data
-def update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=None):
+def update_docs_from_rfc_index(
+ index_data, errata_data, skip_older_than_date=None
+) -> Iterator[tuple[list[str], Document, bool]]:
"""Given parsed data from the RFC Editor index, update the documents in the database
Yields a list of change descriptions for each document, if any.
From 87bd9bbd6d4339d4ab210d9c18ac1f080a7ae273 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 10:36:49 -0300
Subject: [PATCH 03/25] feat: Use rfc doc type when updating from rfc index
---
ietf/sync/rfceditor.py | 181 +++++++++++++++++++++++------------------
1 file changed, 103 insertions(+), 78 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 0e811c731..101b33369 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -7,7 +7,7 @@ import datetime
import re
import requests
-from typing import Iterator
+from typing import Iterator, Optional
from urllib.parse import urlencode
from xml.dom import pulldom, Node
@@ -388,7 +388,7 @@ def update_docs_from_rfc_index(
obsoletes,
obsoleted_by,
also,
- draft,
+ draft_name,
has_errata,
stream,
wg,
@@ -403,66 +403,101 @@ def update_docs_from_rfc_index(
# we assume two things can happen: we get a new RFC, or an
# attribute has been updated at the RFC Editor (RFC Editor
# attributes take precedence over our local attributes)
- events = []
- changes = []
+ rfc_events = []
+ rfc_changes = []
rfc_published = False
+ draft_events = []
+ draft_changes = []
- # make sure we got the document and alias
- doc = None
- name = "rfc%s" % rfc_number
- a = DocAlias.objects.filter(name=name)
- if a:
- doc = a[0].document
- else:
- if draft:
- try:
- doc = Document.objects.get(name=draft)
- except Document.DoesNotExist:
- pass
-
- if not doc:
- changes.append("created document %s" % prettify_std_name(name))
- doc = Document.objects.create(
- name=name, type=DocTypeName.objects.get(slug="draft")
- )
-
- # add alias
- alias, __ = DocAlias.objects.get_or_create(name=name)
+ # Find the document
+ doc, created_rfc = Document.objects.get_or_create(
+ rfc_number=rfc_number,
+ type_id="rfc",
+ defaults={"name": f"rfc{rfc_number}"}
+ )
+ if created_rfc:
+ rfc_changes.append(f"created document {prettify_std_name(doc.name)}")
+ # Create DocAlias (for consistency until we drop DocAlias altogether)
+ alias, _ = DocAlias.objects.get_or_create(name=doc.name)
alias.docs.add(doc)
- changes.append("created alias %s" % prettify_std_name(name))
+ rfc_changes.append("created alias %s" % prettify_std_name(doc.name))
+
+ if draft_name:
+ try:
+ draft = Document.objects.get(name=draft_name)
+ except Document.DoesNotExist:
+ pass
+ else:
+ # Ensure the draft is in the "rfc" state and move its files to the archive
+ # if necessary.
+ if draft.get_state_slug() != "rfc":
+ draft.set_state(State.objects.get(used=True, type="draft", slug="rfc"))
+ move_draft_files_to_archive(draft, draft.rev)
+ draft_changes.append(
+ f"changed state to {draft.get_state()}"
+ )
+
+ # Ensure the draft and rfc are linked with a "became_rfc" relationship
+ r, created_relateddoc = RelatedDocument.objects.get_or_create(
+ source=draft, target=doc, relationship_id="became_rfc"
+ )
+ if created_relateddoc:
+ change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format(
+ rel_name=r.relationship.name.lower(),
+ pretty_draft_name=prettify_std_name(draft),
+ pretty_rfc_name=prettify_std_name(doc),
+ )
+ draft_changes.append(change)
+ rfc_changes.append(change)
+
+ # Ensure draft is in the correct iesg and stream states
+ for t in (
+ "draft-iesg",
+ "draft-stream-iab",
+ "draft-stream-irtf",
+ "draft-stream-ise",
+ ):
+ prev_state = draft.get_state(t)
+ if prev_state is not None:
+ if prev_state.slug not in ("pub", "idexists"):
+ new_state = State.objects.select_related("type").get(
+ used=True, type=t, slug="pub"
+ )
+ draft.set_state(new_state)
+ draft_changes.append(f"changed {new_state.type.label} to {new_state}")
+ e = update_action_holders(draft, prev_state, new_state)
+ if e:
+ draft_events.append(e)
+ elif t == "draft-iesg":
+ draft.set_state(State.objects.get(type_id="draft-iesg", slug="idexists"))
# check attributes
if title != doc.title:
doc.title = title
- changes.append("changed title to '%s'" % doc.title)
+ rfc_changes.append("changed title to '%s'" % doc.title)
if abstract and abstract != doc.abstract:
doc.abstract = abstract
- changes.append("changed abstract to '%s'" % doc.abstract)
+ rfc_changes.append("changed abstract to '%s'" % doc.abstract)
if pages and int(pages) != doc.pages:
doc.pages = int(pages)
- changes.append("changed pages to %s" % doc.pages)
+ rfc_changes.append("changed pages to %s" % doc.pages)
if std_level_mapping[current_status] != doc.std_level:
doc.std_level = std_level_mapping[current_status]
- changes.append("changed standardization level to %s" % doc.std_level)
-
- if doc.get_state_slug() != "rfc":
- doc.set_state(State.objects.get(used=True, type="draft", slug="rfc"))
- move_draft_files_to_archive(doc, doc.rev)
- changes.append("changed state to %s" % doc.get_state())
+ rfc_changes.append("changed standardization level to %s" % doc.std_level)
if doc.stream != stream_mapping[stream]:
doc.stream = stream_mapping[stream]
- changes.append("changed stream to %s" % doc.stream)
+ rfc_changes.append("changed stream to %s" % doc.stream)
if (
not doc.group
): # if we have no group assigned, check if RFC Editor has a suggestion
if wg:
doc.group = Group.objects.get(acronym=wg)
- changes.append("set group to %s" % doc.group)
+ rfc_changes.append("set group to %s" % doc.group)
else:
doc.group = Group.objects.get(
type="individ"
@@ -493,35 +528,13 @@ def update_docs_from_rfc_index(
e.by = system
e.desc = "RFC published"
e.save()
- events.append(e)
+ rfc_events.append(e)
- changes.append(
+ rfc_changes.append(
"added RFC published event at %s" % e.time.strftime("%Y-%m-%d")
)
rfc_published = True
- for t in (
- "draft-iesg",
- "draft-stream-iab",
- "draft-stream-irtf",
- "draft-stream-ise",
- ):
- prev_state = doc.get_state(t)
- if prev_state is not None:
- if prev_state.slug not in ("pub", "idexists"):
- new_state = State.objects.select_related("type").get(
- used=True, type=t, slug="pub"
- )
- doc.set_state(new_state)
- changes.append(
- "changed %s to %s" % (new_state.type.label, new_state)
- )
- e = update_action_holders(doc, prev_state, new_state)
- if e:
- events.append(e)
- elif t == "draft-iesg":
- doc.set_state(State.objects.get(type_id="draft-iesg", slug="idexists"))
-
def parse_relation_list(l):
res = []
for x in l:
@@ -547,7 +560,7 @@ def update_docs_from_rfc_index(
r = RelatedDocument.objects.create(
source=doc, target=x, relationship=relationship_obsoletes
)
- changes.append(
+ rfc_changes.append(
"created %s relation between %s and %s"
% (
r.relationship.name.lower(),
@@ -563,7 +576,7 @@ def update_docs_from_rfc_index(
r = RelatedDocument.objects.create(
source=doc, target=x, relationship=relationship_updates
)
- changes.append(
+ rfc_changes.append(
"created %s relation between %s and %s"
% (
r.relationship.name.lower(),
@@ -577,16 +590,16 @@ def update_docs_from_rfc_index(
a = a.lower()
if not DocAlias.objects.filter(name=a):
DocAlias.objects.create(name=a).docs.add(doc)
- changes.append("created alias %s" % prettify_std_name(a))
+ rfc_changes.append("created alias %s" % prettify_std_name(a))
- doc_errata = errata.get("RFC%04d" % rfc_number, [])
+ doc_errata = errata.get("RFC%04d" % rfc_number, []) # rfc10k problem here
all_rejected = doc_errata and all(
er["errata_status_code"] == "Rejected" for er in doc_errata
)
if has_errata and not all_rejected:
if not doc.tags.filter(pk=tag_has_errata.pk).exists():
doc.tags.add(tag_has_errata)
- changes.append("added Errata tag")
+ rfc_changes.append("added Errata tag")
has_verified_errata = any(
[er["errata_status_code"] == "Verified" for er in doc_errata]
)
@@ -595,34 +608,46 @@ def update_docs_from_rfc_index(
and not doc.tags.filter(pk=tag_has_verified_errata.pk).exists()
):
doc.tags.add(tag_has_verified_errata)
- changes.append("added Verified Errata tag")
+ rfc_changes.append("added Verified Errata tag")
else:
if doc.tags.filter(pk=tag_has_errata.pk):
doc.tags.remove(tag_has_errata)
if all_rejected:
- changes.append("removed Errata tag (all errata rejected)")
+ rfc_changes.append("removed Errata tag (all errata rejected)")
else:
- changes.append("removed Errata tag")
+ rfc_changes.append("removed Errata tag")
if doc.tags.filter(pk=tag_has_verified_errata.pk):
doc.tags.remove(tag_has_verified_errata)
- changes.append("removed Verified Errata tag")
+ rfc_changes.append("removed Verified Errata tag")
- if changes:
- events.append(
+ # Update the draft first
+ if draft_changes:
+ draft_events.append(
+ DocEvent.objects.create(
+ doc=draft,
+ rev=doc.rev,
+ by=system,
+ type="sync_from_rfc_editor",
+ desc="Received changes through RFC Editor sync (%s)"
+ % ", ".join(draft_changes),
+ )
+ )
+ draft.save_with_history(draft_events)
+ yield draft_changes, draft, False # yield changes to the draft
+
+ if rfc_changes:
+ rfc_events.append(
DocEvent.objects.create(
doc=doc,
rev=doc.rev,
by=system,
type="sync_from_rfc_editor",
desc="Received changes through RFC Editor sync (%s)"
- % ", ".join(changes),
+ % ", ".join(rfc_changes),
)
)
-
- doc.save_with_history(events)
-
- if changes:
- yield changes, doc, rfc_published
+ doc.save_with_history(rfc_events)
+ yield rfc_changes, doc, rfc_published # yield changes to the RFC
def post_approved_draft(url, name):
From 4f9a2316ef388c0a6c729dd0202275b956d3f430 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 14:10:49 -0300
Subject: [PATCH 04/25] style: Run Black and use f-strings / str.format()
---
ietf/sync/rfceditor.py | 62 ++++++++++++++++++++----------------------
1 file changed, 30 insertions(+), 32 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 101b33369..5c76100e7 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -411,16 +411,14 @@ def update_docs_from_rfc_index(
# Find the document
doc, created_rfc = Document.objects.get_or_create(
- rfc_number=rfc_number,
- type_id="rfc",
- defaults={"name": f"rfc{rfc_number}"}
+ rfc_number=rfc_number, type_id="rfc", defaults={"name": f"rfc{rfc_number}"}
)
if created_rfc:
rfc_changes.append(f"created document {prettify_std_name(doc.name)}")
# Create DocAlias (for consistency until we drop DocAlias altogether)
alias, _ = DocAlias.objects.get_or_create(name=doc.name)
alias.docs.add(doc)
- rfc_changes.append("created alias %s" % prettify_std_name(doc.name))
+ rfc_changes.append(f"created alias {prettify_std_name(doc.name)}")
if draft_name:
try:
@@ -431,11 +429,11 @@ def update_docs_from_rfc_index(
# Ensure the draft is in the "rfc" state and move its files to the archive
# if necessary.
if draft.get_state_slug() != "rfc":
- draft.set_state(State.objects.get(used=True, type="draft", slug="rfc"))
- move_draft_files_to_archive(draft, draft.rev)
- draft_changes.append(
- f"changed state to {draft.get_state()}"
+ draft.set_state(
+ State.objects.get(used=True, type="draft", slug="rfc")
)
+ move_draft_files_to_archive(draft, draft.rev)
+ draft_changes.append(f"changed state to {draft.get_state()}")
# Ensure the draft and rfc are linked with a "became_rfc" relationship
r, created_relateddoc = RelatedDocument.objects.get_or_create(
@@ -464,40 +462,44 @@ def update_docs_from_rfc_index(
used=True, type=t, slug="pub"
)
draft.set_state(new_state)
- draft_changes.append(f"changed {new_state.type.label} to {new_state}")
+ draft_changes.append(
+ f"changed {new_state.type.label} to {new_state}"
+ )
e = update_action_holders(draft, prev_state, new_state)
if e:
draft_events.append(e)
elif t == "draft-iesg":
- draft.set_state(State.objects.get(type_id="draft-iesg", slug="idexists"))
+ draft.set_state(
+ State.objects.get(type_id="draft-iesg", slug="idexists")
+ )
# check attributes
if title != doc.title:
doc.title = title
- rfc_changes.append("changed title to '%s'" % doc.title)
+ rfc_changes.append(f"changed title to '{doc.title}'")
if abstract and abstract != doc.abstract:
doc.abstract = abstract
- rfc_changes.append("changed abstract to '%s'" % doc.abstract)
+ rfc_changes.append(f"changed abstract to '{doc.abstract}'")
if pages and int(pages) != doc.pages:
doc.pages = int(pages)
- rfc_changes.append("changed pages to %s" % doc.pages)
+ rfc_changes.append(f"changed pages to {doc.pages}")
if std_level_mapping[current_status] != doc.std_level:
doc.std_level = std_level_mapping[current_status]
- rfc_changes.append("changed standardization level to %s" % doc.std_level)
+ rfc_changes.append(f"changed standardization level to {doc.std_level}")
if doc.stream != stream_mapping[stream]:
doc.stream = stream_mapping[stream]
- rfc_changes.append("changed stream to %s" % doc.stream)
+ rfc_changes.append(f"changed stream to {doc.stream}")
if (
not doc.group
): # if we have no group assigned, check if RFC Editor has a suggestion
if wg:
doc.group = Group.objects.get(acronym=wg)
- rfc_changes.append("set group to %s" % doc.group)
+ rfc_changes.append(f"set group to {doc.group}")
else:
doc.group = Group.objects.get(
type="individ"
@@ -531,7 +533,7 @@ def update_docs_from_rfc_index(
rfc_events.append(e)
rfc_changes.append(
- "added RFC published event at %s" % e.time.strftime("%Y-%m-%d")
+ f"added RFC published event at {e.time.strftime('%Y-%m-%d')}"
)
rfc_published = True
@@ -561,11 +563,10 @@ def update_docs_from_rfc_index(
source=doc, target=x, relationship=relationship_obsoletes
)
rfc_changes.append(
- "created %s relation between %s and %s"
- % (
- r.relationship.name.lower(),
- prettify_std_name(r.source.name),
- prettify_std_name(r.target.name),
+ "created {rel_name} relation between {src_name} and {tgt_name}".format(
+ rel_name=r.relationship.name.lower(),
+ src_name=prettify_std_name(r.source.name),
+ tgt_name=prettify_std_name(r.target.name),
)
)
@@ -577,11 +578,10 @@ def update_docs_from_rfc_index(
source=doc, target=x, relationship=relationship_updates
)
rfc_changes.append(
- "created %s relation between %s and %s"
- % (
- r.relationship.name.lower(),
- prettify_std_name(r.source.name),
- prettify_std_name(r.target.name),
+ "created {rel_name} relation between {src_name} and {tgt_name}".format(
+ rel_name=r.relationship.name.lower(),
+ src_name=prettify_std_name(r.source.name),
+ tgt_name=prettify_std_name(r.target.name),
)
)
@@ -590,7 +590,7 @@ def update_docs_from_rfc_index(
a = a.lower()
if not DocAlias.objects.filter(name=a):
DocAlias.objects.create(name=a).docs.add(doc)
- rfc_changes.append("created alias %s" % prettify_std_name(a))
+ rfc_changes.append(f"created alias {prettify_std_name(a)}")
doc_errata = errata.get("RFC%04d" % rfc_number, []) # rfc10k problem here
all_rejected = doc_errata and all(
@@ -628,8 +628,7 @@ def update_docs_from_rfc_index(
rev=doc.rev,
by=system,
type="sync_from_rfc_editor",
- desc="Received changes through RFC Editor sync (%s)"
- % ", ".join(draft_changes),
+ desc=f"Received changes through RFC Editor sync ({', '.join(draft_changes)})",
)
)
draft.save_with_history(draft_events)
@@ -642,8 +641,7 @@ def update_docs_from_rfc_index(
rev=doc.rev,
by=system,
type="sync_from_rfc_editor",
- desc="Received changes through RFC Editor sync (%s)"
- % ", ".join(rfc_changes),
+ desc=f"Received changes through RFC Editor sync ({', '.join(rfc_changes)})",
)
)
doc.save_with_history(rfc_events)
From c37e3b4a397c7746351c285907952458ed88dedc Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 14:29:02 -0300
Subject: [PATCH 05/25] refactor: Consolidate draft handling
---
ietf/sync/rfceditor.py | 31 +++++++++++++++----------------
1 file changed, 15 insertions(+), 16 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 5c76100e7..8e5f782bc 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -406,8 +406,6 @@ def update_docs_from_rfc_index(
rfc_events = []
rfc_changes = []
rfc_published = False
- draft_events = []
- draft_changes = []
# Find the document
doc, created_rfc = Document.objects.get_or_create(
@@ -426,6 +424,9 @@ def update_docs_from_rfc_index(
except Document.DoesNotExist:
pass
else:
+ draft_events = []
+ draft_changes = []
+
# Ensure the draft is in the "rfc" state and move its files to the archive
# if necessary.
if draft.get_state_slug() != "rfc":
@@ -472,6 +473,18 @@ def update_docs_from_rfc_index(
draft.set_state(
State.objects.get(type_id="draft-iesg", slug="idexists")
)
+ if draft_changes:
+ draft_events.append(
+ DocEvent.objects.create(
+ doc=draft,
+ rev=doc.rev,
+ by=system,
+ type="sync_from_rfc_editor",
+ desc=f"Received changes through RFC Editor sync ({', '.join(draft_changes)})",
+ )
+ )
+ draft.save_with_history(draft_events)
+ yield draft_changes, draft, False # yield changes to the draft
# check attributes
if title != doc.title:
@@ -620,20 +633,6 @@ def update_docs_from_rfc_index(
doc.tags.remove(tag_has_verified_errata)
rfc_changes.append("removed Verified Errata tag")
- # Update the draft first
- if draft_changes:
- draft_events.append(
- DocEvent.objects.create(
- doc=draft,
- rev=doc.rev,
- by=system,
- type="sync_from_rfc_editor",
- desc=f"Received changes through RFC Editor sync ({', '.join(draft_changes)})",
- )
- )
- draft.save_with_history(draft_events)
- yield draft_changes, draft, False # yield changes to the draft
-
if rfc_changes:
rfc_events.append(
DocEvent.objects.create(
From eaf8d459ec7d09cca88cf774a892eccda11f4a6e Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 14:58:32 -0300
Subject: [PATCH 06/25] fix: Only match draft docs to rfc index draft name
---
ietf/sync/rfceditor.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 8e5f782bc..e82d645c4 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -420,7 +420,7 @@ def update_docs_from_rfc_index(
if draft_name:
try:
- draft = Document.objects.get(name=draft_name)
+ draft = Document.objects.get(name=draft_name, type_id="draft")
except Document.DoesNotExist:
pass
else:
From 542b87209dc1aa324a3196cf56903a169d8f770c Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 14:59:55 -0300
Subject: [PATCH 07/25] chore: Eliminate python lint
---
ietf/sync/rfceditor.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index e82d645c4..5f6865c98 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -7,7 +7,7 @@ import datetime
import re
import requests
-from typing import Iterator, Optional
+from typing import Iterator
from urllib.parse import urlencode
from xml.dom import pulldom, Node
@@ -18,7 +18,7 @@ from django.utils.encoding import smart_bytes, force_str
import debug # pyflakes:ignore
from ietf.doc.models import ( Document, DocAlias, State, StateType, DocEvent, DocRelationshipName,
- DocTagName, DocTypeName, RelatedDocument )
+ DocTagName, RelatedDocument )
from ietf.doc.expire import move_draft_files_to_archive
from ietf.doc.utils import add_state_change_event, prettify_std_name, update_action_holders
from ietf.group.models import Group
@@ -342,8 +342,8 @@ def update_docs_from_rfc_index(
The skip_older_than_date is a bare date, not a datetime.
"""
-
- errata = {}
+ # Create dict mapping doc-id to list of errata records that apply to it
+ errata: dict[str, list[dict]] = {}
for item in errata_data:
name = item["doc-id"]
if not name in errata:
@@ -443,8 +443,8 @@ def update_docs_from_rfc_index(
if created_relateddoc:
change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format(
rel_name=r.relationship.name.lower(),
- pretty_draft_name=prettify_std_name(draft),
- pretty_rfc_name=prettify_std_name(doc),
+ pretty_draft_name=prettify_std_name(draft_name),
+ pretty_rfc_name=prettify_std_name(doc.name),
)
draft_changes.append(change)
rfc_changes.append(change)
From 4437796bc30eb42fe37c34a2fffaf6229c83f196 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 15:43:10 -0300
Subject: [PATCH 08/25] test: Update test_rfc_index()
---
ietf/sync/tests.py | 75 +++++++++++++++++++++++++++-------------------
1 file changed, 45 insertions(+), 30 deletions(-)
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index 4ad26e3e4..8ca6362c7 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -226,14 +226,14 @@ class RFCSyncTests(TestCase):
def test_rfc_index(self):
area = GroupFactory(type_id='area')
- doc = WgDraftFactory(
+ draft_doc = WgDraftFactory(
group__parent=area,
states=[('draft-iesg','rfcqueue'),('draft-stream-ise','rfc-edit')],
ad=Person.objects.get(user__username='ad'),
)
# it's a bit strange to have draft-stream-ise set when draft-iesg is set
# too, but for testing purposes ...
- doc.action_holders.add(doc.ad) # not normally set, but add to be sure it's cleared
+ draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared
RfcFactory(rfc_number=123)
@@ -298,10 +298,10 @@ class RFCSyncTests(TestCase):
</rfc-entry>
</rfc-index>''' % dict(year=today.strftime("%Y"),
month=today.strftime("%B"),
- name=doc.name,
- rev=doc.rev,
- area=doc.group.parent.acronym,
- group=doc.group.acronym)
+ name=draft_doc.name,
+ rev=draft_doc.rev,
+ area=draft_doc.group.parent.acronym,
+ group=draft_doc.group.acronym)
errata = [{
"errata_id":1,
@@ -321,7 +321,7 @@ class RFCSyncTests(TestCase):
data = rfceditor.parse_index(io.StringIO(t))
self.assertEqual(len(data), 1)
-
+
rfc_number, title, authors, rfc_published_date, current_status, updates, updated_by, obsoletes, obsoleted_by, also, draft, has_errata, stream, wg, file_formats, pages, abstract = data[0]
# currently, we only check what we actually use
@@ -332,44 +332,59 @@ class RFCSyncTests(TestCase):
self.assertEqual(current_status, "Proposed Standard")
self.assertEqual(updates, ["RFC123"])
self.assertEqual(set(also), set(["BCP1", "FYI1", "STD1"]))
- self.assertEqual(draft, doc.name)
- self.assertEqual(wg, doc.group.acronym)
+ self.assertEqual(draft, draft_doc.name)
+ self.assertEqual(wg, draft_doc.group.acronym)
self.assertEqual(has_errata, True)
self.assertEqual(stream, "IETF")
self.assertEqual(pages, "42")
self.assertEqual(abstract, "This is some interesting text.")
- draft_filename = "%s-%s.txt" % (doc.name, doc.rev)
+ draft_filename = "%s-%s.txt" % (draft_doc.name, draft_doc.rev)
self.write_draft_file(draft_filename, 5000)
+ event_count_before = draft_doc.docevent_set.count()
+ draft_title_before = draft_doc.title
+ draft_abstract_before = draft_doc.abstract
+ draft_pages_before = draft_doc.pages
changes = []
for cs, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)):
changes.append(cs)
- doc = Document.objects.get(name=doc.name)
+ draft_doc = Document.objects.get(name=draft_doc.name)
- events = doc.docevent_set.all()
- self.assertEqual(events[0].type, "sync_from_rfc_editor")
- self.assertEqual(events[1].type, "changed_action_holders")
- self.assertEqual(events[2].type, "published_rfc")
- self.assertEqual(events[2].time.astimezone(RPC_TZINFO).date(), today)
- self.assertTrue("errata" in doc.tags.all().values_list("slug", flat=True))
- self.assertTrue(DocAlias.objects.filter(name="rfc1234", docs=doc))
- self.assertTrue(DocAlias.objects.filter(name="bcp1", docs=doc))
- self.assertTrue(DocAlias.objects.filter(name="fyi1", docs=doc))
- self.assertTrue(DocAlias.objects.filter(name="std1", docs=doc))
- self.assertTrue(RelatedDocument.objects.filter(source=doc, target__name="rfc123", relationship="updates").exists())
- self.assertEqual(doc.title, "A Testing RFC")
- self.assertEqual(doc.abstract, "This is some interesting text.")
- self.assertEqual(doc.get_state_slug(), "rfc")
- self.assertEqual(doc.get_state_slug("draft-iesg"), "pub")
- self.assertCountEqual(doc.action_holders.all(), [])
- self.assertEqual(doc.get_state_slug("draft-stream-ise"), "pub")
- self.assertEqual(doc.std_level_id, "ps")
- self.assertEqual(doc.pages, 42)
+ draft_events = draft_doc.docevent_set.all()
+ self.assertEqual(len(draft_events) - event_count_before, 2)
+ self.assertEqual(draft_events[0].type, "sync_from_rfc_editor")
+ self.assertEqual(draft_events[1].type, "changed_action_holders")
+ self.assertEqual(draft_doc.get_state_slug(), "rfc")
+ self.assertEqual(draft_doc.get_state_slug("draft-iesg"), "pub")
+ self.assertCountEqual(draft_doc.action_holders.all(), [])
+ self.assertEqual(draft_doc.get_state_slug("draft-stream-ise"), "pub")
+ self.assertEqual(draft_doc.title, draft_title_before)
+ self.assertEqual(draft_doc.abstract, draft_abstract_before)
+ self.assertEqual(draft_doc.pages, draft_pages_before)
self.assertTrue(not os.path.exists(os.path.join(settings.INTERNET_DRAFT_PATH, draft_filename)))
self.assertTrue(os.path.exists(os.path.join(settings.INTERNET_DRAFT_ARCHIVE_DIR, draft_filename)))
+ rfc_doc = Document.objects.filter(rfc_number=1234, type_id="rfc").first()
+ self.assertIsNotNone(rfc_doc, "RFC document should have been created")
+ rfc_events = rfc_doc.docevent_set.all()
+ self.assertEqual(len(rfc_events), 2)
+ self.assertEqual(rfc_events[0].type, "sync_from_rfc_editor")
+ self.assertEqual(rfc_events[1].type, "published_rfc")
+ self.assertEqual(rfc_events[1].time.astimezone(RPC_TZINFO).date(), today)
+ self.assertTrue("errata" in rfc_doc.tags.all().values_list("slug", flat=True))
+ self.assertTrue(DocAlias.objects.filter(name="rfc1234", docs=rfc_doc))
+ self.assertTrue(DocAlias.objects.filter(name="bcp1", docs=rfc_doc))
+ self.assertTrue(DocAlias.objects.filter(name="fyi1", docs=rfc_doc))
+ self.assertTrue(DocAlias.objects.filter(name="std1", docs=rfc_doc))
+ self.assertTrue(RelatedDocument.objects.filter(source=rfc_doc, target__name="rfc123", relationship="updates").exists())
+ self.assertTrue(RelatedDocument.objects.filter(source=draft_doc, target=rfc_doc, relationship="became_rfc").exists())
+ self.assertEqual(rfc_doc.title, "A Testing RFC")
+ self.assertEqual(rfc_doc.abstract, "This is some interesting text.")
+ self.assertEqual(rfc_doc.std_level_id, "ps")
+ self.assertEqual(rfc_doc.pages, 42)
+
# make sure we can apply it again with no changes
changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)))
self.assertEqual(len(changed), 0)
From 112ac906dcae2c2af798750b7b993f6e1b2fac22 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 16:01:34 -0300
Subject: [PATCH 09/25] test: Confirm changes are returned for draft and rfc
---
ietf/sync/tests.py | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index 8ca6362c7..345bf6eb6 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -347,11 +347,10 @@ class RFCSyncTests(TestCase):
draft_abstract_before = draft_doc.abstract
draft_pages_before = draft_doc.pages
changes = []
- for cs, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)):
- changes.append(cs)
+ for _, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)):
+ changes.append({"doc_pk": d.pk, "rfc_published": rfc_published}) # we ignore the actual change list
draft_doc = Document.objects.get(name=draft_doc.name)
-
draft_events = draft_doc.docevent_set.all()
self.assertEqual(len(draft_events) - event_count_before, 2)
self.assertEqual(draft_events[0].type, "sync_from_rfc_editor")
@@ -385,6 +384,13 @@ class RFCSyncTests(TestCase):
self.assertEqual(rfc_doc.std_level_id, "ps")
self.assertEqual(rfc_doc.pages, 42)
+ # check that we got the expected changes
+ self.assertEqual(len(changes), 2)
+ self.assertEqual(changes[0]["doc_pk"], draft_doc.pk)
+ self.assertEqual(changes[0]["rfc_published"], False)
+ self.assertEqual(changes[1]["doc_pk"], rfc_doc.pk)
+ self.assertEqual(changes[1]["rfc_published"], True)
+
# make sure we can apply it again with no changes
changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)))
self.assertEqual(len(changed), 0)
From e898e43bc809642d8977497858c183f3fc622df3 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 16:09:53 -0300
Subject: [PATCH 10/25] fix: Only consider rfcs for updates/obsoletes targets
---
ietf/sync/rfceditor.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 5f6865c98..e7e500696 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -561,7 +561,7 @@ def update_docs_from_rfc_index(
# # sensibly; otherwise we'll have to ignore them
# l = DocAlias.objects.filter(name__startswith="rfc", docs__docalias__name=x.lower())
# else:
- l = Document.objects.filter(name=x.lower())
+ l = Document.objects.filter(name=x.lower(), type_id="rfc")
for a in l:
if a not in res:
From fd1d425db6687fd5e2b657d877630c7efd8a903e Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 16:12:01 -0300
Subject: [PATCH 11/25] feat: Log warning if rfc index refers to unknown draft
---
ietf/sync/rfceditor.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index e7e500696..b2e8a15c2 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -422,7 +422,7 @@ def update_docs_from_rfc_index(
try:
draft = Document.objects.get(name=draft_name, type_id="draft")
except Document.DoesNotExist:
- pass
+ log(f"Warning: RFC index for {rfc_number} referred to unknown draft {draft_name}")
else:
draft_events = []
draft_changes = []
From 7535df3eeb67dddcb9756f17761828e7d98c27bf Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 17:00:08 -0300
Subject: [PATCH 12/25] feat: Set more fields on rfc doc
---
ietf/sync/rfceditor.py | 149 ++++++++++++++++++++++++-----------------
1 file changed, 86 insertions(+), 63 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index b2e8a15c2..7f668c496 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -374,6 +374,7 @@ def update_docs_from_rfc_index(
tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata")
relationship_obsoletes = DocRelationshipName.objects.get(slug="obs")
relationship_updates = DocRelationshipName.objects.get(slug="updates")
+ rfc_published_state = State.objects.get(type_id="rfc", slug="published")
system = Person.objects.get(name="(System)")
@@ -407,9 +408,34 @@ def update_docs_from_rfc_index(
rfc_changes = []
rfc_published = False
- # Find the document
+ # Find the draft, if any
+ draft = None
+ if draft_name:
+ try:
+ draft = Document.objects.get(name=draft_name, type_id="draft")
+ except Document.DoesNotExist:
+ log(f"Warning: RFC index for {rfc_number} referred to unknown draft {draft_name}")
+
+ # Find or create the RFC document
+ creation_args = {"name": f"rfc{rfc_number}"}
+ if draft:
+ creation_args.update(
+ {
+ "title": draft.title,
+ "stream": draft.stream,
+ "group": draft.group,
+ "abstract": draft.abstract,
+ "pages": draft.pages,
+ "words": draft.words,
+ "std_level": draft.std_level,
+ "ad": draft.ad,
+ "external_url": draft.external_url,
+ "uploaded_filename": draft.uploaded_filename,
+ "note": draft.note,
+ }
+ )
doc, created_rfc = Document.objects.get_or_create(
- rfc_number=rfc_number, type_id="rfc", defaults={"name": f"rfc{rfc_number}"}
+ rfc_number=rfc_number, type_id="rfc", defaults=creation_args
)
if created_rfc:
rfc_changes.append(f"created document {prettify_std_name(doc.name)}")
@@ -417,74 +443,71 @@ def update_docs_from_rfc_index(
alias, _ = DocAlias.objects.get_or_create(name=doc.name)
alias.docs.add(doc)
rfc_changes.append(f"created alias {prettify_std_name(doc.name)}")
+ if draft:
+ doc.formal_languages.set(draft.formal_languages.all())
- if draft_name:
- try:
- draft = Document.objects.get(name=draft_name, type_id="draft")
- except Document.DoesNotExist:
- log(f"Warning: RFC index for {rfc_number} referred to unknown draft {draft_name}")
- else:
- draft_events = []
- draft_changes = []
+ if draft:
+ draft_events = []
+ draft_changes = []
- # Ensure the draft is in the "rfc" state and move its files to the archive
- # if necessary.
- if draft.get_state_slug() != "rfc":
- draft.set_state(
- State.objects.get(used=True, type="draft", slug="rfc")
- )
- move_draft_files_to_archive(draft, draft.rev)
- draft_changes.append(f"changed state to {draft.get_state()}")
-
- # Ensure the draft and rfc are linked with a "became_rfc" relationship
- r, created_relateddoc = RelatedDocument.objects.get_or_create(
- source=draft, target=doc, relationship_id="became_rfc"
+ # Ensure the draft is in the "rfc" state and move its files to the archive
+ # if necessary.
+ if draft.get_state_slug() != "rfc":
+ draft.set_state(
+ State.objects.get(used=True, type="draft", slug="rfc")
)
- if created_relateddoc:
- change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format(
- rel_name=r.relationship.name.lower(),
- pretty_draft_name=prettify_std_name(draft_name),
- pretty_rfc_name=prettify_std_name(doc.name),
- )
- draft_changes.append(change)
- rfc_changes.append(change)
+ move_draft_files_to_archive(draft, draft.rev)
+ draft_changes.append(f"changed state to {draft.get_state()}")
- # Ensure draft is in the correct iesg and stream states
- for t in (
- "draft-iesg",
- "draft-stream-iab",
- "draft-stream-irtf",
- "draft-stream-ise",
- ):
- prev_state = draft.get_state(t)
- if prev_state is not None:
- if prev_state.slug not in ("pub", "idexists"):
- new_state = State.objects.select_related("type").get(
- used=True, type=t, slug="pub"
- )
- draft.set_state(new_state)
- draft_changes.append(
- f"changed {new_state.type.label} to {new_state}"
- )
- e = update_action_holders(draft, prev_state, new_state)
- if e:
- draft_events.append(e)
- elif t == "draft-iesg":
- draft.set_state(
- State.objects.get(type_id="draft-iesg", slug="idexists")
+ # Ensure the draft and rfc are linked with a "became_rfc" relationship
+ r, created_relateddoc = RelatedDocument.objects.get_or_create(
+ source=draft, target=doc, relationship_id="became_rfc"
+ )
+ if created_relateddoc:
+ change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format(
+ rel_name=r.relationship.name.lower(),
+ pretty_draft_name=prettify_std_name(draft_name),
+ pretty_rfc_name=prettify_std_name(doc.name),
+ )
+ draft_changes.append(change)
+ rfc_changes.append(change)
+
+ # Ensure draft is in the correct iesg and stream states
+ for t in (
+ "draft-iesg",
+ "draft-stream-iab",
+ "draft-stream-irtf",
+ "draft-stream-ise",
+ ):
+ prev_state = draft.get_state(t)
+ if prev_state is not None:
+ if prev_state.slug not in ("pub", "idexists"):
+ new_state = State.objects.select_related("type").get(
+ used=True, type=t, slug="pub"
)
- if draft_changes:
- draft_events.append(
- DocEvent.objects.create(
- doc=draft,
- rev=doc.rev,
- by=system,
- type="sync_from_rfc_editor",
- desc=f"Received changes through RFC Editor sync ({', '.join(draft_changes)})",
+ draft.set_state(new_state)
+ draft_changes.append(
+ f"changed {new_state.type.label} to {new_state}"
)
+ e = update_action_holders(draft, prev_state, new_state)
+ if e:
+ draft_events.append(e)
+ elif t == "draft-iesg":
+ draft.set_state(
+ State.objects.get(type_id="draft-iesg", slug="idexists")
)
- draft.save_with_history(draft_events)
- yield draft_changes, draft, False # yield changes to the draft
+ if draft_changes:
+ draft_events.append(
+ DocEvent.objects.create(
+ doc=draft,
+ rev=doc.rev,
+ by=system,
+ type="sync_from_rfc_editor",
+ desc=f"Received changes through RFC Editor sync ({', '.join(draft_changes)})",
+ )
+ )
+ draft.save_with_history(draft_events)
+ yield draft_changes, draft, False # yield changes to the draft
# check attributes
if title != doc.title:
From 39ccea92f9c6e243ce560f1b9c3f0d1dc965f7cb Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 17:01:06 -0300
Subject: [PATCH 13/25] feat: Set published state on rfc doc
---
ietf/sync/rfceditor.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 7f668c496..8b0fc5f5e 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -443,6 +443,7 @@ def update_docs_from_rfc_index(
alias, _ = DocAlias.objects.get_or_create(name=doc.name)
alias.docs.add(doc)
rfc_changes.append(f"created alias {prettify_std_name(doc.name)}")
+ doc.set_state(rfc_published_state)
if draft:
doc.formal_languages.set(draft.formal_languages.all())
@@ -530,9 +531,12 @@ def update_docs_from_rfc_index(
doc.stream = stream_mapping[stream]
rfc_changes.append(f"changed stream to {doc.stream}")
- if (
- not doc.group
- ): # if we have no group assigned, check if RFC Editor has a suggestion
+ if doc.get_state(rfc_published_state.type) != rfc_published_state:
+ doc.set_state(rfc_published_state)
+ rfc_changes.append(f"changed {rfc_published_state.type.label} to {rfc_published_state}")
+
+ # if we have no group assigned, check if RFC Editor has a suggestion
+ if not doc.group:
if wg:
doc.group = Group.objects.get(acronym=wg)
rfc_changes.append(f"set group to {doc.group}")
From 8b636216d0b34ba5578e6e90996ffd15b168cc09 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 17:35:12 -0300
Subject: [PATCH 14/25] feat: describe change as "set" or "changed" as
appropriate
---
ietf/sync/rfceditor.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 8b0fc5f5e..d924fb522 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -511,29 +511,30 @@ def update_docs_from_rfc_index(
yield draft_changes, draft, False # yield changes to the draft
# check attributes
+ verbed = "set" if created_rfc else "changed"
if title != doc.title:
doc.title = title
- rfc_changes.append(f"changed title to '{doc.title}'")
+ rfc_changes.append(f"{verbed} title to '{doc.title}'")
if abstract and abstract != doc.abstract:
doc.abstract = abstract
- rfc_changes.append(f"changed abstract to '{doc.abstract}'")
+ rfc_changes.append(f"{verbed} abstract to '{doc.abstract}'")
if pages and int(pages) != doc.pages:
doc.pages = int(pages)
- rfc_changes.append(f"changed pages to {doc.pages}")
+ rfc_changes.append(f"{verbed} pages to {doc.pages}")
if std_level_mapping[current_status] != doc.std_level:
doc.std_level = std_level_mapping[current_status]
- rfc_changes.append(f"changed standardization level to {doc.std_level}")
+ rfc_changes.append(f"{verbed} standardization level to {doc.std_level}")
if doc.stream != stream_mapping[stream]:
doc.stream = stream_mapping[stream]
- rfc_changes.append(f"changed stream to {doc.stream}")
+ rfc_changes.append(f"{verbed} stream to {doc.stream}")
if doc.get_state(rfc_published_state.type) != rfc_published_state:
doc.set_state(rfc_published_state)
- rfc_changes.append(f"changed {rfc_published_state.type.label} to {rfc_published_state}")
+ rfc_changes.append(f"{verbed} {rfc_published_state.type.label} to {rfc_published_state}")
# if we have no group assigned, check if RFC Editor has a suggestion
if not doc.group:
From ab99972d768ec3b30ef04812c847149208fb3157 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 17:44:37 -0300
Subject: [PATCH 15/25] fix: Check state correctly
---
ietf/sync/rfceditor.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index d924fb522..a88e420f6 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -532,7 +532,7 @@ def update_docs_from_rfc_index(
doc.stream = stream_mapping[stream]
rfc_changes.append(f"{verbed} stream to {doc.stream}")
- if doc.get_state(rfc_published_state.type) != rfc_published_state:
+ if doc.get_state() != rfc_published_state:
doc.set_state(rfc_published_state)
rfc_changes.append(f"{verbed} {rfc_published_state.type.label} to {rfc_published_state}")
From 92ad44e70ca51eb828773970f4ec96f626d85b26 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 17:53:35 -0300
Subject: [PATCH 16/25] test: Test more fields on the rfc doc
---
ietf/sync/tests.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index 345bf6eb6..2706c7057 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -230,6 +230,8 @@ class RFCSyncTests(TestCase):
group__parent=area,
states=[('draft-iesg','rfcqueue'),('draft-stream-ise','rfc-edit')],
ad=Person.objects.get(user__username='ad'),
+ external_url="http://my-external-url.example.com",
+ note="this is a note",
)
# it's a bit strange to have draft-stream-ise set when draft-iesg is set
# too, but for testing purposes ...
@@ -372,6 +374,7 @@ class RFCSyncTests(TestCase):
self.assertEqual(rfc_events[0].type, "sync_from_rfc_editor")
self.assertEqual(rfc_events[1].type, "published_rfc")
self.assertEqual(rfc_events[1].time.astimezone(RPC_TZINFO).date(), today)
+ self.assertEqual(rfc_doc.get_state_slug(), "published")
self.assertTrue("errata" in rfc_doc.tags.all().values_list("slug", flat=True))
self.assertTrue(DocAlias.objects.filter(name="rfc1234", docs=rfc_doc))
self.assertTrue(DocAlias.objects.filter(name="bcp1", docs=rfc_doc))
@@ -383,6 +386,12 @@ class RFCSyncTests(TestCase):
self.assertEqual(rfc_doc.abstract, "This is some interesting text.")
self.assertEqual(rfc_doc.std_level_id, "ps")
self.assertEqual(rfc_doc.pages, 42)
+ self.assertEqual(rfc_doc.stream, draft_doc.stream)
+ self.assertEqual(rfc_doc.group, draft_doc.group)
+ self.assertEqual(rfc_doc.words, draft_doc.words)
+ self.assertEqual(rfc_doc.ad, draft_doc.ad)
+ self.assertEqual(rfc_doc.external_url, draft_doc.external_url)
+ self.assertEqual(rfc_doc.note, draft_doc.note)
# check that we got the expected changes
self.assertEqual(len(changes), 2)
From a66457bfaf2fe67f23397a2288be471ac23d5f16 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Thu, 10 Aug 2023 18:01:11 -0300
Subject: [PATCH 17/25] test: Remove python lint
---
ietf/sync/rfceditor.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index a88e420f6..1e3e0bf73 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -7,7 +7,7 @@ import datetime
import re
import requests
-from typing import Iterator
+from typing import Iterator, Optional, Union
from urllib.parse import urlencode
from xml.dom import pulldom, Node
@@ -417,7 +417,7 @@ def update_docs_from_rfc_index(
log(f"Warning: RFC index for {rfc_number} referred to unknown draft {draft_name}")
# Find or create the RFC document
- creation_args = {"name": f"rfc{rfc_number}"}
+ creation_args: dict[str, Optional[Union[str, int]]] = {"name": f"rfc{rfc_number}"}
if draft:
creation_args.update(
{
From aff4c2d0db6aecdf3ccb8c8e1124167974262bef Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Fri, 11 Aug 2023 12:04:15 -0300
Subject: [PATCH 18/25] fix: Drop leading 0s from RFC doc-id
---
ietf/sync/rfceditor.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 1e3e0bf73..56501cb0f 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -346,6 +346,8 @@ def update_docs_from_rfc_index(
errata: dict[str, list[dict]] = {}
for item in errata_data:
name = item["doc-id"]
+ if name.upper().startswith("RFC"):
+ name = f"RFC{int(name[3:])}" # removes leading 0s on the rfc number
if not name in errata:
errata[name] = []
errata[name].append(item)
@@ -633,7 +635,7 @@ def update_docs_from_rfc_index(
DocAlias.objects.create(name=a).docs.add(doc)
rfc_changes.append(f"created alias {prettify_std_name(a)}")
- doc_errata = errata.get("RFC%04d" % rfc_number, []) # rfc10k problem here
+ doc_errata = errata.get(f"RFC{rfc_number}", [])
all_rejected = doc_errata and all(
er["errata_status_code"] == "Rejected" for er in doc_errata
)
From f9ca1bc9d615ce8003fc663061ce303c94c3475b Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 14 Aug 2023 12:15:09 -0300
Subject: [PATCH 19/25] test: Test behavior of verified-errata tag
Also add comments flagging the near-miss between doc-id
and RFC number in the test_rfc_index() test data. It's
unclear whether these were intended to match, but the
test was self-consistent so I am not changing it.
---
ietf/sync/tests.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index 2706c7057..e1ea1522e 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -307,7 +307,7 @@ class RFCSyncTests(TestCase):
errata = [{
"errata_id":1,
- "doc-id":"RFC123",
+ "doc-id":"RFC123", # n.b. this is not the same RFC as in the above index XML!
"errata_status_code":"Verified",
"errata_type_code":"Editorial",
"section": "4.1",
@@ -375,7 +375,11 @@ class RFCSyncTests(TestCase):
self.assertEqual(rfc_events[1].type, "published_rfc")
self.assertEqual(rfc_events[1].time.astimezone(RPC_TZINFO).date(), today)
self.assertEqual(rfc_doc.get_state_slug(), "published")
- self.assertTrue("errata" in rfc_doc.tags.all().values_list("slug", flat=True))
+ # Should have an "errata" tag because there is an errata-url in the index XML, but no "verified-errata" tag
+ # because there is no verified item in the errata JSON with doc-id matching the RFC document.
+ tag_slugs = rfc_doc.tags.values_list("slug", flat=True)
+ self.assertTrue("errata" in tag_slugs)
+ self.assertFalse("verified-errata" in tag_slugs)
self.assertTrue(DocAlias.objects.filter(name="rfc1234", docs=rfc_doc))
self.assertTrue(DocAlias.objects.filter(name="bcp1", docs=rfc_doc))
self.assertTrue(DocAlias.objects.filter(name="fyi1", docs=rfc_doc))
From 92460319d141d8a210c114f4ecc85a50e7ff8e23 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 14 Aug 2023 13:06:48 -0300
Subject: [PATCH 20/25] refactor: Refactor handling of draft stream states
---
ietf/sync/rfceditor.py | 52 +++++++++++++++++++++++-------------------
1 file changed, 29 insertions(+), 23 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 56501cb0f..f49ce4e96 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -475,30 +475,36 @@ def update_docs_from_rfc_index(
draft_changes.append(change)
rfc_changes.append(change)
- # Ensure draft is in the correct iesg and stream states
- for t in (
- "draft-iesg",
- "draft-stream-iab",
- "draft-stream-irtf",
- "draft-stream-ise",
- ):
- prev_state = draft.get_state(t)
- if prev_state is not None:
- if prev_state.slug not in ("pub", "idexists"):
- new_state = State.objects.select_related("type").get(
- used=True, type=t, slug="pub"
- )
- draft.set_state(new_state)
- draft_changes.append(
- f"changed {new_state.type.label} to {new_state}"
- )
- e = update_action_holders(draft, prev_state, new_state)
- if e:
- draft_events.append(e)
- elif t == "draft-iesg":
- draft.set_state(
- State.objects.get(type_id="draft-iesg", slug="idexists")
+ # Draft should be in the "pub" draft-iesg state - complain otherwise
+ iesg_state = draft.get_state("draft-iesg")
+ if iesg_state is None:
+ log(f'Warning while processing {doc.name}: {draft.name} has no "draft-iesg" state')
+ elif iesg_state.slug != "pub":
+ log(
+ 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "pub")'.format(
+ doc.name, draft.name, iesg_state.slug
)
+ )
+
+ # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain.
+ if draft.stream != doc.stream:
+ log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format(
+ doc.name, draft.name, draft.stream, doc.stream
+ ))
+ elif draft.stream.slug in ["draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"]:
+ stream_slug = draft.stream.slug
+ prev_state = draft.get_state(stream_slug)
+ if prev_state is None:
+ log(f"Warning while processing {doc.name}: draft {draft.name} stream state was not set")
+ elif prev_state.slug != "pub":
+ new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub")
+ draft.set_state(new_state)
+ draft_changes.append(
+ f"changed {new_state.type.label} to {new_state}"
+ )
+ e = update_action_holders(draft, prev_state, new_state)
+ if e:
+ draft_events.append(e)
if draft_changes:
draft_events.append(
DocEvent.objects.create(
From 06adb25e43c375c89f9839962d41fc5614300076 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 14 Aug 2023 13:37:24 -0300
Subject: [PATCH 21/25] fix: Set draft-iesg state to "pub" when a draft is
published
---
ietf/sync/rfceditor.py | 23 +++++++++++++++--------
1 file changed, 15 insertions(+), 8 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index f49ce4e96..08ae16f33 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -475,16 +475,23 @@ def update_docs_from_rfc_index(
draft_changes.append(change)
rfc_changes.append(change)
- # Draft should be in the "pub" draft-iesg state - complain otherwise
- iesg_state = draft.get_state("draft-iesg")
- if iesg_state is None:
+ # Draft should be in the "pub" or "approved" draft-iesg state - complain otherwise
+ prev_iesg_state = draft.get_state("draft-iesg")
+ if prev_iesg_state is None:
log(f'Warning while processing {doc.name}: {draft.name} has no "draft-iesg" state')
- elif iesg_state.slug != "pub":
- log(
- 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "pub")'.format(
- doc.name, draft.name, iesg_state.slug
+ elif prev_iesg_state.slug != "pub":
+ if prev_iesg_state.slug != "rfcqueue":
+ log(
+ 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format(
+ doc.name, draft.name, prev_iesg_state.slug
+ )
)
- )
+ new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub")
+ draft.set_state(new_iesg_state)
+ draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}")
+ e = update_action_holders(draft, prev_iesg_state, new_iesg_state)
+ if e:
+ draft_events.append(e)
# If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain.
if draft.stream != doc.stream:
From cf470a46806c3c99cdd638ea4b9e34b420744e31 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 14 Aug 2023 14:08:48 -0300
Subject: [PATCH 22/25] fix: Use correct stream slugs
---
ietf/sync/rfceditor.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 08ae16f33..2e6861959 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -498,12 +498,12 @@ def update_docs_from_rfc_index(
log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format(
doc.name, draft.name, draft.stream, doc.stream
))
- elif draft.stream.slug in ["draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"]:
- stream_slug = draft.stream.slug
+ elif draft.stream.slug in ["iab", "irtf", "ise"]:
+ stream_slug = f"draft-stream-{draft.stream.slug}"
prev_state = draft.get_state(stream_slug)
if prev_state is None:
log(f"Warning while processing {doc.name}: draft {draft.name} stream state was not set")
- elif prev_state.slug != "pub":
+ if prev_state.slug != "pub":
new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub")
draft.set_state(new_state)
draft_changes.append(
From b565f104d854089bc8147eeea36f0a6af5fd5baf Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 14 Aug 2023 14:09:37 -0300
Subject: [PATCH 23/25] test: Don't mix stream states in test_rfc_index()
Make the test a better simulation of actual practice.
---
ietf/sync/tests.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index e1ea1522e..d5e380672 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -228,13 +228,11 @@ class RFCSyncTests(TestCase):
area = GroupFactory(type_id='area')
draft_doc = WgDraftFactory(
group__parent=area,
- states=[('draft-iesg','rfcqueue'),('draft-stream-ise','rfc-edit')],
+ states=[('draft-iesg','rfcqueue')],
ad=Person.objects.get(user__username='ad'),
external_url="http://my-external-url.example.com",
note="this is a note",
)
- # it's a bit strange to have draft-stream-ise set when draft-iesg is set
- # too, but for testing purposes ...
draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared
RfcFactory(rfc_number=123)
@@ -360,7 +358,6 @@ class RFCSyncTests(TestCase):
self.assertEqual(draft_doc.get_state_slug(), "rfc")
self.assertEqual(draft_doc.get_state_slug("draft-iesg"), "pub")
self.assertCountEqual(draft_doc.action_holders.all(), [])
- self.assertEqual(draft_doc.get_state_slug("draft-stream-ise"), "pub")
self.assertEqual(draft_doc.title, draft_title_before)
self.assertEqual(draft_doc.abstract, draft_abstract_before)
self.assertEqual(draft_doc.pages, draft_pages_before)
From 80a361c3c821cce96873b6c0ec8e49e8eed5d30e Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Mon, 14 Aug 2023 14:27:26 -0300
Subject: [PATCH 24/25] test: Check that no logs are emitted in test_rfc_index
---
ietf/sync/tests.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py
index d5e380672..a3e857768 100644
--- a/ietf/sync/tests.py
+++ b/ietf/sync/tests.py
@@ -6,6 +6,7 @@ import os
import io
import json
import datetime
+import mock
import quopri
from django.conf import settings
@@ -347,9 +348,11 @@ class RFCSyncTests(TestCase):
draft_abstract_before = draft_doc.abstract
draft_pages_before = draft_doc.pages
changes = []
- for _, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)):
- changes.append({"doc_pk": d.pk, "rfc_published": rfc_published}) # we ignore the actual change list
-
+ with mock.patch("ietf.sync.rfceditor.log") as mock_log:
+ for _, d, rfc_published in rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30)):
+ changes.append({"doc_pk": d.pk, "rfc_published": rfc_published}) # we ignore the actual change list
+ self.assertFalse(mock_log.called, "No log messages expected")
+
draft_doc = Document.objects.get(name=draft_doc.name)
draft_events = draft_doc.docevent_set.all()
self.assertEqual(len(draft_events) - event_count_before, 2)
From 4706ac2a37bd4b6d6cbb77d37c2dd55326139114 Mon Sep 17 00:00:00 2001
From: Jennifer Richards <jennifer@staff.ietf.org>
Date: Wed, 16 Aug 2023 11:53:58 -0300
Subject: [PATCH 25/25] fix: Restore old handling of draft-iesg state
Missing state -> idexists, idexists -> idexists, anything
else becomes pub.
---
ietf/sync/rfceditor.py | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py
index 2e6861959..ce0cf2da2 100644
--- a/ietf/sync/rfceditor.py
+++ b/ietf/sync/rfceditor.py
@@ -475,11 +475,20 @@ def update_docs_from_rfc_index(
draft_changes.append(change)
rfc_changes.append(change)
- # Draft should be in the "pub" or "approved" draft-iesg state - complain otherwise
+ # Always set the "draft-iesg" state. This state should be set for all drafts, so
+ # log a warning if it is not set. What should happen here is that ietf stream
+ # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index.
+ # Other stream documents should normally be "idexists" and be left that way. The
+ # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub",
+ # and changes any other state to "pub". If unset, it changes it to "idexists".
+ # This reflects historical behavior and should probably be updated, but a migration
+ # of existing drafts (and validation of the change) is needed before we change the
+ # handling.
prev_iesg_state = draft.get_state("draft-iesg")
if prev_iesg_state is None:
log(f'Warning while processing {doc.name}: {draft.name} has no "draft-iesg" state')
- elif prev_iesg_state.slug != "pub":
+ new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists")
+ elif prev_iesg_state.slug not in ("pub", "idexists"):
if prev_iesg_state.slug != "rfcqueue":
log(
'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format(
@@ -487,6 +496,10 @@ def update_docs_from_rfc_index(
)
)
new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub")
+ else:
+ new_iesg_state = prev_iesg_state
+
+ if new_iesg_state != prev_iesg_state:
draft.set_state(new_iesg_state)
draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}")
e = update_action_holders(draft, prev_iesg_state, new_iesg_state)