275 lines
12 KiB
Python
275 lines
12 KiB
Python
# Copyright The IETF Trust 2024, All Rights Reserved
|
|
|
|
import debug # pyflakes:ignore
|
|
|
|
import csv
|
|
import datetime
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
|
|
from collections import namedtuple, Counter
|
|
from pathlib import Path
|
|
|
|
from django.conf import settings
|
|
from django.core.management.base import BaseCommand
|
|
|
|
from ietf.doc.models import Document, DocEvent, State
|
|
from ietf.utils.text import xslugify
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "Performs a one-time import of IESG statements"
|
|
|
|
def handle(self, *args, **options):
|
|
if Document.objects.filter(type="statement", group__acronym="iesg").exists():
|
|
self.stdout.write("IESG statement documents already exist - exiting")
|
|
exit(-1)
|
|
tmpdir = tempfile.mkdtemp()
|
|
process = subprocess.Popen(
|
|
["git", "clone", "https://github.com/kesara/iesg-scraper.git", tmpdir],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
sub_stdout, sub_stderr = process.communicate()
|
|
if not Path(tmpdir).joinpath("iesg_statements", "2000-08-29-0.md").exists():
|
|
self.stdout.write(
|
|
"Git clone of the iesg-scraper directory did not go as expected"
|
|
)
|
|
self.stdout.write("stdout:", sub_stdout)
|
|
self.stdout.write("stderr:", sub_stderr)
|
|
self.stdout.write(f"Clean up {tmpdir} manually")
|
|
exit(-1)
|
|
|
|
redirects = []
|
|
for item in self.get_work_items():
|
|
replaced = item.title.endswith(
|
|
" SUPERSEDED"
|
|
) or item.doc_time.date() == datetime.date(2007, 7, 30)
|
|
title = item.title
|
|
if title.endswith(" - SUPERSEDED"):
|
|
title = title[: -len(" - SUPERSEDED")]
|
|
name = f"statement-iesg-{xslugify(title)}-{item.doc_time:%Y%m%d}"
|
|
dest_filename = f"{name}-00.md"
|
|
# Create Document
|
|
doc = Document.objects.create(
|
|
name=name,
|
|
type_id="statement",
|
|
title=title,
|
|
group_id=2, # The IESG group
|
|
rev="00",
|
|
uploaded_filename=dest_filename,
|
|
)
|
|
doc.set_state(
|
|
State.objects.get(
|
|
type_id="statement",
|
|
slug="replaced" if replaced else "active",
|
|
)
|
|
)
|
|
e1 = DocEvent.objects.create(
|
|
time=item.doc_time,
|
|
type="published_statement",
|
|
doc=doc,
|
|
rev="00",
|
|
by_id=1, # (System)
|
|
desc="Statement published (note: The exact time of day is inaccurate - the actual time of day is not known)",
|
|
)
|
|
e2 = DocEvent.objects.create(
|
|
type="added_comment",
|
|
doc=doc,
|
|
rev="00",
|
|
by_id=1, # (System)
|
|
desc="Statement moved into datatracker from www.ietf.org",
|
|
)
|
|
doc.save_with_history([e1, e2])
|
|
|
|
# Put file in place
|
|
source = Path(tmpdir).joinpath("iesg_statements", item.source_filename)
|
|
dest = Path(settings.DOCUMENT_PATH_PATTERN.format(doc=doc)).joinpath(
|
|
dest_filename
|
|
)
|
|
if dest.exists():
|
|
self.stdout.write(
|
|
f"WARNING: {dest} already exists - not overwriting it."
|
|
)
|
|
else:
|
|
os.makedirs(dest.parent, exist_ok=True)
|
|
shutil.copy(source, dest)
|
|
|
|
redirects.append(
|
|
[
|
|
f"www.ietf.org/about/groups/iesg/statements/{item.slug}",
|
|
f"https://datatracker.ietf.org/doc/{name}",
|
|
302,
|
|
]
|
|
)
|
|
|
|
shutil.rmtree(tmpdir)
|
|
with open("iesg_statement_redirects.csv", "w", newline="") as f:
|
|
csvwriter = csv.writer(f)
|
|
for row in redirects:
|
|
csvwriter.writerow(row)
|
|
|
|
def get_work_items(self):
|
|
Item = namedtuple("Item", "doc_time source_filename title slug")
|
|
items = []
|
|
dressed_rows = " ".join(
|
|
self.cut_paste_from_www().expandtabs(1).split(" ")
|
|
).split("\n")
|
|
old_slugs = self.get_old_slugs()
|
|
# Rube-Goldberg-esque dance to deal with conflicting directions of the scrape and
|
|
# what order we want the result to sort to
|
|
dressed_rows.reverse()
|
|
old_slugs.reverse()
|
|
total_times_date_seen = Counter([row.split(" ")[0] for row in dressed_rows])
|
|
count_date_seen_so_far = Counter()
|
|
for row, slug in zip(dressed_rows, old_slugs):
|
|
date_part = row.split(" ")[0]
|
|
title_part = row[len(date_part) + 1 :]
|
|
datetime_args = list(map(int, date_part.replace("-0", "-").split("-")))
|
|
# Use the minutes in timestamps to preserve order of statements
|
|
# on the same day as they currently appear at www.ietf.org
|
|
datetime_args.extend([12, count_date_seen_so_far[date_part]])
|
|
count_date_seen_so_far[date_part] += 1
|
|
doc_time = datetime.datetime(*datetime_args, tzinfo=datetime.timezone.utc)
|
|
items.append(
|
|
Item(
|
|
doc_time,
|
|
f"{date_part}-{total_times_date_seen[date_part] - count_date_seen_so_far[date_part]}.md",
|
|
title_part,
|
|
slug,
|
|
)
|
|
)
|
|
return items
|
|
|
|
def cut_paste_from_www(self):
|
|
return """2023-08-24 Support Documents in IETF Working Groups
|
|
2023-08-14 Guidance on In-Person and Online Interim Meetings
|
|
2023-05-01 IESG Statement on EtherTypes
|
|
2023-03-15 Second Report on the RFC 8989 Experiment
|
|
2023-01-27 Guidance on In-Person and Online Interim Meetings - SUPERSEDED
|
|
2022-10-31 Statement on Restricting Access to IETF IT Systems
|
|
2022-01-21 Handling Ballot Positions
|
|
2021-09-01 Report on the RFC 8989 experiment
|
|
2021-07-21 IESG Statement on Allocation of Email Addresses in the ietf.org Domain
|
|
2021-05-11 IESG Statement on Inclusive Language
|
|
2021-05-10 IESG Statement on Internet-Draft Authorship
|
|
2021-05-07 IESG Processing of RFC Errata for the IETF Stream
|
|
2021-04-16 Last Call Guidance to the Community
|
|
2020-07-23 IESG Statement On Oppressive or Exclusionary Language
|
|
2020-05-01 Guidance on Face-to-Face and Virtual Interim Meetings - SUPERSEDED
|
|
2018-03-16 IETF Meeting Photography Policy
|
|
2018-01-11 Guidance on Face-to-Face and Virtual Interim Meetings - SUPERSEDED
|
|
2017-02-09 License File for Open Source Repositories
|
|
2016-11-13 Support Documents in IETF Working Groups - SUPERSEDED
|
|
2016-02-05 Guidance on Face-to-Face and Virtual Interim Meetings - SUPERSEDED
|
|
2016-01-11 Guidance on Face-to-Face and Virtual Interim Meetings - SUPERSEDED
|
|
2015-08-20 IESG Statement on Maximizing Encrypted Access To IETF Information
|
|
2015-06-11 IESG Statement on Internet-Draft Authorship - SUPERSEDED
|
|
2014-07-20 IESG Statement on Designating RFCs as Historic
|
|
2014-05-07 DISCUSS Criteria in IESG Review
|
|
2014-03-02 Writable MIB Module IESG Statement
|
|
2013-11-03 IETF Anti-Harassment Policy
|
|
2012-10-25 IESG Statement on Ethertypes - SUPERSEDED
|
|
2012-10-25 IESG Statement on Removal of an Internet-Draft from the IETF Web Site
|
|
2011-10-20 IESG Statement on Designating RFCs as Historic - SUPERSEDED
|
|
2011-06-27 IESG Statement on Designating RFCs as Historic - SUPERSEDED
|
|
2011-06-13 IESG Statement on IESG Processing of RFC Errata concerning RFC Metadata
|
|
2010-10-11 IESG Statement on Document Shepherds
|
|
2010-05-24 IESG Statement on the Usage of Assignable Codepoints, Addresses and Names in Specification Examples
|
|
2010-05-24 IESG Statement on NomCom Eligibility and Day Passes
|
|
2009-09-08 IESG Statement on Copyright
|
|
2009-01-20 IESG Statement on Proposed Status for IETF Documents Reserving Resources for Example Purposes
|
|
2008-09-02 Guidance on Interim Meetings, Conference Calls and Jabber Sessions - SUPERSEDED
|
|
2008-07-30 IESG Processing of RFC Errata for the IETF Stream
|
|
2008-04-14 IESG Statement on Spam Control on IETF Mailing Lists
|
|
2008-03-03 IESG Statement on Registration Requests for URIs Containing Telephone Numbers
|
|
2008-02-27 IESG Statement on RFC3406 and URN Namespaces Registry Review
|
|
2008-01-23 Advice for WG Chairs Dealing with Off-Topic Postings
|
|
2007-10-04 On Appeals of IESG and Area Director Actions and Decisions
|
|
2007-07-05 Experimental Specification of New Congestion Control Algorithms
|
|
2007-03-20 Guidance on Area Director Sponsoring of Documents
|
|
2007-01-15 Last Call Guidance to the Community - SUPERSEDED
|
|
2006-04-19 IESG Statement: Normative and Informative References
|
|
2006-02-17 IESG Statement on Disruptive Posting
|
|
2006-01-09 Guidance for Spam Control on IETF Mailing Lists - SUPERSEDED
|
|
2006-01-05 IESG Statement on AUTH48 State
|
|
2005-05-12 Syntax for Format Definitions
|
|
2003-02-11 IESG Statement on IDN
|
|
2002-11-27 Copyright Statement in MIB and PIB Modules
|
|
2002-03-13 Guidance for Spam Control on IETF Mailing Lists - SUPERSEDED
|
|
2001-12-21 On Design Teams
|
|
2001-10-01 Guidelines for the Use of Formal Languages in IETF Specifications
|
|
2001-03-21 Establishment of Temporary Sub-IP Area
|
|
2000-12-06 Plans to Organize "Sub-IP" Technologies in the IETF
|
|
2000-11-20 A New IETF Work Area
|
|
2000-08-29 Guidance on Interim IETF Working Group Meetings and Conference Calls - SUPERSEDED
|
|
2000-08-29 IESG Guidance on the Moderation of IETF Working Group Mailing Lists"""
|
|
|
|
def get_old_slugs(self):
|
|
return [
|
|
"support-documents",
|
|
"interim-meetings-guidance",
|
|
"ethertypes",
|
|
"second-report-on-the-rfc-8989-experiment",
|
|
"interim-meetings-guidance-2023-01-27",
|
|
"statement-on-restricting-access",
|
|
"handling-ballot-positions",
|
|
"report-on-rfc8989-experiment",
|
|
"email-addresses-ietf-domain",
|
|
"on-inclusive-language",
|
|
"internet-draft-authorship",
|
|
"processing-errata-ietf-stream",
|
|
"last-call-guidance",
|
|
"statement-on-oppressive-exclusionary-language",
|
|
"interim-meetings-guidance-2020-05-01",
|
|
"meeting-photography-policy",
|
|
"interim-meetings-guidance-2018-01-11",
|
|
"open-source-repositories-license",
|
|
"support-documents-2016-11-13",
|
|
"interim-meetings-guidance-2016-02-05",
|
|
"interim-meetings-guidance-2016-01-11",
|
|
"maximizing-encrypted-access",
|
|
"internet-draft-authorship-2015-06-11",
|
|
"designating-rfcs-historic",
|
|
"iesg-discuss-criteria",
|
|
"writable-mib-module",
|
|
"anti-harassment-policy",
|
|
"ethertypes-2012-10-25",
|
|
"internet-draft-removal",
|
|
"designating-rfcs-historic-2011-10-20",
|
|
"designating-rfcs-historic-2011-06-27",
|
|
"rfc-metadata-errata",
|
|
"document-shepherds",
|
|
"assignable-codepoints-addresses-names",
|
|
"nomcom-eligibility-day-passes",
|
|
"copyright-2009-09-08",
|
|
"reserving-resources-examples",
|
|
"interim-meetings-guidance-2008-09-02",
|
|
"processing-rfc-errata",
|
|
"spam-control-2008-04-14",
|
|
"registration-requests-uris",
|
|
"urn-namespaces-registry",
|
|
"off-topic-postings",
|
|
"appeals-actions-decisions",
|
|
"experimental-congestion-control",
|
|
"area-director-sponsoring-documents",
|
|
"last-call-guidance-2007-01-15",
|
|
"normative-informative-references",
|
|
"disruptive-posting",
|
|
"spam-control-2006-01-09",
|
|
"auth48",
|
|
"syntax-format-definitions",
|
|
"idn",
|
|
"copyright-2002-11-27",
|
|
"spam-control-2002-03-13",
|
|
"design-teams",
|
|
"formal-languages-use",
|
|
"sub-ip-area-2001-03-21",
|
|
"sub-ip-area-2000-11-20",
|
|
"sub-ip-area-2000-12-06",
|
|
"interim-meetings-guidance-2000-08-29",
|
|
"mailing-lists-moderation",
|
|
]
|