chore: moved away code that should be kept present as documentation, but not as live ammunition
This commit is contained in:
Robert Sparks
parent
5489fea9f7
commit
08257e0d2e
|
|
@ -1,151 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
# Copyright The IETF Trust 2017-2019, All Rights Reserved
|
|
||||||
|
|
||||||
import datetime
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
from contextlib import closing
|
|
||||||
|
|
||||||
os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
|
|
||||||
|
|
||||||
import django
|
|
||||||
django.setup()
|
|
||||||
|
|
||||||
from django.conf import settings
|
|
||||||
from django.core.validators import validate_email, ValidationError
|
|
||||||
from ietf.utils.draft import PlaintextDraft
|
|
||||||
from ietf.submit.utils import update_authors
|
|
||||||
from ietf.utils.timezone import date_today
|
|
||||||
|
|
||||||
import debug # pyflakes:ignore
|
|
||||||
|
|
||||||
from ietf.doc.models import Document, NewRevisionDocEvent, DocEvent, State
|
|
||||||
from ietf.person.models import Person
|
|
||||||
|
|
||||||
system = Person.objects.get(name="(System)")
|
|
||||||
expired = State.objects.get(type='draft',slug='expired')
|
|
||||||
|
|
||||||
names = set()
|
|
||||||
print 'collecting draft names ...'
|
|
||||||
versions = 0
|
|
||||||
for p in Path(settings.INTERNET_DRAFT_PATH).glob('draft*.txt'):
|
|
||||||
n = str(p).split('/')[-1].split('-')
|
|
||||||
if n[-1][:2].isdigit():
|
|
||||||
name = '-'.join(n[:-1])
|
|
||||||
if '--' in name or '.txt' in name or '[' in name or '=' in name or '&' in name:
|
|
||||||
continue
|
|
||||||
if name.startswith('draft-draft-'):
|
|
||||||
continue
|
|
||||||
if name == 'draft-ietf-trade-iotp-v1_0-dsig':
|
|
||||||
continue
|
|
||||||
if len(n[-1]) != 6:
|
|
||||||
continue
|
|
||||||
if name.startswith('draft-mlee-'):
|
|
||||||
continue
|
|
||||||
names.add('-'.join(n[:-1]))
|
|
||||||
|
|
||||||
count=0
|
|
||||||
print 'iterating through names ...'
|
|
||||||
for name in sorted(names):
|
|
||||||
if not Document.objects.filter(name=name).exists():
|
|
||||||
paths = list(Path(settings.INTERNET_DRAFT_PATH).glob('%s-??.txt'%name))
|
|
||||||
paths.sort()
|
|
||||||
doc = None
|
|
||||||
for p in paths:
|
|
||||||
n = str(p).split('/')[-1].split('-')
|
|
||||||
rev = n[-1][:2]
|
|
||||||
with open(str(p)) as txt_file:
|
|
||||||
raw = txt_file.read()
|
|
||||||
try:
|
|
||||||
text = raw.decode('utf8')
|
|
||||||
except UnicodeDecodeError:
|
|
||||||
text = raw.decode('latin1')
|
|
||||||
try:
|
|
||||||
draft = PlaintextDraft(text, txt_file.name, name_from_source=True)
|
|
||||||
except Exception as e:
|
|
||||||
print name, rev, "Can't parse", p,":",e
|
|
||||||
continue
|
|
||||||
if draft.errors and draft.errors.keys()!=['draftname',]:
|
|
||||||
print "Errors - could not process", name, rev, datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc), draft.errors, draft.get_title().encode('utf8')
|
|
||||||
else:
|
|
||||||
time = datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc)
|
|
||||||
if not doc:
|
|
||||||
doc = Document.objects.create(name=name,
|
|
||||||
time=time,
|
|
||||||
type_id='draft',
|
|
||||||
title=draft.get_title(),
|
|
||||||
abstract=draft.get_abstract(),
|
|
||||||
rev = rev,
|
|
||||||
pages=draft.get_pagecount(),
|
|
||||||
words=draft.get_wordcount(),
|
|
||||||
expires=time+datetime.timedelta(settings.INTERNET_DRAFT_DAYS_TO_EXPIRE),
|
|
||||||
)
|
|
||||||
DocAlias.objects.create(name=doc.name).docs.add(doc)
|
|
||||||
doc.states.add(expired)
|
|
||||||
# update authors
|
|
||||||
authors = []
|
|
||||||
for author in draft.get_author_list():
|
|
||||||
full_name, first_name, middle_initial, last_name, name_suffix, email, country, company = author
|
|
||||||
|
|
||||||
author_name = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip()
|
|
||||||
|
|
||||||
if email:
|
|
||||||
try:
|
|
||||||
validate_email(email)
|
|
||||||
except ValidationError:
|
|
||||||
email = ""
|
|
||||||
|
|
||||||
def turn_into_unicode(s):
|
|
||||||
if s is None:
|
|
||||||
return u""
|
|
||||||
|
|
||||||
if isinstance(s, unicode):
|
|
||||||
return s
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
return s.decode("utf-8")
|
|
||||||
except UnicodeDecodeError:
|
|
||||||
try:
|
|
||||||
return s.decode("latin-1")
|
|
||||||
except UnicodeDecodeError:
|
|
||||||
return ""
|
|
||||||
|
|
||||||
author_name = turn_into_unicode(author_name)
|
|
||||||
email = turn_into_unicode(email)
|
|
||||||
company = turn_into_unicode(company)
|
|
||||||
|
|
||||||
authors.append({
|
|
||||||
"name": author_name,
|
|
||||||
"email": email,
|
|
||||||
"affiliation": company,
|
|
||||||
"country": country
|
|
||||||
})
|
|
||||||
dummysubmission=type('', (), {})() #https://stackoverflow.com/questions/19476816/creating-an-empty-object-in-python
|
|
||||||
dummysubmission.authors = authors
|
|
||||||
update_authors(doc,dummysubmission)
|
|
||||||
|
|
||||||
# add a docevent with words explaining where this came from
|
|
||||||
events = []
|
|
||||||
e = NewRevisionDocEvent.objects.create(
|
|
||||||
type="new_revision",
|
|
||||||
doc=doc,
|
|
||||||
rev=rev,
|
|
||||||
by=system,
|
|
||||||
desc="New version available: <b>%s-%s.txt</b>" % (doc.name, doc.rev),
|
|
||||||
time=time,
|
|
||||||
)
|
|
||||||
events.append(e)
|
|
||||||
e = DocEvent.objects.create(
|
|
||||||
type="comment",
|
|
||||||
doc = doc,
|
|
||||||
rev = rev,
|
|
||||||
by = system,
|
|
||||||
desc = "Revision added from id-archive on %s by %s"%(date_today(),sys.argv[0]),
|
|
||||||
time=time,
|
|
||||||
)
|
|
||||||
events.append(e)
|
|
||||||
doc.time = time
|
|
||||||
doc.rev = rev
|
|
||||||
doc.save_with_history(events)
|
|
||||||
print "Added",name, rev
|
|
||||||
155
dev/legacy/add-old-drafts-from-archive.py
Executable file
155
dev/legacy/add-old-drafts-from-archive.py
Executable file
|
|
@ -0,0 +1,155 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# Copyright The IETF Trust 2017-2019, All Rights Reserved
|
||||||
|
|
||||||
|
## Holding on to this as documentation - it was originally in bin
|
||||||
|
## If resuscitated, it will need to bemake python3, learn about the "rfc" document type
|
||||||
|
## and made into a management command
|
||||||
|
|
||||||
|
# import datetime
|
||||||
|
# import os
|
||||||
|
# import sys
|
||||||
|
# from pathlib import Path
|
||||||
|
# from contextlib import closing
|
||||||
|
|
||||||
|
# os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
|
||||||
|
|
||||||
|
# import django
|
||||||
|
# django.setup()
|
||||||
|
|
||||||
|
# from django.conf import settings
|
||||||
|
# from django.core.validators import validate_email, ValidationError
|
||||||
|
# from ietf.utils.draft import PlaintextDraft
|
||||||
|
# from ietf.submit.utils import update_authors
|
||||||
|
# from ietf.utils.timezone import date_today
|
||||||
|
|
||||||
|
# import debug # pyflakes:ignore
|
||||||
|
|
||||||
|
# from ietf.doc.models import Document, NewRevisionDocEvent, DocEvent, State
|
||||||
|
# from ietf.person.models import Person
|
||||||
|
|
||||||
|
# system = Person.objects.get(name="(System)")
|
||||||
|
# expired = State.objects.get(type='draft',slug='expired')
|
||||||
|
|
||||||
|
# names = set()
|
||||||
|
# print 'collecting draft names ...'
|
||||||
|
# versions = 0
|
||||||
|
# for p in Path(settings.INTERNET_DRAFT_PATH).glob('draft*.txt'):
|
||||||
|
# n = str(p).split('/')[-1].split('-')
|
||||||
|
# if n[-1][:2].isdigit():
|
||||||
|
# name = '-'.join(n[:-1])
|
||||||
|
# if '--' in name or '.txt' in name or '[' in name or '=' in name or '&' in name:
|
||||||
|
# continue
|
||||||
|
# if name.startswith('draft-draft-'):
|
||||||
|
# continue
|
||||||
|
# if name == 'draft-ietf-trade-iotp-v1_0-dsig':
|
||||||
|
# continue
|
||||||
|
# if len(n[-1]) != 6:
|
||||||
|
# continue
|
||||||
|
# if name.startswith('draft-mlee-'):
|
||||||
|
# continue
|
||||||
|
# names.add('-'.join(n[:-1]))
|
||||||
|
|
||||||
|
# count=0
|
||||||
|
# print 'iterating through names ...'
|
||||||
|
# for name in sorted(names):
|
||||||
|
# if not Document.objects.filter(name=name).exists():
|
||||||
|
# paths = list(Path(settings.INTERNET_DRAFT_PATH).glob('%s-??.txt'%name))
|
||||||
|
# paths.sort()
|
||||||
|
# doc = None
|
||||||
|
# for p in paths:
|
||||||
|
# n = str(p).split('/')[-1].split('-')
|
||||||
|
# rev = n[-1][:2]
|
||||||
|
# with open(str(p)) as txt_file:
|
||||||
|
# raw = txt_file.read()
|
||||||
|
# try:
|
||||||
|
# text = raw.decode('utf8')
|
||||||
|
# except UnicodeDecodeError:
|
||||||
|
# text = raw.decode('latin1')
|
||||||
|
# try:
|
||||||
|
# draft = PlaintextDraft(text, txt_file.name, name_from_source=True)
|
||||||
|
# except Exception as e:
|
||||||
|
# print name, rev, "Can't parse", p,":",e
|
||||||
|
# continue
|
||||||
|
# if draft.errors and draft.errors.keys()!=['draftname',]:
|
||||||
|
# print "Errors - could not process", name, rev, datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc), draft.errors, draft.get_title().encode('utf8')
|
||||||
|
# else:
|
||||||
|
# time = datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc)
|
||||||
|
# if not doc:
|
||||||
|
# doc = Document.objects.create(name=name,
|
||||||
|
# time=time,
|
||||||
|
# type_id='draft',
|
||||||
|
# title=draft.get_title(),
|
||||||
|
# abstract=draft.get_abstract(),
|
||||||
|
# rev = rev,
|
||||||
|
# pages=draft.get_pagecount(),
|
||||||
|
# words=draft.get_wordcount(),
|
||||||
|
# expires=time+datetime.timedelta(settings.INTERNET_DRAFT_DAYS_TO_EXPIRE),
|
||||||
|
# )
|
||||||
|
# DocAlias.objects.create(name=doc.name).docs.add(doc)
|
||||||
|
# doc.states.add(expired)
|
||||||
|
# # update authors
|
||||||
|
# authors = []
|
||||||
|
# for author in draft.get_author_list():
|
||||||
|
# full_name, first_name, middle_initial, last_name, name_suffix, email, country, company = author
|
||||||
|
|
||||||
|
# author_name = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip()
|
||||||
|
|
||||||
|
# if email:
|
||||||
|
# try:
|
||||||
|
# validate_email(email)
|
||||||
|
# except ValidationError:
|
||||||
|
# email = ""
|
||||||
|
|
||||||
|
# def turn_into_unicode(s):
|
||||||
|
# if s is None:
|
||||||
|
# return u""
|
||||||
|
|
||||||
|
# if isinstance(s, unicode):
|
||||||
|
# return s
|
||||||
|
# else:
|
||||||
|
# try:
|
||||||
|
# return s.decode("utf-8")
|
||||||
|
# except UnicodeDecodeError:
|
||||||
|
# try:
|
||||||
|
# return s.decode("latin-1")
|
||||||
|
# except UnicodeDecodeError:
|
||||||
|
# return ""
|
||||||
|
|
||||||
|
# author_name = turn_into_unicode(author_name)
|
||||||
|
# email = turn_into_unicode(email)
|
||||||
|
# company = turn_into_unicode(company)
|
||||||
|
|
||||||
|
# authors.append({
|
||||||
|
# "name": author_name,
|
||||||
|
# "email": email,
|
||||||
|
# "affiliation": company,
|
||||||
|
# "country": country
|
||||||
|
# })
|
||||||
|
# dummysubmission=type('', (), {})() #https://stackoverflow.com/questions/19476816/creating-an-empty-object-in-python
|
||||||
|
# dummysubmission.authors = authors
|
||||||
|
# update_authors(doc,dummysubmission)
|
||||||
|
|
||||||
|
# # add a docevent with words explaining where this came from
|
||||||
|
# events = []
|
||||||
|
# e = NewRevisionDocEvent.objects.create(
|
||||||
|
# type="new_revision",
|
||||||
|
# doc=doc,
|
||||||
|
# rev=rev,
|
||||||
|
# by=system,
|
||||||
|
# desc="New version available: <b>%s-%s.txt</b>" % (doc.name, doc.rev),
|
||||||
|
# time=time,
|
||||||
|
# )
|
||||||
|
# events.append(e)
|
||||||
|
# e = DocEvent.objects.create(
|
||||||
|
# type="comment",
|
||||||
|
# doc = doc,
|
||||||
|
# rev = rev,
|
||||||
|
# by = system,
|
||||||
|
# desc = "Revision added from id-archive on %s by %s"%(date_today(),sys.argv[0]),
|
||||||
|
# time=time,
|
||||||
|
# )
|
||||||
|
# events.append(e)
|
||||||
|
# doc.time = time
|
||||||
|
# doc.rev = rev
|
||||||
|
# doc.save_with_history(events)
|
||||||
|
# print "Added",name, rev
|
||||||
Loading…
Reference in a new issue