chore: make some old things kept for documentation a little safer (#6260)
* chore: make some old things kept for documentation a little safer * fix: sh syntax
This commit is contained in:
Robert Sparks
GitHub
parent
f2dc32a310
commit
87f0781ff3
274
bin/add-old-drafts-from-archive.py
Executable file → Normal file
274
bin/add-old-drafts-from-archive.py
Executable file → Normal file
|
|
@ -1,151 +1,157 @@
|
|||
#!/usr/bin/env python
|
||||
# Copyright The IETF Trust 2017-2019, All Rights Reserved
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from contextlib import closing
|
||||
|
||||
os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
|
||||
print("This is only here as documention - please read the file")
|
||||
sys.exit(0)
|
||||
|
||||
import django
|
||||
django.setup()
|
||||
# #!/usr/bin/env python
|
||||
# # Copyright The IETF Trust 2017-2019, All Rights Reserved
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.validators import validate_email, ValidationError
|
||||
from ietf.utils.draft import PlaintextDraft
|
||||
from ietf.submit.utils import update_authors
|
||||
from ietf.utils.timezone import date_today
|
||||
# import datetime
|
||||
# import os
|
||||
# import sys
|
||||
# from pathlib import Path
|
||||
# from contextlib import closing
|
||||
|
||||
import debug # pyflakes:ignore
|
||||
# os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings"
|
||||
|
||||
from ietf.doc.models import Document, NewRevisionDocEvent, DocEvent, State
|
||||
from ietf.person.models import Person
|
||||
# import django
|
||||
# django.setup()
|
||||
|
||||
system = Person.objects.get(name="(System)")
|
||||
expired = State.objects.get(type='draft',slug='expired')
|
||||
# from django.conf import settings
|
||||
# from django.core.validators import validate_email, ValidationError
|
||||
# from ietf.utils.draft import PlaintextDraft
|
||||
# from ietf.submit.utils import update_authors
|
||||
# from ietf.utils.timezone import date_today
|
||||
|
||||
names = set()
|
||||
print 'collecting draft names ...'
|
||||
versions = 0
|
||||
for p in Path(settings.INTERNET_DRAFT_PATH).glob('draft*.txt'):
|
||||
n = str(p).split('/')[-1].split('-')
|
||||
if n[-1][:2].isdigit():
|
||||
name = '-'.join(n[:-1])
|
||||
if '--' in name or '.txt' in name or '[' in name or '=' in name or '&' in name:
|
||||
continue
|
||||
if name.startswith('draft-draft-'):
|
||||
continue
|
||||
if name == 'draft-ietf-trade-iotp-v1_0-dsig':
|
||||
continue
|
||||
if len(n[-1]) != 6:
|
||||
continue
|
||||
if name.startswith('draft-mlee-'):
|
||||
continue
|
||||
names.add('-'.join(n[:-1]))
|
||||
# import debug # pyflakes:ignore
|
||||
|
||||
count=0
|
||||
print 'iterating through names ...'
|
||||
for name in sorted(names):
|
||||
if not Document.objects.filter(name=name).exists():
|
||||
paths = list(Path(settings.INTERNET_DRAFT_PATH).glob('%s-??.txt'%name))
|
||||
paths.sort()
|
||||
doc = None
|
||||
for p in paths:
|
||||
n = str(p).split('/')[-1].split('-')
|
||||
rev = n[-1][:2]
|
||||
with open(str(p)) as txt_file:
|
||||
raw = txt_file.read()
|
||||
try:
|
||||
text = raw.decode('utf8')
|
||||
except UnicodeDecodeError:
|
||||
text = raw.decode('latin1')
|
||||
try:
|
||||
draft = PlaintextDraft(text, txt_file.name, name_from_source=True)
|
||||
except Exception as e:
|
||||
print name, rev, "Can't parse", p,":",e
|
||||
continue
|
||||
if draft.errors and draft.errors.keys()!=['draftname',]:
|
||||
print "Errors - could not process", name, rev, datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc), draft.errors, draft.get_title().encode('utf8')
|
||||
else:
|
||||
time = datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc)
|
||||
if not doc:
|
||||
doc = Document.objects.create(name=name,
|
||||
time=time,
|
||||
type_id='draft',
|
||||
title=draft.get_title(),
|
||||
abstract=draft.get_abstract(),
|
||||
rev = rev,
|
||||
pages=draft.get_pagecount(),
|
||||
words=draft.get_wordcount(),
|
||||
expires=time+datetime.timedelta(settings.INTERNET_DRAFT_DAYS_TO_EXPIRE),
|
||||
)
|
||||
DocAlias.objects.create(name=doc.name).docs.add(doc)
|
||||
doc.states.add(expired)
|
||||
# update authors
|
||||
authors = []
|
||||
for author in draft.get_author_list():
|
||||
full_name, first_name, middle_initial, last_name, name_suffix, email, country, company = author
|
||||
# from ietf.doc.models import Document, NewRevisionDocEvent, DocEvent, State
|
||||
# from ietf.person.models import Person
|
||||
|
||||
author_name = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip()
|
||||
# system = Person.objects.get(name="(System)")
|
||||
# expired = State.objects.get(type='draft',slug='expired')
|
||||
|
||||
if email:
|
||||
try:
|
||||
validate_email(email)
|
||||
except ValidationError:
|
||||
email = ""
|
||||
# names = set()
|
||||
# print 'collecting draft names ...'
|
||||
# versions = 0
|
||||
# for p in Path(settings.INTERNET_DRAFT_PATH).glob('draft*.txt'):
|
||||
# n = str(p).split('/')[-1].split('-')
|
||||
# if n[-1][:2].isdigit():
|
||||
# name = '-'.join(n[:-1])
|
||||
# if '--' in name or '.txt' in name or '[' in name or '=' in name or '&' in name:
|
||||
# continue
|
||||
# if name.startswith('draft-draft-'):
|
||||
# continue
|
||||
# if name == 'draft-ietf-trade-iotp-v1_0-dsig':
|
||||
# continue
|
||||
# if len(n[-1]) != 6:
|
||||
# continue
|
||||
# if name.startswith('draft-mlee-'):
|
||||
# continue
|
||||
# names.add('-'.join(n[:-1]))
|
||||
|
||||
def turn_into_unicode(s):
|
||||
if s is None:
|
||||
return u""
|
||||
# count=0
|
||||
# print 'iterating through names ...'
|
||||
# for name in sorted(names):
|
||||
# if not Document.objects.filter(name=name).exists():
|
||||
# paths = list(Path(settings.INTERNET_DRAFT_PATH).glob('%s-??.txt'%name))
|
||||
# paths.sort()
|
||||
# doc = None
|
||||
# for p in paths:
|
||||
# n = str(p).split('/')[-1].split('-')
|
||||
# rev = n[-1][:2]
|
||||
# with open(str(p)) as txt_file:
|
||||
# raw = txt_file.read()
|
||||
# try:
|
||||
# text = raw.decode('utf8')
|
||||
# except UnicodeDecodeError:
|
||||
# text = raw.decode('latin1')
|
||||
# try:
|
||||
# draft = PlaintextDraft(text, txt_file.name, name_from_source=True)
|
||||
# except Exception as e:
|
||||
# print name, rev, "Can't parse", p,":",e
|
||||
# continue
|
||||
# if draft.errors and draft.errors.keys()!=['draftname',]:
|
||||
# print "Errors - could not process", name, rev, datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc), draft.errors, draft.get_title().encode('utf8')
|
||||
# else:
|
||||
# time = datetime.datetime.fromtimestamp(p.stat().st_mtime, datetime.timezone.utc)
|
||||
# if not doc:
|
||||
# doc = Document.objects.create(name=name,
|
||||
# time=time,
|
||||
# type_id='draft',
|
||||
# title=draft.get_title(),
|
||||
# abstract=draft.get_abstract(),
|
||||
# rev = rev,
|
||||
# pages=draft.get_pagecount(),
|
||||
# words=draft.get_wordcount(),
|
||||
# expires=time+datetime.timedelta(settings.INTERNET_DRAFT_DAYS_TO_EXPIRE),
|
||||
# )
|
||||
# DocAlias.objects.create(name=doc.name).docs.add(doc)
|
||||
# doc.states.add(expired)
|
||||
# # update authors
|
||||
# authors = []
|
||||
# for author in draft.get_author_list():
|
||||
# full_name, first_name, middle_initial, last_name, name_suffix, email, country, company = author
|
||||
|
||||
if isinstance(s, unicode):
|
||||
return s
|
||||
else:
|
||||
try:
|
||||
return s.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
try:
|
||||
return s.decode("latin-1")
|
||||
except UnicodeDecodeError:
|
||||
return ""
|
||||
# author_name = full_name.replace("\n", "").replace("\r", "").replace("<", "").replace(">", "").strip()
|
||||
|
||||
author_name = turn_into_unicode(author_name)
|
||||
email = turn_into_unicode(email)
|
||||
company = turn_into_unicode(company)
|
||||
# if email:
|
||||
# try:
|
||||
# validate_email(email)
|
||||
# except ValidationError:
|
||||
# email = ""
|
||||
|
||||
authors.append({
|
||||
"name": author_name,
|
||||
"email": email,
|
||||
"affiliation": company,
|
||||
"country": country
|
||||
})
|
||||
dummysubmission=type('', (), {})() #https://stackoverflow.com/questions/19476816/creating-an-empty-object-in-python
|
||||
dummysubmission.authors = authors
|
||||
update_authors(doc,dummysubmission)
|
||||
# def turn_into_unicode(s):
|
||||
# if s is None:
|
||||
# return u""
|
||||
|
||||
# if isinstance(s, unicode):
|
||||
# return s
|
||||
# else:
|
||||
# try:
|
||||
# return s.decode("utf-8")
|
||||
# except UnicodeDecodeError:
|
||||
# try:
|
||||
# return s.decode("latin-1")
|
||||
# except UnicodeDecodeError:
|
||||
# return ""
|
||||
|
||||
# author_name = turn_into_unicode(author_name)
|
||||
# email = turn_into_unicode(email)
|
||||
# company = turn_into_unicode(company)
|
||||
|
||||
# authors.append({
|
||||
# "name": author_name,
|
||||
# "email": email,
|
||||
# "affiliation": company,
|
||||
# "country": country
|
||||
# })
|
||||
# dummysubmission=type('', (), {})() #https://stackoverflow.com/questions/19476816/creating-an-empty-object-in-python
|
||||
# dummysubmission.authors = authors
|
||||
# update_authors(doc,dummysubmission)
|
||||
|
||||
# add a docevent with words explaining where this came from
|
||||
events = []
|
||||
e = NewRevisionDocEvent.objects.create(
|
||||
type="new_revision",
|
||||
doc=doc,
|
||||
rev=rev,
|
||||
by=system,
|
||||
desc="New version available: <b>%s-%s.txt</b>" % (doc.name, doc.rev),
|
||||
time=time,
|
||||
)
|
||||
events.append(e)
|
||||
e = DocEvent.objects.create(
|
||||
type="comment",
|
||||
doc = doc,
|
||||
rev = rev,
|
||||
by = system,
|
||||
desc = "Revision added from id-archive on %s by %s"%(date_today(),sys.argv[0]),
|
||||
time=time,
|
||||
)
|
||||
events.append(e)
|
||||
doc.time = time
|
||||
doc.rev = rev
|
||||
doc.save_with_history(events)
|
||||
print "Added",name, rev
|
||||
# # add a docevent with words explaining where this came from
|
||||
# events = []
|
||||
# e = NewRevisionDocEvent.objects.create(
|
||||
# type="new_revision",
|
||||
# doc=doc,
|
||||
# rev=rev,
|
||||
# by=system,
|
||||
# desc="New version available: <b>%s-%s.txt</b>" % (doc.name, doc.rev),
|
||||
# time=time,
|
||||
# )
|
||||
# events.append(e)
|
||||
# e = DocEvent.objects.create(
|
||||
# type="comment",
|
||||
# doc = doc,
|
||||
# rev = rev,
|
||||
# by = system,
|
||||
# desc = "Revision added from id-archive on %s by %s"%(date_today(),sys.argv[0]),
|
||||
# time=time,
|
||||
# )
|
||||
# events.append(e)
|
||||
# doc.time = time
|
||||
# doc.rev = rev
|
||||
# doc.save_with_history(events)
|
||||
# print "Added",name, rev
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue