Import some attributes, aliases and relationships from the rfc_index_mirror table

- Legacy-Id: 2737
This commit is contained in:
Ole Laursen 2011-01-05 19:26:09 +00:00
parent daf08da04b
commit 34d9f5f890
7 changed files with 216 additions and 111 deletions

View file

@ -287,10 +287,10 @@ class RfcWrapper:
self.draft_name = self._rfcindex.draft
def _rfc_doc_list(self, name):
if (not self._rfcindex) or (not self._rfcindex.__dict__[name]):
if (not self._rfcindex) or (not getattr(self._rfcindex, name)):
return None
else:
s = self._rfcindex.__dict__[name]
s = getattr(self._rfcindex, name)
s = s.replace(",", ", ")
s = re.sub("([A-Z])([0-9])", "\\1 \\2", s)
return s

View file

@ -96,4 +96,8 @@ class DraftVersions(models.Model):
return "DraftVersions"+self.filename+self.revision+str(self.revision_date)
class Meta:
db_table = "draft_versions_mirror"
from django.conf import settings
if settings.USE_DB_REDESIGN_PROXY_CLASSES:
from redesign.doc.proxy import RfcIndex

View file

@ -149,7 +149,8 @@ def _get_history(doc, versions):
results = []
if settings.USE_DB_REDESIGN_PROXY_CLASSES:
versions = [] # clear versions
for e in doc._draft.event_set.all().select_related('by').order_by('-time'):
event_holder = doc._draft if hasattr(doc, "_draft") else doc._rfcindex
for e in event_holder.event_set.all().select_related('by').order_by('-time'):
info = {}
if e.type == "new_revision":
filename = u"%s-%s" % (e.doc.name, e.newrevision.rev)

View file

@ -10,7 +10,7 @@ import datetime
class DocumentInfo(models.Model):
"""Any kind of document. Draft, RFC, Charter, IPR Statement, Liaison Statement"""
time = models.DateTimeField() # should probably have auto_now=True
time = models.DateTimeField(default=datetime.datetime.now) # should probably have auto_now=True
# Document related
type = models.ForeignKey(DocTypeName, blank=True, null=True) # Draft, Agenda, Minutes, Charter, Discuss, Guideline, Email, Review, Issue, Wiki, External ...
title = models.CharField(max_length=255)
@ -103,8 +103,8 @@ class Document(DocumentInfo):
super(Document, self).save(force_insert, force_update)
class RelatedDocHistory(models.Model):
document = models.ForeignKey('DocHistory')
doc_alias = models.ForeignKey('DocAlias', related_name="reversely_related_document_history_set")
document = models.ForeignKey('DocHistory') # source
doc_alias = models.ForeignKey('DocAlias', related_name="reversely_related_document_history_set") # target
relationship = models.ForeignKey(DocRelationshipName)
def __unicode__(self):
return u"%s %s %s" % (self.document.name, self.relationship.name.lower(), self.doc_alias.name)
@ -127,7 +127,8 @@ class DocAlias(models.Model):
return "%s-->%s" % (self.name, self.document.name)
document_link = admin_link("document")
class Meta:
verbose_name_plural = "aliases"
verbose_name = "document alias"
verbose_name_plural = "document aliases"
class SendQueue(models.Model):
time = models.DateTimeField() # Scheduled at this time
@ -165,6 +166,7 @@ EVENT_TYPES = [
("expired_document", "Expired document"),
("requested_resurrect", "Requested resurrect"),
("completed_resurrect", "Completed resurrect"),
("published_rfc", "Published RFC"),
# IESG events
("sent_ballot_announcement", "Sent ballot announcement"),

View file

@ -10,7 +10,8 @@ import glob, os
class InternetDraft(Document):
objects = TranslatingManager(dict(filename="name",
id_document_tag="id",
status="state"))
status="state",
rfc_number=lambda v: ("docalias__name", "rfc%s" % v)))
DAYS_TO_EXPIRE=185
@ -46,7 +47,7 @@ class InternetDraft(Document):
#file_type = models.CharField(max_length=20)
@property
def file_type(self):
matches = glob.glob(os.path.join(settings.INTERNET_DRAFT_PATH, self.filename + "*.*"))
matches = glob.glob(os.path.join(settings.INTERNET_DRAFT_PATH, self.name + "*.*"))
possible_types = [".txt", ".pdf", ".xml", ".ps"]
res = set()
for m in matches:
@ -133,7 +134,7 @@ class InternetDraft(Document):
#replaced_by = models.ForeignKey('self', db_column='replaced_by', blank=True, null=True, related_name='replaces_set')
@property
def replaced_by(self):
r = InternetDraft.objects.filter(related__document=self, related__relateddocument__relationship="replaces")
r = InternetDraft.objects.filter(relateddocument__doc_alias__document=self, relateddocument__relationship="replaces")
return r[0] if r else None
#replaces = FKAsOneToOne('replaces', reverse=True)
@ -144,7 +145,7 @@ class InternetDraft(Document):
@property
def replaces_set(self):
return InternetDraft.objects.filter(docalias__relateddocument__relationship="replaces", docalias__relateddocument__document=self)
return InternetDraft.objects.filter(docalias__relateddocument__document=self, docalias__relateddocument__relationship="replaces")
#review_by_rfc_editor = models.BooleanField()
@property
@ -548,6 +549,71 @@ class InternetDraft(Document):
return answer.rstrip()
# things from RfcIndex
#rfc_number = models.IntegerField(primary_key=True) # already taken care of
#title = models.CharField(max_length=250) # same name
#authors = models.CharField(max_length=250) FIXME
#rfc_published_date = models.DateField()
@property
def rfc_published_date(self):
e = self.latest_event(type="published_rfc")
return e.time.date() if e else None
#current_status = models.CharField(max_length=50,null=True)
@property
def current_status(self):
return self.std_level
#updates = models.CharField(max_length=200,blank=True,null=True)
@property
def updates(self):
return ",".join(sorted("RFC%s" % d.rfc_number for d in InternetDraft.objects.filter(docalias__relateddocument__document=self, docalias__relateddocument__relationship="updates")))
#updated_by = models.CharField(max_length=200,blank=True,null=True)
@property
def updated_by(self):
return ",".join(sorted("RFC%s" % d.rfc_number for d in InternetDraft.objects.filter(relateddocument__doc_alias__document=self, relateddocument__relationship="updates")))
#obsoletes = models.CharField(max_length=200,blank=True,null=True)
@property
def obsoletes(self):
return ",".join(sorted("RFC%s" % d.rfc_number for d in InternetDraft.objects.filter(docalias__relateddocument__document=self, docalias__relateddocument__relationship="obs")))
#obsoleted_by = models.CharField(max_length=200,blank=True,null=True)
@property
def obsoleted_by(self):
return ",".join(sorted("RFC%s" % d.rfc_number for d in InternetDraft.objects.filter(relateddocument__doc_alias__document=self, relateddocument__relationship="obs")))
#also = models.CharField(max_length=50,blank=True,null=True)
@property
def also(self):
aliases = self.docalias_set.filter(models.Q(name__startswith="bcp") |
models.Q(name__startswith="std") |
models.Q(name__startswith="bcp"))
return aliases[0].name.upper() if aliases else None
#draft = models.CharField(max_length=200,null=True)
@property
def draft(self):
if not self.name.startswith("rfc"):
return self.name
else:
return None
#has_errata = models.BooleanField() FIXME
#stream = models.CharField(max_length=15,blank=True,null=True)
@property
def stream(self):
return super(InternetDraft, self).stream.name
#wg = models.CharField(max_length=15,blank=True,null=True) FIXME
#file_formats = models.CharField(max_length=20,blank=True,null=True)
@property
def file_formats(self):
return self.file_type.replace(".", "").replace("txt", "ascii")
class Meta:
proxy = True
RfcIndex = InternetDraft

View file

@ -14,14 +14,14 @@ management.setup_environ(settings)
from redesign.doc.models import *
from redesign.group.models import *
from redesign.name.models import *
from ietf.idtracker.models import InternetDraft, IESGLogin, DocumentComment, PersonOrOrgInfo
from ietf.idrfc.models import DraftVersions
from ietf.idtracker.models import InternetDraft, IESGLogin, DocumentComment, PersonOrOrgInfo, RfcObsolete
from ietf.idrfc.models import RfcIndex, DraftVersions
import sys
draft_name_to_import = None
document_name_to_import = None
if len(sys.argv) > 1:
draft_name_to_import = sys.argv[1]
document_name_to_import = sys.argv[1]
# assumptions:
# - groups have been imported
@ -33,7 +33,8 @@ if len(sys.argv) > 1:
# objects, we just import the comments as events.
# imports InternetDraft, IDInternal, BallotInfo, Position,
# IESGComment, IESGDiscuss, DocumentComment, idrfc.DraftVersions
# IESGComment, IESGDiscuss, DocumentComment, RfcObsolete,
# idrfc.RfcIndex, idrfc.DraftVersions
def name(name_class, slug, name, desc=""):
# create if it doesn't exist, set name and desc
@ -43,10 +44,17 @@ def name(name_class, slug, name, desc=""):
obj.save()
return obj
def alias_doc(name, doc):
DocAlias.objects.filter(name=name).exclude(document=doc).delete()
alias, _ = DocAlias.objects.get_or_create(name=name, document=doc)
return alias
type_draft = name(DocTypeName, "draft", "Draft")
stream_ietf = name(DocStreamName, "ietf", "IETF")
relationship_replaces = name(DocRelationshipName, "replaces", "Replaces")
relationship_updates = name(DocRelationshipName, "updates", "Updates")
relationship_obsoletes = name(DocRelationshipName, "obs", "Obsoletes")
intended_status_mapping = {
"BCP": name(IntendedStatusName, "bcp", "Best Current Practice"),
@ -165,7 +173,7 @@ def iesg_login_to_email(l):
except Email.DoesNotExist:
print "MISSING IESG LOGIN", l.person.email()
return None
# regexps for parsing document comments
date_re_str = "(?P<year>[0-9][0-9][0-9][0-9])-(?P<month>[0-9][0-9])-(?P<day>[0-9][0-9])"
@ -192,8 +200,8 @@ re_state_change_notice = re.compile(r"State Change Notice email list (have been
re_area_acronym_changed = re.compile(r"Area acronymn? has been changed to \w+ from \w+(<b>)?")
all_drafts = InternetDraft.objects.all().select_related()
if draft_name_to_import:
all_drafts = all_drafts.filter(filename=draft_name_to_import)
if document_name_to_import:
all_drafts = all_drafts.filter(filename=document_name_to_import)
#all_drafts = all_drafts[all_drafts.count() - 1000:]
for o in all_drafts:
@ -212,14 +220,14 @@ for o in all_drafts:
d.iesg_state = iesg_state_mapping[o.idinternal.cur_state.state if o.idinternal else None]
# we currently ignore the previous IESG state prev_state
d.iana_state = None
# d.rfc_state =
# d.rfc_state = # FIXME
d.rev = o.revision
d.abstract = o.abstract
d.pages = o.txt_page_count
d.intended_std_level = intended_status_mapping[o.intended_status.intended_status]
# d.std_level =
# d.std_level = # FIXME
# d.authors =
# d.related =
# d.related = # FIXME
d.ad = iesg_login_to_email(o.idinternal.job_owner) if o.idinternal else None
d.shepherd = None
d.notify = o.idinternal.state_change_notice_to or "" if o.idinternal else ""
@ -229,8 +237,7 @@ for o in all_drafts:
d.save()
# make sure our alias is updated
DocAlias.objects.filter(name=d.name).exclude(document=d).delete()
d_alias, _ = DocAlias.objects.get_or_create(name=d.name, document=d)
alias_doc(d.name, d)
# clear already imported events
d.event_set.all().delete()
@ -601,101 +608,110 @@ for o in all_drafts:
sync_tag(o.idinternal and o.idinternal.approved_in_minute, tag_approved_in_minute)
# RFC alias
if o.rfc_number:
rfc_name = "rfc%s" % o.rfc_number
DocAlias.objects.get_or_create(document=d, name=rfc_name)
alias_doc("rfc%s" % o.rfc_number, d)
# FIXME: some RFCs seem to be called rfc1234bis?
if o.replaced_by:
replacement, _ = Document.objects.get_or_create(name=o.replaced_by.filename)
RelatedDocument.objects.get_or_create(document=replacement, doc_alias=d_alias, relationship=relationship_replaces)
# the RFC-related attributes are imported when we handle the RFCs below
print "imported", d.name, " - ", d.iesg_state
# checklist of attributes below: handled attributes are commented out
# now process RFCs
def get_or_create_rfc_document(rfc_number):
name = "rfc%s" % rfc_number
# try to find a draft that can form the base of the document
draft = None
ids = InternetDraft.objects.filter(rfc_number=rfc_number)[:1]
if ids:
draft = ids[0]
else:
r = RfcIndex.objects.get(rfc_number=rfc_number)
# rfcindex occasionally includes drafts that were not
# really submitted to IETF (e.g. April 1st)
if r.draft:
ids = InternetDraft.objects.filter(filename=r.draft)[:1]
if ids:
draft = ids[0]
if draft:
name = draft.filename
d, _ = Document.objects.get_or_create(name=name)
if not name.startswith('rfc'):
# make sure draft also got an alias
alias_doc(name, d)
alias = alias_doc("rfc%s" % rfc_number, d)
return (d, alias)
all_rfcs = RfcIndex.objects.all()
if all_drafts.count() != InternetDraft.objects.count():
if document_name_to_import.startswith("rfc"):
# we wanted to import just an RFC, great
all_rfcs = all_rfcs.filter(rfc_number=document_name_to_import[3:])
else:
# if we didn't process all drafts, limit the RFCs to the ones we
# did process
all_rfcs = all_rfcs.filter(rfc_number__in=set(d.rfc_number for d in all_drafts if d.rfc_number))
for o in all_rfcs:
d, d_alias = get_or_create_rfc_document(o.rfc_number)
# import obsoletes/updates
def make_relation(other_rfc, rel_type, reverse):
other_number = int(other_rfc.replace("RFC", ""))
other, other_alias = get_or_create_rfc_document(other_number)
if reverse:
RelatedDocument.objects.get_or_create(document=other, doc_alias=d_alias, relationship=rel_type)
else:
RelatedDocument.objects.get_or_create(document=d, doc_alias=other_alias, relationship=rel_type)
if o.obsoletes:
for x in o.obsoletes.split(','):
make_relation(x, relationship_obsoletes, False)
if o.obsoleted_by:
for x in o.obsoleted_by.split(','):
make_relation(x, relationship_obsoletes, True)
if o.updates:
for x in o.updates.split(','):
make_relation(x, relationship_updates, False)
if o.updated_by:
for x in o.updated_by.split(','):
make_relation(x, relationship_updates, True)
if o.also:
print o.also
alias_doc(o.also.lower(), d)
print "imported", d_alias.name, " - ", d.rfc_state
sys.exit(0)
#class CheckListInternetDraft(models.Model):
# id_document_tag = models.AutoField(primary_key=True)
# title = models.CharField(max_length=255, db_column='id_document_name')
# id_document_key = models.CharField(max_length=255, editable=False)
# group = models.ForeignKey(Acronym, db_column='group_acronym_id')
# filename = models.CharField(max_length=255, unique=True)
# revision = models.CharField(max_length=2)
# revision_date = models.DateField()
# file_type = models.CharField(max_length=20)
# txt_page_count = models.IntegerField()
# local_path = models.CharField(max_length=255, blank=True, null=True)
# start_date = models.DateField()
# expiration_date = models.DateField(null=True)
# abstract = models.TextField()
# dunn_sent_date = models.DateField(null=True, blank=True)
# extension_date = models.DateField(null=True, blank=True)
# status = models.ForeignKey(IDStatus)
# intended_status = models.ForeignKey(IDIntendedStatus)
# lc_sent_date = models.DateField(null=True, blank=True)
# lc_changes = models.CharField(max_length=3,null=True)
# lc_expiration_date = models.DateField(null=True, blank=True)
# b_sent_date = models.DateField(null=True, blank=True)
# b_discussion_date = models.DateField(null=True, blank=True)
# b_approve_date = models.DateField(null=True, blank=True)
# wgreturn_date = models.DateField(null=True, blank=True)
# rfc_number = models.IntegerField(null=True, blank=True, db_index=True)
# comments = models.TextField(blank=True,null=True)
# last_modified_date = models.DateField()
# replaced_by = BrokenForeignKey('self', db_column='replaced_by', blank=True, null=True, related_name='replaces_set')
# replaces = FKAsOneToOne('replaces', reverse=True)
# review_by_rfc_editor = models.BooleanField()
# expired_tombstone = models.BooleanField()
# idinternal = FKAsOneToOne('idinternal', reverse=True, query=models.Q(rfc_flag = 0))
#class CheckListIDInternal(models.Model):
# draft = models.ForeignKey(InternetDraft, primary_key=True, unique=True, db_column='id_document_tag')
# rfc_flag = models.IntegerField(null=True)
# ballot = models.ForeignKey('BallotInfo', related_name='drafts', db_column="ballot_id")
# primary_flag = models.IntegerField(blank=True, null=True)
# group_flag = models.IntegerField(blank=True, default=0)
# token_name = models.CharField(blank=True, max_length=25)
# token_email = models.CharField(blank=True, max_length=255)
# note = models.TextField(blank=True)
# status_date = models.DateField(blank=True,null=True)
# email_display = models.CharField(blank=True, max_length=50)
# agenda = models.IntegerField(null=True, blank=True)
# cur_state = models.ForeignKey(IDState, db_column='cur_state', related_name='docs')
# prev_state = models.ForeignKey(IDState, db_column='prev_state', related_name='docs_prev')
# assigned_to = models.CharField(blank=True, max_length=25)
# mark_by = models.ForeignKey('IESGLogin', db_column='mark_by', related_name='marked')
# job_owner = models.ForeignKey(IESGLogin, db_column='job_owner', related_name='documents')
# event_date = models.DateField(null=True)
# area_acronym = models.ForeignKey('Area')
# cur_sub_state = BrokenForeignKey('IDSubState', related_name='docs', null=True, blank=True, null_values=(0, -1))
# prev_sub_state = BrokenForeignKey('IDSubState', related_name='docs_prev', null=True, blank=True, null_values=(0, -1))
# returning_item = models.IntegerField(null=True, blank=True)
# telechat_date = models.DateField(null=True, blank=True)
# via_rfc_editor = models.IntegerField(null=True, blank=True)
# state_change_notice_to = models.CharField(blank=True, max_length=255)
# dnp = models.IntegerField(null=True, blank=True)
# dnp_date = models.DateField(null=True, blank=True)
# noproblem = models.IntegerField(null=True, blank=True)
# resurrect_requested_by = BrokenForeignKey('IESGLogin', db_column='resurrect_requested_by', related_name='docsresurrected', null=True, blank=True)
# approved_in_minute = models.IntegerField(null=True, blank=True)
#class CheckListBallotInfo(models.Model):
# ballot = models.AutoField(primary_key=True, db_column='ballot_id')
# active = models.BooleanField()
# an_sent = models.BooleanField()
# an_sent_date = models.DateField(null=True, blank=True)
# an_sent_by = models.ForeignKey('IESGLogin', db_column='an_sent_by', related_name='ansent', null=True)
# defer = models.BooleanField(blank=True)
# defer_by = models.ForeignKey('IESGLogin', db_column='defer_by', related_name='deferred', null=True)
# defer_date = models.DateField(null=True, blank=True)
# approval_text = models.TextField(blank=True)
# last_call_text = models.TextField(blank=True)
# ballot_writeup = models.TextField(blank=True)
# ballot_issued = models.IntegerField(null=True, blank=True)
class RfcIndex(models.Model):
# rfc_number = models.IntegerField(primary_key=True)
title = models.CharField(max_length=250)
authors = models.CharField(max_length=250)
rfc_published_date = models.DateField()
current_status = models.CharField(max_length=50,null=True)
# updates = models.CharField(max_length=200,blank=True,null=True)
# updated_by = models.CharField(max_length=200,blank=True,null=True)
# obsoletes = models.CharField(max_length=200,blank=True,null=True)
# obsoleted_by = models.CharField(max_length=200,blank=True,null=True)
# also = models.CharField(max_length=50,blank=True,null=True)
draft = models.CharField(max_length=200,null=True)
has_errata = models.BooleanField()
stream = models.CharField(max_length=15,blank=True,null=True)
wg = models.CharField(max_length=15,blank=True,null=True)
file_formats = models.CharField(max_length=20,blank=True,null=True)

View file

@ -4,7 +4,17 @@ from django.db.models.query import QuerySet
class TranslatingQuerySet(QuerySet):
def translated_kwargs(self, kwargs):
trans = self.translated_attrs
return dict((trans[k], v) if k in trans else (k, v) for k, v in kwargs.iteritems())
res = dict()
for k, v in kwargs.iteritems():
if k in trans:
t = trans[k]
if callable(t):
t, v = t(v)
res[t] = v
else:
res[k] = v
return res
# overridden methods
def _clone(self, *args, **kwargs):
@ -109,6 +119,12 @@ class TranslatingQuerySet(QuerySet):
return super(self.__class__, self)._update(values, **kwargs)
class TranslatingManager(Manager):
"""Translates keyword arguments for the ORM, for use in proxy
wrapping, e.g. given trans={'foo': 'bar'} it will transform a
lookup of the field foo to a lookup on the field bar. The right
hand side can either be a string or a function which is called
with the right-hand side to transform it."""
def __init__(self, trans):
super(self.__class__, self).__init__()
self.translated_attrs = trans