Import more history and fix import/proxy bugs (enough to get draft-arkko-townsley-coexistence working), more proxying for group/person related things (enough to get left base menu working)

- Legacy-Id: 2714

ietf/idrfc/idrfc_wrapper.py

@@ -622,7 +622,7 @@ class BallotWrapper:
                 p = dict(ad_name=pos.ad.get_name(),
                          ad_username="", # FIXME: don't seem to have username at the moment
                          position=pos.pos.name,
-                         is_old_ad=pos.ad in active_ads,
+                         is_old_ad=pos.ad not in active_ads,
                          old_positions=[])
 
                 if pos.pos.slug == "discuss":

ietf/idrfc/views_doc.py

@@ -148,14 +148,30 @@ def document_main(request, name):
 def _get_history(doc, versions):
     results = []
     if settings.USE_DB_REDESIGN_PROXY_CLASSES:
+        versions = [] # clear versions
         for e in doc._draft.event_set.all().select_related('by').order_by('-time'):
             info = {}
+            if e.type == "new_revision":
+                filename = u"%s-%s" % (e.doc.name, e.newrevision.rev)
+                e.desc = 'New version available: <a href="http://tools.ietf.org/id/%s.txt">%s</a>' % (filename, filename)
+                if int(e.newrevision.rev) != 0:
+                    e.desc += ' (<a href="http://tools.ietf.org/rfcdiff?url2=%s">diff from -%02d</a>)' % (filename, int(e.newrevision.rev) - 1)
+                info["dontmolest"] = True
+                    
             info['text'] = e.desc
             info['by'] = e.by.get_name()
             info['textSnippet'] = truncatewords_html(format_textarea(fill(info['text'], 80)), 25)
-            info['snipped'] = info['textSnippet'][-3:] == "..."
-            e.version = e.doc.rev
+            info['snipped'] = info['textSnippet'][-3:] == "..." and e.type != "new_revision"
             results.append({'comment':e, 'info':info, 'date':e.time, 'is_com':True})
+
+        prev_rev = "00"
+        for o in reversed(results):
+            e = o["comment"]
+            if e.type == "new_revision":
+                e.version = e.newrevision.rev
+            else:
+                e.version = prev_rev
+            prev_rev = e.version
     else:
         if doc.is_id_wrapper:
             comments = DocumentComment.objects.filter(document=doc.tracker_id).exclude(rfc_flag=1)

ietf/idrfc/views_edit.py

@@ -236,7 +236,7 @@ def edit_info(request, name):
                     setattr(obj, attr, r[attr])
 
             diff(doc, 'intended_status', "Intended Status")
-            diff(doc.idinternal, 'status_date', "Status Date")
+            diff(doc.idinternal, 'status_date', "Status date")
             if 'area_acronym' in r and r['area_acronym']:
                 diff(doc.idinternal, 'area_acronym', 'Area acronym')
             diff(doc.idinternal, 'job_owner', 'Responsible AD')

ietf/templates/idrfc/doc_history.html

@@ -65,8 +65,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 {{ c.info.text|fill:"80"|safe|urlize|linebreaksbr|keep_spacing|sanitize_html|safe }}
 </div>
 {% else %}
+{% if c.info.dontmolest %}
+{{ c.info.text|safe }}
+{% else %}
 {{ c.info.text|fill:"80"|safe|urlize|linebreaksbr|keep_spacing|sanitize_html|safe }}
 {% endif %}
+{% endif %}
 </td>
 {% endif %}
 

redesign/doc/admin.py

@@ -40,6 +40,7 @@ admin.site.register(Event, EventAdmin)
 admin.site.register(Message, EventAdmin)
 admin.site.register(Text, EventAdmin)
 admin.site.register(BallotPosition, EventAdmin)
+admin.site.register(Status, EventAdmin)
 admin.site.register(Expiration, EventAdmin)
 admin.site.register(Telechat, EventAdmin)
 

redesign/doc/models.py

@@ -6,6 +6,7 @@ from redesign.name.models import *
 from redesign.person.models import Email
 from redesign.util import admin_link
 
+import datetime
 
 class RelatedDoc(models.Model):
     relationship = models.ForeignKey(DocRelationshipName)
@@ -49,12 +50,13 @@ class DocumentInfo(models.Model):
         abstract = True
     def author_list(self):
         return ", ".join(email.address for email in self.authors.all())
-    def latest_event(self, **filter_args):
-        e = self.event_set.filter(**filter_args).order_by('-time')[:1]
-        if e:
-            return e[0]
-        else:
-            return None
+    def latest_event(self, *args, **filter_args):
+        """Get latest event with specific requirements, e.g.
+        d.latest_event(type="xyz") returns an Event while
+        d.latest_event(Status, type="xyz") returns a Status event."""
+        model = args[0] if args else Event
+        e = model.objects.filter(doc=self).filter(**filter_args).order_by('-time')[:1]
+        return e[0] if e else None
 
 class Document(DocumentInfo):
     name = models.CharField(max_length=255, primary_key=True)           # immutable
@@ -150,6 +152,7 @@ EVENT_TYPES = [
     ("changed_document", "Changed document metadata"),
     
     # misc document events
+    ("added_comment", "Added comment"),
     ("added_tombstone", "Added tombstone"),
     ("requested_resurrect", "Requested resurrect"),
     
@@ -163,6 +166,9 @@ EVENT_TYPES = [
 
     ("changed_last_call_text", "Changed last call text"),
     ("sent_last_call", "Sent last call"),
+    ("requested_last_call", "Requested last call"),
+    
+    ("changed_status_date", "Changed status date"),
     
     ("scheduled_for_telechat", "Scheduled for telechat"),
 
@@ -174,9 +180,9 @@ EVENT_TYPES = [
 
 class Event(models.Model):
     """An occurrence in connection with a document."""
-    time = models.DateTimeField()
+    time = models.DateTimeField(default=datetime.datetime.now, help_text="When the event happened")
     type = models.CharField(max_length=50, choices=EVENT_TYPES)
-    by = models.ForeignKey(Email, blank=True, null=True)
+    by = models.ForeignKey(Email, blank=True, null=True) # FIXME: make NOT NULL?
     doc = models.ForeignKey('doc.Document')
     desc = models.TextField()
 
@@ -192,7 +198,10 @@ class Message(Event):
 
 class Text(Event):
     content = models.TextField(blank=True)
-    
+
+class NewRevision(Event):
+    rev = models.CharField(max_length=16)
+   
 # IESG events
 class BallotPosition(Event):
     ad = models.ForeignKey(Email)
@@ -202,6 +211,9 @@ class BallotPosition(Event):
     comment = models.TextField(help_text="Optional comment", blank=True)
     comment_time = models.DateTimeField(help_text="Time optional comment was written", blank=True, null=True)
     
+class Status(Event):
+    date = models.DateField()
+
 class Expiration(Event):
     expires = models.DateTimeField()
     

redesign/doc/proxy.py

@@ -2,6 +2,10 @@ from models import *
 from redesign.person.models import Email 
 from redesign.proxy_utils import TranslatingManager
 
+from django.conf import settings
+
+import glob, os
+
 
 class InternetDraft(Document):
     objects = TranslatingManager(dict(filename="name",
@@ -12,53 +16,61 @@ class InternetDraft(Document):
 
     # things from InternetDraft
     
-#     id_document_tag = models.AutoField(primary_key=True)
+    #id_document_tag = models.AutoField(primary_key=True)
     @property
     def id_document_tag(self):
         return self.name              # Will only work for some use cases
-#     title = models.CharField(max_length=255, db_column='id_document_name') # same name
-#     id_document_key = models.CharField(max_length=255, editable=False)
+    #title = models.CharField(max_length=255, db_column='id_document_name') # same name
+    #id_document_key = models.CharField(max_length=255, editable=False)
     @property
     def id_document_key(self):
         return self.title.upper()
-#     group = models.ForeignKey(Acronym, db_column='group_acronym_id')
+    #group = models.ForeignKey(Acronym, db_column='group_acronym_id')
     @property
     def group(self):
-        return super(self.__class__, self).group
-#     filename = models.CharField(max_length=255, unique=True)
+        from group.proxy import Acronym as AcronymProxy
+        return AcronymProxy(super(self.__class__, self).group)
+    #filename = models.CharField(max_length=255, unique=True)
     @property
     def filename(self):
         return self.name
-#     revision = models.CharField(max_length=2)
+    #revision = models.CharField(max_length=2)
     @property
     def revision(self):
         return self.rev
-#     revision_date = models.DateField()
+    #revision_date = models.DateField()
     @property
     def revision_date(self):
         e = self.latest_event(type="new_revision")
-        return e.time if e else None
-#     file_type = models.CharField(max_length=20)
+        return e.time.date() if e else None
+    #file_type = models.CharField(max_length=20)
     @property
     def file_type(self):
-        return ".txt"                   # FIXME XXX should look in the repository to see what's available
-#     txt_page_count = models.IntegerField()
+        matches = glob.glob(os.path.join(settings.INTERNET_DRAFT_PATH, self.filename + "*.*"))
+        possible_types = [".txt", ".pdf", ".xml", ".ps"]
+        res = set()
+        for m in matches:
+            for t in possible_types:
+                if m.endswith(t):
+                    res.add(t)
+        return ",".join(res)
+    #txt_page_count = models.IntegerField()
     @property
     def txt_page_count(self):
         return self.pages
-#     local_path = models.CharField(max_length=255, blank=True) # unused
-#     start_date = models.DateField()
+    #local_path = models.CharField(max_length=255, blank=True) # unused
+    #start_date = models.DateField()
     @property
     def start_date(self):
         return self.dochistory_set.dates("time","day","ASC")[0]
-#     expiration_date = models.DateField()
+    #expiration_date = models.DateField()
     @property
     def expiration_date(self):
         return self.expiration()
-#     abstract = models.TextField() # same name
-#     dunn_sent_date = models.DateField(null=True, blank=True) # unused
-#     extension_date = models.DateField(null=True, blank=True) # unused
-#     status = models.ForeignKey(IDStatus)
+    #abstract = models.TextField() # same name
+    #dunn_sent_date = models.DateField(null=True, blank=True) # unused
+    #extension_date = models.DateField(null=True, blank=True) # unused
+    #status = models.ForeignKey(IDStatus)
     @property
     def status(self):
         from redesign.name.proxy import IDStatus
@@ -69,42 +81,42 @@ class InternetDraft(Document):
         from redesign.name.proxy import IDStatus
         return { 'active': 1, 'repl': 5, 'expired': 2, 'rfc': 3, 'auth-rm': 4, 'ietf-rm': 6 }[self.state_id]
         
-#     intended_status = models.ForeignKey(IDIntendedStatus)
+    #intended_status = models.ForeignKey(IDIntendedStatus)
     @property
     def intended_status(self):
         return self.intended_std_level
         
-#     lc_sent_date = models.DateField(null=True, blank=True)
+    #lc_sent_date = models.DateField(null=True, blank=True)
     @property
     def lc_sent_date(self):
         e = self.latest_event(type="sent_last_call")
         return e.time if e else None
         
-#     lc_changes = models.CharField(max_length=3) # used in DB, unused in Django code?
+    #lc_changes = models.CharField(max_length=3) # used in DB, unused in Django code?
         
-#     lc_expiration_date = models.DateField(null=True, blank=True)
+    #lc_expiration_date = models.DateField(null=True, blank=True)
     @property
     def lc_expiration_date(self):
         e = self.latest_event(type="sent_last_call")
         return e.expiration.expires if e else None
         
-#     b_sent_date = models.DateField(null=True, blank=True)
+    #b_sent_date = models.DateField(null=True, blank=True)
     @property
     def b_sent_date(self):
         e = self.latest_event(type="sent_ballot_announcement")
         return e.time if e else None
         
-#     b_discussion_date = models.DateField(null=True, blank=True) # unused
+    #b_discussion_date = models.DateField(null=True, blank=True) # unused
         
-#     b_approve_date = models.DateField(null=True, blank=True)
+    #b_approve_date = models.DateField(null=True, blank=True)
     @property
     def b_approve_date(self):
         e = self.latest_event(type="approved_ballot")
         return e.time if e else None
         
-#     wgreturn_date = models.DateField(null=True, blank=True) # unused
+    #wgreturn_date = models.DateField(null=True, blank=True) # unused
         
-#     rfc_number = models.IntegerField(null=True, blank=True, db_index=True)
+    #rfc_number = models.IntegerField(null=True, blank=True, db_index=True)
     @property
     def rfc_number(self):
         try:
@@ -112,20 +124,20 @@ class InternetDraft(Document):
         except IndexError:
             return None
         
-#     comments = models.TextField(blank=True) # unused
+    #comments = models.TextField(blank=True) # unused
         
-#     last_modified_date = models.DateField()
+    #last_modified_date = models.DateField()
     @property
     def last_modified_date(self):
         return self.time
         
-#     replaced_by = models.ForeignKey('self', db_column='replaced_by', blank=True, null=True, related_name='replaces_set')
+    #replaced_by = models.ForeignKey('self', db_column='replaced_by', blank=True, null=True, related_name='replaces_set')
     @property
     def replaced_by(self):
         r = InternetDraft.objects.filter(docalias__relateddoc__relationship="replaces", docalias__relateddoc__related_document_set=self)
         return r[0] if r else None
         
-#     replaces = FKAsOneToOne('replaces', reverse=True)
+    #replaces = FKAsOneToOne('replaces', reverse=True)
     @property
     def replaces(self):
         r = InternetDraft.objects.filter(related__doc_alias__document=self, related__relationship="replaces")
@@ -137,17 +149,17 @@ class InternetDraft(Document):
         # this is replaced_by
         return InternetDraft.objects.filter(docalias__relateddoc__relationship="replaces", docalias__relateddoc__related_document_set=self)
         
-#     review_by_rfc_editor = models.BooleanField()
+    #review_by_rfc_editor = models.BooleanField()
     @property
     def review_by_rfc_editor(self): raise NotImplemented # should use tag
         
-#     expired_tombstone = models.BooleanField()
+    #expired_tombstone = models.BooleanField()
     @property
     def expired_tombstone(self):
         # FIXME: this is probably not perfect, what happens when we delete it again
         return self.latest_event(type="added_tombstone")
         
-#     idinternal = FKAsOneToOne('idinternal', reverse=True, query=models.Q(rfc_flag = 0))
+    #idinternal = FKAsOneToOne('idinternal', reverse=True, query=models.Q(rfc_flag = 0))
     @property
     def idinternal(self):
         return self if self.iesg_state else None
@@ -246,7 +258,9 @@ class InternetDraft(Document):
     
     #status_date = models.DateField(blank=True,null=True)
     @property
-    def status_date(self): raise NotImplemented # FIXME
+    def status_date(self):
+        e = self.latest_event(Status, type="changed_status_date")
+        return e.date if e else None
 
     #email_display = models.CharField(blank=True, max_length=50) # unused
     #agenda = models.IntegerField(null=True, blank=True)
@@ -276,7 +290,8 @@ class InternetDraft(Document):
     # job_owner = models.ForeignKey(IESGLogin, db_column='job_owner', related_name='documents')
     @property
     def job_owner(self):
-        return self.ad
+        from person.proxy import IESGLogin as IESGLoginProxy
+        return IESGLoginProxy(self.ad)
     
     #event_date = models.DateField(null=True)
     @property
@@ -485,7 +500,7 @@ class InternetDraft(Document):
         
 	yes = noobj = discuss = recuse = 0
 	for position in positions:
-            p = position.pos_id
+            p = position.ballotposition.pos_id
             if p == "yes":
                 yes += 1
             if p == "noobj":
@@ -505,7 +520,7 @@ class InternetDraft(Document):
 	if standardsTrack:
 	    # For standards-track, need positions from 2/3 of the
 	    # non-recused current IESG.
-	    needed = (len(ads) - recuse) * 2 / 3
+	    needed = int((len(ads) - recuse) * 2 / 3)
 	else:
 	    # Info and experimental only need one position.
 	    needed = 1

redesign/group/proxy.py

@@ -1,6 +1,12 @@
 from models import *
 
 class Acronym(Group):
+    class LazyIndividualSubmitter(object):
+        def __get__(self, obj, type=None):
+            return Group.objects.get(acronym="none").id
+    
+    INDIVIDUAL_SUBMITTER = LazyIndividualSubmitter()
+    
     def __init__(self, base):
         for f in base._meta.fields:
             setattr(self, f.name, getattr(base, f.name))
@@ -15,6 +21,12 @@ class Acronym(Group):
     @property
     def name_key(self):
         return self.name.upper()
+
+    def __str__(self):
+        return self.acronym
+
+    def __unicode__(self):
+        return self.acronym
     
     class Meta:
         proxy = True
@@ -35,12 +47,91 @@ class Area(Group):
 
     #def additional_urls(self):
     #    return AreaWGURL.objects.filter(name=self.area_acronym.name)
-    #def active_wgs(self):
-    #    return IETFWG.objects.filter(group_type=1,status=IETFWG.ACTIVE,areagroup__area=self).order_by('group_acronym__acronym')
+    def active_wgs(self):
+        return IETFWG.objects.filter(type="wg", state="active", parent=self).order_by("acronym")
     
     @staticmethod
     def active_areas():
-        return Area.objects.filter(type="area", state="active")
+        return Area.objects.filter(type="area", state="active").order_by('acronym')
+    
+    class Meta:
+        proxy = True
+
+
+class IETFWG(Group):
+    ACTIVE=1
+    #group_acronym = models.OneToOneField(Acronym, primary_key=True, editable=False)
+    @property
+    def group_acronym(self):
+        return Acronym(self)
+    
+    #group_type = models.ForeignKey(WGType)
+    #proposed_date = models.DateField(null=True, blank=True)
+    #start_date = models.DateField(null=True, blank=True)
+    #dormant_date = models.DateField(null=True, blank=True)
+    #concluded_date = models.DateField(null=True, blank=True)
+    #status = models.ForeignKey(WGStatus)
+    #area_director = models.ForeignKey(AreaDirector, null=True)
+    #meeting_scheduled = models.CharField(blank=True, max_length=3)
+    #email_address = models.CharField(blank=True, max_length=60)
+    #email_subscribe = models.CharField(blank=True, max_length=120)
+    #email_keyword = models.CharField(blank=True, max_length=50)
+    #email_archive = models.CharField(blank=True, max_length=95)
+    #comments = models.TextField(blank=True)
+    #last_modified_date = models.DateField()
+    #meeting_scheduled_old = models.CharField(blank=True, max_length=3)
+    #area = FKAsOneToOne('areagroup', reverse=True)
+    def __str__(self):
+	return self.group_acronym.acronym
+
+    def __unicode__(self):
+	return self.group_acronym.acronym
+
+    # everything below here is not fixed yet
+    def active_drafts(self):
+	return self.group_acronym.internetdraft_set.all().filter(status__status="Active")
+    def choices():
+	return [(wg.group_acronym_id, wg.group_acronym.acronym) for wg in IETFWG.objects.all().filter(group_type__type='WG').select_related().order_by('acronym.acronym')]
+    choices = staticmethod(choices)
+    def area_acronym(self):
+        areas = AreaGroup.objects.filter(group__exact=self.group_acronym)
+        if areas:
+            return areas[areas.count()-1].area.area_acronym
+        else:
+            return None
+    def area_directors(self):
+        areas = AreaGroup.objects.filter(group__exact=self.group_acronym)
+        if areas:
+            return areas[areas.count()-1].area.areadirector_set.all()
+        else:
+            return None
+    def chairs(self): # return a set of WGChair objects for this work group
+        return WGChair.objects.filter(group_acronym__exact=self.group_acronym)
+    def secretaries(self): # return a set of WGSecretary objects for this group
+        return WGSecretary.objects.filter(group_acronym__exact=self.group_acronym)
+    def milestones(self): # return a set of GoalMilestone objects for this group
+        return GoalMilestone.objects.filter(group_acronym__exact=self.group_acronym)
+    def rfcs(self): # return a set of Rfc objects for this group
+        return Rfc.objects.filter(group_acronym__exact=self.group_acronym)
+    def drafts(self): # return a set of Rfc objects for this group
+        return InternetDraft.objects.filter(group__exact=self.group_acronym)
+    def charter_text(self): # return string containing WG description read from file
+        # get file path from settings. Syntesize file name from path, acronym, and suffix
+        try:
+            filename = os.path.join(settings.IETFWG_DESCRIPTIONS_PATH, self.group_acronym.acronym) + ".desc.txt"
+            desc_file = open(filename)
+            desc = desc_file.read()
+        except BaseException:    
+            desc =  'Error Loading Work Group Description'
+        return desc
+
+    def additional_urls(self):
+        return AreaWGURL.objects.filter(name=self.group_acronym.acronym)        
+    def clean_email_archive(self):
+        x = self.email_archive
+        # remove "current/" and "maillist.html"
+        x = re.sub("^(http://www\.ietf\.org/mail-archive/web/)([^/]+/)(current/)?([a-z]+\.html)?$", "\\1\\2", x)
+        return x
     
     class Meta:
         proxy = True

redesign/import-document-state.py

@@ -14,14 +14,27 @@ management.setup_environ(settings)
 from redesign.doc.models import *
 from redesign.group.models import *
 from redesign.name.models import *
-from ietf.idtracker.models import InternetDraft, IESGLogin, DocumentComment
+from ietf.idtracker.models import InternetDraft, IESGLogin, DocumentComment, PersonOrOrgInfo
+from ietf.idrfc.models import DraftVersions
+
+import sys
+
+draft_name_to_import = None
+if len(sys.argv) > 1:
+    draft_name_to_import = sys.argv[1]
 
 # assumptions:
 # - groups have been imported
-# - roles have been imported
+# - IESG login emails/roles have been imported
 
 # FIXME: what about RFCs
 
+# Regarding history, we currently don't try to create DocumentHistory
+# objects, we just import the comments as events.
+
+# imports InternetDraft, IDInternal, BallotInfo, Position,
+# IESGComment, IESGDiscuss, DocumentComment, idrfc.DraftVersions
+
 def name(name_class, slug, name, desc=""):
     # create if it doesn't exist, set name
     obj, _ = name_class.objects.get_or_create(slug=slug)
@@ -85,16 +98,6 @@ ballot_position_mapping = {
     None: name(BallotPositionName, 'norecord', 'No record'),
     }
 
-# regexps for parsing document comments
-
-date_re_str = "(?P<year>[0-9][0-9][0-9][0-9])-(?P<month>[0-9][0-9])-(?P<day>[0-9][0-9])"
-def date_in_match(match):
-    return datetime.date(int(match.group('year')), int(match.group('month')), int(match.group('day')))
-
-re_telechat_agenda = re.compile(r"(Placed on|Removed from) agenda for telechat - %s by" % date_re_str)
-re_ballot_position = re.compile(r"\[Ballot Position Update\] (New position, (?P<position>.*), has been recorded (|for (?P<for>.*) )|Position (|for (?P<for2>.*) )has been changed to (?P<position2>.*) from .*)by (?P<by>.*)")
-re_ballot_issued = re.compile(r"Ballot has been issued by")
-re_state_changed = re.compile(r"(State (changed|Changes) to <b>(?P<to>.*)</b> from <b>(?P<from>.*)</b> by|Sub state has been changed to (?P<tosub>.*) from (?P<fromsub>.*))")
 
 # helpers for events
 
@@ -102,10 +105,11 @@ def save_event(doc, event, comment):
     event.time = comment.datetime()
     event.by = iesg_login_to_email(comment.created_by)
     event.doc = doc
-    event.desc = comment.comment_text # FIXME: consider unquoting here
+    if not event.desc:
+        event.desc = comment.comment_text # FIXME: consider unquoting here
     event.save()
 
-iesg_login_cache = {}
+buggy_iesg_logins_cache = {}
 
 # make sure system email exists
 system_email, _ = Email.objects.get_or_create(address="(System)")
@@ -116,13 +120,18 @@ def iesg_login_to_email(l):
     else:
         # fix logins without the right person
         if not l.person:
-            if l.id not in iesg_login_cache:
+            if l.id not in buggy_iesg_logins_cache:
                 logins = IESGLogin.objects.filter(first_name=l.first_name, last_name=l.last_name).exclude(id=l.id)
                 if logins:
-                    iesg_login_cache[l.id] = logins[0]
+                    buggy_iesg_logins_cache[l.id] = logins[0]
                 else:
-                    iesg_login_cache[l.id] = None
-            l = iesg_login_cache[l.id]
+                    persons = PersonOrOrgInfo.objects.filter(first_name=l.first_name, last_name=l.last_name)
+                    if persons:
+                        l.person = persons[0]
+                        buggy_iesg_logins_cache[l.id] = l
+                    else:
+                        buggy_iesg_logins_cache[l.id] = None
+            l = buggy_iesg_logins_cache[l.id]
             
         try:
             return Email.objects.get(address=l.person.email()[1])
@@ -130,9 +139,25 @@ def iesg_login_to_email(l):
             print "MISSING IESG LOGIN", l.person.email()
             return None
     
+# regexps for parsing document comments
+
+date_re_str = "(?P<year>[0-9][0-9][0-9][0-9])-(?P<month>[0-9][0-9])-(?P<day>[0-9][0-9])"
+def date_in_match(match):
+    return datetime.date(int(match.group('year')), int(match.group('month')), int(match.group('day')))
+
+re_telechat_agenda = re.compile(r"(Placed on|Removed from) agenda for telechat - %s by" % date_re_str)
+re_ballot_position = re.compile(r"\[Ballot Position Update\] (New position, (?P<position>.*), has been recorded (|for (?P<for>.*) )|Position (|for (?P<for2>.*) )has been changed to (?P<position2>.*) from .*)by (?P<by>.*)")
+re_ballot_issued = re.compile(r"Ballot has been issued by")
+re_state_changed = re.compile(r"(State (changed|Changes) to <b>(?P<to>.*)</b> from <b>(?P<from>.*)</b> by|Sub state has been changed to (?P<tosub>.*) from (?P<fromsub>.*))")
+re_note_field_cleared = re.compile(r"Note field has been cleared by")
+re_draft_added = re.compile(r"Draft [Aa]dded (by .*)? in state (?P<state>.*)")
+re_last_call_requested = re.compile(r"Last Call was requested")
+re_status_date_changed = re.compile(r"Status [dD]ate has been changed to (<b>)?" + date_re_str)
+
         
 all_drafts = InternetDraft.objects.all().select_related()
-all_drafts = all_drafts.filter(filename="draft-arkko-townsley-coexistence")
+if draft_name_to_import:
+    all_drafts = all_drafts.filter(filename=draft_name_to_import)
 #all_drafts = all_drafts[all_drafts.count() - 1000:]
 
 for o in all_drafts:
@@ -198,6 +223,7 @@ for o in all_drafts:
                 e = BallotPosition()
                 e.type = "changed_ballot_position"
                 e.ad = iesg_login_to_email(c.created_by)
+                e.desc = "[Ballot Position Update] New position, Yes, has been recorded by %s" % e.ad.get_name()
                 last_pos = d.latest_event(type="changed_ballot_position", ballotposition__ad=e.ad)
                 e.pos = ballot_position_mapping["Yes"]
                 e.discuss = last_pos.ballotposition.discuss if last_pos else ""
@@ -251,21 +277,96 @@ for o in all_drafts:
                     c.comment_text = "[Ballot comment]\n" + c.comment_text
                 save_event(d, e, c)
                 handled = True
-                
+
+            # last call requested
+            match = re_last_call_requested.search(c.comment_text)
+            if match:
+                e = Event(type="requested_last_call")
+                save_event(d, e, c)
+                handled = True
+
             # state changes
             match = re_state_changed.search(c.comment_text)
             if match:
-                # we currently don't recreate DocumentHistory
                 e = Event(type="changed_document")
                 save_event(d, e, c)
                 handled = True
 
+            # note cleared
+            match = re_note_field_cleared.search(c.comment_text)
+            if match:
+                e = Event(type="changed_document")
+                save_event(d, e, c)
+                handled = True
+
+            # note cleared
+            match = re_draft_added.search(c.comment_text)
+            if match:
+                e = Event(type="changed_document")
+                save_event(d, e, c)
+                handled = True
+
+            # status date changed
+            match = re_status_date_changed.search(c.comment_text)
+            if match:
+                # FIXME: handle multiple comments in one
+                e = Status(type="changed_status_date", date=date_in_match(match))
+                save_event(d, e, c)
+                handled = True
                 
+            # new version
+            if c.comment_text == "New version available":
+                e = NewRevision(type="new_revision", rev=c.version)
+                save_event(d, e, c)
+                handled = True
+
+            # all others are added as comments
+            if not handled:
+                e = Event(type="added_comment")
+                save_event(d, e, c)
+
+                # stop typical comments from being output
+                typical_comments = ["Who is the Document Shepherd for this document",
+                                    "We understand that this document doesn't require any IANA actions"]
+                for t in typical_comments:
+                    if t in c.comment_text:
+                        handled = True
+                        break
+            
             if not handled:
                 print "couldn't handle %s '%s'" % (c.id, c.comment_text.replace("\n", "").replace("\r", ""))
+
+        # import new revision changes from DraftVersions
+        known_revisions = set(e.newrevision.rev for e in d.event_set.filter(type="new_revision").select_related('newrevision'))
+        for v in DraftVersions.objects.filter(filename=d.name).order_by("revision"):
+            if v.revision not in known_revisions:
+                e = NewRevision(type="new_revision")
+                e.rev = v.revision
+                # we don't have time information in this source, so
+                # hack the seconds to include the revision to ensure
+                # they're ordered correctly
+                e.time = datetime.datetime.combine(v.revision_date, datetime.time(0, 0, int(v.revision)))
+                e.by = system_email
+                e.doc = d
+                e.desc = "New version available"
+                e.save()
+                known_revisions.add(v.revision)
         
-    
-    print "imported", d.name, "state", d.iesg_state
+        # import events that might be missing, we don't know where to
+        # place them but if we don't generate them, we'll be missing
+        # the information completely
+        e = d.latest_event(Status, type="changed_status_date")
+        status_date = e.date if e else None
+        if o.idinternal.status_date != status_date:
+            e = Status(type="changed_status_date", date=o.idinternal.status_date)
+            e.by = system_email
+            e.doc = d
+            e.desc = "Status date has been changed to <b>%s</b> from <b>%s</b>" % (o.idinternal.status_date, status_date)
+            e.save()
+
+        # FIXME: import writeups
+            
+    print "imported", d.name, "S:", d.iesg_state
 
     
 
@@ -316,7 +417,7 @@ class CheckListIDInternal(models.Model):
     token_name = models.CharField(blank=True, max_length=25)
     token_email = models.CharField(blank=True, max_length=255)
 #    note = models.TextField(blank=True)
-    status_date = models.DateField(blank=True,null=True)
+#    status_date = models.DateField(blank=True,null=True)
     email_display = models.CharField(blank=True, max_length=50)
     agenda = models.IntegerField(null=True, blank=True)
 #    cur_state = models.ForeignKey(IDState, db_column='cur_state', related_name='docs')

redesign/import-groups.py

@@ -6,6 +6,8 @@ basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
 sys.path = [ basedir ] + sys.path
 
 from ietf import settings
+settings.USE_DB_REDESIGN_PROXY_CLASSES = False
+
 from django.core import management
 management.setup_environ(settings)
 
@@ -31,6 +33,7 @@ GroupTypeName.objects.get_or_create(slug="area", name="Area")
 GroupTypeName.objects.get_or_create(slug="wg", name="WG")
 GroupTypeName.objects.get_or_create(slug="rg", name="RG")
 GroupTypeName.objects.get_or_create(slug="team", name="Team")
+GroupTypeName.objects.get_or_create(slug="individ", name="Individual")
 # FIXME: what about AG (area group?)?
 
 
@@ -53,7 +56,11 @@ for o in Area.objects.all():
     
 # IETFWG, AreaGroup
 for o in IETFWG.objects.all():
-    group, _ = Group.objects.get_or_create(acronym=o.group_acronym.acronym)
+    try:
+        group = Group.objects.get(acronym=o.group_acronym.acronym)
+    except Group.DoesNotExist:
+        group = Group(acronym=o.group_acronym.acronym)
+        
     group.name = o.group_acronym.name
     # state
     if o.group_type.type == "BOF":
@@ -68,14 +75,25 @@ for o in IETFWG.objects.all():
         s = GroupStateName.objects.get(slug="conclude")
     group.state = s
     # type
-    if o.group_type.type == "team":
+    if o.group_type.type == "TEAM":
         group.type = GroupTypeName.objects.get(slug="team")
-    else:
+    elif o.group_type.type == "AG":
+        # this contains groups like
+        #apptsv, none, saag, iesg, iab, tsvdir, apples, usac, secdir, apparea, null, opsarea, rtgarea, usvarea, genarea, tsvarea, raiarea, dirdir
+        # which we currently just ignore
+        if o.group_acronym.acronym == "none":
+            group.type = GroupTypeName.objects.get(slug="individ")
+        else:
+            if group.id:
+                group.delete()
+            continue
+    else: # PWG/BOF/WG
         group.type = GroupTypeName.objects.get(slug="wg")
 
     if o.area:
-        print "no area for", group.acronym, group.name, group.type, group.state
         group.parent = Group.objects.get(acronym=o.area.area.area_acronym.acronym)
+    else:
+        print "no area for", group.acronym, group.name, group.type, group.state
         
     # FIXME: missing fields from old: proposed_date, start_date, dormant_date, concluded_date, meeting_scheduled, email_address, email_subscribe, email_keyword, email_archive, comments, last_modified_date, meeting_scheduled_old
     

redesign/import-roles.py

@@ -14,7 +14,7 @@ management.setup_environ(settings)
 from redesign.person.models import *
 from redesign.group.models import *
 from redesign.name.models import *
-from ietf.idtracker.models import IESGLogin, AreaDirector
+from ietf.idtracker.models import IESGLogin, AreaDirector, PersonOrOrgInfo
 
 # assumptions:
 #  - groups have been imported
@@ -39,7 +39,7 @@ area_director_role = name(RoleName, "ad", "Area Director")
 
 
 # helpers for creating the objects
-def get_or_create_email(o, old_person):
+def get_or_create_email(o):
     email = o.person.email()[1]
     if not email:
         print "NO EMAIL FOR %s %s" % (o.__class__, o.id)
@@ -67,10 +67,14 @@ for o in IESGLogin.objects.all():
     print "importing IESGLogin", o.id, o.first_name, o.last_name
     
     if not o.person:
-        print "NO PERSON", o.person_id
-        continue
+        persons = PersonOrOrgInfo.objects.filter(first_name=o.first_name, last_name=o.last_name)
+        if persons:
+            o.person = persons[0]
+        else:
+            print "NO PERSON", o.person_id
+            continue
 
-    email = get_or_create_email(o, o.person)
+    email = get_or_create_email(o)
 
     # FIXME: import o.user_level
     # FIXME: import o.login_name, o.user_level
@@ -80,7 +84,7 @@ for o in IESGLogin.objects.all():
 Role.objects.filter(name=area_director_role).delete()
 for o in AreaDirector.objects.all():
     print "importing AreaDirector", o.area, o.person
-    email = get_or_create_email(o, o.person)
+    email = get_or_create_email(o)
     if not o.area:
         print "NO AREA", o.area_id
         continue

redesign/person/proxy.py

@@ -0,0 +1,48 @@
+from models import *
+
+class IESGLogin(Email):
+    def __init__(self, base):
+        for f in base._meta.fields:
+            setattr(self, f.name, getattr(base, f.name))
+    SECRETARIAT_LEVEL = 0
+    AD_LEVEL = 1
+    INACTIVE_AD_LEVEL = 2
+
+    #login_name = models.CharField(blank=True, max_length=255)
+    @property
+    def login_name(self): raise NotImplemented
+    #password = models.CharField(max_length=25)
+    @property
+    def password(self): raise NotImplemented
+    #user_level = models.IntegerField(choices=USER_LEVEL_CHOICES)
+    @property
+    def user_level(self): raise NotImplemented
+    
+    #first_name = models.CharField(blank=True, max_length=25)
+    @property
+    def first_name(self):
+        return self.get_name().split(" ")[0]
+    
+    #last_name = models.CharField(blank=True, max_length=25)
+    @property
+    def last_name(self):
+        return self.get_name().split(" ")[1]
+
+    # FIXME: person isn't wrapped yet
+    #person = BrokenForeignKey(PersonOrOrgInfo, db_column='person_or_org_tag', unique=True, null_values=(0, 888888), null=True)
+
+    # apparently unused
+    #pgp_id = models.CharField(blank=True, null=True, max_length=20)
+    #default_search = models.NullBooleanField()
+    
+    def __str__(self):
+        return self.get_name()
+    def is_current_ad(self):
+	return self in Email.objects.filter(role__name="ad", role__group__state="active")
+    @staticmethod
+    def active_iesg():
+        raise NotImplemented
+	#return IESGLogin.objects.filter(user_level=1,id__gt=1).order_by('last_name')
+    
+    class Meta:
+        proxy = True