From 814e1315b50e4d1aec6f2002611d0446279d31a7 Mon Sep 17 00:00:00 2001
From: Henrik Levkowetz <henrik@levkowetz.com>
Date: Mon, 20 Mar 2017 14:08:52 +0000
Subject: [PATCH] Added a new document tab for some documents, showing htmlized
 drafts and RFCs using the htmlization code previously developed for
 tools.ietf.org.  As the generation of the htmlized page is a bit too costly
 to do on the fly for often-referenced drafts and RFCs, the part of each page
 which contains the htmlized document is cached on file with a cache time of 2
 weeks.

Changed all links which pointed to the htmlized version on tools to instead
point at the datatracker htmlized document.

Tweaked some URLs which didn't permit retrieval of intermediate-rev-charters.

Narrowed the pattern for document names to disallow dots in names, and
instead explicitly enumerated the few historical draftw with dots in the
name.

Added a file-system cache for the htmlized documents, and specified a
max_entries value for caches, overriding the default 300 entries.

Tweaked the code for new author email entries to provide a time if missing
in an updated entry.

Changed links in various email templates which pointed at tools.ietf.org
pages to instead point at datatracker pages, where appropriate.

Changed the search result rows to provide links to both the current meta-
information document pages (with a (i) info symbol) and to the new htmlized
document pages.
 - Legacy-Id: 13040
---
 ietf/doc/templatetags/ietf_filters.py         |   2 +-
 ietf/doc/tests.py                             |   9 ++
 ietf/doc/urls.py                              |   5 +-
 ietf/doc/urls_charter.py                      |   4 +-
 ietf/doc/utils.py                             | 129 ++++++++++++++++++
 ietf/doc/views_doc.py                         |  42 +++++-
 ietf/settings.py                              |  34 ++++-
 ietf/settings_testcrawl.py                    |  13 +-
 ietf/static/ietf/css/ietf.css                 | 116 +++++++++++++++-
 ietf/submit/utils.py                          |   2 +
 ietf/templates/base.html                      |  12 +-
 ietf/templates/doc/document_bibtex.bib        |   2 +-
 ietf/templates/doc/document_draft.html        |  20 ++-
 ietf/templates/doc/document_html.html         |  69 ++++++++++
 ietf/templates/doc/document_top.html          |   2 +-
 .../doc/search/search_result_row.html         |   4 +-
 ietf/templates/group/customize_workflow.html  |   2 +-
 .../templates/submit/announce_new_version.txt |   2 +-
 ietf/templates/submit/announce_to_authors.txt |   4 +-
 ietf/templates/submit/announce_to_lists.txt   |   4 +-
 20 files changed, 444 insertions(+), 33 deletions(-)
 create mode 100644 ietf/templates/doc/document_html.html

diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py
index 1deca4466..c5ee9b15e 100644
--- a/ietf/doc/templatetags/ietf_filters.py
+++ b/ietf/doc/templatetags/ietf_filters.py
@@ -197,7 +197,7 @@ def rfclink(string):
     URL for that RFC.
     """
     string = str(string);
-    return "https://tools.ietf.org/html/rfc" + string;
+    return "/doc/html/rfc" + string;
 
 @register.filter(name='urlize_ietf_docs', is_safe=True, needs_autoescape=True)
 def urlize_ietf_docs(string, autoescape=None):
diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py
index 81de8b8ce..a4b629620 100644
--- a/ietf/doc/tests.py
+++ b/ietf/doc/tests.py
@@ -493,6 +493,15 @@ Man                    Expires September 22, 2015               [Page 3]
         self.assertTrue("Show full document text" in unicontent(r))
         self.assertFalse("Deimos street" in unicontent(r))
 
+        r = self.client.get(urlreverse("ietf.doc.views_doc.document_html", kwargs=dict(name=draft.name)))
+        self.assertEqual(r.status_code, 200)
+        self.assertTrue("Versions:" in unicontent(r))
+        self.assertTrue("Deimos street" in unicontent(r))
+        q = PyQuery(r.content)
+        self.assertEqual(len(q('.rfcmarkup pre')), 4)
+        self.assertEqual(len(q('span.h1')), 2)
+        self.assertEqual(len(q('a[href]')), 116)
+
         # expired draft
         draft.set_state(State.objects.get(type="draft", slug="expired"))
 
diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py
index 69fa7ad55..f5ca91a87 100644
--- a/ietf/doc/urls.py
+++ b/ietf/doc/urls.py
@@ -59,13 +59,14 @@ urlpatterns = [
     url(r'^stats/newrevisiondocevent/data/?$', views_stats.chart_data_newrevisiondocevent),
     url(r'^stats/person/(?P<id>[0-9]+)/drafts/conf/?$', views_stats.chart_conf_person_drafts),
     url(r'^stats/person/(?P<id>[0-9]+)/drafts/data/?$', views_stats.chart_data_person_drafts),
+    url(r'^html/%(name)s(?:-%(rev)s)?(\.txt|\.html)?$' % settings.URL_REGEXPS, views_doc.document_html),
 
     url(r'^all/$', views_search.index_all_drafts),
     url(r'^active/$', views_search.index_active_drafts),
     url(r'^select2search/(?P<model_name>(document|docalias))/(?P<doc_type>draft)/$', views_search.ajax_select2_search_docs),
 
-    url(r'^%(name)s/(?:%(rev)s/)?$' % settings.URL_REGEXPS, views_doc.document_main),
-    url(r'^%(name)s/(?:%(rev)s/)?bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex),
+    url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main),
+    url(r'^%(name)s(?:/%(rev)s)?/bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex),
     url(r'^%(name)s/history/$' % settings.URL_REGEXPS, views_doc.document_history),
     url(r'^%(name)s/writeup/$' % settings.URL_REGEXPS, views_doc.document_writeup),
     url(r'^%(name)s/email/$' % settings.URL_REGEXPS, views_doc.document_email),
diff --git a/ietf/doc/urls_charter.py b/ietf/doc/urls_charter.py
index 5daf4092d..9e0f4bb1f 100644
--- a/ietf/doc/urls_charter.py
+++ b/ietf/doc/urls_charter.py
@@ -1,7 +1,5 @@
 # Copyright The IETF Trust 2011, All Rights Reserved
 
-from django.conf import settings
-
 from ietf.doc import views_charter, views_doc
 from ietf.utils.urls import url
 
@@ -17,5 +15,5 @@ urlpatterns = [
     url(r'^ballotwriteupnotes/$', views_charter.ballot_writeupnotes),
     url(r'^approve/$', views_charter.approve),
     url(r'^submit/(?:(?P<option>initcharter|recharter)/)?$', views_charter.submit),
-    url(r'^withmilestones-%(rev)s.txt$' % settings.URL_REGEXPS, views_charter.charter_with_milestones_txt),
+    url(r'^withmilestones-(?P<rev>[0-9-]{2,5}).txt$', views_charter.charter_with_milestones_txt),
 ]
diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py
index 2e002b851..81223da11 100644
--- a/ietf/doc/utils.py
+++ b/ietf/doc/utils.py
@@ -12,6 +12,8 @@ from django.forms import ValidationError
 from django.utils.html import escape
 from django.core.urlresolvers import reverse as urlreverse
 
+import debug                            # pyflakes:ignore
+
 from ietf.doc.models import Document, DocHistory, State, DocumentAuthor, DocHistoryAuthor
 from ietf.doc.models import DocAlias, RelatedDocument, RelatedDocHistory, BallotType, DocReminder
 from ietf.doc.models import DocEvent, ConsensusDocEvent, BallotDocEvent, NewRevisionDocEvent, StateDocEvent
@@ -705,3 +707,130 @@ def get_search_cache_key(params):
     key = "doc:document:search:" + hashlib.sha512(json.dumps(kwargs, sort_keys=True)).hexdigest()
     return key
     
+def label_wrap(label, items, joiner=',', max=50):
+    lines = []
+    if not items:
+        return lines
+    line = '%s: %s' % (label, items[0])
+    for item in items[1:]:
+        if len(line)+len(joiner+' ')+len(item) > max:
+            lines.append(line+joiner)
+            line = ' '*(len(label)+len(': ')) + item
+        else:
+            line += joiner+' '+item
+    if line:
+        lines.append(line)
+    return lines
+
+def join_justified(left, right, width=72):
+    count = max(len(left), len(right))
+    left = left + ['']*(count-len(left))
+    right = right + ['']*(count-len(right))
+    lines = []
+    i = 0
+    while True:
+        l = left[i]
+        r = right[i]
+        if len(l)+1+len(r) > width:
+            left = left + ['']
+            right = right[:i] + [''] + right[i:]
+            r = right[i]
+            count += 1
+        lines.append( l + ' ' + r.rjust(width-len(l)-1) )
+        i += 1
+        if i >= count:
+            break
+    return lines
+
+def build_doc_meta_block(doc, path):
+    def add_markup(path, doc, lines):
+        is_hst = doc.is_dochistory()
+        rev = doc.rev
+        if is_hst:
+            doc = doc.doc
+        name = doc.name
+        rfcnum = doc.rfc_number()
+        errata_url = settings.RFC_EDITOR_ERRATA_URL.format(rfc_number=rfcnum) if not is_hst else ""
+        ipr_url = "%s?submit=draft&amp;id=%s" % (urlreverse('ietf.ipr.views.search'), name)
+        for i, line in enumerate(lines):
+            # add draft links
+            line = re.sub(r'\b(draft-[-a-z0-9]+)\b', '<a href="%s/\g<1>">\g<1></a>'%(path, ), line)
+            # add rfcXXXX to RFC links
+            line = re.sub(r' (rfc[0-9]+)\b', ' <a href="%s/\g<1>">\g<1></a>'%(path, ), line)
+            # add XXXX to RFC links
+            line = re.sub(r' ([0-9]{3,5})\b', ' <a href="%s/rfc\g<1>">\g<1></a>'%(path, ), line)
+            # add draft revision links
+            line = re.sub(r' ([0-9]{2})\b', ' <a href="%s/%s-\g<1>">\g<1></a>'%(path, name, ), line)
+            if rfcnum:
+                # add errata link
+                line = re.sub(r'Errata exist', '<a class="text-warning" href="%s">Errata exist</a>'%(errata_url, ), line)
+            if is_hst or not rfcnum:
+                # make current draft rev bold
+                line = re.sub(r'>(%s)<'%rev, '><b>\g<1></b><', line)
+            line = re.sub(r'IPR declarations', '<a class="text-warning" href="%s">IPR declarations</a>'%(ipr_url, ), line)
+            line = line.replace(r'[txt]', '[<a href="%s">txt</a>]' % doc.href())
+            lines[i] = line
+        return lines
+    #
+    now = datetime.datetime.now()
+    draft_state = doc.get_state('draft')
+    block = ''
+    meta = {}
+    if doc.type_id == 'draft':
+        revisions = []
+        ipr = doc.related_ipr()
+        if ipr:
+            meta['ipr'] = [ "IPR declarations" ]
+        if doc.is_rfc() and not doc.is_dochistory():
+            if not doc.name.startswith('rfc'):
+                meta['from'] = [ "%s-%s"%(doc.name, doc.rev) ]
+            meta['errata'] = [ "Errata exist" ] if doc.tags.filter(slug='errata').exists() else []
+            meta['obsoletedby'] = [ alias.document.rfc_number() for alias in doc.related_that('obs') ]
+            meta['obsoletedby'].sort()
+            meta['updatedby'] = [ alias.document.rfc_number() for alias in doc.related_that('updates') ]
+            meta['updatedby'].sort()
+            meta['stdstatus'] = [ doc.std_level.name ]
+        else:
+            dd = doc.doc if doc.is_dochistory() else doc
+            revisions += [ '(%s)%s'%(d.name, ' '*(2-((len(d.name)-1)%3))) for d in dd.replaces() ]
+            revisions += doc.revisions()
+            if doc.is_dochistory() and doc.doc.is_rfc():
+                revisions += [ doc.doc.canonical_name() ]
+            else:
+                revisions += [ d.name for d in doc.replaced_by() ]
+            meta['versions'] = revisions
+            if not doc.is_dochistory and draft_state.slug == 'active' and now > doc.expires:
+                # Active past expiration date
+                meta['active'] = [ 'Document is active' ]
+                meta['state' ] = [ doc.friendly_state() ]
+            intended_std = doc.intended_std_level if doc.intended_std_level else None
+            if intended_std:
+                if intended_std.slug in ['ps', 'ds', 'std']:
+                    meta['stdstatus'] = [ "Standards Track" ]
+                else:
+                    meta['stdstatus'] = [ intended_std.name ]
+    elif doc.type_id == 'charter':
+        meta['versions'] = doc.revisions()
+    #
+    # Add markup to items that needs it.
+    if 'versions' in meta:
+        meta['versions'] = label_wrap('Versions', meta['versions'], joiner="")
+    for label in ['Obsoleted by', 'Updated by', 'From' ]:
+        item = label.replace(' ','').lower()
+        if item in meta and meta[item]:
+            meta[item] = label_wrap(label, meta[item])
+    #
+    left = []
+    right = []
+    #right = [ '[txt]']
+    for item in [ 'from', 'versions', 'obsoletedby', 'updatedby', ]:
+        if item in meta and meta[item]:
+            left += meta[item]
+    for item in ['stdstatus', 'active', 'state', 'ipr', 'errata', ]:
+        if item in meta and meta[item]:
+            right += meta[item]
+    lines = join_justified(left, right)
+    block = '\n'.join(add_markup(path, doc, lines))
+    #
+    return block
+
diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py
index 50fa41097..75202f440 100644
--- a/ietf/doc/views_doc.py
+++ b/ietf/doc/views_doc.py
@@ -50,7 +50,7 @@ from ietf.doc.utils import ( add_links_in_new_revision_events, augment_events_wi
     can_adopt_draft, get_chartering_type, get_document_content, get_tags_for_stream_id,
     needed_ballot_positions, nice_consensus, prettify_std_name, update_telechat, has_same_ballot,
     get_initial_notify, make_notify_changed_event, crawl_history, default_consensus,
-    add_events_message_info, get_unicode_document_content)
+    add_events_message_info, get_unicode_document_content, build_doc_meta_block)
 from ietf.community.utils import augment_docs_with_tracking_info
 from ietf.group.models import Role
 from ietf.group.utils import can_manage_group_type, can_manage_materials
@@ -67,9 +67,13 @@ from ietf.review.models import ReviewRequest
 from ietf.review.utils import can_request_review_of_doc, review_requests_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
 
+
 def render_document_top(request, doc, tab, name):
     tabs = []
-    tabs.append(("Document", "document", urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=name)), True, None))
+    tabs.append(("Status", "status", urlreverse("ietf.doc.views_doc.document_main", kwargs=dict(name=name)), True, None))
+
+    if doc.type_id in ["draft", "charter", ]:
+        tabs.append(("Document", "document", urlreverse("ietf.doc.views_doc.document_html", kwargs=dict(name=name)), True, None))
 
     ballot = doc.latest_event(BallotDocEvent, type="created_ballot")
     if doc.type_id in ("draft","conflrev", "statchg"):
@@ -141,7 +145,7 @@ def document_main(request, name, rev=None):
     # set this after we've found the right doc instance
     group = doc.group
 
-    top = render_document_top(request, doc, "document", name)
+    top = render_document_top(request, doc, "status", name)
 
 
     telechat = doc.latest_event(TelechatDocEvent, type="scheduled_for_telechat")
@@ -597,9 +601,39 @@ def document_main(request, name, rev=None):
                            other_reviews=other_reviews,
                       ))
 
-    raise Http404
+    raise Http404("Document not found: %s" % (name + ("-%s"%rev if rev else "")))
 
 
+def document_html(request, name, rev=None):
+    if name.startswith('rfc0'):
+        name = "rfc" + name[3:].lstrip('0')
+    if name.startswith('review-') and re.search('-\d\d\d\d-\d\d$', name):
+        name = "%s-%s" % (name, rev)
+    docs = Document.objects.filter(docalias__name=name)
+    if not docs.exists():
+        # handle some special cases, like draft-ietf-tsvwg-ieee-802-11
+        name = '%s-%s' % (name, rev)
+        rev=None
+        docs = Document.objects.filter(docalias__name=name)
+    doc = docs.get()
+
+    if not os.path.exists(doc.get_file_name()):
+        raise Http404("Document not found: %s" % doc.get_base_name())
+
+    top = render_document_top(request, doc, "document", name)
+    if not rev and not name.startswith('rfc'):
+        rev = doc.rev
+    if rev:
+        docs = DocHistory.objects.filter(doc=doc, rev=rev)
+        if docs.exists():
+            doc = docs.first()
+        else:
+            doc = doc.fake_history_obj(rev)
+    if doc.type_id in ['draft',]:
+        doc.meta = build_doc_meta_block(doc, settings.HTMLIZER_URL_PREFIX)
+
+    return render(request, "doc/document_html.html", {"doc":doc, "top":top, "navbar_mode":"navbar-static-top",  })
+
 def check_doc_email_aliases():
     pattern = re.compile('^expand-(.*?)(\..*?)?@.*? +(.*)$')
     good_count = 0
diff --git a/ietf/settings.py b/ietf/settings.py
index 13d253d52..b8b6e9713 100644
--- a/ietf/settings.py
+++ b/ietf/settings.py
@@ -61,7 +61,7 @@ ALLOWED_HOSTS = [".ietf.org", ".ietf.org.", "209.208.19.216", "4.31.198.44", ]
 TOOLS_SERVER = 'tools.' + IETF_DOMAIN
 TOOLS_SERVER_URL = 'https://' + TOOLS_SERVER
 TOOLS_ID_PDF_URL = TOOLS_SERVER_URL + '/pdf/'
-TOOLS_ID_HTML_URL = TOOLS_SERVER_URL + '/html/'
+TOOLS_ID_HTML_URL = '/doc/html/'
 
 # Override this in the settings_local.py file:
 SERVER_EMAIL = 'Django Server <django-project@' + TOOLS_SERVER + '>'
@@ -524,12 +524,14 @@ GROUP_STATES_WITH_EXTRA_PROCESSING = ["sub-pub", "rfc-edit", ]
 DATE_FORMAT = "Y-m-d"
 DATETIME_FORMAT = "Y-m-d H:i T"
 
+
+DRAFT_NAMES_WITH_DOT = "(draft-[a-z-]+-(ion-sig-uni4.0|pilc-2.5g3g|trade-iotp-v1.0-[a-z]+|msword-template-v2.0))"
 URL_REGEXPS = {
     "acronym": r"(?P<acronym>[-a-z0-9]+)",
     "charter": r"(?P<name>charter-[-a-z0-9]+)",
     "date": r"(?P<date>\d{4}-\d{2}-\d{2})",
-    "name": r"(?P<name>[A-Za-z0-9._+-]+)",
-    "rev": r"(?P<rev>[0-9-]+)",
+    "name": r"(?P<name>([A-Za-z0-9_+-]+?|%s))" % DRAFT_NAMES_WITH_DOT,
+    "rev": r"(?P<rev>[0-9-]{2})",
     "owner": r"(?P<owner>[-A-Za-z0-9\'+._]+@[A-Za-z0-9-._]+)",
     "schedule_name": r"(?P<name>[A-Za-z0-9-:_]+)",
 }
@@ -595,9 +597,22 @@ CACHES = {
     'default': {
         'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
         'LOCATION': '127.0.0.1:11211',
-    }
+        'OPTIONS': {
+            'MAX_ENTRIES': 10000,       # 10,000
+        },
+    },
+    'htmlized': {
+        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+        'LOCATION': '/var/cache/datatracker/htmlized',
+        'OPTIONS': {
+            'MAX_ENTRIES': 100000,      # 100,000
+        },
+    },
 }
 
+HTMLIZER_VERSION = 1
+HTMLIZER_URL_PREFIX = "/doc/html"
+
 IPR_EMAIL_FROM = 'ietf-ipr@ietf.org'
 AUDIO_IMPORT_EMAIL = ['agenda@ietf.org']
 IANA_EVAL_EMAIL = "drafts-eval@icann.org"
@@ -613,6 +628,7 @@ RFC_EDITOR_SYNC_PASSWORD="secret"
 RFC_EDITOR_SYNC_NOTIFICATION_URL = "https://www.rfc-editor.org/parser/parser.php"
 RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml"
 RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml"
+RFC_EDITOR_ERRATA_URL = "https://www.rfc-editor.org/errata_search.php?rfc={rfc_number}&amp;rec_status=0"
 
 # NomCom Tool settings
 ROLODEX_URL = ""
@@ -889,7 +905,15 @@ if SERVER_MODE != 'production':
     CACHES = {
          'default': {
              'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
-         }
+         },
+        'htmlized': {
+            'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
+            #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+            'LOCATION': '/var/cache/datatracker/htmlized',
+            'OPTIONS': {
+                'MAX_ENTRIES': 1000,
+            },
+        }
     }
     SESSION_ENGINE = "django.contrib.sessions.backends.db"
 
diff --git a/ietf/settings_testcrawl.py b/ietf/settings_testcrawl.py
index 4bc439ce1..36533014f 100644
--- a/ietf/settings_testcrawl.py
+++ b/ietf/settings_testcrawl.py
@@ -19,7 +19,18 @@ TEMPLATES[0]['OPTIONS']['loaders'] = (
 CACHES = {
     'default': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-    }
+        'OPTIONS': {
+            'MAX_ENTRIES': 10000,
+        },
+    },
+    'htmlized': {
+        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
+        #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
+        'LOCATION': '/var/cache/datatracker/htmlized',
+        'OPTIONS': {
+            'MAX_ENTRIES': 100000,
+        },
+    },
 }
 
 PASSWORD_HASHERS = ( 'django.contrib.auth.hashers.MD5PasswordHasher', )
diff --git a/ietf/static/ietf/css/ietf.css b/ietf/static/ietf/css/ietf.css
index d7053b7dd..489c5c87e 100644
--- a/ietf/static/ietf/css/ietf.css
+++ b/ietf/static/ietf/css/ietf.css
@@ -6,7 +6,7 @@
  */
 
 /* Passing for static navbar; see http://getbootstrap.com/components/#navbar-fixed-top */
-body { padding-top: 70px; }
+body { padding-top: 72px; }
 
 /* Browse Happy prompt */
 .browsehappy {
@@ -101,7 +101,12 @@ body { padding-top: 70px; }
 h2.anchor-target:before,
 h3.anchor-target:before,
 h4.anchor-target:before,
+.h1.anchor-target:before,
+.h2.anchor-target:before,
+.h3.anchor-target:before,
+.h4.anchor-target:before,
 tr th.anchor-target:before,
+span.anchor-target:before,
 div.anchor-target:before {
   content: '';
   display: block;
@@ -110,7 +115,10 @@ div.anchor-target:before {
   height: 65px;
   margin-top: -65px;
 }
-div.anchor-target { z-index: 0; }
+div.anchor-target {
+  z-index: 0;
+  height: 0px;
+}
 
 /* Make the panel title font normally large */
 .panel-title { font-size: 14px }
@@ -715,3 +723,107 @@ blockquote {
   padding-top: 4px;
   padding-right: 4px;
 } 
+
+
+.symbol-link,
+.symbol-link a:link,
+.symbol-link a:visited {
+  padding-right: 0.5em;
+}
+
+
+.rfcmarkup div {
+  margin-top: 1em;
+}
+.rfcmarkup pre {
+	font-size: 10.5pt;
+	margin-right: 0;
+	margin-left: 0;
+	border: 0;
+	margin: 0;
+	padding: 0;
+	padding-bottom: 1em;
+	background-color: white;
+	line-height: 1.12;
+}
+
+.rfcmarkup pre span.h1,
+.rfcmarkup pre span.h2,
+.rfcmarkup pre span.h3,
+.rfcmarkup pre span.h4,
+.rfcmarkup pre span.h5,
+.rfcmarkup pre span.h6 {
+	font-weight: bold;
+	line-height: 0pt;
+	display: inline;
+	white-space: pre;
+	font-family: monospace;
+	font-size: 1em;
+	font-weight: bold;
+
+}
+.rfcmarkup pre span.invisible {
+	text-decoration: none;
+	color: white;
+}
+
+.rfcmarkup pre a { text-decoration: underline; }
+
+.rfcmarkup pre .grey,
+.rfcmarkup pre .grey a:link,
+.rfcmarkup pre .grey a:visited {
+	color: #777;
+}
+
+.rfcmarkup pre.meta-info {
+  padding: 0.5em;
+  margin-left: -0.5em;
+  margin-bottom: 0.5em;
+  background-color: #f8f8f8;
+  border: 1px solid #e0e0e0;
+  width: 89ex;
+}
+
+.rfcmarkup hr {
+	margin: 0;
+	width: 80ex;
+}
+
+.rfcmarkup .text-warning,
+.rfcmarkup a.text-warning,
+.rfcmarkup a.text-warning:focus,
+.rfcmarkup a.text-warning:active,
+.rfcmarkup a.text-warning:visited,
+.rfcmarkup a.text-warning:hover
+{
+  color: #d9534f;			/* brand-danger colour */
+}
+
+@media print {
+  .rfcmarkup .noprint { display: none; }
+  .rfcmarkup a,
+  .rfcmarkup a:visited,
+  .rfcmarkup pre a,
+  .rfcmarkup pre a:visited {
+    text-decoration: none;
+  }
+  .rfcmarkup a[href]:after {
+    content: "";
+  }
+  .rfcmarkup abbr[title]:after {
+    content: "";
+  }
+  .rfcmarkup pre {
+    font-size: 10pt;
+
+    margin-top: 0;
+    margin-bottom: 0;
+    margin-left: auto;
+    margin-right: auto;
+    
+
+  }
+  .rfcmarkup .newpage {
+    page-break-before: always;
+  }
+}
diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py
index 59f6c850c..84878359d 100644
--- a/ietf/submit/utils.py
+++ b/ietf/submit/utils.py
@@ -430,6 +430,8 @@ def ensure_person_email_info_exists(name, email):
             email.active = active
 
         email.person = person
+        if email.time is None:
+            email.time = datetime.datetime.now()
         email.save()
 
     return email
diff --git a/ietf/templates/base.html b/ietf/templates/base.html
index 2718d2b97..0d22e2d86 100644
--- a/ietf/templates/base.html
+++ b/ietf/templates/base.html
@@ -44,7 +44,7 @@
   </head>
 
   <body {% block bodyAttrs %}{%endblock%} data-group-menu-data-url="{% url 'ietf.group.views_ajax.group_menu_data' %}">
-      <nav class="navbar {% if server_mode and server_mode != "production" %}navbar-default{% else %}navbar-inverse{% endif %} navbar-fixed-top">
+      <nav class="navbar {% if server_mode and server_mode != "production" %}navbar-default{% else %}navbar-inverse{% endif %} {% if navbar_mode %}{{ navbar_mode }}{% else %}navbar-fixed-top{% endif %}">
         <div class="container-fluid">
 	  <div class="navbar-header">
 	    <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-collapse">
@@ -57,7 +57,7 @@
 	       <img alt="IETF Logo" src="{% static 'ietf/images/ietflogo-small-transparent.png' %}">
 	       {% if not user.is_authenticated %}
 		  {% if server_mode and server_mode != "production" %}
-		  <b><i>Development mode</i></b>
+		  <b><i><small>Development mode</small></i></b>
 		  {% else %}
 		  Datatracker
 		  {% endif %}
@@ -115,6 +115,7 @@
           </div>
         {% endif %}
 
+	{% block footer %}
 	<hr>
 	  <div class="col-md-12">
 	    <div class="text-center padded">
@@ -131,6 +132,7 @@
 	    </div>
 	  </div>
 
+
         <footer class="row">
 	  <div class="col-md-12">
 	    <div class="text-center">
@@ -151,8 +153,12 @@
 	    </div>
 	  </div>
         </footer>
-        {% include "debug.html" %}
+	{% endblock %}
+
       </div>
+
+      {% include "debug.html" %}
+
       <script src="{% static 'jquery/jquery.min.js' %}"></script>
       {% comment %}
       <!-- Remove the *-nojs attributes if we are running js.  This depends on jQuery's removeClass(): -->
diff --git a/ietf/templates/doc/document_bibtex.bib b/ietf/templates/doc/document_bibtex.bib
index 489abf3ec..d316b9ef9 100644
--- a/ietf/templates/doc/document_bibtex.bib
+++ b/ietf/templates/doc/document_bibtex.bib
@@ -23,7 +23,7 @@
 	institution =	{% templatetag openbrace %}Internet Engineering Task Force{% templatetag closebrace %},
 	publisher =	{% templatetag openbrace %}Internet Engineering Task Force{% templatetag closebrace %},
 	note =		{% templatetag openbrace %}Work in Progress{% templatetag closebrace %},
-	url =		{% templatetag openbrace %}https://tools.ietf.org/html/{{doc.name}}-{{doc.rev}}{% templatetag closebrace %},{% endif %}
+	url =		{% templatetag openbrace %}https://datatracker.ietf.org/doc/html/{{doc.name}}-{{doc.rev}}{% templatetag closebrace %},{% endif %}
 	author =	{% templatetag openbrace %}{% for entry in doc.authors.all %}{% with entry.person as author %}{{author.name}}{% endwith %}{% if not forloop.last %} and {% endif %}{% endfor %}{% templatetag closebrace %},
 	title =		{% templatetag openbrace %}{% templatetag openbrace %}{{doc.title}}{% templatetag closebrace %}{% templatetag closebrace %},
 	pagetotal =	{{ doc.pages }},
diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html
index 8e1032c53..804480e74 100644
--- a/ietf/templates/doc/document_draft.html
+++ b/ietf/templates/doc/document_draft.html
@@ -3,6 +3,7 @@
 {% load origin %}
 {% load staticfiles %}
 {% load ietf_filters %}
+{% load cache %}
 
 {% block pagehead %}
   <link rel="alternate" type="application/atom+xml" title="Document changes" href="/feed/document-changes/{{ name }}/">
@@ -547,10 +548,24 @@
   </div>
 
   {% if doc.get_state_slug == "active" or doc.get_state_slug == "rfc" %}
-    {{ content|safe }}
 
     {% if split_content %}
+      {{ content|safe }}
       <a class="btn btn-default btn-block" href="?include_text=1"><span class="fa fa-caret-down"></span> Show full document text</a>
+    {% else %}
+    <div class="col-md-12">
+      <div class="col-md-2"></div>
+      <div class="col-md-8 rfcmarkup">
+      {% with 1209600 as two_weeks %}
+	{% cache two_weeks htmlized doc.name doc.rev using="htmlized" %}
+	  <div>
+	  {{ doc.htmlized|default:"Generation of htmlized text failed"|safe }}
+	  </div>
+	{% endcache %}
+      {% endwith %}
+      </div>
+      <div class="col-md-2"></div>
+    </div>
     {% endif %}
 
   {% else %}
@@ -559,7 +574,8 @@
       <div class="panel-heading">
 	 This Internet-Draft is no longer active. A copy of
 	 the expired Internet-Draft can be found at<br/>
-          <a href="{{doc.href}}">{{doc.href}}</a>
+	 {% url 'ietf.doc.views_doc.document_html' name=doc.name rev=doc.rev as html_url %}
+	 <a href="{{ html_url }}">{{ html_url }}</a>
       </div>
       <div class="panel-body">
         <p>
diff --git a/ietf/templates/doc/document_html.html b/ietf/templates/doc/document_html.html
new file mode 100644
index 000000000..4aba74599
--- /dev/null
+++ b/ietf/templates/doc/document_html.html
@@ -0,0 +1,69 @@
+{% extends "base.html" %}
+{# Copyright The IETF Trust 2016, All Rights Reserved #}
+{% load origin %}
+{% load staticfiles %}
+{% load ietf_filters %}
+{% load cache %}
+
+{% block pagehead %}
+  <link rel="alternate" type="application/atom+xml" title="Document changes" href="/feed/document-changes/{{ doc.name }}/">
+  <meta name="description" content="{{ doc.title }} {% if doc.get_state_slug == "rfc" %}(RFC {{ rfc_number }}{% if published %}, {{ published.time|date:"F Y" }}{% endif %}{% if obsoleted_by %}; obsoleted by {{ obsoleted_by|join:", " }}{% endif %}){% else %}(Internet-Draft, {{ doc.time|date:"Y" }}){% endif %}">
+  <script src="{% static 'd3/d3.min.js' %}"></script>
+  <script src="{% static 'jquery/jquery.min.js' %}"></script>
+{% endblock %}
+
+{% block morecss %}
+.inline	{ display: inline; }
+{% endblock %}
+
+{% block title %}
+  {% if doc.get_state_slug == "rfc" %}
+    {{ doc.canonical_name }}
+  {% else %}
+    {{ doc.name }}-{{ doc.rev }}
+  {% endif %}
+{% endblock %}
+
+{% block bodyAttrs %}style="padding-top: 0;"{% endblock %}
+
+{% block content %}
+  {% origin %}
+  <div class="hidden-print">
+  {{ top | safe }}
+  </div>
+
+{#  {% include "doc/revisions_list.html" %} #}
+  <div class="col-md-2"></div>
+  <div class="col-md-8 rfcmarkup">
+    {% if doc.meta %}
+      <div class="hidden-print">
+	 <pre class="meta-info">{{ doc.meta|safe }}</pre>
+      </div>
+    {% endif %}
+
+    {% comment %}
+    {% if doc.is_dochistory %}
+      {% if doc.rev != doc.doc.rev %}
+        <pre class="meta-info alert-warning text-center">A newer version of the document below exists</pre>
+      {% elif doc.doc.is_rfc %}
+        <pre class="meta-info alert-info text-center">The draft below has been published as <a href="{% url 'ietf.doc.views_doc.document_html' name=doc.doc.canonical_name %}">RFC {{doc.doc.rfc_number}}</a></pre>
+      {% endif %}
+    {% endif %}
+    {% endcomment %}
+
+    {% with 1209600 as two_weeks %}
+      {% cache two_weeks htmlized doc.name doc.rev using="htmlized" %}
+        <div>
+	{{ doc.htmlized|default:"Generation of htmlized text failed"|safe }}
+	</div>
+      {% endcache %}
+    {% endwith %}
+  </div>
+  <div class="col-md-1"></div>
+  <div class="col-md-1"></div>
+
+{% endblock %}
+
+{% block footer %}
+<div></div>
+{% endblock %}
diff --git a/ietf/templates/doc/document_top.html b/ietf/templates/doc/document_top.html
index e74b12d5e..9be668f5e 100644
--- a/ietf/templates/doc/document_top.html
+++ b/ietf/templates/doc/document_top.html
@@ -1,6 +1,6 @@
 {# Copyright The IETF Trust 2015, All Rights Reserved #}{% load origin %}{% origin %}
 {% load ietf_filters %}
-<h1>{{ doc.title }}<br><small>{{ name }}</small></h1>
+<h2>{{ doc.title }}<br><small>{{ name }}</small></h2>
 
 <ul class="nav nav-tabs" role="tablist">
   {% for name, t, url, active, tooltip in tabs %}
diff --git a/ietf/templates/doc/search/search_result_row.html b/ietf/templates/doc/search/search_result_row.html
index 797e25865..dab801ad4 100644
--- a/ietf/templates/doc/search/search_result_row.html
+++ b/ietf/templates/doc/search/search_result_row.html
@@ -24,8 +24,8 @@
 
   <td class="doc">
     <div>
-      <a href="{{ doc.get_absolute_url }}">{% if doc.get_state_slug == "rfc" %}RFC {{ doc.rfc_number }}{% else %}{{ doc.name }}-{{ doc.rev }}{% endif %}</a>
-
+      <a class="symbol-link" href="{{ doc.get_absolute_url }}"><span class="fa fa-info-circle fa-lg"></span></a>
+      <a href="{% url 'ietf.doc.views_doc.document_html' name=doc.canonical_name %}">{% if doc.get_state_slug == "rfc" %}RFC {{ doc.rfc_number }}{% else %}{{ doc.name }}-{{ doc.rev }}{% endif %}</a>
       {% if doc.get_state_slug == "rfc" and "draft" in doc.name %}
         <i>(was {{ doc.name }})</i>
       {% endif %}
diff --git a/ietf/templates/group/customize_workflow.html b/ietf/templates/group/customize_workflow.html
index 34376e701..c60683168 100644
--- a/ietf/templates/group/customize_workflow.html
+++ b/ietf/templates/group/customize_workflow.html
@@ -20,7 +20,7 @@
 
     {% if group.type_id == "wg" %}
       <p class="help-block">You can see the default Working Group I-D State Diagram
-        in <a href="https://tools.ietf.org/html/rfc6174#section-4.1">Section 4.1 of RFC6174</a>.</p>
+        in <a href="/doc/html/rfc6174#section-4.1">Section 4.1 of RFC6174</a>.</p>
     {% endif %}
 
     <h3>States</h3>
diff --git a/ietf/templates/submit/announce_new_version.txt b/ietf/templates/submit/announce_new_version.txt
index ba078c6e0..8f7f98c01 100644
--- a/ietf/templates/submit/announce_new_version.txt
+++ b/ietf/templates/submit/announce_new_version.txt
@@ -12,7 +12,7 @@ Diff from previous version:
 {{rfcdiff_base_url}}?url2={{ submission.name }}-{{ submission.rev }}
 
 Please note that it may take a couple of minutes from the time of submission
-until the htmlized version and diff are available at tools.ietf.org.
+until the diff is available at tools.ietf.org.
 
 IETF Secretariat.
 {% endautoescape %}
diff --git a/ietf/templates/submit/announce_to_authors.txt b/ietf/templates/submit/announce_to_authors.txt
index 5be3e0344..60f5c7ef0 100644
--- a/ietf/templates/submit/announce_to_authors.txt
+++ b/ietf/templates/submit/announce_to_authors.txt
@@ -11,7 +11,7 @@ Group:		{{ group }}
 Pages:		{{ submission.pages }}
 URL:            https://www.ietf.org/internet-drafts/{{ submission.name }}-{{ submission.rev }}.txt
 Status:         https://datatracker.ietf.org/doc/{{ submission.name }}/
-Htmlized:       https://tools.ietf.org/html/{{ submission.name }}-{{ submission.rev }}
+Htmlized:       https://datatracker.ietf.org/doc/html/{{ submission.name }}-{{ submission.rev }}
 {% if submission.rev != "00" %}Diff:           {{rfcdiff_base_url}}?url2={{ submission.name }}-{{ submission.rev }}{% endif %}
 
 Abstract:
@@ -20,7 +20,7 @@ Abstract:
 {{ submission.note|default:"" }}
 
 Please note that it may take a couple of minutes from the time of submission
-until the htmlized version and diff are available at tools.ietf.org.
+until the diff is available at tools.ietf.org.
 
 The IETF Secretariat
 {% endautoescape %}
diff --git a/ietf/templates/submit/announce_to_lists.txt b/ietf/templates/submit/announce_to_lists.txt
index 0d9817d57..18f068b03 100644
--- a/ietf/templates/submit/announce_to_lists.txt
+++ b/ietf/templates/submit/announce_to_lists.txt
@@ -16,14 +16,14 @@ The IETF datatracker status page for this draft is:
 https://datatracker.ietf.org/doc/{{ submission.name }}/
 
 There's also a htmlized version available at:
-https://tools.ietf.org/html/{{ submission.name }}-{{ submission.rev }}
+https://datatracker.ietf.org/doc/html/{{ submission.name }}-{{ submission.rev }}
 {% if submission.rev != "00" %}
 A diff from the previous version is available at:
 {{settings.RFCDIFF_BASE_URL}}?url2={{ submission.name }}-{{ submission.rev }}
 {% endif %}
 
 Please note that it may take a couple of minutes from the time of submission
-until the htmlized version and diff are available at tools.ietf.org.
+until the diff is available at tools.ietf.org.
 
 Internet-Drafts are also available by anonymous FTP at:
 ftp://ftp.ietf.org/internet-drafts/