chore: Replace markup_txt with rfc2html (#6632)

* chore: Replace markup_txt with rfc2html

* Suppress HTML issues

* Remove comment

* Suppress harder

* Suppress even more

* Fix test

* Ignore vnu "duplicate ID" errors

* Undo accidental commit

* Fix RFCs

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>

ietf/doc/views_doc.py

@@ -51,6 +51,8 @@ from django.conf import settings
 from django import forms
 from django.contrib.staticfiles import finders
 
+import rfc2html
+
 import debug                            # pyflakes:ignore
 
 from ietf.doc.models import ( Document, DocHistory, DocEvent, BallotDocEvent, BallotType,
@@ -80,7 +82,7 @@ from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions,
 from ietf.review.models import ReviewAssignment
 from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs
 from ietf.review.utils import no_review_from_teams_on_doc
-from ietf.utils import markup_txt, log, markdown
+from ietf.utils import log, markdown
 from ietf.utils.draft import PlaintextDraft
 from ietf.utils.response import permission_denied
 from ietf.utils.text import maybe_split
@@ -264,7 +266,9 @@ def document_main(request, name, rev=None, document_html=False):
 
         file_urls, found_types = build_file_urls(doc)
         content = doc.text_or_error() # pyflakes:ignore
-        content = markup_txt.markup(maybe_split(content, split=split_content))
+        content = maybe_split(content, split=split_content)
+        if content:
+            content = rfc2html.markup(content)
 
         if not found_types:
             content = "This RFC is not currently available online."
@@ -400,7 +404,9 @@ def document_main(request, name, rev=None, document_html=False):
 
         file_urls, found_types = build_file_urls(doc)
         content = doc.text_or_error() # pyflakes:ignore
-        content = markup_txt.markup(maybe_split(content, split=split_content))
+        content = maybe_split(content, split=split_content)
+        if content:
+            content = rfc2html.markup(content)
 
         latest_revision = doc.latest_event(NewRevisionDocEvent, type="new_revision")
 
@@ -775,7 +781,7 @@ def document_main(request, name, rev=None, document_html=False):
             content = "A conflict review response has not yet been proposed."
         else:     
             content = doc.text_or_error() # pyflakes:ignore
-            content = markup_txt.markup(content)
+            content = markdown.markdown(content)
 
         ballot_summary = None
         if doc.get_state_slug() in ("iesgeval", ) and doc.active_ballot():
@@ -890,8 +896,6 @@ def document_main(request, name, rev=None, document_html=False):
         basename = "{}.txt".format(doc.name)
         pathname = os.path.join(doc.get_file_path(), basename)
         content = get_unicode_document_content(basename, pathname)
-        # If we want to go back to using markup_txt.markup_unicode, call it explicitly here like this:
-        # content = markup_txt.markup_unicode(content, split=False, width=80)
        
         assignments = ReviewAssignment.objects.filter(review__name=doc.name)
         review_assignment = assignments.first()

ietf/templates/doc/document_draft.html

@@ -15,7 +15,15 @@
     <meta name="description"
           content="{{ doc.title }} {% if doc.type_id == 'rfc' and not snapshot %}(RFC {{ rfc_number }}{% if published %}, {{ doc.pub_date|date:'F Y' }}{% endif %}{% if obsoleted_by %}; obsoleted by {% for rel in obsoleted_by %}{{ rel.source.name|prettystdname}}{% if not forloop.last%}, {% endif %}{% endfor %}{% endif %}){% endif %}">
 {% endblock %}
-{% block morecss %}.inline { display: inline; }{% endblock %}
+{% block morecss %}
+    .rfcmarkup :is(h1, h2, h3, h4, h5, h6),
+    .rfcmarkup :is(.h1, .h2, .h3, .h4, .h5, .h6) {
+        font-weight: bold;
+        font-size: 1em;
+    }
+    .rfcmarkup .newpage { margin-top: -1.5em; }
+    .rfcmarkup .grey, hr { opacity: .25; }
+{% endblock %}
 {% block title %}
     {% if doc.type_id == "rfc" and not snapshot %}
         RFC {{ rfc_number }} - {{ doc.title }}
@@ -658,7 +666,7 @@
                 </i>
                 Nits
             </a>
-            <div class="dropdown inline">
+            <div class="dropdown d-inline">
                 <button class="btn btn-primary btn-sm dropdown-toggle"
                         type="button"
                         id="ddSearchMenu"
@@ -760,8 +768,9 @@
                         {{ name }}-{{ doc.rev }}
                     {% endif %}
                 </div>
-                <div class="card-body">
+                <div class="card-body rfcmarkup">
-                    <pre>{{ content|sanitize|safe|default:"(Unavailable)" }}</pre>
+                    <!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level, no-dup-id, valid-id -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
+                    {{ content|safe|default:"(Unavailable)" }}
                 </div>
             </div>
             {% if split_content %}

ietf/templates/doc/document_html.html

@@ -165,7 +165,7 @@
                 {% else %}
                     <div class="rfcmarkup">
                         <br class="noprint">
-                        <!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
+                        <!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level, no-dup-id, valid-id -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
                         {{ doc.htmlized|default:"Generation of htmlized text failed"|safe }}
                     </div>
                 {% endif %}

ietf/templates/doc/document_rfc.html

@@ -15,7 +15,15 @@
     <meta name="description"
           content="{{ doc.title }} (RFC {{ doc.rfc_number }}, {{ doc.pub_date|date:'F Y' }}{% if obsoleted_by %}; obsoleted by {% for rel in obsoleted_by %}{{ rel.source.name|prettystdname}}{% if not forloop.last%}, {% endif %}{% endfor %}{% endif %})">
 {% endblock %}
-{% block morecss %}.inline { display: inline; }{% endblock %}
+{% block morecss %}
+    .rfcmarkup :is(h1, h2, h3, h4, h5, h6),
+    .rfcmarkup :is(.h1, .h2, .h3, .h4, .h5, .h6) {
+        font-weight: bold;
+        font-size: 1em;
+    }
+    .rfcmarkup .newpage { margin-top: -1.5em; }
+    .rfcmarkup .grey, hr { opacity: .25; }
+{% endblock %}
 {% block title %}
         RFC {{ doc.rfc_number }} - {{ doc.title }}
 {% endblock %}
@@ -155,8 +163,9 @@
             <div class="card-header">
                     RFC {{ doc.rfc_number }}
             </div>
-            <div class="card-body">
+                <div class="card-body rfcmarkup">
-                <pre>{{ content|sanitize|safe|default:"(Unavailable)" }}</pre>
+                    <!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level, no-dup-id, valid-id -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
+                    {{ content|safe|default:"(Unavailable)" }}
             </div>
         </div>
         {% if split_content %}

ietf/utils/markup_txt.py

@@ -1,69 +0,0 @@
-# Copyright The IETF Trust 2009-2020, All Rights Reserved
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
-# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-#  * Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-#
-#  * Redistributions in binary form must reproduce the above
-#    copyright notice, this list of conditions and the following
-#    disclaimer in the documentation and/or other materials provided
-#    with the distribution.
-#
-#  * Neither the name of the Nokia Corporation and/or its
-#    subsidiary(-ies) nor the names of its contributors may be used
-#    to endorse or promote products derived from this software
-#    without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-import re
-
-from django.utils.html import escape
-
-from ietf.utils import log
-from ietf.utils.text import wordwrap
-
-def markup(content, width=None):
-    log.assertion('isinstance(content, str)')
-    # normalize line endings to LF only
-    content = content.replace("\r\n", "\n")
-    content = content.replace("\r", "\n")
-
-    # remove leading white space
-    content = content.lstrip()
-    # remove runs of blank lines
-    content = re.sub("\n\n\n+", "\n\n", content)
-
-    # maybe wordwrap.  This must be done before the escaping below.
-    if width:
-        content = wordwrap(content, width)
-
-    # expand tabs + escape 
-    content = escape(content.expandtabs())
-
-    content = re.sub(r"\n(.+\[Page \d+\])\n\f\n(.+)\n", r"""\n<span class="m_ftr">\g<1></span>\n<span class="m_hdr">\g<2></span>\n""", content)
-    content = re.sub(r"\n(.+\[Page \d+\])\n\s*$", r"""\n<span class="m_ftr">\g<1></span>\n""", content)
-    # remove remaining FFs (to be valid XHTML)
-    content = content.replace("\f","\n")
-
-    content = re.sub(r"\n\n([0-9]+\\.|[A-Z]\\.[0-9]|Appendix|Status of|Abstract|Table of|Full Copyright|Copyright|Intellectual Property|Acknowled|Author|Index)(.*)(?=\n\n)", r"""\n\n<span class="m_h">\g<1>\g<2></span>""", content)
-
-    return content

ietf/utils/test_runner.py

@@ -196,6 +196,7 @@ def vnu_filter_message(msg, filter_db_issues, filter_test_issues):
     if filter_test_issues and re.search(
         r"""Ceci\ n'est\ pas\ une\ URL|
             ^The\ '\w+'\ attribute\ on\ the\ '\w+'\ element\ is\ obsolete|
+            ^Duplicate\ ID|^The\ first\ occurrence\ of\ ID|
             ^Section\ lacks\ heading""",
         msg["message"],
         flags=re.VERBOSE,