altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 4 Packages Projects Releases Wiki Activity

chore: Replace markup_txt with rfc2html (#6632)

Browse source 
* chore: Replace markup_txt with rfc2html

* Suppress HTML issues

* Remove comment

* Suppress harder

* Suppress even more

* Fix test

* Ignore vnu "duplicate ID" errors

* Undo accidental commit

* Fix RFCs

---------

Co-authored-by: Robert Sparks <rjsparks@nostrum.com>
This commit is contained in:
Lars Eggert 2024-01-05 20:29:28 +02:00 committed by GitHub
parent caa1d5f462
commit e24fb601db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 39 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
ietf/doc/views_doc.py
View file

@ -51,6 +51,8 @@ from django.conf import settings
from django import forms
from django.contrib.staticfiles import finders
import rfc2html
import debug # pyflakes:ignore
from ietf.doc.models import ( Document, DocHistory, DocEvent, BallotDocEvent, BallotType,
@ -80,7 +82,7 @@ from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions,
from ietf.review.models import ReviewAssignment
from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs
from ietf.review.utils import no_review_from_teams_on_doc
from ietf.utils import markup_txt, log, markdown
from ietf.utils import log, markdown
from ietf.utils.draft import PlaintextDraft
from ietf.utils.response import permission_denied
from ietf.utils.text import maybe_split
@ -264,7 +266,9 @@ def document_main(request, name, rev=None, document_html=False):
file_urls, found_types = build_file_urls(doc)
content = doc.text_or_error() # pyflakes:ignore
content = markup_txt.markup(maybe_split(content, split=split_content))
content = maybe_split(content, split=split_content)
if content:
content = rfc2html.markup(content)
if not found_types:
content = "This RFC is not currently available online."
@ -400,7 +404,9 @@ def document_main(request, name, rev=None, document_html=False):
file_urls, found_types = build_file_urls(doc)
content = doc.text_or_error() # pyflakes:ignore
content = markup_txt.markup(maybe_split(content, split=split_content))
content = maybe_split(content, split=split_content)
if content:
content = rfc2html.markup(content)
latest_revision = doc.latest_event(NewRevisionDocEvent, type="new_revision")
@ -775,7 +781,7 @@ def document_main(request, name, rev=None, document_html=False):
content = "A conflict review response has not yet been proposed."
else:
content = doc.text_or_error() # pyflakes:ignore
content = markup_txt.markup(content)
content = markdown.markdown(content)
ballot_summary = None
if doc.get_state_slug() in ("iesgeval", ) and doc.active_ballot():
@ -890,8 +896,6 @@ def document_main(request, name, rev=None, document_html=False):
basename = "{}.txt".format(doc.name)
pathname = os.path.join(doc.get_file_path(), basename)
content = get_unicode_document_content(basename, pathname)
# If we want to go back to using markup_txt.markup_unicode, call it explicitly here like this:
# content = markup_txt.markup_unicode(content, split=False, width=80)
assignments = ReviewAssignment.objects.filter(review__name=doc.name)
review_assignment = assignments.first()

17
ietf/templates/doc/document_draft.html
View file

@ -15,7 +15,15 @@
<meta name="description"
content="{{ doc.title }} {% if doc.type_id == 'rfc' and not snapshot %}(RFC {{ rfc_number }}{% if published %}, {{ doc.pub_date|date:'F Y' }}{% endif %}{% if obsoleted_by %}; obsoleted by {% for rel in obsoleted_by %}{{ rel.source.name|prettystdname}}{% if not forloop.last%}, {% endif %}{% endfor %}{% endif %}){% endif %}">
{% endblock %}
{% block morecss %}.inline { display: inline; }{% endblock %}
{% block morecss %}
.rfcmarkup :is(h1, h2, h3, h4, h5, h6),
.rfcmarkup :is(.h1, .h2, .h3, .h4, .h5, .h6) {
font-weight: bold;
font-size: 1em;
}
.rfcmarkup .newpage { margin-top: -1.5em; }
.rfcmarkup .grey, hr { opacity: .25; }
{% endblock %}
{% block title %}
{% if doc.type_id == "rfc" and not snapshot %}
RFC {{ rfc_number }} - {{ doc.title }}
@ -658,7 +666,7 @@
</i>
Nits
</a>
<div class="dropdown inline">
<div class="dropdown d-inline">
<button class="btn btn-primary btn-sm dropdown-toggle"
type="button"
id="ddSearchMenu"
@ -760,8 +768,9 @@
{{ name }}-{{ doc.rev }}
{% endif %}
</div>
<div class="card-body">
<pre>{{ content|sanitize|safe|default:"(Unavailable)" }}</pre>
<div class="card-body rfcmarkup">
<!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level, no-dup-id, valid-id -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
{{ content|safe|default:"(Unavailable)" }}
</div>
</div>
{% if split_content %}

4
ietf/templates/doc/document_html.html
View file

@ -165,7 +165,7 @@
{% else %}
<div class="rfcmarkup">
<br class="noprint">
<!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
<!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level, no-dup-id, valid-id -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
{{ doc.htmlized|default:"Generation of htmlized text failed"|safe }}
</div>
{% endif %}
@ -332,4 +332,4 @@
</div>
{% analytical_body_bottom %}
</body>
</html>
</html>

17
ietf/templates/doc/document_rfc.html
View file

@ -15,7 +15,15 @@
<meta name="description"
content="{{ doc.title }} (RFC {{ doc.rfc_number }}, {{ doc.pub_date|date:'F Y' }}{% if obsoleted_by %}; obsoleted by {% for rel in obsoleted_by %}{{ rel.source.name|prettystdname}}{% if not forloop.last%}, {% endif %}{% endfor %}{% endif %})">
{% endblock %}
{% block morecss %}.inline { display: inline; }{% endblock %}
{% block morecss %}
.rfcmarkup :is(h1, h2, h3, h4, h5, h6),
.rfcmarkup :is(.h1, .h2, .h3, .h4, .h5, .h6) {
font-weight: bold;
font-size: 1em;
}
.rfcmarkup .newpage { margin-top: -1.5em; }
.rfcmarkup .grey, hr { opacity: .25; }
{% endblock %}
{% block title %}
RFC {{ doc.rfc_number }} - {{ doc.title }}
{% endblock %}
@ -155,8 +163,9 @@
<div class="card-header">
RFC {{ doc.rfc_number }}
</div>
<div class="card-body">
<pre>{{ content|sanitize|safe|default:"(Unavailable)" }}</pre>
<div class="card-body rfcmarkup">
<!-- [html-validate-disable-block attr-quotes, void-style, element-permitted-content, heading-level, no-dup-id, valid-id -- FIXME: rfcmarkup/rfc2html generates HTML with issues] -->
{{ content|safe|default:"(Unavailable)" }}
</div>
</div>
{% if split_content %}
@ -173,4 +182,4 @@
</script>
<script src="{% static 'ietf/js/document_timeline.js' %}">
</script>
{% endblock %}
{% endblock %}

69
ietf/utils/markup_txt.py
View file

@ -1,69 +0,0 @@
# Copyright The IETF Trust 2009-2020, All Rights Reserved
# -*- coding: utf-8 -*-
#
# Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
# All rights reserved. Contact: Pasi Eronen <pasi.eronen@nokia.com>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided
# with the distribution.
#
# * Neither the name of the Nokia Corporation and/or its
# subsidiary(-ies) nor the names of its contributors may be used
# to endorse or promote products derived from this software
# without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import re
from django.utils.html import escape
from ietf.utils import log
from ietf.utils.text import wordwrap
def markup(content, width=None):
log.assertion('isinstance(content, str)')
# normalize line endings to LF only
content = content.replace("\r\n", "\n")
content = content.replace("\r", "\n")
# remove leading white space
content = content.lstrip()
# remove runs of blank lines
content = re.sub("\n\n\n+", "\n\n", content)
# maybe wordwrap. This must be done before the escaping below.
if width:
content = wordwrap(content, width)
# expand tabs + escape
content = escape(content.expandtabs())
content = re.sub(r"\n(.+\[Page \d+\])\n\f\n(.+)\n", r"""\n<span class="m_ftr">\g<1></span>\n<span class="m_hdr">\g<2></span>\n""", content)
content = re.sub(r"\n(.+\[Page \d+\])\n\s*$", r"""\n<span class="m_ftr">\g<1></span>\n""", content)
# remove remaining FFs (to be valid XHTML)
content = content.replace("\f","\n")
content = re.sub(r"\n\n([0-9]+\\.|[A-Z]\\.[0-9]|Appendix|Status of|Abstract|Table of|Full Copyright|Copyright|Intellectual Property|Acknowled|Author|Index)(.*)(?=\n\n)", r"""\n\n<span class="m_h">\g<1>\g<2></span>""", content)
return content

1
ietf/utils/test_runner.py
View file

@ -196,6 +196,7 @@ def vnu_filter_message(msg, filter_db_issues, filter_test_issues):
if filter_test_issues and re.search(
r"""Ceci\ n'est\ pas\ une\ URL|
^The\ '\w+'\ attribute\ on\ the\ '\w+'\ element\ is\ obsolete|
^Duplicate\ ID|^The\ first\ occurrence\ of\ ID|
^Section\ lacks\ heading""",
msg["message"],
flags=re.VERBOSE,