altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

fix: add more HTML validation & fixes (#3891)

Browse source 
* Update vnu.jar

* Fix py2 -> py3 issue

* Run pyupgrade

* test: Add default-jdk to images

* test: Add option to also validate HTML with vnu.jar

Since it's already installed in bin. Don't do this by default, since it
increases the time needed for tests by ~50%.

* fix: Stop the urlizer from urlizing in linkified mailto: text

* More HTML fixes

* More HTML validation fixes

* And more HTML fixes

* Fix floating badge

* Ignore unicode errors

* Only URLize docs that are existing

* Final fixes

* Don't URLize everything during test-crawl

* Feed HTML into vnu using python rather than Java to speed things up

* Allow test-crawl to start vnu on a different port

* Increase retry count to vnu. Restore batch size to 30.

* More HTML validation fixes

* Use urllib3 to make requests to vnu, since overriding requests_mock is tricky

* Undo commit of unmodified file

* Also urlize ftp links

* Fix matching of file name

* More HTML fixes

* Add `is_valid_url` filter

* weekday -> data-weekday

* urlencode URLs

* Add and use vnu_fmt_message. Bump vnu max buffer.

* Simplify doc_exists

* Don't add tab link to mail archive if the URL is invalid

* Run urlize_ietf_docs before linkify

Reduces the possibility of generating incorrect HTML

* Undo superfluous change

* Runner fixes

* Consolidate vnu message filtering into vnu_filter_message

* Correctly handle multiple persons with same name

* Minimze diff

* Fix HTML nits

* Print source snippet in vnu_fmt_message

* Only escape if there is something to escape

* Fix snippet

* Skip crufty old IPR declarations

* Only include modal when needed. Add handles.

* Fix wordwrap+linkification

* Update ietf/doc/templatetags/ietf_filters.py

* Update ietf/doc/templatetags/tests_ietf_filters.py

* Don't right-align second column
This commit is contained in:
Lars Eggert 2022-05-03 21:55:48 +03:00 committed by GitHub
parent f778058005
commit 5598762608
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
55 changed files with 479 additions and 609 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

493
bin/test-crawl
View file

@ -3,7 +3,7 @@
# Copyright The IETF Trust 2013-2019, All Rights Reserved # Copyright The IETF Trust 2013-2019, All Rights Reserved
import os, sys, re, datetime, argparse, traceback, json, subprocess import os, sys, re, datetime, argparse, traceback, json
import html5lib import html5lib
import random import random
@ -31,7 +31,7 @@ parser.add_argument('--no-follow', dest='follow', action='store_false', default=
parser.add_argument('--validator-nu', dest='validator_nu', action='store_true', parser.add_argument('--validator-nu', dest='validator_nu', action='store_true',
help='Use validator.nu instead of html5lib for HTML validation') help='Use validator.nu instead of html5lib for HTML validation')
parser.add_argument('--pedantic', action='store_true', parser.add_argument('--pedantic', action='store_true',
help='Stop the crawl on the first HTML validation issue') help='Stop the crawl on the first error or warning')
parser.add_argument('--random', action='store_true', parser.add_argument('--random', action='store_true',
help='Crawl URLs randomly') help='Crawl URLs randomly')
parser.add_argument('--validate-all', dest='validate_all', action='store_true', default=False, parser.add_argument('--validate-all', dest='validate_all', action='store_true', default=False,
@ -44,6 +44,7 @@ args = parser.parse_args()
# Import Django, call setup() # Import Django, call setup()
os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_testcrawl") os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_testcrawl")
os.environ["DJANGO_URLIZE_IETF_DOCS_PRODUCTION"] = "1"
import django import django
import django.test import django.test
@ -65,6 +66,7 @@ import debug # pyflakes:ignore
from ietf.name.models import DocTypeName from ietf.name.models import DocTypeName
from ietf.utils.html import unescape from ietf.utils.html import unescape
from ietf.utils.test_utils import unicontent from ietf.utils.test_utils import unicontent
from ietf.utils.test_runner import start_vnu_server, vnu_validate, vnu_fmt_message, vnu_filter_message
# --- Constants --- # --- Constants ---
@ -156,55 +158,38 @@ def check_html_valid(url, response, args):
key = re.sub("/submit/status/nnnn/[0-9a-f]+/", "/submit/status/nnnn/bar/", key) key = re.sub("/submit/status/nnnn/[0-9a-f]+/", "/submit/status/nnnn/bar/", key)
key = re.sub("/team/[a-z0-9-]+/", "/team/foo/", key) key = re.sub("/team/[a-z0-9-]+/", "/team/foo/", key)
key = re.sub("/wg/[a-z0-9-]+/", "/wg/foo/", key) key = re.sub("/wg/[a-z0-9-]+/", "/wg/foo/", key)
key = re.sub("\?.*$", "", key) key = re.sub(r"\?.*$", "", key)
for slug in doc_types: for slug in doc_types:
key = re.sub("/%s-.*/"%slug, "/%s-nnnn/"%slug, key) key = re.sub("/%s-.*/"%slug, "/%s-nnnn/"%slug, key)
if not key in validated_urls: if not key in validated_urls:
note('Validate: %-32s: %s' % (url[:32], key)) note('Validate: %-32s: %s' % (url[:32], key))
# These URLs have known issues, skip them until those are fixed
for pattern in (
'/secr',
'admin/',
'/doc/.*/edit/info/',
'rfc542$',
'rfc776$',
'draft-leroux-pce-pcecp-interarea-reqs',
'draft-fujiwara-dnsop-resolver-update',
):
if re.search(pattern, url):
validated_urls[key] = True
log("%s blacklisted; skipping HTML validation" % url)
return
if hasattr(response, "content"): if hasattr(response, "content"):
content = response.content content = response.content
else: else:
content = response.streaming_content content = response.streaming_content
validated_urls[key] = True validated_urls[key] = True
if args.validator_nu: if args.validator_nu:
v = subprocess.Popen(["java", "-jar", basedir + "/bin/vnu.jar", ret = vnu_validate(content, response["Content-Type"], port=8887)
"--format", "json", "-"], assert ret
stdin=subprocess.PIPE, stderr=subprocess.PIPE) for m in json.loads(ret)["messages"]:
for m in json.loads(v.communicate(content)[1])["messages"]: if "lastLine" not in m:
t = m["subType"] if m["type"] == "info" else m["type"] tag = m # just dump the raw JSON for now
tags.append("\n%s\tLine %d: %s" % else:
(t.upper(), m["lastLine"], m["message"])) tag = vnu_fmt_message(url, m, content.decode())
tags.append("\n\t%s" % m["extract"].replace('\n', ' '))
tags.append("\n\t%s%s" %
(" " * m["hiliteStart"], "^" * m["hiliteLength"]))
# disregard some HTML issues that are (usually) due to invalid # disregard some HTML issues that are (usually) due to invalid
# database content # database content
if not re.search('Forbidden code point|Bad value|seamless|The first child|Duplicate ID|The first occurrence of ID', m["message"]): if not vnu_filter_message(m, True, False):
warnings += 1 warnings += 1
tags.append(tag)
else: else:
try: try:
parser.parse(content) parser.parse(content)
except Exception as e: except Exception as e:
for err in parser.errors: for err in parser.errors:
pos, code, data = err pos, _, _ = err
tags.append(u"WARN invalid html at line, pos %s: %s" % (pos, e)) tags.append("WARN invalid html at line, pos {}: {}".format(pos, e))
warnings += 1 warnings += 1
def skip_extract_from(url): def skip_extract_from(url):
@ -219,410 +204,24 @@ def skip_extract_from(url):
def skip_url(url): def skip_url(url):
for pattern in ( for pattern in (
"^/community/[0-9]+/remove_document/", r"^/community/[0-9]+/remove_document/",
"^/community/personal/", r"^/community/personal/",
# Skip most of the slow pdf composite generation urls and svg urls # Skip most of the slow pdf composite generation urls and svg urls
"^/meeting/[0-9]+/agenda/[0-9b-z].*-drafts\\.pdf", r"^/meeting/[0-9]+/agenda/[0-9b-z].*-drafts\\.pdf",
"^/wg/[a-z0-9-]+/deps/svg/", r"^/wg/[a-z0-9-]+/deps/svg/",
# This bad url occurs in an uploaded html agenda: # Skip other bad urls
r"/site/ietfdhcwg/_/rsrc/1311005436000/system/app/css/overlay.css\?cb=simple100%250150goog-ws-left", r"^/dir/tsvdir/reviews/",
r"/dir/tsvdir/reviews/", r"^/ipr/\d{,3}/history/",
r"draft-touch-msword-template-v2\.0",
# Skip most html conversions, not worth the time # Skip most html conversions, not worth the time
"^/doc/html/draft-[0-9ac-z]", r"^/doc/html/draft-[0-9ac-z]",
"^/doc/html/draft-b[0-9b-z]", r"^/doc/html/draft-b[0-9b-z]",
"^/doc/pdf/draft-[0-9ac-z]", r"^/doc/pdf/draft-[0-9ac-z]",
"^/doc/pdf/draft-b[0-9b-z]", r"^/doc/pdf/draft-b[0-9b-z]",
"^/doc/html/charter-.*", r"^/doc/html/charter-.*",
"^/doc/html/status-.*", r"^/doc/html/status-.*",
"^/doc/html/rfc.*", r"^/doc/html/rfc.*",
# These will always 404, but include only those not excluded above
# r"^/doc/html/charter-ietf-cicm",
# r"^/doc/html/charter-ietf-dcon",
# r"^/doc/html/charter-ietf-fun",
# r"^/doc/html/charter-ietf-multrans",
# r"^/doc/html/charter-ietf-sdn",
# r"^/doc/html/charter-ietf-woes",
# r"^/doc/html/draft-allan-mpls-loadbal-06",
# r"^/doc/html/draft-allocchio-mail11-00",
# r"^/doc/html/draft-almquist-leak-01",
# r"^/doc/html/draft-almquist-nexthop-01",
# r"^/doc/html/draft-armitage-ion-mars-scsp-06",
r"^/doc/html/draft-balakrishnan-cm-03",
r"^/doc/html/draft-ballardie-cbt-02",
# r"^/doc/html/draft-bellovin-ipng-shape-of-bits-00",
# r"^/doc/html/draft-bellovin-itrace-04",
# r"^/doc/html/draft-bhaskar-pim-ss-04",
# r"^/doc/html/draft-bhattach-diot-pimso-04",
# r"^/doc/html/draft-bierman-rmonmib-apmcaps-04",
# r"^/doc/html/draft-blaze-ipsp-trustmgt-04",
# r"^/doc/html/draft-blumenthal-snmpv2a-community-00",
# r"^/doc/html/draft-borenstein-agc-spec-00",
# r"^/doc/html/draft-borenstein-kidcode-00",
# r"^/doc/html/draft-borenstein-mailcap-00",
# r"^/doc/html/draft-borenstein-pgp-mime-00",
# r"^/doc/html/draft-brockhaus-lamps-cmp-updates-00",
# r"^/doc/html/draft-brockhaus-lamps-cmp-updates-03",
# r"^/doc/html/draft-brockhaus-lamps-lightweight-cmp-profile-00",
# r"^/doc/html/draft-brockhaus-lamps-lightweight-cmp-profile-03",
# r"^/doc/html/draft-brown-supplpkgs-04",
# r"^/doc/html/draft-brownlee-acct-arch-report-03",
# r"^/doc/html/draft-cain-igmp-00",
# r"^/doc/html/draft-calhoun-aaa-diameter-comp-01",
# r"^/doc/html/draft-calhoun-mobileip-fa-tokens-04",
# r"^/doc/html/draft-calhoun-mobileip-min-lat-handoff-02",
# r"^/doc/html/draft-callon-addflow-support-clnp-00",
# r"^/doc/html/draft-callon-routing-00",
# r"^/doc/html/draft-carpenter-ipng-nosi-00",
# r"^/doc/html/draft-carpenter-percep-00",
# r"^/doc/html/draft-casati-gtp-03",
# r"^/doc/html/draft-ceuppens-mpls-optical-04",
# r"^/doc/html/draft-chapin-clnp-ISO8473-00",
# r"^/doc/html/draft-cheng-modular-ikmp-00",
# r"^/doc/html/draft-cole-appm-01",
# r"^/doc/html/draft-conta-ipv6-icmp-igmp-00",
# r"^/doc/html/draft-coya-test-01",
# r"^/doc/html/draft-crocker-cidrd-myth-00",
# r"^/doc/html/draft-crocker-pci-00",
# r"^/doc/html/draft-crocker-stif-00",
# r"^/doc/html/draft-daigle-napstr-05",
# r"^/doc/html/draft-davie-intserv-compress-01",
# r"^/doc/html/draft-davie-tag-switching-atm-04",
# r"^/doc/html/draft-davin-qosrep-00",
# r"^/doc/html/draft-davin-rsvfms-00",
# r"^/doc/html/draft-dckmtr-proxy-00",
# r"^/doc/html/draft-doolan-tdp-spec-04",
# r"^/doc/html/draft-duerst-ruby-04",
# r"^/doc/html/draft-dusse-pem-message-00",
# r"^/doc/html/draft-dutt-rap-rsvp-proxy-01",
# r"^/doc/html/draft-eastlake-muse-05",
# r"^/doc/html/draft-elmalki-soliman-hmipv4v6-04",
# r"^/doc/html/draft-ema-vpim-simplev3-02",
# r"^/doc/html/draft-esaki-co-cl-ip-forw-atm-01",
# r"^/doc/html/draft-etf-marid-protocol-00",
# r"^/doc/html/draft-faltstrom-macmime-00",
# r"^/doc/html/draft-faltstrom-whois-05",
# r"^/doc/html/draft-fielding-http-spec-01",
# r"^/doc/html/draft-flick-interfaces-mib-00",
# r"^/doc/html/draft-flick-repeater-dev-mib-00",
# r"^/doc/html/draft-floyd-cc-alt",
# r"^/doc/html/draft-ford-bigten-bt-format-00",
# r"^/doc/html/draft-ford-sdrp-sipp16-format-00",
# r"^/doc/html/draft-freed-ftbp-00",
# r"^/doc/html/draft-freed-newenc-01",
# r"^/doc/html/draft-gellens-imapext-annotate-01",
# r"^/doc/html/draft-gharai-hdtv-video-04",
# r"^/doc/html/draft-glenn-layer-security-00",
# r"^/doc/html/draft-hares-idrp-familytree-00",
# r"^/doc/html/draft-harrington-control-mib-00",
# r"^/doc/html/draft-harrington-data-filter-mib-00",
# r"^/doc/html/draft-haskin-intra-route-server-00",
# r"^/doc/html/draft-helenius-ppp-subnet-04",
# r"^/doc/html/draft-hinden-ipng-addr-00",
# r"^/doc/html/draft-holbrook-ssm-04",
# r"^/doc/html/draft-hollenbeck-epp-tcp-02",
# r"^/doc/html/draft-houldsworth-ip6-nsap-use-00",
# r"^/doc/html/draft-houldsworth-sc6-hot-finland-00",
# r"^/doc/html/draft-huitema-shipworm-01",
# r"^/doc/html/draft-iab-liaisons-00",
# r"^/doc/html/draft-iab-mou2jtc1-03",
# r"^/doc/html/draft-iab-standards-processv3-00",
# r"^/doc/html/draft-ietf-aft-socks-md5-auth-00",
# r"^/doc/html/draft-ietf-bgpdepl-minutes-93feb-00",
# r"^/doc/html/draft-ietf-bmwg-overallperf-00",
# r"^/doc/html/draft-ietf-bridge-sr-obj-00",
# r"^/doc/html/draft-ietf-cat-altftp-00",
# r"^/doc/html/draft-ietf-cip-apisocket-00",
# r"^/doc/html/draft-ietf-cipso-ipsec-option-00",
# r"^/doc/html/draft-ietf-decnetiv-mib-implement-00",
# r"^/doc/html/draft-ietf-dhc-problem-stmt-00",
# r"^/doc/html/draft-ietf-dns-idpr-02",
# r"^/doc/html/draft-ietf-dns-ixfr-01",
# r"^/doc/html/draft-ietf-dnsind-dynDNS-arch-00",
# r"^/doc/html/draft-ietf-dnsind-dynDNS-impl-00",
# r"^/doc/html/draft-ietf-dtn-tcpclv4-00",
# r"^/doc/html/draft-ietf-dtn-tcpclv4-15",
# r"^/doc/html/draft-ietf-dtn-tcpclv4-18",
# r"^/doc/html/draft-ietf-dtn-tcpclv4-19",
# r"^/doc/html/draft-ietf-ethermib-objects-00",
# r"^/doc/html/draft-ietf-fax-tiff-f-reg-01",
# r"^/doc/html/draft-ietf-geopriv-dhcp-lo-option-01",
# r"^/doc/html/draft-ietf-html-charset-harmful-00",
# r"^/doc/html/draft-ietf-iafa-templates-00",
# r"^/doc/html/draft-ietf-idmr-mtree-00",
# r"^/doc/html/draft-ietf-idmr-pim-dense-spec-00",
# r"^/doc/html/draft-ietf-idmr-pim-dm-spec-08",
# r"^/doc/html/draft-ietf-idr-bgp-tcp-md5bad-01",
# r"^/doc/html/draft-ietf-idr-community-00",
# r"^/doc/html/draft-ietf-idr-rifs-00",
# r"^/doc/html/draft-ietf-ids-iwps-design-spec-01",
# r"^/doc/html/draft-ietf-ids-pilots-00",
# r"^/doc/html/draft-ietf-iesg-evolutionplan-00",
# r"^/doc/html/draft-ietf-iiir-html-01",
# r"^/doc/html/draft-ietf-iiir-http-00",
# r"^/doc/html/draft-ietf-ipae-new-ip-00",
# r"^/doc/html/draft-ietf-ipidrp-sip-01",
# r"^/doc/html/draft-ietf-iplpdn-multi-isdn-02",
# r"^/doc/html/draft-ietf-iplpdn-para-negotiation-02",
# r"^/doc/html/draft-ietf-iplpdn-shortcutrouting-02",
# r"^/doc/html/draft-ietf-iplpdn-simple-multi-01",
# r"^/doc/html/draft-ietf-ipp-indp-04",
# r"^/doc/html/draft-ietf-ipsec-ike-base-mode-03",
# r"^/doc/html/draft-ietf-ipsec-intragkm-03",
# r"^/doc/html/draft-ietf-ipsp-spsl-04",
# r"^/doc/html/draft-ietf-ipsra-pic-07",
# r"^/doc/html/draft-ietf-isis-atipx-00",
# r"^/doc/html/draft-ietf-isis-multilevel-routing-00",
# r"^/doc/html/draft-ietf-isis-nbma-00",
# r"^/doc/html/draft-ietf-isis-tcpip-01",
# r"^/doc/html/draft-ietf-isn-aup-01",
# r"^/doc/html/draft-ietf-lsma-scenarios-03",
# r"^/doc/html/draft-ietf-mailext-lang-char-00",
# r"^/doc/html/draft-ietf-mhsds-822dir-03",
# r"^/doc/html/draft-ietf-mhsds-convert-01",
# r"^/doc/html/draft-ietf-mhsds-mhsprofile-04",
# r"^/doc/html/draft-ietf-mhsds-mhsuse-03",
# r"^/doc/html/draft-ietf-mmusic-agree-00",
# r"^/doc/html/draft-ietf-mobileip-aaa-req-00",
# r"^/doc/html/draft-ietf-mobileip-addr-ext-00",
# r"^/doc/html/draft-ietf-mobileip-integrated-00",
# r"^/doc/html/draft-ietf-mobileip-mib-fa-01",
# r"^/doc/html/draft-ietf-mobileip-mib-ha-01",
# r"^/doc/html/draft-ietf-mobileip-mib-mn-01",
# r"^/doc/html/draft-ietf-mobileip-mib-sec-01",
# r"^/doc/html/draft-ietf-msi-api-03",
# r"^/doc/html/draft-ietf-nasreq-nasrequirements-01",
# r"^/doc/html/draft-ietf-netdata-implement-03",
# r"^/doc/html/draft-ietf-netdata-netdata-04",
# r"^/doc/html/draft-ietf-nimrod-dns-01",
# r"^/doc/html/draft-ietf-nisi-nicdoc-00",
# r"^/doc/html/draft-ietf-nisi-nics-00",
# r"^/doc/html/draft-ietf-nntp-news-01",
# r"^/doc/html/draft-ietf-oncrpc-remote-06",
# r"^/doc/html/draft-ietf-osids-dirtree-00",
# r"^/doc/html/draft-ietf-osids-dsanaming-02",
# r"^/doc/html/draft-ietf-osids-requirements-00",
# r"^/doc/html/draft-ietf-osids-simple-stack-00",
# r"^/doc/html/draft-ietf-osids-treestructure-00",
# r"^/doc/html/draft-ietf-osinsap-format-01",
# r"^/doc/html/draft-ietf-osix500-directories-01",
# r"^/doc/html/draft-ietf-ospf-extattr-00",
# r"^/doc/html/draft-ietf-ospf-ipv6-ext-00",
# r"^/doc/html/draft-ietf-ospf-pmp-if-00",
# r"^/doc/html/draft-ietf-otp-ver-03",
# r"^/doc/html/draft-ietf-pana-aaa-interworking-00",
# r"^/doc/html/draft-ietf-pem-notary-00",
# r"^/doc/html/draft-ietf-pim-ipv6-04",
# r"^/doc/html/draft-ietf-pim-simplekmp-02",
# r"^/doc/html/draft-ietf-pint-conf-02",
# r"^/doc/html/draft-ietf-pip-vector-00",
# r"^/doc/html/draft-ietf-poised-nomcomm-00",
# r"^/doc/html/draft-ietf-pppext-aha-auth-00",
# r"^/doc/html/draft-ietf-pppext-ipcp-network-04",
# r"^/doc/html/draft-ietf-pppext-kap-auth-00",
# r"^/doc/html/draft-ietf-pppext-kapv4-auth-00",
# r"^/doc/html/draft-ietf-ripv2-ripng-00",
# r"^/doc/html/draft-ietf-rmon-trap-00",
# r"^/doc/html/draft-ietf-rmonmib-rmon2hc-01",
# r"^/doc/html/draft-ietf-roamops-actng-08",
# r"^/doc/html/draft-ietf-rohc-rtp-rocco-performance-01",
# r"^/doc/html/draft-ietf-rohc-rtp-rocco-video-01",
# r"^/doc/html/draft-ietf-rolc-nhrp-mib-00",
# r"^/doc/html/draft-ietf-rreq-iprouters-04",
# r"^/doc/html/draft-ietf-rsvp-policy-ext-05",
# r"^/doc/html/draft-ietf-rsvp-state-compression-04",
# r"^/doc/html/draft-ietf-sdr-IPv6-pack-00",
# r"^/doc/html/draft-ietf-sdr-pl-00",
# r"^/doc/html/draft-ietf-sdr-route-construction-01",
# r"^/doc/html/draft-ietf-sdr-route-setup-00",
# r"^/doc/html/draft-ietf-sdr-speakers-attribute-00",
# r"^/doc/html/draft-ietf-sip-64bit-plan-00",
# r"^/doc/html/draft-ietf-sip-dnss-00",
# r"^/doc/html/draft-ietf-sip-ospf-00",
# r"^/doc/html/draft-ietf-sip-rip-01",
# r"^/doc/html/draft-ietf-sip-unicast-addr-00",
# r"^/doc/html/draft-ietf-sipp-auto-addr-00",
# r"^/doc/html/draft-ietf-sipp-bsd-api-02",
# r"^/doc/html/draft-ietf-sipp-dhcpopt-01",
# r"^/doc/html/draft-ietf-sipp-discovery-04",
# r"^/doc/html/draft-ietf-sipp-discovery-formats-00",
# r"^/doc/html/draft-ietf-sipp-dns-01",
# r"^/doc/html/draft-ietf-sipp-dns-ext-00",
# r"^/doc/html/draft-ietf-sipp-icmp-igmp-00",
# r"^/doc/html/draft-ietf-sipp-routing-addr-02",
# r"^/doc/html/draft-ietf-sipp-sst-overview-00",
# r"^/doc/html/draft-ietf-sipping-overload-design",
# r"^/doc/html/draft-ietf-smime-certdist-06",
# r"^/doc/html/draft-ietf-smtpext-pipeline-02",
# r"^/doc/html/draft-ietf-snmp-isdn-cisco-00",
# r"^/doc/html/draft-ietf-snmpsec-m2mv2-01",
# r"^/doc/html/draft-ietf-snmpsec-mibv2-00",
# r"^/doc/html/draft-ietf-snmpsec-protov2-01",
# r"^/doc/html/draft-ietf-snmpsec-tmv2-00",
# r"^/doc/html/draft-ietf-stjohns-ipso-00",
# r"^/doc/html/draft-ietf-svrloc-discovery-11",
# r"^/doc/html/draft-ietf-tcplw-extensions-00",
# r"^/doc/html/draft-ietf-tcplw-high-performance-01",
# r"^/doc/html/draft-ietf-telnet-authker-v5-01",
# r"^/doc/html/draft-ietf-telnet-compression-00",
# r"^/doc/html/draft-ietf-telnet-encryption-02",
# r"^/doc/html/draft-ietf-tewg-measure-07",
# r"^/doc/html/draft-ietf-thinosi-profile-00",
# r"^/doc/html/draft-ietf-tnfs-spec-03",
# r"^/doc/html/draft-ietf-ucp-connectivity-01",
# r"^/doc/html/draft-ietf-udlr-life-03",
# r"^/doc/html/draft-ietf-ufdl-spec-01",
# r"^/doc/html/draft-ietf-uri-roy-urn-urc-00",
# r"^/doc/html/draft-ietf-uri-urc-00",
# r"^/doc/html/draft-ietf-uri-urc-sgml-00",
# r"^/doc/html/draft-ietf-uri-urc-spec-00",
# r"^/doc/html/draft-ietf-uri-urc-trivial-00",
# r"^/doc/html/draft-ietf-uri-urn-issues-00",
# r"^/doc/html/draft-ietf-uri-urn-madsen-critique-00",
# r"^/doc/html/draft-ietf-uri-urn-res-descript-00",
# r"^/doc/html/draft-ietf-uri-urn-res-thoughts-00",
# r"^/doc/html/draft-ietf-uri-urn-syntax-00",
# r"^/doc/html/draft-ietf-uri-urn-x-dns-2-00",
# r"^/doc/html/draft-ietf-uri-urn2urc-00",
# r"^/doc/html/draft-ietf-uri-yaurn-00",
# r"^/doc/html/draft-ietf-userdoc2-fyi-biblio-00",
# r"^/doc/html/draft-ietf-uswg-fyi1-02",
# r"^/doc/html/draft-ietf-whip-reqs-summary-01",
# r"^/doc/html/draft-ietf-x400ops-admd-03",
# r"^/doc/html/draft-ietf-x400ops-dnsx400rout-02",
# r"^/doc/html/draft-ietf-x400ops-tbl-dist-00",
# r"^/doc/html/draft-ietf-x400ops-tbl-dist-part1-01",
# r"^/doc/html/draft-ietf-x400ops-tbl-dist-part2-01",
# r"^/doc/html/draft-ipsec-isakmp-mode-cfg-02",
# r"^/doc/html/draft-johnson-imhp-00",
# r"^/doc/html/draft-jseng-utf5-02",
# r"^/doc/html/draft-just-ldapv3-rescodes-03",
# r"^/doc/html/draft-karrenberg-proposal-00",
# r"^/doc/html/draft-kastenholz-loki-00",
# r"^/doc/html/draft-kempf-scope-rules-04",
# r"^/doc/html/draft-klyne-conneg-feature-match-03",
# r"^/doc/html/draft-koch-dnsind-local-compression-01",
# r"^/doc/html/draft-kzm-rap-sppi-04",
# r"^/doc/html/draft-kzm-snmpv2-adminv2-alt-00",
# r"^/doc/html/draft-kzm-snmpv2-coex-alt-00",
# r"^/doc/html/draft-kzm-snmpv2-conf-alt-00",
# r"^/doc/html/draft-kzm-snmpv2-intro-alt-00",
# r"^/doc/html/draft-kzm-snmpv2-mib-alt-00",
# r"^/doc/html/draft-kzm-snmpv2-smi-alt-00",
# r"^/doc/html/draft-kzm-snmpv2-usec-conf-alt-00",
# r"^/doc/html/draft-larson-bad-dns-res-01",
# r"^/doc/html/draft-lear-foglamps-03",
# r"^/doc/html/draft-leech-socks-protocol-v4-01",
# r"^/doc/html/draft-levi-snmp-mid-level-mgr-00",
# r"^/doc/html/draft-levi-snmp-script-language-00",
# r"^/doc/html/draft-levinson-sgml-02",
# r"^/doc/html/draft-li-bigten-addr-format-00",
# r"^/doc/html/draft-li-tap-ipv7-00",
# r"^/doc/html/draft-lloyd-ip6-iso-itu-reg-00",
# r"^/doc/html/draft-macker-mdp-framework-05",
# r"^/doc/html/draft-mahoney-snmpv2-features-00",
# r"^/doc/html/draft-mahoney-snmpv2-proto-alt-00",
# r"^/doc/html/draft-martensson-rocco-video-04",
# r"^/doc/html/draft-mccann-mobileip-sessionid-04",
# r"^/doc/html/draft-megginson-ldup-lcup-01",
# r"^/doc/html/draft-metzger-ah-sha-00",
# r"^/doc/html/draft-mpls-rsvpte-attributes-00",
# r"^/doc/html/draft-myers-imap-imsp-01",
# r"^/doc/html/draft-myers-imap-mbox-00",
# r"^/doc/html/draft-myers-smtp-mult-01",
# r"^/doc/html/draft-nelson-model-mailext-00",
# r"^/doc/html/draft-newman-imap-annotate-00",
# r"^/doc/html/draft-nguyen-bgp-ipv6-vpn-03",
# r"^/doc/html/draft-nordmark-ipv6-aaa-hooks-04",
# r"^/doc/html/draft-nyckelgard-isl-arch-04",
# r"^/doc/html/draft-ohta-address-allocation-01",
# r"^/doc/html/draft-ohta-dynamic-dns-00",
# r"^/doc/html/draft-ohta-ip-over-atm-02",
# r"^/doc/html/draft-ohta-mime-charset-names-00",
# r"^/doc/html/draft-ohta-shared-media-02",
# r"^/doc/html/draft-ohta-simple-dns-01",
# r"^/doc/html/draft-ohta-text-encoding-01",
# r"^/doc/html/draft-ohta-translation-instr-01",
# r"^/doc/html/draft-ooms-cl-multicast-03",
# r"^/doc/html/draft-ops-rfc2011-update-01",
# r"^/doc/html/draft-ouldbrahim-bgpvpn-auto-03",
# r"^/doc/html/draft-palme-autosub-06",
# r"^/doc/html/draft-pan-diffserv-mib-01",
# r"^/doc/html/draft-perkins-cnlp-support-00",
# r"^/doc/html/draft-perkins-homeaddr-dhcpopt-00",
# r"^/doc/html/draft-perkins-opaque-04",
# r"^/doc/html/draft-polk-slp-loc-auth-server-04",
# r"^/doc/html/draft-popp-cnrp-goals-01",
# r"^/doc/html/draft-pusateri-igmp-mib-00",
# r"^/doc/html/draft-pusateri-ipmulti-mib-00",
# r"^/doc/html/draft-reddy-opsawg-mud-tls-00",
# r"^/doc/html/draft-reddy-opsawg-mud-tls-03",
# r"^/doc/html/draft-reichmeyer-polterm-terminology-04",
# r"^/doc/html/draft-rekhter-arch-sipp16-addr-00",
# r"^/doc/html/draft-rekhter-bigten-addr-arch-00",
# r"^/doc/html/draft-rekhter-direct-provider-01",
# r"^/doc/html/draft-rekhter-idr-over-atm-00",
# r"^/doc/html/draft-rekhter-lsr-mobile-hosts-00",
# r"^/doc/html/draft-rekhter-select-providers-02",
# r"^/doc/html/draft-rekhter-sops-02",
# r"^/doc/html/draft-rekhter-stratum-aggregation-01",
# r"^/doc/html/draft-renwick-hippiarp-01",
# r"^/doc/html/draft-renwick-hippimib-01",
# r"^/doc/html/draft-rfced-info-corson-00",
# r"^/doc/html/draft-rfced-info-katsube-oops-00",
# r"^/doc/html/draft-rfced-info-perkins-05",
# r"^/doc/html/draft-rfced-info-pi-vs-pa-addrspac-00",
# r"^/doc/html/draft-rfced-info-senie-00",
# r"^/doc/html/draft-ronc-domain-phb-set-ldap-rep-04",
# r"^/doc/html/draft-ronc-domain-phb-set-specification-04",
# r"^/doc/html/draft-rose-limit-01",
# r"^/doc/html/draft-rose-smxp-spec-00",
# r"^/doc/html/draft-rosen-ppvpn-l2vpn-01",
# r"^/doc/html/draft-rosen-tag-stack-05",
# r"^/doc/html/draft-rosenberg-mmusic-sdp-offer-answer-01",
# r"^/doc/html/draft-rosenberg-sip-tunnels-01",
# r"^/doc/html/draft-salzr-ldap-repsig-01",
# r"^/doc/html/draft-sandick-pimsm-ssmrules-04",
# r"^/doc/html/draft-schroeppel-dnsind-ecc-04",
# r"^/doc/html/draft-simpson-exchanges-00",
# r"^/doc/html/draft-simpson-ipv6-deploy-00",
# r"^/doc/html/draft-simpson-ipv6-discovery-req-00",
# r"^/doc/html/draft-simpson-ipv6-hc-00",
# r"^/doc/html/draft-simpson-sipp-64-bit-plan-00",
# r"^/doc/html/draft-sinnreich-interdomain-sip-qos-osp-02",
# r"^/doc/html/draft-slutsman-aicd-02",
# r"^/doc/html/draft-speer-avt-layered-video-05",
# r"^/doc/html/draft-stein-green-commerce-model-00",
# r"^/doc/html/draft-svanbro-rohc-lower-layer-guidelines-04",
# r"^/doc/html/draft-templin-atn-aero-interface-00",
# r"^/doc/html/draft-templin-atn-aero-interface-21",
# r"^/doc/html/draft-teraoka-ipv6-mobility-sup-07",
# r"^/doc/html/draft-thayer-seccomp-04",
# r"^/doc/html/draft-traina-bgp-confed-00",
# r"^/doc/html/draft-treese-class-desc-00",
# r"^/doc/html/draft-vaudreuil-binaryheaders-01",
# r"^/doc/html/draft-vaudreuil-enum-e164dir-05",
# r"^/doc/html/draft-veizades-ipng-svrloc-00",
# r"^/doc/html/draft-villamizar-isis-omp-01",
# r"^/doc/html/draft-waldbusser-conventions-01",
# r"^/doc/html/draft-waldbusser-rmonmib-apm-04",
# r"^/doc/html/draft-waldbusser-ssecimpl-01",
# r"^/doc/html/draft-waters-snmpv1-sec-mech-00",
# r"^/doc/html/draft-weider-comindex-00",
# r"^/doc/html/draft-wijnen-snmpv2-snmpv2t-00",
# r"^/doc/html/draft-woundy-dhcpleasequery-04",
# r"^/doc/html/draft-wright-policy-mpls-04",
# r"^/doc/html/draft-yu-asn1-pitfalls-04",
# r"^/doc/html/draft-yu-rpd-00",
# r"^/doc/html/draft-zaccone-nat-rsip-gen-arch-02",
# r"^/doc/html/draft-zaccone-nat-transp-fram-02",
# r"^/doc/html/draft-zaccone-nat-transport-03",
# r"^/doc/html/status-change-icmpv6-dns-ipv6-to-internet-standard",
r"^/static/coverage/", r"^/static/coverage/",
r"^/meeting/6[0-4]/agenda", r"^/meeting/\d{,2}/agenda", # no agendas < 100
r"^https?://www.ietf.org/",
): ):
if re.search(pattern, url): if re.search(pattern, url):
return True return True
@ -691,8 +290,16 @@ logfile = None
if args.logfile: if args.logfile:
logfile = open(args.logfile, "w") logfile = open(args.logfile, "w")
vnu = None
# --- Main --- # --- Main ---
def do_exit(code):
if vnu:
vnu.terminate()
sys.exit(code)
if __name__ == "__main__": if __name__ == "__main__":
if (args.user): if (args.user):
# log in as user, to have the respective HTML generated by the templates # log in as user, to have the respective HTML generated by the templates
@ -702,7 +309,7 @@ if __name__ == "__main__":
if (response.status_code != 200): if (response.status_code != 200):
log("Could not log in as %s, HTML response %d" % log("Could not log in as %s, HTML response %d" %
(args.user, response.status_code)) (args.user, response.status_code))
sys.exit(1) do_exit(1)
# Run django system checks and checks from ietf.checks: # Run django system checks and checks from ietf.checks:
error_list = django.core.checks.run_checks() error_list = django.core.checks.run_checks()
@ -718,10 +325,13 @@ if __name__ == "__main__":
for entry in error_list: for entry in error_list:
print(entry) print(entry)
if args.validator_nu:
vnu = start_vnu_server(port=8887)
while urls: while urls:
if args.random: if args.random:
# popitem() is documented to be random, but really isn't # popitem() is documented to be random, but really isn't
url = random.choice(urls.keys()) url = random.choice(list(urls.keys()))
referrer = urls.pop(url) referrer = urls.pop(url)
else: else:
url, referrer = urls.popitem() url, referrer = urls.popitem()
@ -746,10 +356,10 @@ if __name__ == "__main__":
elapsed = datetime.datetime.now() - request_start elapsed = datetime.datetime.now() - request_start
except KeyboardInterrupt: except KeyboardInterrupt:
log(" ... was fetching %s" % url) log(" ... was fetching %s" % url)
sys.exit(1) do_exit(1)
except: except:
elapsed = datetime.datetime.now() - request_start elapsed = datetime.datetime.now() - request_start
tags = [ u"FAIL (from [ %s ])" % (",\n\t".join(get_referrers(url))) ] tags = [ "FAIL (from [ %s ])" % (",\n\t".join(get_referrers(url))) ]
log("%2d:%02d:%02d %7d %6d %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), 500, elapsed.total_seconds(), url, " ".join(tags))) log("%2d:%02d:%02d %7d %6d %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), 500, elapsed.total_seconds(), url, " ".join(tags)))
log("=============") log("=============")
log(traceback.format_exc()) log(traceback.format_exc())
@ -760,7 +370,7 @@ if __name__ == "__main__":
if r.status_code in (301, 302): if r.status_code in (301, 302):
u = strip_url(r["Location"]) u = strip_url(r["Location"])
if u not in visited and u not in urls: if not url.startswith("/") and u not in visited and u not in urls:
urls[u] = referrer # referrer is original referrer, not redirected url urls[u] = referrer # referrer is original referrer, not redirected url
referrers[u] = referrer referrers[u] = referrer
@ -799,8 +409,9 @@ if __name__ == "__main__":
log("=============") log("=============")
else: else:
tags.append(u"FAIL (from %s)" % (referrer, )) tags.append("FAIL (from {})".format(referrer))
errors += 1 if not url.startswith("/person/"): # FIXME: those fail sometimes
errors += 1
if elapsed.total_seconds() > slow_threshold: if elapsed.total_seconds() > slow_threshold:
tags.append("SLOW") tags.append("SLOW")
@ -816,7 +427,7 @@ if __name__ == "__main__":
log("%2d:%02d:%02d %7d %6d %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), r.status_code, elapsed.total_seconds(), url, " ".join(tags))) log("%2d:%02d:%02d %7d %6d %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), r.status_code, elapsed.total_seconds(), url, " ".join(tags)))
if ((errors or warnings) and args.pedantic): if ((errors or warnings) and args.pedantic):
sys.exit(1) do_exit(1)
if logfile: if logfile:
logfile.close() logfile.close()
@ -824,7 +435,7 @@ if __name__ == "__main__":
if errors > 0: if errors > 0:
sys.stderr.write("Found %s errors, grep output for FAIL for details\n" % errors) sys.stderr.write("Found %s errors, grep output for FAIL for details\n" % errors)
sys.exit(1) do_exit(1)
else: else:
sys.stderr.write("Found no errors.\n") sys.stderr.write("Found no errors.\n")
if warnings > 0: if warnings > 0:

BIN
bin/vnu.jar
View file

Binary file not shown.

1
dev/tests/Dockerfile
View file

@ -23,6 +23,7 @@ RUN apt-get install -qy \
bash \ bash \
build-essential \ build-essential \
curl \ curl \
default-jdk \
docker-ce-cli \ docker-ce-cli \
enscript \ enscript \
gawk \ gawk \

1
docker/app.Dockerfile
View file

@ -32,6 +32,7 @@ RUN apt-get install -qy \
bash \ bash \
build-essential \ build-essential \
curl \ curl \
default-jdk \
docker-ce-cli \ docker-ce-cli \
enscript \ enscript \
fish \ fish \

107
ietf/doc/templatetags/ietf_filters.py
View file

@ -4,6 +4,7 @@
import datetime import datetime
import re import re
import os
from urllib.parse import urljoin from urllib.parse import urljoin
from email.utils import parseaddr from email.utils import parseaddr
@ -17,10 +18,13 @@ from django.utils.html import strip_tags
from django.utils.encoding import force_text from django.utils.encoding import force_text
from django.utils.encoding import force_str # pyflakes:ignore force_str is used in the doctests from django.utils.encoding import force_str # pyflakes:ignore force_str is used in the doctests
from django.urls import reverse as urlreverse from django.urls import reverse as urlreverse
from django.core.cache import cache
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError
import debug # pyflakes:ignore import debug # pyflakes:ignore
from ietf.doc.models import BallotDocEvent from ietf.doc.models import BallotDocEvent, DocAlias
from ietf.doc.models import ConsensusDocEvent from ietf.doc.models import ConsensusDocEvent
from ietf.utils.html import sanitize_fragment from ietf.utils.html import sanitize_fragment
from ietf.utils import log from ietf.utils import log
@ -184,49 +188,113 @@ def rfceditor_info_url(rfcnum : str):
"""Link to the RFC editor info page for an RFC""" """Link to the RFC editor info page for an RFC"""
return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}') return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}')
def doc_exists(name):
"""Check whether a given document exists"""
def find_unique(n):
key = hash(n)
found = cache.get(key)
if not found:
exact = DocAlias.objects.filter(name=n).first()
found = exact.name if exact else "_"
cache.set(key, found)
return None if found == "_" else found
# all documents exist when tests are running
if settings.SERVER_MODE == 'test':
# unless we are running test-crawl, which would otherwise 404
if "DJANGO_URLIZE_IETF_DOCS_PRODUCTION" not in os.environ:
return True
# chop away extension
extension_split = re.search(r"^(.+)\.(txt|ps|pdf)$", name)
if extension_split:
name = extension_split.group(1)
if find_unique(name):
return True
# check for embedded rev - this may be ambiguous, so don't
# chop it off if we don't find a match
rev_split = re.search("^(.+)-([0-9]{2,})$", name)
if rev_split:
name = rev_split.group(1)
if find_unique(name):
return True
return False
def link_charter_doc_match1(match):
if not doc_exists(match[0]):
return match[0]
return f'<a href="/doc/{match[1][:-1]}/{match[2]}/">{match[0]}</a>'
def link_charter_doc_match2(match):
if not doc_exists(match[0]):
return match[0]
return f'<a href="/doc/{match[1][:-1]}/{match[2]}/">{match[0]}</a>'
def link_non_charter_doc_match(match): def link_non_charter_doc_match(match):
if len(match[3])==2 and match[3].isdigit(): if not doc_exists(match[0]):
return match[0]
if len(match[3]) == 2 and match[3].isdigit():
return f'<a href="/doc/{match[2][:-1]}/{match[3]}/">{match[0]}</a>' return f'<a href="/doc/{match[2][:-1]}/{match[3]}/">{match[0]}</a>'
else: else:
return f'<a href="/doc/{match[2]}{match[3]}/">{match[0]}</a>' return f'<a href="/doc/{match[2]}{match[3]}/">{match[0]}</a>'
@register.filter(name='urlize_ietf_docs', is_safe=True, needs_autoescape=True)
def link_other_doc_match(match):
# there may be whitespace in the match
doc = re.sub(r"\s+", "", match[0])
if not doc_exists(doc):
return match[0]
return f'<a href="/doc/{match[2].strip().lower()}{match[3]}/">{match[1]}</a>'
@register.filter(name="urlize_ietf_docs", is_safe=True, needs_autoescape=True)
def urlize_ietf_docs(string, autoescape=None): def urlize_ietf_docs(string, autoescape=None):
""" """
Make occurrences of RFC NNNN and draft-foo-bar links to /doc/. Make occurrences of RFC NNNN and draft-foo-bar links to /doc/.
""" """
if autoescape and not isinstance(string, SafeData): if autoescape and not isinstance(string, SafeData):
string = escape(string) if "<" in string:
exp1 = r"\b(charter-(?:[\d\w\.+]+-)*)(\d\d-\d\d)(\.txt)?\b" string = escape(string)
exp2 = r"\b(charter-(?:[\d\w\.+]+-)*)(\d\d)(\.txt)?\b" else:
string = mark_safe(string)
exp1 = r"\b(?<![/\-:=#])(charter-(?:[\d\w\.+]+-)*)(\d\d-\d\d)(\.txt)?\b"
exp2 = r"\b(?<![/\-:=#])(charter-(?:[\d\w\.+]+-)*)(\d\d)(\.txt)?\b"
if re.search(exp1, string): if re.search(exp1, string):
string = re.sub( string = re.sub(
exp1, exp1,
lambda x: f'<a href="/doc/{x[1][:-1]}/{x[2]}/">{x[0]}</a>', link_charter_doc_match1,
string, string,
flags=re.IGNORECASE | re.ASCII, flags=re.IGNORECASE | re.ASCII,
) )
elif re.search(exp2, string): elif re.search(exp2, string):
string = re.sub( string = re.sub(
exp2, exp2,
lambda x: f'<a href="/doc/{x[1][:-1]}/{x[2]}/">{x[0]}</a>', link_charter_doc_match2,
string, string,
flags=re.IGNORECASE | re.ASCII, flags=re.IGNORECASE | re.ASCII,
) )
string = re.sub( string = re.sub(
r"\b(?<![/-])(((?:draft-|bofreq-|conflict-review-|status-change-)(?:[\d\w\.+]+-)*)([\d\w\.+]+?)(\.txt)?)\b", r"\b(?<![/\-:=#])(((?:draft-|bofreq-|conflict-review-|status-change-)(?:[\d\w\.+]+-)*)([\d\w\.+]+?)(\.txt)?)\b(?![-@])",
link_non_charter_doc_match, link_non_charter_doc_match,
string, string,
flags=re.IGNORECASE | re.ASCII, flags=re.IGNORECASE | re.ASCII,
) )
string = re.sub( string = re.sub(
# r"\b((RFC|BCP|STD|FYI|(?:draft-|bofreq-|conflict-review-|status-change-|charter-)[-\d\w.+]+)\s*0*(\d+))\b", # r"\b((RFC|BCP|STD|FYI|(?:draft-|bofreq-|conflict-review-|status-change-|charter-)[-\d\w.+]+)\s*0*(\d+))\b",
r"\b(?<!-)((RFC|BCP|STD|FYI)\s*0*(\d+))\b", r"\b(?<![/\-:=#])((RFC|BCP|STD|FYI)\s*0*(\d+))\b",
lambda x: f'<a href="/doc/{x[2].strip().lower()}{x[3]}/">{x[1]}</a>', link_other_doc_match,
string, string,
flags=re.IGNORECASE | re.ASCII, flags=re.IGNORECASE | re.ASCII,
) )
return mark_safe(string) return mark_safe(string)
urlize_ietf_docs = stringfilter(urlize_ietf_docs) urlize_ietf_docs = stringfilter(urlize_ietf_docs)
@register.filter(name='urlize_related_source_list', is_safe=True, needs_autoescape=True) @register.filter(name='urlize_related_source_list', is_safe=True, needs_autoescape=True)
@ -444,7 +512,7 @@ def format_snippet(text, trunc_words=25):
@register.simple_tag @register.simple_tag
def doc_edit_button(url_name, *args, **kwargs): def doc_edit_button(url_name, *args, **kwargs):
"""Given URL name/args/kwargs, looks up the URL just like "url" tag and returns a properly formatted button for the document material tables.""" """Given URL name/args/kwargs, looks up the URL just like "url" tag and returns a properly formatted button for the document material tables."""
return mark_safe('<a class="btn btn-primary btn-sm" type="button" href="%s">Edit</a>' % (urlreverse(url_name, args=args, kwargs=kwargs))) return mark_safe('<a class="btn btn-primary btn-sm" href="%s">Edit</a>' % (urlreverse(url_name, args=args, kwargs=kwargs)))
@register.filter @register.filter
def textify(text): def textify(text):
@ -765,3 +833,16 @@ def absurl(viewname, **kwargs):
Uses settings.IDTRACKER_BASE_URL as the base. Uses settings.IDTRACKER_BASE_URL as the base.
""" """
return urljoin(settings.IDTRACKER_BASE_URL, urlreverse(viewname, kwargs=kwargs)) return urljoin(settings.IDTRACKER_BASE_URL, urlreverse(viewname, kwargs=kwargs))
@register.filter
def is_valid_url(url):
"""
Check if the given URL is syntactically valid
"""
validate_url = URLValidator()
try:
validate_url(url)
except ValidationError:
return False
return True

14
ietf/doc/templatetags/tests_ietf_filters.py
View file

@ -46,14 +46,22 @@ class IetfFiltersTests(TestCase):
), ),
( (
"draft-madanapalli-nd-over-802.16-problems", "draft-madanapalli-nd-over-802.16-problems",
'<a href="/doc/draft-madanapalli-nd-over-802.16-problems/">draft-madanapalli-nd-over-802.16-problems</a>' '<a href="/doc/draft-madanapalli-nd-over-802.16-problems/">draft-madanapalli-nd-over-802.16-problems</a>'
), ),
( (
"draft-madanapalli-nd-over-802.16-problems-02.txt", "draft-madanapalli-nd-over-802.16-problems-02.txt",
'<a href="/doc/draft-madanapalli-nd-over-802.16-problems/02/">draft-madanapalli-nd-over-802.16-problems-02.txt</a>' '<a href="/doc/draft-madanapalli-nd-over-802.16-problems/02/">draft-madanapalli-nd-over-802.16-problems-02.txt</a>'
),
(
'<a href="mailto:draft-ietf-some-names@ietf.org">draft-ietf-some-names@ietf.org</a>',
'<a href="mailto:draft-ietf-some-names@ietf.org">draft-ietf-some-names@ietf.org</a>',
),
(
"http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf",
"http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf"
) )
] ]
# Some edge cases scraped from existing old draft names # Some edge cases scraped from existing old draft names
for name in [ for name in [
# "draft-odell-8+8", # This fails since + matches the right side of \b # "draft-odell-8+8", # This fails since + matches the right side of \b

4
ietf/group/utils.py
View file

@ -23,6 +23,7 @@ from ietf.person.models import Email
from ietf.review.utils import can_manage_review_requests_for_team from ietf.review.utils import can_manage_review_requests_for_team
from ietf.utils import log from ietf.utils import log
from ietf.utils.history import get_history_object_for, copy_many_to_many_for_history from ietf.utils.history import get_history_object_for, copy_many_to_many_for_history
from ietf.doc.templatetags.ietf_filters import is_valid_url
from functools import reduce from functools import reduce
def save_group_in_history(group): def save_group_in_history(group):
@ -208,7 +209,8 @@ def construct_group_menu_context(request, group, selected, group_type, others):
entries.append(("Photos", urlreverse("ietf.group.views.group_photos", kwargs=kwargs))) entries.append(("Photos", urlreverse("ietf.group.views.group_photos", kwargs=kwargs)))
entries.append(("Email expansions", urlreverse("ietf.group.views.email", kwargs=kwargs))) entries.append(("Email expansions", urlreverse("ietf.group.views.email", kwargs=kwargs)))
if group.list_archive.startswith("http:") or group.list_archive.startswith("https:") or group.list_archive.startswith("ftp:"): if group.list_archive.startswith("http:") or group.list_archive.startswith("https:") or group.list_archive.startswith("ftp:"):
entries.append((mark_safe("List archive &raquo;"), group.list_archive)) if is_valid_url(group.list_archive):
entries.append((mark_safe("List archive &raquo;"), group.list_archive))
# actions # actions

4
ietf/meeting/forms.py
View file

@ -35,7 +35,7 @@ from ietf.utils.validators import ( validate_file_size, validate_mime_type,
# need to insert empty option for use in ChoiceField # need to insert empty option for use in ChoiceField
# countries.insert(0, ('', '-'*9 )) # countries.insert(0, ('', '-'*9 ))
countries.insert(0, ('', '')) countries.insert(0, ('', '-' * 9))
timezones.insert(0, ('', '-' * 9)) timezones.insert(0, ('', '-' * 9))
# ------------------------------------------------- # -------------------------------------------------
@ -827,4 +827,4 @@ def sessiondetailsformset_factory(min_num=1, max_num=3):
min_num=min_num, min_num=min_num,
max_num=max_num, max_num=max_num,
extra=max_num, # only creates up to max_num total extra=max_num, # only creates up to max_num total
) )

4
ietf/secr/proceedings/forms.py
View file

@ -22,7 +22,9 @@ VALID_BLUESHEET_EXTENSIONS = ('.pdf','.jpg','.jpeg')
class RecordingForm(forms.Form): class RecordingForm(forms.Form):
external_url = forms.URLField(label='Url') external_url = forms.URLField(label='Url')
session = forms.ModelChoiceField(queryset=Session.objects,empty_label='') session = forms.ModelChoiceField(queryset=Session.objects)
session.widget.attrs['class'] = "select2-field"
session.widget.attrs['data-minimum-input-length'] = 0
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.meeting = kwargs.pop('meeting') self.meeting = kwargs.pop('meeting')

6
ietf/secr/static/js/proceedings-recording.js
View file

@ -1,6 +0,0 @@
/* proceedings-recordings.js - utility functions */
$(document).ready(function() {
$('#id_session').select2({ placeholder: 'Type group acronym or part of session name', width: '450px' });;
});

3
ietf/secr/templates/proceedings/recording.html
View file

@ -11,7 +11,6 @@
{% block extrahead %}{{ block.super }} {% block extrahead %}{{ block.super }}
<script src="{% static 'ietf/js/jquery-ui.js' %}"></script> <script src="{% static 'ietf/js/jquery-ui.js' %}"></script>
<script src="{% static 'ietf/js/select2.js' %}"></script> <script src="{% static 'ietf/js/select2.js' %}"></script>
<script src="{% static 'secr/js/proceedings-recording.js' %}"></script>
{% endblock %} {% endblock %}
{% block breadcrumbs %}{{ block.super }} {% block breadcrumbs %}{{ block.super }}
@ -119,4 +118,4 @@
{% block footer-extras %} {% block footer-extras %}
{% include "includes/upload_footer.html" %} {% include "includes/upload_footer.html" %}
{% endblock %} {% endblock %}

2
ietf/static/js/agenda_timezone.js
View file

@ -25,7 +25,7 @@
.utc(); .utc();
item.end_ts = moment.unix(this.getAttribute("data-end-time")) item.end_ts = moment.unix(this.getAttribute("data-end-time"))
.utc(); .utc();
if (this.hasAttribute("weekday")) { if (this.hasAttribute("data-weekday")) {
item.format = 2; item.format = 2;
} else { } else {
item.format = 1; item.format = 1;

6
ietf/templates/doc/ballot/send_ballot_comment.html
View file

@ -32,8 +32,8 @@
<input id="subject" class="form-control" type="text" placeholder="{{ subject }}" disabled> <input id="subject" class="form-control" type="text" placeholder="{{ subject }}" disabled>
</div> </div>
<div class="mb-3"> <div class="mb-3">
<label class="form-label" for="body">Body</label> <p class="form-label">Body</p>
<pre id="body" class="border p-3">{{ body|maybewordwrap }}</pre> <pre class="border p-3">{{ body|maybewordwrap }}</pre>
</div> </div>
<button type="submit" class="btn btn-danger">Send</button> <button type="submit" class="btn btn-danger">Send</button>
<a class="btn btn-secondary float-end" <a class="btn btn-secondary float-end"
@ -41,4 +41,4 @@
Back Back
</a> </a>
</form> </form>
{% endblock %} {% endblock %}

8
ietf/templates/doc/document_ballot_content.html
View file

@ -122,7 +122,7 @@
</div> </div>
</div> </div>
<div class="card-body"> <div class="card-body">
<pre class="ballot pasted">{{ p.discuss|linkify|urlize_ietf_docs }}</pre> <pre class="ballot pasted">{{ p.discuss|urlize_ietf_docs|linkify }}</pre>
</div> </div>
</div> </div>
{% endif %} {% endif %}
@ -148,7 +148,7 @@
</div> </div>
</div> </div>
<div class="card-body"> <div class="card-body">
<pre class="ballot pasted">{{ p.comment|linkify|urlize_ietf_docs }}</pre> <pre class="ballot pasted">{{ p.comment|urlize_ietf_docs|linkify }}</pre>
</div> </div>
</div> </div>
{% endif %} {% endif %}
@ -199,11 +199,11 @@
</div> </div>
{% if p.pos.blocking and p.discuss %} {% if p.pos.blocking and p.discuss %}
<div class="card-body"> <div class="card-body">
<pre class="ballot pasted">{{ p.discuss|linkify|urlize_ietf_docs }}</pre> <pre class="ballot pasted">{{ p.discuss|urlize_ietf_docs|linkify }}</pre>
</div> </div>
{% else %} {% else %}
<div class="card-body"> <div class="card-body">
<pre class="ballot pasted">{{ p.comment|linkify|urlize_ietf_docs }}</pre> <pre class="ballot pasted">{{ p.comment|urlize_ietf_docs|linkify }}</pre>
</div> </div>
{% endif %} {% endif %}
</div> </div>

6
ietf/templates/doc/document_bofreq.html
View file

@ -107,13 +107,13 @@
{% if resources %} {% if resources %}
{% for resource in resources|dictsort:"display_name" %} {% for resource in resources|dictsort:"display_name" %}
{% if resource.name.type.slug == 'url' or resource.name.type.slug == 'email' %} {% if resource.name.type.slug == 'url' or resource.name.type.slug == 'email' %}
<a href="{{ resource.value }}" title="{{ resource.name.name }}"> <a href="{{ resource.value|urlencode }}" title="{{ resource.name.name }}">
{% firstof resource.display_name resource.name.name %} {% firstof resource.display_name resource.name.name %}
</a> </a>
<br> <br>
{# Maybe make how a resource displays itself a method on the class so templates aren't doing this switching #} {# Maybe make how a resource displays itself a method on the class so templates aren't doing this switching #}
{% else %} {% else %}
<span title="{{ resource.name.name }}">{% firstof resource.display_name resource.name.name %}: {{ resource.value }}</span> <span title="{{ resource.name.name }}">{% firstof resource.display_name resource.name.name %}: {{ resource.value|escape }}</span>
<br> <br>
{% endif %} {% endif %}
{% endfor %} {% endfor %}
@ -164,4 +164,4 @@
</script> </script>
<script src="{% static 'ietf/js/document_timeline.js' %}"> <script src="{% static 'ietf/js/document_timeline.js' %}">
</script> </script>
{% endblock %} {% endblock %}

2
ietf/templates/doc/document_charter.html
View file

@ -230,7 +230,7 @@
{{ doc.canonical_name }}-{{ doc.rev }} {{ doc.canonical_name }}-{{ doc.rev }}
</div> </div>
<div class="card-body"> <div class="card-body">
<pre>{{ content|maybewordwrap|linkify|urlize_ietf_docs }}</pre> <pre>{{ content|maybewordwrap|urlize_ietf_docs|linkify }}</pre>
</div> </div>
</div> </div>
{% endif %} {% endif %}

2
ietf/templates/doc/document_conflict_review.html
View file

@ -139,7 +139,7 @@
{{ doc.name }}-{{ doc.rev }} {{ doc.name }}-{{ doc.rev }}
</div> </div>
<div class="card-body"> <div class="card-body">
<pre>{{ content|maybewordwrap|linkify|urlize_ietf_docs }}</pre> <pre>{{ content|maybewordwrap|urlize_ietf_docs|linkify }}</pre>
</div> </div>
</div> </div>
{% endif %} {% endif %}

19
ietf/templates/doc/document_draft.html
View file

@ -366,23 +366,30 @@
{% if resources or doc.group and doc.group.list_archive %} {% if resources or doc.group and doc.group.list_archive %}
{% for resource in resources|dictsort:"display_name" %} {% for resource in resources|dictsort:"display_name" %}
{% if resource.name.type.slug == 'url' or resource.name.type.slug == 'email' %} {% if resource.name.type.slug == 'url' or resource.name.type.slug == 'email' %}
<a href="{{ resource.value }}" title="{{ resource.name.name }}"> <a href="{{ resource.value|urlencode }}" title="{{ resource.name.name }}">
{% firstof resource.display_name resource.name.name %} {% firstof resource.display_name resource.name.name %}
</a> </a>
<br> <br>
{# Maybe make how a resource displays itself a method on the class so templates aren't doing this switching #} {# Maybe make how a resource displays itself a method on the class so templates aren't doing this switching #}
{% else %} {% else %}
<span title="{{ resource.name.name }}"> <span title="{{ resource.name.name }}">
{% firstof resource.display_name resource.name.name %}: {{ resource.value }} {% firstof resource.display_name resource.name.name %}: {{ resource.value|escape }}
</span> </span>
<br> <br>
{% endif %} {% endif %}
{% endfor %} {% endfor %}
{% if doc.group and doc.group.list_archive %} {% if doc.group and doc.group.list_archive %}
<a href="{{ doc.group.list_archive }}?q={{ doc.name }}"> {% if doc.group.list_archive|startswith:settings.MAILING_LIST_ARCHIVE_URL %}
Mailing list discussion <a href="{{ doc.group.list_archive }}?q={{ doc.name }}">
</a> Mailing list discussion
<br> </a>
{% elif doc.group.list_archive|is_valid_url %}
<a href="{{ doc.group.list_archive }}">
Mailing list discussion
</a>
{% else %}
{{ doc.group.list_archive|urlencode }}
{% endif %}
{% endif %} {% endif %}
{% endif %} {% endif %}
</td> </td>

8
ietf/templates/doc/document_material.html
View file

@ -3,7 +3,7 @@
{% load origin %} {% load origin %}
{% load static %} {% load static %}
{% load ietf_filters textfilters %} {% load ietf_filters textfilters %}
{% block title %}{{ doc.title }}{% endblock %} {% block title %}{{ doc.title|default:"Untitled" }}{% endblock %}
{% block content %} {% block content %}
{% origin %} {% origin %}
{{ top|safe }} {{ top|safe }}
@ -36,7 +36,7 @@
{% doc_edit_button 'ietf.doc.views_material.edit_material' name=doc.name action="title" %} {% doc_edit_button 'ietf.doc.views_material.edit_material' name=doc.name action="title" %}
{% endif %} {% endif %}
</td> </td>
<th scope="row">{{ doc.title }}</th> <th scope="row">{{ doc.title|default:'<span class="text-muted">(None)</span>' }}</th>
</tr> </tr>
{% if doc.abstract or doc.type_id == 'slides' and can_manage_material and not snapshot %} {% if doc.abstract or doc.type_id == 'slides' and can_manage_material and not snapshot %}
<tr> <tr>
@ -122,7 +122,7 @@
{% if content_is_html %} {% if content_is_html %}
{{ content|sanitize|safe }} {{ content|sanitize|safe }}
{% else %} {% else %}
<pre>{{ content|maybewordwrap|linkify|urlize_ietf_docs }}</pre> <pre>{{ content|maybewordwrap|urlize_ietf_docs|linkify }}</pre>
{% endif %} {% endif %}
{% else %} {% else %}
Not available as plain text. Not available as plain text.
@ -136,4 +136,4 @@
{% block js %} {% block js %}
<script src="{% static 'ietf/js/d3.js' %}"></script> <script src="{% static 'ietf/js/d3.js' %}"></script>
<script src="{% static 'ietf/js/document_timeline.js' %}"></script> <script src="{% static 'ietf/js/document_timeline.js' %}"></script>
{% endblock %} {% endblock %}

2
ietf/templates/doc/document_review.html
View file

@ -47,7 +47,7 @@
<div class="card mt-5"> <div class="card mt-5">
<div class="card-header">{{ doc.name }}-{{ doc.rev }}</div> <div class="card-header">{{ doc.name }}-{{ doc.rev }}</div>
<div class="card-body"> <div class="card-body">
<pre class="pasted">{{ content|maybewordwrap|linkify|urlize_ietf_docs }}</pre> <pre class="pasted">{{ content|maybewordwrap|urlize_ietf_docs|linkify }}</pre>
</div> </div>
</div> </div>
{% endif %} {% endif %}

2
ietf/templates/doc/document_status_change.html
View file

@ -162,7 +162,7 @@
{{ doc.name }}-{{ doc.rev }} {{ doc.name }}-{{ doc.rev }}
</div> </div>
<div class="card-body"> <div class="card-body">
<pre>{{ content|maybewordwrap|linkify|urlize_ietf_docs }}</pre> <pre>{{ content|maybewordwrap|urlize_ietf_docs|linkify }}</pre>
</div> </div>
</div> </div>
{% endif %} {% endif %}

4
ietf/templates/doc/document_top.html
View file

@ -3,7 +3,7 @@
{% origin %} {% origin %}
{% load ietf_filters %} {% load ietf_filters %}
<h1> <h1>
{{ doc.title }} {{ doc.title|default:"(Untitled)" }}
<br> <br>
<small class="text-muted">{{ name }}</small> <small class="text-muted">{{ name }}</small>
</h1> </h1>
@ -16,4 +16,4 @@
</a> </a>
</li> </li>
{% endfor %} {% endfor %}
</ul> </ul>

4
ietf/templates/doc/document_writeup.html
View file

@ -17,7 +17,7 @@
</h2> </h2>
{% for name, text, url in writeups %} {% for name, text, url in writeups %}
{% if name %}<h3 class="mt-5">{{ name }}</h3>{% endif %} {% if name %}<h3 class="mt-5">{{ name }}</h3>{% endif %}
{% if text %}<pre class="border p-3 my-3">{{ text|linkify|urlize_ietf_docs }}</pre>{% endif %} {% if text %}<pre class="border p-3 my-3">{{ text|urlize_ietf_docs|linkify }}</pre>{% endif %}
{% if can_edit %} {% if can_edit %}
<p> <p>
<a href="{{ url }}" class="btn btn-primary"> <a href="{{ url }}" class="btn btn-primary">
@ -32,4 +32,4 @@
{% endif %} {% endif %}
{% endfor %} {% endfor %}
{% endfor %} {% endfor %}
{% endblock %} {% endblock %}

4
ietf/templates/doc/downref.html
View file

@ -18,7 +18,7 @@
<thead> <thead>
<tr> <tr>
<th scope="col" data-sort="ref">Referenced RFC</th> <th scope="col" data-sort="ref">Referenced RFC</th>
<th scope="col" data-sort="id">Internet-Draft making the reference</th> <th scope="col" data-sort="refby">Internet-Draft making the reference</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@ -41,4 +41,4 @@
{% endblock %} {% endblock %}
{% block js %} {% block js %}
<script src="{% static "ietf/js/list.js" %}"></script> <script src="{% static "ietf/js/list.js" %}"></script>
{% endblock %} {% endblock %}

2
ietf/templates/doc/drafts_in_iesg_process.html
View file

@ -60,7 +60,7 @@
{% endif %} {% endif %}
{% if doc.note %} {% if doc.note %}
<br> <br>
<i>Note: {{ doc.note|linkify|urlize_ietf_docs|linebreaksbr }}</i> <i>Note: {{ doc.note|urlize_ietf_docs|linkify|linebreaksbr }}</i>
{% endif %} {% endif %}
</td> </td>
<td> <td>

14
ietf/templates/doc/review/request_info.html
View file

@ -19,7 +19,7 @@
</tr> </tr>
<tr> <tr>
<td></td> <td></td>
<th scope="row">Requested rev.</th> <th scope="row">Requested revision</th>
<td> <td>
{% if review_req.requested_rev %} {% if review_req.requested_rev %}
{{ review_req.requested_rev }} {{ review_req.requested_rev }}
@ -299,12 +299,10 @@
<th scope="row"> <th scope="row">
</th> </th>
<th scope="row"> <th scope="row">
Reviewed rev. Reviewed revision
</th> </th>
<td> <td>
<a href="{% url "ietf.doc.views_doc.document_main" name=review_req.doc.name rev=assignment.reviewed_rev %}"> <a href="{% url "ietf.doc.views_doc.document_main" name=review_req.doc.name rev=assignment.reviewed_rev %}">{{ assignment.reviewed_rev }}</a>
{{ assignment.reviewed_rev }}
</a>
{% if assignment.reviewed_rev != review_req.doc.rev %}(document currently at {{ review_req.doc.rev }}){% endif %} {% if assignment.reviewed_rev != review_req.doc.rev %}(document currently at {{ review_req.doc.rev }}){% endif %}
</td> </td>
</tr> </tr>
@ -314,7 +312,7 @@
<th scope="row"> <th scope="row">
</th> </th>
<th scope="row"> <th scope="row">
Review result Result
</th> </th>
<td class="{% if assignment.result.name|slice:5 == 'Ready' %}text-success{% else %}text-danger{% endif %}"> <td class="{% if assignment.result.name|slice:5 == 'Ready' %}text-success{% else %}text-danger{% endif %}">
{{ assignment.result.name }} {{ assignment.result.name }}
@ -326,7 +324,7 @@
<th scope="row"> <th scope="row">
</th> </th>
<th scope="row"> <th scope="row">
Review completed: Completed
</th> </th>
<td> <td>
{{ assignment.completed_on|date:"Y-m-d" }} {{ assignment.completed_on|date:"Y-m-d" }}
@ -343,4 +341,4 @@
</i> </i>
Assign reviewer Assign reviewer
</a> </a>
{% endif %} {% endif %}

4
ietf/templates/doc/shepherd_writeup.html
View file

@ -11,11 +11,11 @@
<br> <br>
<small class="text-muted">{{ doc.canonical_name }}-{{ doc.rev }}</small> <small class="text-muted">{{ doc.canonical_name }}-{{ doc.rev }}</small>
</h1> </h1>
<pre class="border p-3 my-3">{{writeup|linkify|urlize_ietf_docs|wordwrap:"80"}}</pre> <pre class="border p-3 my-3">{{writeup|maybewordwrap|urlize_ietf_docs|linkify}}</pre>
{% if can_edit %} {% if can_edit %}
<a class="btn btn-primary" <a class="btn btn-primary"
href="{% url 'ietf.doc.views_draft.edit_shepherd_writeup' name=doc.name %}">Edit</a> href="{% url 'ietf.doc.views_draft.edit_shepherd_writeup' name=doc.name %}">Edit</a>
{% endif %} {% endif %}
<a class="btn btn-secondary float-end" <a class="btn btn-secondary float-end"
href="{% url "ietf.doc.views_doc.document_main" name=doc.name %}">Back</a> href="{% url "ietf.doc.views_doc.document_main" name=doc.name %}">Back</a>
{% endblock %} {% endblock %}

2
ietf/templates/group/active_areas.html
View file

@ -50,7 +50,7 @@
</h2> </h2>
{% if area.description %} {% if area.description %}
<p> <p>
{{ area.description|linkify|urlize_ietf_docs|safe }} {{ area.description|urlize_ietf_docs|linkify|safe }}
</p> </p>
{% endif %} {% endif %}
{% endfor %} {% endfor %}

4
ietf/templates/group/all_status.html
View file

@ -22,7 +22,7 @@
{{ rpt.time|date:"Y-m-d" }} {{ rpt.time|date:"Y-m-d" }}
</div> </div>
<div class="card-body"> <div class="card-body">
<pre class="pasted">{{ rpt.desc|default:"(none)"|linkify|urlize_ietf_docs }}</pre> <pre class="pasted">{{ rpt.desc|default:"(none)"|urlize_ietf_docs|linkify }}</pre>
</div> </div>
</div> </div>
{% endfor %} {% endfor %}
@ -38,7 +38,7 @@
{{ rpt.time|date:"Y-m-d" }} {{ rpt.time|date:"Y-m-d" }}
</div> </div>
<div class="card-body"> <div class="card-body">
<pre class="pasted">{{ rpt.desc|default:"(none)"|linkify|urlize_ietf_docs }}</pre> <pre class="pasted">{{ rpt.desc|default:"(none)"|urlize_ietf_docs|linkify }}</pre>
</div> </div>
</div> </div>
{% endfor %} {% endfor %}

8
ietf/templates/group/group_about.html
View file

@ -148,10 +148,10 @@
{% for resource in resources|dictsort:"display_name" %} {% for resource in resources|dictsort:"display_name" %}
{# Maybe make how a resource displays itself a method on the class so templates aren't doing this switching #} {# Maybe make how a resource displays itself a method on the class so templates aren't doing this switching #}
{% if resource.name.type.slug == 'url' or resource.name.type.slug == 'email' %} {% if resource.name.type.slug == 'url' or resource.name.type.slug == 'email' %}
<a href="{{ resource.value }}" title="{{ resource.name.name }}"> <a href="{{ resource.value|urlencode }}" title="{{ resource.name.name }}">
{% firstof resource.display_name resource.name.name %}</a>{% else %} {% firstof resource.display_name resource.name.name %}</a>{% else %}
<span title="{{ resource.name.name }}"> <span title="{{ resource.name.name }}">
{% firstof resource.display_name resource.name.name %}: {{ resource.value }} {% firstof resource.display_name resource.name.name %}: {{ resource.value|escape }}
</span>{% endif %}{% if not forloop.last %}{% if resource.display_name|length < 15 and resource.name.name|length < 15 and resources|length <= 3 %},{% else %}<br>{% endif %}{% endif %} </span>{% endif %}{% if not forloop.last %}{% if resource.display_name|length < 15 and resource.name.name|length < 15 and resources|length <= 3 %},{% else %}<br>{% endif %}{% endif %}
{% endfor %} {% endfor %}
{% endif %} {% endif %}
@ -309,7 +309,7 @@
{{ group.type.desc.title }} {{ group.type.desc.title }}
</h2> </h2>
{# the linebreaks filter adds <p>, no surrounding <p> necessary: #} {# the linebreaks filter adds <p>, no surrounding <p> necessary: #}
{{ group.charter_text|linkify|urlize_ietf_docs|linebreaks }} {{ group.charter_text|urlize_ietf_docs|linkify|linebreaks }}
{% else %} {% else %}
<h2 class="mt-3"> <h2 class="mt-3">
{% if requested_close or group.state_id == "conclude" %}Final{% endif %} {% if requested_close or group.state_id == "conclude" %}Final{% endif %}
@ -333,4 +333,4 @@
</p> </p>
{% endif %} {% endif %}
{% endif %} {% endif %}
{% endblock %} {% endblock %}

2
ietf/templates/group/group_about_status.html
View file

@ -28,7 +28,7 @@
</p> </p>
</div> </div>
{% endif %} {% endif %}
<pre class="border p-3 my-3 pasted">{{ status_update.desc|default:"(none)"|linkify|urlize_ietf_docs }}</pre> <pre class="border p-3 my-3 pasted">{{ status_update.desc|default:"(none)"|urlize_ietf_docs|linkify }}</pre>
{% if can_provide_status_update %} {% if can_provide_status_update %}
<a id="edit_button" <a id="edit_button"
class="btn btn-primary" class="btn btn-primary"

2
ietf/templates/group/group_about_status_meeting.html
View file

@ -9,7 +9,7 @@
{% block content %} {% block content %}
{% origin %} {% origin %}
<h1>Status update for {{ group.type.name }} {{ group.acronym }} at {{ meeting }}</h1> <h1>Status update for {{ group.type.name }} {{ group.acronym }} at {{ meeting }}</h1>
<pre class="border p-3 my-3 pasted">{{ status_update.desc|default:"(none)"|linkify|urlize_ietf_docs }}</pre> <pre class="border p-3 my-3 pasted">{{ status_update.desc|default:"(none)"|urlize_ietf_docs|linkify }}</pre>
<a class="btn btn-secondary float-end" <a class="btn btn-secondary float-end"
href="{% url "ietf.meeting.views.proceedings" num=meeting.number %}">Back</a> href="{% url "ietf.meeting.views.proceedings" num=meeting.number %}">Back</a>
{% endblock %} {% endblock %}

10
ietf/templates/group/group_base.html
View file

@ -9,12 +9,12 @@
{% block content %} {% block content %}
{% origin %} {% origin %}
<h1> <h1>
{{ group.name }} ({{ group.acronym }})
{% if group.state_id == "dormant" or group.state_id == "conclude" %} {% if group.state_id == "dormant" or group.state_id == "conclude" %}
<span class="badge bg-warning float-end">Concluded {{ group.type.name }}</span> <span class="badge bg-warning float-end ms-3">Concluded {{ group.type.name }}</span>
{% endif %} {% endif %}
{% if group.state_id == "replaced" %}<span class="badge bg-warning float-end">Replaced {{ group.type.name }}</span>{% endif %} {% if group.state_id == "replaced" %}<span class="badge bg-warning float-end ms-3">Replaced {{ group.type.name }}</span>{% endif %}
{% if group.state_id == "proposed" %}<span class="badge bg-info float-end">Proposed {{ group.type.name }}</span>{% endif %} {% if group.state_id == "proposed" %}<span class="badge bg-info float-end ms-3">Proposed {{ group.type.name }}</span>{% endif %}
{{ group.name }} ({{ group.acronym }})
</h1> </h1>
<ul class="nav nav-tabs my-3"> <ul class="nav nav-tabs my-3">
{% for name, url in menu_entries %} {% for name, url in menu_entries %}
@ -40,4 +40,4 @@
{% endblock %} {% endblock %}
{% block js %} {% block js %}
<script src="{% static "ietf/js/list.js" %}"></script> <script src="{% static "ietf/js/list.js" %}"></script>
{% endblock %} {% endblock %}

4
ietf/templates/group/manage_review_requests.html
View file

@ -51,7 +51,7 @@
<div class="row request-metadata"> <div class="row request-metadata">
<div class="col-sm-6"> <div class="col-sm-6">
<p class="lead"> <p class="lead">
{{ r.doc.title|linkify|urlize_ietf_docs }} {{ r.doc.title|urlize_ietf_docs|linkify }}
</p> </p>
{% if r.pk != None %} {% if r.pk != None %}
<span class="fw-bold">Requested:</span> <span class="fw-bold">Requested:</span>
@ -143,7 +143,7 @@
</div> </div>
<div class="col-sm-6"> <div class="col-sm-6">
<span class="fw-bold">Abstract:</span> <span class="fw-bold">Abstract:</span>
{{ r.doc.abstract|linkify|urlize_ietf_docs }} {{ r.doc.abstract|urlize_ietf_docs|linkify }}
</div> </div>
</div> </div>
{% if r.form.non_field_errors %} {% if r.form.non_field_errors %}

4
ietf/templates/group/review_requests.html
View file

@ -30,7 +30,7 @@
</thead> </thead>
<tbody> <tbody>
<tr class="table-info" id="unassigned-review-requests"> <tr class="table-info" id="unassigned-review-requests">
<th scope="col" colspan="7">Unassigned review requests</th> <th scope="col" colspan="6">Unassigned review requests</th>
</tr> </tr>
</tbody> </tbody>
<tbody> <tbody>
@ -149,7 +149,7 @@
</thead> </thead>
<tbody> <tbody>
<tr class="table-info"> <tr class="table-info">
<th scope="col" colspan="8"> <th scope="col" colspan="6">
Closed review requests Closed review requests
</th> </th>
</tr> </tr>

6
ietf/templates/iesg/agenda.html
View file

@ -1,7 +1,7 @@
{% extends "base.html" %} {% extends "base.html" %}
{# Copyright The IETF Trust 2015, All Rights Reserved #} {# Copyright The IETF Trust 2015, All Rights Reserved #}
{% load origin %} {% load origin %}
{% load ietf_filters %} {% load ietf_filters textfilters %}
{% block pagehead %}<link rel="alternate" type="application/atom+xml" href="/feed/iesg-agenda/">{% endblock %} {% block pagehead %}<link rel="alternate" type="application/atom+xml" href="/feed/iesg-agenda/">{% endblock %}
{% block title %}IESG agenda: {{ date }}{% endblock %} {% block title %}IESG agenda: {{ date }}{% endblock %}
{% block content %} {% block content %}
@ -107,8 +107,8 @@
{% endif %} {% endif %}
{% endif %} {% endif %}
{% if num|startswith:"6." and user|has_role:"Area Director,IAB Chair,Secretariat" %} {% if num|startswith:"6." and user|has_role:"Area Director,IAB Chair,Secretariat" %}
<pre class="border p-3">{{ section.text|wordwrap:"80" }}</pre> <pre class="border p-3">{{ section.text|maybewordwrap|urlize_ietf_docs|linkify }}</pre>
{% endif %} {% endif %}
{% endfor %} {% endfor %}
{% endblock %} {% endblock %}
{% block js %}<script>// automatically generate a right-hand navigation tab for long pages</script>{% endblock %} {% block js %}<script>// automatically generate a right-hand navigation tab for long pages</script>{% endblock %}

10
ietf/templates/ietfauth/apikeys.html
View file

@ -15,6 +15,7 @@
<small class="text-muted">{{ user.username }}</small> <small class="text-muted">{{ user.username }}</small>
</h1> </h1>
{% csrf_token %} {% csrf_token %}
{% if person.apikeys.all %}
<table class="table table-sm tablesorter"> <table class="table table-sm tablesorter">
<thead> <thead>
<tr > <tr >
@ -49,13 +50,12 @@
{% endif %} {% endif %}
</td> </td>
</tr> </tr>
{% empty %}
<tr>
<td>You have no personal API keys.</td>
</tr>
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
{% else %}
<p>You have no personal API keys.</p>
{% endif %}
<a href="{% url 'ietf.ietfauth.views.apikey_create' %}" <a href="{% url 'ietf.ietfauth.views.apikey_create' %}"
class="btn btn-primary add-apikey my-3"> class="btn btn-primary add-apikey my-3">
Get a new personal API key Get a new personal API key
@ -63,4 +63,4 @@
{% endblock %} {% endblock %}
{% block js %} {% block js %}
<script src="{% static "ietf/js/list.js" %}"></script> <script src="{% static "ietf/js/list.js" %}"></script>
{% endblock %} {% endblock %}

5
ietf/templates/ipr/details_history.html
View file

@ -52,7 +52,8 @@
<td> <td>
{% if e.message %} {% if e.message %}
{% if e.response_due %}<span class="badge bg-info">Response due {{ e.response_due|date:"Y-m-d" }}</span>{% endif %} {% if e.response_due %}<span class="badge bg-info">Response due {{ e.response_due|date:"Y-m-d" }}</span>{% endif %}
<pre>{{ e.message|render_message_for_history|format_history_text:"100" }}</pre> {# FIXME: can't do format_history_text, because that inserts a <div> into the <pre>, which is illegal. Need to rework the snippeting. #}
<pre>{{ e.message|render_message_for_history|urlize_ietf_docs|linkify }}</pre>
{% else %} {% else %}
{{ e.desc|format_history_text }} {{ e.desc|format_history_text }}
{% endif %} {% endif %}
@ -65,4 +66,4 @@
{% endblock %} {% endblock %}
{% block js %} {% block js %}
<script src="{% static "ietf/js/list.js" %}"></script> <script src="{% static "ietf/js/list.js" %}"></script>
{% endblock %} {% endblock %}

2
ietf/templates/liaisons/detail.html
View file

@ -201,7 +201,7 @@
Body Body
</th> </th>
<td> <td>
<pre>{{ liaison.body|maybewordwrap:"80"|linkify|urlize_ietf_docs }}</pre> <pre>{{ liaison.body|maybewordwrap:"80"|urlize_ietf_docs|linkify }}</pre>
</td> </td>
</tr> </tr>
{% endif %} {% endif %}

11
ietf/templates/liaisons/edit.html
View file

@ -52,7 +52,14 @@
{% for fieldset in form.fieldsets %} {% for fieldset in form.fieldsets %}
<h2>{{ fieldset.name }}</h2> <h2>{{ fieldset.name }}</h2>
{% for field in fieldset %} {% for field in fieldset %}
{% bootstrap_field field layout="horizontal" %} {% if field.id_for_label != "id_attachments" %}
{% bootstrap_field field layout="horizontal" %}
{% else %}
<div class="row mb-3">
<p class="col-md-2 fw-bold col-form-label">{{ field.label }}</p>
<div class="col-md-10">{{ field }}</div>
</div>
{% endif %}
{% endfor %} {% endfor %}
{% endfor %} {% endfor %}
<a class="btn btn-danger float-end" <a class="btn btn-danger float-end"
@ -70,4 +77,4 @@
{% block js %} {% block js %}
{{ form.media.js }} {{ form.media.js }}
<script src="{% static 'ietf/js/liaisons.js' %}"></script> <script src="{% static 'ietf/js/liaisons.js' %}"></script>
{% endblock %} {% endblock %}

32
ietf/templates/meeting/agenda.html
View file

@ -57,7 +57,7 @@
{{ schedule.meeting.agenda_info_note|removetags:"h1"|safe }} {{ schedule.meeting.agenda_info_note|removetags:"h1"|safe }}
</p> </p>
{% endif %} {% endif %}
{% include 'meeting/tz-display.html' with meeting_timezone=timezone only %} {% include 'meeting/tz-display.html' with id_suffix="" meeting_timezone=timezone only %}
{% include "meeting/agenda_filter.html" with filter_categories=filter_categories customize_button_text="Filter this agenda view..." always_show=personalize %} {% include "meeting/agenda_filter.html" with filter_categories=filter_categories customize_button_text="Filter this agenda view..." always_show=personalize %}
{% include "meeting/agenda_personalize_buttonlist.html" with meeting=schedule.meeting personalize=personalize only %} {% include "meeting/agenda_personalize_buttonlist.html" with meeting=schedule.meeting personalize=personalize only %}
<div class="input-group mb-3"> <div class="input-group mb-3">
@ -84,7 +84,7 @@
{% endfor %} {% endfor %}
</ul> </ul>
<a class="btn btn-outline-primary {% if non_area_keywords|length == 0 %}disabled{% endif %}" <a class="btn btn-outline-primary {% if non_area_keywords|length == 0 %}disabled{% endif %}"
href="{% url "ietf.meeting.views.agenda_ical" num=schedule.meeting.number %}?show={{ non_area_keywords|join:',' }}"> href="{% if non_area_keywords %}{% url 'ietf.meeting.views.agenda_ical' num=schedule.meeting.number %}?show={{ non_area_keywords|join:',' }}{% else %}#{% endif %}">
Download non-area events Download non-area events
</a> </a>
</div> </div>
@ -181,18 +181,20 @@
{% if item.session.current_status == 'canceled' %} {% if item.session.current_status == 'canceled' %}
<span class="badge bg-danger float-end">CANCELLED</span> <span class="badge bg-danger float-end">CANCELLED</span>
{% else %} {% else %}
<div class="float-end ps-2"> {% if item.slot_type.slug == 'other' %}
{% if item.slot_type.slug == 'other' %} {% if item.session.agenda or item.session.remote_instructions or item.session.agenda_note %}
{% if item.session.agenda or item.session.remote_instructions or item.session.agenda_note %} <div class="float-end ps-2">
{% include "meeting/session_buttons_include.html" with show_agenda=True item=item schedule=schedule %} {% include "meeting/session_buttons_include.html" with show_agenda=True item=item schedule=schedule %}
{% else %} </div>
{% for slide in item.session.slides %} {% else %}
<a href="{{ slide.get_href }}">{{ slide.title|clean_whitespace }}</a> <div>
<br> {% for slide in item.session.slides %}
{% endfor %} <a href="{{ slide.get_href }}">{{ slide.title|clean_whitespace }}</a>
{% endif %} <br>
{% endfor %}
</div>
{% endif %} {% endif %}
</div> {% endif %}
{% endif %} {% endif %}
</td> </td>
</tr> </tr>
@ -300,7 +302,7 @@
<div class="timetooltip reschedtimetooltip"> <div class="timetooltip reschedtimetooltip">
<div data-start-time="{{ item.session.rescheduled_to.utc_start_time|date:"U" }}" <div data-start-time="{{ item.session.rescheduled_to.utc_start_time|date:"U" }}"
data-end-time="{{ item.session.rescheduled_to.utc_end_time|date:"U" }}" data-end-time="{{ item.session.rescheduled_to.utc_end_time|date:"U" }}"
{% if item.timeslot.time|date:"l" != item.session.rescheduled_to.time|date:"l" %} weekday="1"{% endif %}> {% if item.timeslot.time|date:"l" != item.session.rescheduled_to.time|date:"l" %} data-weekday="1"{% endif %}>
{% if "-utc" in request.path %} {% if "-utc" in request.path %}
{{ item.session.rescheduled_to.utc_start_time|date:"l G:i"|upper }}-{{ item.session.rescheduled_to.utc_end_time|date:"G:i" }} {{ item.session.rescheduled_to.utc_start_time|date:"l G:i"|upper }}-{{ item.session.rescheduled_to.utc_end_time|date:"G:i" }}
{% else %} {% else %}
@ -313,7 +315,7 @@
{% endif %} {% endif %}
{% if item.session.agenda_note|first_url|conference_url %} {% if item.session.agenda_note|first_url|conference_url %}
<br> <br>
<a href={{ item.session.agenda_note|first_url }}>{{ item.session.agenda_note|slice:":23" }} <a href="{{ item.session.agenda_note|first_url }}">{{ item.session.agenda_note|slice:":23" }}
</a> </a>
{% elif item.session.agenda_note %} {% elif item.session.agenda_note %}
<br> <br>

4
ietf/templates/meeting/group_materials.html
View file

@ -17,7 +17,7 @@
{% endif %} {% endif %}
</td> </td>
{% if session.all_meeting_sessions_cancelled %} {% if session.all_meeting_sessions_cancelled %}
<td colspan="{% if user|has_role:'Secretariat' or user_groups %}7{% else %}6{% endif %}"> <td colspan="{% if user|has_role:'Secretariat' or user_groups %}6{% else %}5{% endif %}">
<span class="badge bg-danger">Session cancelled</span> <span class="badge bg-danger">Session cancelled</span>
</td> </td>
{% else %} {% else %}
@ -121,4 +121,4 @@
</td> </td>
{% endif %} {% endif %}
{% endif %} {% endif %}
</tr> </tr>

2
ietf/templates/meeting/important-dates.html
View file

@ -42,7 +42,7 @@
{% endif %} {% endif %}
</td> </td>
<td> <td>
{{ d.name.desc|linkify|urlize_ietf_docs }} {{ d.name.desc|urlize_ietf_docs|linkify }}
{% if first and d.name.slug == 'openreg' or first and d.name.slug == 'earlybird' %} {% if first and d.name.slug == 'openreg' or first and d.name.slug == 'earlybird' %}
<a href="https://www.ietf.org/how/meetings/register/">Register here</a>. <a href="https://www.ietf.org/how/meetings/register/">Register here</a>.
{% endif %} {% endif %}

4
ietf/templates/meeting/interim_session_buttons.html
View file

@ -4,9 +4,9 @@
{% load textfilters %} {% load textfilters %}
{% origin %} {% origin %}
{% with item=session.official_timeslotassignment acronym=session.historic_group.acronym %} {% with item=session.official_timeslotassignment acronym=session.historic_group.acronym %}
{% include "meeting/session_agenda_include.html" with slug=item.slug session=session timeslot=item.timeslot only %}
<div role="group" class="btn-group btn-group-sm"> <div role="group" class="btn-group btn-group-sm">
{% if session.agenda and show_agenda %} {% if session.agenda and show_agenda %}
{% include "meeting/session_agenda_include.html" with slug=item.slug session=session timeslot=item.timeslot only %}
{# Note: if called with show_agenda=True, calling template must load agenda_materials.js, needed by session_agenda_include.html #} {# Note: if called with show_agenda=True, calling template must load agenda_materials.js, needed by session_agenda_include.html #}
{# agenda pop-up button #} {# agenda pop-up button #}
<a class="btn btn-outline-primary" <a class="btn btn-outline-primary"
@ -157,4 +157,4 @@
{% endwith %} {% endwith %}
{% endif %} {% endif %}
</div> </div>
{% endwith %} {% endwith %}

7
ietf/templates/meeting/session_details_panel.html
View file

@ -45,7 +45,7 @@
{% endif %} {% endif %}
{% endif %} {% endif %}
{% if meeting.type.slug == 'interim' and session.remote_instructions %} {% if meeting.type.slug == 'interim' and session.remote_instructions %}
<div> <div class="my-3">
<b>Remote instructions:</b> <b>Remote instructions:</b>
{% if session.agenda_note|first_url|conference_url %} {% if session.agenda_note|first_url|conference_url %}
<a href="{{ session.agenda_note|first_url }}" title="Online conference"> <a href="{{ session.agenda_note|first_url }}" title="Online conference">
@ -125,6 +125,7 @@
{% for pres in session.filtered_slides %} {% for pres in session.filtered_slides %}
<tr data-name="{{ pres.document.name }}" title="Drag to reorder."> <tr data-name="{{ pres.document.name }}" title="Drag to reorder.">
{% url 'ietf.doc.views_doc.document_main' name=pres.document.name as url %} {% url 'ietf.doc.views_doc.document_main' name=pres.document.name as url %}
<td><i class="bi bi-grip-vertical"></i></td>
<td> <td>
<a href="{{ pres.document.get_href }}">{{ pres.document.title }}</a> <a href="{{ pres.document.get_href }}">{{ pres.document.title }}</a>
<a href="{{ url }}">({{ pres.document.name }})</a> <a href="{{ url }}">({{ pres.document.name }})</a>
@ -158,7 +159,7 @@
</a> </a>
{% endif %} {% endif %}
{% if can_manage_materials %}<div class="small">Drag-and-drop to reorder slides</div>{% endif %} {% if can_manage_materials %}<div class="form-text">Drag-and-drop to reorder slides</div>{% endif %}
<h3 class="mt-4">Drafts</h3> <h3 class="mt-4">Drafts</h3>
<table class="table table-sm table-striped"> <table class="table table-sm table-striped">
{% if session.filtered_drafts %} {% if session.filtered_drafts %}
@ -192,4 +193,4 @@
</a> </a>
{% endif %} {% endif %}
{% endwith %} {% endwith %}
{% endfor %} {% endfor %}

4
ietf/templates/meeting/upcoming.html
View file

@ -29,7 +29,7 @@
{% endfor %} {% endfor %}
</div> </div>
{% endif %} {% endif %}
{% include 'meeting/tz-display.html' with meeting_timezone=None only %} {% include 'meeting/tz-display.html' with id_suffix="" meeting_timezone=None only %}
{% include 'meeting/agenda_filter.html' with filter_categories=filter_categories customize_button_text="Customize the meeting list..." only %} {% include 'meeting/agenda_filter.html' with filter_categories=filter_categories customize_button_text="Customize the meeting list..." only %}
{% cache 600 upcoming-meetings entries.count %} {% cache 600 upcoming-meetings entries.count %}
{% if entries %} {% if entries %}
@ -152,4 +152,4 @@
agenda_filter.enable(); agenda_filter.enable();
}); });
</script> </script>
{% endblock %} {% endblock %}

2
ietf/templates/message/message.html
View file

@ -36,5 +36,5 @@
{{ message.subject|linkify }} {{ message.subject|linkify }}
</dd> </dd>
</dl> </dl>
<pre>{{ message.body|linkify|urlize_ietf_docs }}</pre> <pre>{{ message.body|urlize_ietf_docs|linkify }}</pre>
{% endblock %} {% endblock %}

2
ietf/templates/nomcom/questionnaires.html
View file

@ -21,7 +21,7 @@
{% for position in positions %} {% for position in positions %}
<div role="tabpanel" class="tab-pane {% if forloop.first %} active{% endif %}" <div role="tabpanel" class="tab-pane {% if forloop.first %} active{% endif %}"
id="{{ position.name|slugify }}"> id="{{ position.name|slugify }}">
{{ position.get_questionnaire|linkify|urlize_ietf_docs|linebreaks }} {{ position.get_questionnaire|urlize_ietf_docs|linkify|linebreaks }}
</div> </div>
{% endfor %} {% endfor %}
</div> </div>

2
ietf/templates/nomcom/show_template.html
View file

@ -49,7 +49,7 @@
</dd> </dd>
{% endif %} {% endif %}
</dl> </dl>
<pre class="border p-3 pasted">{{ template.content|linkify|urlize_ietf_docs|linebreaksbr }}</pre> <pre class="border p-3 pasted">{{ template.content|urlize_ietf_docs|linkify|linebreaksbr }}</pre>
<a class="btn btn-secondary my-3" <a class="btn btn-secondary my-3"
href="{% if return_url %}{{ return_url }}{% else %}../{% endif %}">Back</a> href="{% if return_url %}{{ return_url }}{% else %}../{% endif %}">Back</a>
{% endblock %} {% endblock %}

64
ietf/templates/person/profile.html
View file

@ -13,7 +13,7 @@
{% origin %} {% origin %}
{% if persons|length > 1 %} {% if persons|length > 1 %}
<p class="alert alert-warning my-3"> <p class="alert alert-warning my-3">
More than one person with this name has been found. Showing all: More than one person with this name has been found. Showing all.
</p> </p>
{% endif %} {% endif %}
{% for person in persons %} {% for person in persons %}
@ -29,15 +29,18 @@
{% if person.photo %} {% if person.photo %}
<div class="float-end ms-3 mb-3">{% include "person/photo.html" with person=person %}</div> <div class="float-end ms-3 mb-3">{% include "person/photo.html" with person=person %}</div>
{% endif %} {% endif %}
{{ person.biography|linkify|urlize_ietf_docs|apply_markup:"restructuredtext" }} {{ person.biography|apply_markup:"restructuredtext"|urlize_ietf_docs|linkify }}
</div> </div>
{% if person.role_set.exists %} {% if person.role_set.exists %}
<h2 class="mt-5" id="roles">Roles</h2> <h2 class="mt-5" id="roles-{{ forloop.counter }}">Roles</h2>
{% if person.role_set.all|active_roles %}
<table class="table table-sm table-striped tablesorter"> <table class="table table-sm table-striped tablesorter">
<thead> <thead>
<th scope="col" data-sort="role">Role</th> <tr>
<th scope="col" data-sort="group">Group</th> <th scope="col" data-sort="role">Role</th>
<th scope="col" data-sort="email">Email</th> <th scope="col" data-sort="group">Group</th>
<th scope="col" data-sort="email">Email</th>
</tr>
</thead> </thead>
<tbody> <tbody>
{% for role in person.role_set.all|active_roles %} {% for role in person.role_set.all|active_roles %}
@ -51,18 +54,21 @@
<a href="mailto:{{ role.email.address }}">{{ role.email.address }}</a> <a href="mailto:{{ role.email.address }}">{{ role.email.address }}</a>
</td> </td>
</tr> </tr>
{% empty %}
{{ person.first_name }} has no active roles as of {{ today }}.
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
{% else %}
<p>{{ person.first_name }} has no active roles as of {{ today }}.</p>
{% endif %}
{% endif %} {% endif %}
{% if person.personextresource_set.exists %} {% if person.personextresource_set.exists %}
<h2 class="mt-5" id="extresources">External Resources</h2> <h2 class="mt-5" id="extresources-{{ forloop.counter }}">External Resources</h2>
<table class="table table-sm table-striped tablesorter"> <table class="table table-sm table-striped tablesorter">
<thead> <thead>
<th scope="col" data-sort="name">Name</th> <tr>
<th scope="col" data-sort="value">Value</th> <th scope="col" data-sort="name">Name</th>
<th scope="col" data-sort="value">Value</th>
</tr>
</thead> </thead>
<tbody> <tbody>
{% for extres in person.personextresource_set.all %} {% for extres in person.personextresource_set.all %}
@ -76,16 +82,18 @@
</tbody> </tbody>
</table> </table>
{% endif %} {% endif %}
<h2 class="mt-5" id="rfcs"> <h2 class="mt-5" id="rfcs-{{ forloop.counter }}">
RFCs <small class="text-muted">({{ person.rfcs|length }})</small> RFCs <small class="text-muted">({{ person.rfcs|length }})</small>
</h2> </h2>
{% if person.rfcs %} {% if person.rfcs %}
<table class="table table-sm table-striped tablesorter"> <table class="table table-sm table-striped tablesorter">
<thead> <thead>
<th scope="col" data-sort="rfc">RFC</th> <tr>
<th scope="col" data-sort="date">Date</th> <th scope="col" data-sort="rfc">RFC</th>
<th scope="col" data-sort="title">Title</th> <th scope="col" data-sort="date">Date</th>
<th scope="col" data-sort="citedby">Cited by</th> <th scope="col" data-sort="title">Title</th>
<th scope="col" data-sort="citedby">Cited by</th>
</tr>
</thead> </thead>
<tbody> <tbody>
{% for doc in person.rfcs %} {% for doc in person.rfcs %}
@ -113,7 +121,7 @@
{% else %} {% else %}
{{ person.first_name }} has no RFCs as of {{ today }}. {{ person.first_name }} has no RFCs as of {{ today }}.
{% endif %} {% endif %}
<h2 class="mt-5" id="drafts"> <h2 class="mt-5" id="drafts-{{ forloop.counter }}">
Active Drafts <small class="text-muted">({{ person.active_drafts|length }})</small> Active Drafts <small class="text-muted">({{ person.active_drafts|length }})</small>
</h2> </h2>
{% if person.active_drafts.exists %} {% if person.active_drafts.exists %}
@ -146,11 +154,11 @@
{% else %} {% else %}
{{ person.first_name }} has no expired drafts as of {{ today }}. {{ person.first_name }} has no expired drafts as of {{ today }}.
{% endif %} {% endif %}
{% if persons|length == 1 and person.has_drafts %} {% if person.has_drafts %}
<h2 class="mt-5"> <h2 class="mt-5">
Draft Activity Draft Activity
</h2> </h2>
<div id="chart"> <div id="chart-{{ forloop.counter }}">
</div> </div>
{% endif %} {% endif %}
{% endfor %} {% endfor %}
@ -161,15 +169,17 @@
<script src="{% static 'ietf/js/highcharts-exporting.js' %}"></script> <script src="{% static 'ietf/js/highcharts-exporting.js' %}"></script>
<script> <script>
$(function () { $(function () {
$.getJSON('{% url "ietf.doc.views_stats.chart_conf_person_drafts" id=persons.0.id %}', function (conf) { {% for person in persons %}
// Create the chart $.getJSON('{% url "ietf.doc.views_stats.chart_conf_person_drafts" id=person.id %}', function (conf) {
chart = Highcharts.stockChart('chart', conf); // Create the chart
chart.showLoading(); chart = Highcharts.stockChart('chart-{{ forloop.counter }}', conf);
$.getJSON('{% url "ietf.doc.views_stats.chart_data_person_drafts" id=persons.0.id %}', function (data) { chart.showLoading();
chart.series[0].setData(data); $.getJSON('{% url "ietf.doc.views_stats.chart_data_person_drafts" id=person.id %}', function (data) {
chart.hideLoading(); chart.series[0].setData(data);
chart.hideLoading();
});
}); });
}); {% endfor %}
}); });
</script> </script>
{% endblock %} {% endblock %}

2
ietf/templates/submit/submission_email.html
View file

@ -35,7 +35,7 @@
Message Message
</dt> </dt>
<dd class="col-sm-10"> <dd class="col-sm-10">
<pre>{{ message.message.body|linkify|urlize_ietf_docs|linebreaksbr }}</pre> <pre>{{ message.message.body|urlize_ietf_docs|linkify|linebreaksbr }}</pre>
</dd> </dd>
<dt class="col-sm-2"> <dt class="col-sm-2">
Attachment Attachment

2
ietf/templates/submit/submission_status.html
View file

@ -94,7 +94,7 @@
aria-label="Close"></button> aria-label="Close"></button>
</div> </div>
<div class="modal-body" id="{{ check.checker|slugify }}-message"> <div class="modal-body" id="{{ check.checker|slugify }}-message">
<pre>{{ check.message|linkify|urlize_ietf_docs }}</pre> <pre>{{ check.message|urlize_ietf_docs|linkify }}</pre>
</div> </div>
<div class="modal-footer"> <div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button> <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>

160
ietf/utils/test_runner.py
View file

@ -51,6 +51,8 @@ import subprocess
import tempfile import tempfile
import copy import copy
import factory.random import factory.random
import urllib3
from urllib.parse import urlencode
from fnmatch import fnmatch from fnmatch import fnmatch
from pathlib import Path from pathlib import Path
@ -95,6 +97,108 @@ template_coverage_collection = None
code_coverage_collection = None code_coverage_collection = None
url_coverage_collection = None url_coverage_collection = None
def start_vnu_server(port=8888):
"Start a vnu validation server on the indicated port"
vnu = subprocess.Popen(
[
"java",
"-Dnu.validator.servlet.bind-address=127.0.0.1",
"-Dnu.validator.servlet.max-file-size=16777216",
"-cp",
"bin/vnu.jar",
"nu.validator.servlet.Main",
f"{port}",
],
stdout=subprocess.DEVNULL,
)
print(" Waiting for vnu server to start up...", end="")
while vnu_validate(b"", content_type="", port=port) is None:
print(".", end="")
time.sleep(1)
print()
return vnu
http = urllib3.PoolManager(retries=urllib3.Retry(99, redirect=False))
def vnu_validate(html, content_type="text/html", port=8888):
"Pass the HTML to the vnu server running on the indicated port"
gzippeddata = gzip.compress(html)
try:
req = http.request(
"POST",
f"http://127.0.0.1:{port}/?"
+ urlencode({"out": "json", "asciiquotes": "yes"}),
headers={
"Content-Type": content_type,
"Accept-Encoding": "gzip",
"Content-Encoding": "gzip",
"Content-Length": str(len(gzippeddata)),
},
body=gzippeddata,
)
except (
urllib3.exceptions.NewConnectionError,
urllib3.exceptions.MaxRetryError,
ConnectionRefusedError,
):
return None
assert req.status == 200
return req.data.decode("utf-8")
def vnu_fmt_message(file, msg, content):
"Convert a vnu JSON message into a printable string"
ret = f"\n{file}:\n"
if "extract" in msg:
ret += msg["extract"].replace("\n", " ") + "\n"
ret += " " * msg["hiliteStart"]
ret += "^" * msg["hiliteLength"] + "\n"
ret += " " * msg["hiliteStart"]
ret += f"{msg['type']}: {msg['message']}\n"
if "firstLine" in msg and "lastLine" in msg:
ret += f'Source snippet, lines {msg["firstLine"]-5} to {msg["lastLine"]+4}:\n'
lines = content.splitlines()
for line in range(msg["firstLine"] - 5, msg["lastLine"] + 5):
ret += f"{line}: {lines[line]}\n"
return ret
def vnu_filter_message(msg, filter_db_issues, filter_test_issues):
"True if the vnu message is a known false positive"
if filter_db_issues and re.search(
r"""^Forbidden\ code\ point\ U\+|
'href'\ on\ element\ 'a':\ Percentage\ \("%"\)\ is\ not\ followed|
^Saw\ U\+\d+\ in\ stream|
^Document\ uses\ the\ Unicode\ Private\ Use\ Area""",
msg["message"],
flags=re.VERBOSE,
):
return True
if filter_test_issues and re.search(
r"""Ceci\ n'est\ pas\ une\ URL|
^The\ '\w+'\ attribute\ on\ the\ '\w+'\ element\ is\ obsolete|
^Section\ lacks\ heading|
is\ not\ in\ Unicode\ Normalization\ Form\ C""",
msg["message"],
flags=re.VERBOSE,
):
return True
return re.search(
r"""document\ is\ not\ mappable\ to\ XML\ 1|
^Attribute\ 'required'\ not\ allowed\ on\ element\ 'div'|
^The\ 'type'\ attribute\ is\ unnecessary\ for\ JavaScript""",
msg["message"],
flags=re.VERBOSE,
)
def load_and_run_fixtures(verbosity): def load_and_run_fixtures(verbosity):
loadable = [f for f in settings.GLOBAL_TEST_FIXTURES if "." not in f] loadable = [f for f in settings.GLOBAL_TEST_FIXTURES if "." not in f]
call_command('loaddata', *loadable, verbosity=int(verbosity)-1, database="default") call_command('loaddata', *loadable, verbosity=int(verbosity)-1, database="default")
@ -196,7 +300,7 @@ class ValidatingTemplate(Template):
return content return content
fingerprint = hash(content) + sys.maxsize + 1 # make hash positive fingerprint = hash(content) + sys.maxsize + 1 # make hash positive
if fingerprint in self.backend.validation_cache: if not settings.validate_html_harder and fingerprint in self.backend.validation_cache:
# already validated this HTML fragment, skip it # already validated this HTML fragment, skip it
# as an optimization, make page a bit smaller by not returning HTML for the menus # as an optimization, make page a bit smaller by not returning HTML for the menus
# FIXME: figure out why this still includes base/menu.html # FIXME: figure out why this still includes base/menu.html
@ -590,8 +694,11 @@ class IetfTestRunner(DiscoverRunner):
parser.add_argument('--no-validate-html', parser.add_argument('--no-validate-html',
action='store_false', dest="validate_html", default=True, action='store_false', dest="validate_html", default=True,
help='Do not validate all generated HTML with html-validate.org') help='Do not validate all generated HTML with html-validate.org')
parser.add_argument('--validate-html-harder',
action='store_true', dest="validate_html_harder", default=False,
help='Validate all generated HTML with additional validators (slow)')
def __init__(self, skip_coverage=False, save_version_coverage=None, html_report=None, permit_mixed_migrations=None, show_logging=None, validate_html=None, **kwargs): def __init__(self, skip_coverage=False, save_version_coverage=None, html_report=None, permit_mixed_migrations=None, show_logging=None, validate_html=None, validate_html_harder=None, **kwargs):
# #
self.check_coverage = not skip_coverage self.check_coverage = not skip_coverage
self.save_version_coverage = save_version_coverage self.save_version_coverage = save_version_coverage
@ -599,6 +706,7 @@ class IetfTestRunner(DiscoverRunner):
self.permit_mixed_migrations = permit_mixed_migrations self.permit_mixed_migrations = permit_mixed_migrations
self.show_logging = show_logging self.show_logging = show_logging
settings.validate_html = self if validate_html else None settings.validate_html = self if validate_html else None
settings.validate_html_harder = self if validate_html and validate_html_harder else None
settings.show_logging = show_logging settings.show_logging = show_logging
# #
self.root_dir = os.path.dirname(settings.BASE_DIR) self.root_dir = os.path.dirname(settings.BASE_DIR)
@ -718,7 +826,7 @@ class IetfTestRunner(DiscoverRunner):
print(" Not validating any generated HTML; " print(" Not validating any generated HTML; "
"please do so at least once before committing changes") "please do so at least once before committing changes")
else: else:
print(" Validating all HTML generated during the tests") print(" Validating all HTML generated during the tests", end="")
self.batches = {"doc": [], "frag": []} self.batches = {"doc": [], "frag": []}
# keep the html-validate configs here, so they can be kept in sync easily # keep the html-validate configs here, so they can be kept in sync easily
@ -768,6 +876,13 @@ class IetfTestRunner(DiscoverRunner):
self.config_file[kind].flush() self.config_file[kind].flush()
Path(self.config_file[kind].name).chmod(0o644) Path(self.config_file[kind].name).chmod(0o644)
if not settings.validate_html_harder:
print("")
self.vnu = None
else:
print(" (extra pedantically)")
self.vnu = start_vnu_server()
super(IetfTestRunner, self).setup_test_environment(**kwargs) super(IetfTestRunner, self).setup_test_environment(**kwargs)
def teardown_test_environment(self, **kwargs): def teardown_test_environment(self, **kwargs):
@ -789,12 +904,18 @@ class IetfTestRunner(DiscoverRunner):
else: else:
with open(self.coverage_file, "w") as file: with open(self.coverage_file, "w") as file:
json.dump(self.coverage_master, file, indent=2, sort_keys=True) json.dump(self.coverage_master, file, indent=2, sort_keys=True)
super(IetfTestRunner, self).teardown_test_environment(**kwargs)
if settings.validate_html: if settings.validate_html:
for kind in self.batches: for kind in self.batches:
self.validate(kind) try:
self.validate(kind)
except Exception:
pass
self.config_file[kind].close() self.config_file[kind].close()
if self.vnu:
self.vnu.terminate()
super(IetfTestRunner, self).teardown_test_environment(**kwargs)
def validate(self, kind): def validate(self, kind):
if not self.batches[kind]: if not self.batches[kind]:
@ -843,9 +964,10 @@ class IetfTestRunner(DiscoverRunner):
errors += ( errors += (
f'\n{result["filePath"]}:\n' f'\n{result["filePath"]}:\n'
+ "".join(source_lines[line - 5 : line]) + "".join(source_lines[line - 5 : line])
+ "-" * (msg["column"] - 1) + " " * (msg["column"] - 1)
+ "^" * msg["size"] + "^" * msg["size"] + "\n"
+ f' {msg["ruleId"]}: {msg["message"]} ' + " " * (msg["column"] - 1)
+ f'{msg["ruleId"]}: {msg["message"]} '
+ f'on line {line}:{msg["column"]}\n' + f'on line {line}:{msg["column"]}\n'
+ "".join(source_lines[line : line + 5]) + "".join(source_lines[line : line + 5])
+ "\n" + "\n"
@ -853,6 +975,28 @@ class IetfTestRunner(DiscoverRunner):
if errors: if errors:
testcase.fail(errors) testcase.fail(errors)
if settings.validate_html_harder:
if kind == "frag":
return
files = [
os.path.join(d, f)
for d, dirs, files in os.walk(tmpdir.name)
for f in files
]
for file in files:
with open(file, "rb") as f:
content = f.read()
result = vnu_validate(content)
assert result
for msg in json.loads(result)["messages"]:
if vnu_filter_message(msg, False, True):
continue
errors = vnu_fmt_message(file, msg, content)
if errors:
testcase.fail(errors)
tmpdir.cleanup() tmpdir.cleanup()
def get_test_paths(self, test_labels): def get_test_paths(self, test_labels):

4
ietf/utils/text.py
View file

@ -17,8 +17,10 @@ import debug # pyflakes:ignore
from .texescape import init as texescape_init, tex_escape_map from .texescape import init as texescape_init, tex_escape_map
tlds_sorted = sorted(tlds.tld_set, key=len, reverse=True) tlds_sorted = sorted(tlds.tld_set, key=len, reverse=True)
protocols = bleach.sanitizer.ALLOWED_PROTOCOLS
protocols.append("ftp") # we still have some ftp links
bleach_linker = bleach.Linker( bleach_linker = bleach.Linker(
url_re=bleach.linkifier.build_url_re(tlds=tlds_sorted), url_re=bleach.linkifier.build_url_re(tlds=tlds_sorted, protocols=protocols),
email_re=bleach.linkifier.build_email_re(tlds=tlds_sorted), # type: ignore email_re=bleach.linkifier.build_email_re(tlds=tlds_sorted), # type: ignore
parse_email=True parse_email=True
) )

1
package.json
View file

@ -152,7 +152,6 @@
"ietf/secr/static/images/tooltag-arrowright.webp", "ietf/secr/static/images/tooltag-arrowright.webp",
"ietf/secr/static/images/tooltag-arrowright_over.webp", "ietf/secr/static/images/tooltag-arrowright_over.webp",
"ietf/secr/static/js/dynamic_inlines.js", "ietf/secr/static/js/dynamic_inlines.js",
"ietf/secr/static/js/proceedings-recording.js",
"ietf/secr/static/js/session_form.js", "ietf/secr/static/js/session_form.js",
"ietf/secr/static/js/sessions.js", "ietf/secr/static/js/sessions.js",
"ietf/secr/static/js/utils.js" "ietf/secr/static/js/utils.js"