diff --git a/bin/test-crawl b/bin/test-crawl index 3bf5f838d..8ad475aea 100755 --- a/bin/test-crawl +++ b/bin/test-crawl @@ -3,7 +3,7 @@ # Copyright The IETF Trust 2013-2019, All Rights Reserved -import os, sys, re, datetime, argparse, traceback, json, subprocess +import os, sys, re, datetime, argparse, traceback, json import html5lib import random @@ -31,7 +31,7 @@ parser.add_argument('--no-follow', dest='follow', action='store_false', default= parser.add_argument('--validator-nu', dest='validator_nu', action='store_true', help='Use validator.nu instead of html5lib for HTML validation') parser.add_argument('--pedantic', action='store_true', - help='Stop the crawl on the first HTML validation issue') + help='Stop the crawl on the first error or warning') parser.add_argument('--random', action='store_true', help='Crawl URLs randomly') parser.add_argument('--validate-all', dest='validate_all', action='store_true', default=False, @@ -44,6 +44,7 @@ args = parser.parse_args() # Import Django, call setup() os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_testcrawl") +os.environ["DJANGO_URLIZE_IETF_DOCS_PRODUCTION"] = "1" import django import django.test @@ -65,6 +66,7 @@ import debug # pyflakes:ignore from ietf.name.models import DocTypeName from ietf.utils.html import unescape from ietf.utils.test_utils import unicontent +from ietf.utils.test_runner import start_vnu_server, vnu_validate, vnu_fmt_message, vnu_filter_message # --- Constants --- @@ -156,55 +158,38 @@ def check_html_valid(url, response, args): key = re.sub("/submit/status/nnnn/[0-9a-f]+/", "/submit/status/nnnn/bar/", key) key = re.sub("/team/[a-z0-9-]+/", "/team/foo/", key) key = re.sub("/wg/[a-z0-9-]+/", "/wg/foo/", key) - key = re.sub("\?.*$", "", key) + key = re.sub(r"\?.*$", "", key) for slug in doc_types: key = re.sub("/%s-.*/"%slug, "/%s-nnnn/"%slug, key) if not key in validated_urls: note('Validate: %-32s: %s' % (url[:32], key)) - # These URLs have known issues, skip them until those are fixed - for pattern in ( - '/secr', - 'admin/', - '/doc/.*/edit/info/', - 'rfc542$', - 'rfc776$', - 'draft-leroux-pce-pcecp-interarea-reqs', - 'draft-fujiwara-dnsop-resolver-update', - ): - if re.search(pattern, url): - validated_urls[key] = True - log("%s blacklisted; skipping HTML validation" % url) - return - if hasattr(response, "content"): content = response.content else: content = response.streaming_content validated_urls[key] = True if args.validator_nu: - v = subprocess.Popen(["java", "-jar", basedir + "/bin/vnu.jar", - "--format", "json", "-"], - stdin=subprocess.PIPE, stderr=subprocess.PIPE) - for m in json.loads(v.communicate(content)[1])["messages"]: - t = m["subType"] if m["type"] == "info" else m["type"] - tags.append("\n%s\tLine %d: %s" % - (t.upper(), m["lastLine"], m["message"])) - tags.append("\n\t%s" % m["extract"].replace('\n', ' ')) - tags.append("\n\t%s%s" % - (" " * m["hiliteStart"], "^" * m["hiliteLength"])) + ret = vnu_validate(content, response["Content-Type"], port=8887) + assert ret + for m in json.loads(ret)["messages"]: + if "lastLine" not in m: + tag = m # just dump the raw JSON for now + else: + tag = vnu_fmt_message(url, m, content.decode()) # disregard some HTML issues that are (usually) due to invalid # database content - if not re.search('Forbidden code point|Bad value|seamless|The first child|Duplicate ID|The first occurrence of ID', m["message"]): + if not vnu_filter_message(m, True, False): warnings += 1 + tags.append(tag) else: try: parser.parse(content) except Exception as e: for err in parser.errors: - pos, code, data = err - tags.append(u"WARN invalid html at line, pos %s: %s" % (pos, e)) + pos, _, _ = err + tags.append("WARN invalid html at line, pos {}: {}".format(pos, e)) warnings += 1 def skip_extract_from(url): @@ -219,410 +204,24 @@ def skip_extract_from(url): def skip_url(url): for pattern in ( - "^/community/[0-9]+/remove_document/", - "^/community/personal/", + r"^/community/[0-9]+/remove_document/", + r"^/community/personal/", # Skip most of the slow pdf composite generation urls and svg urls - "^/meeting/[0-9]+/agenda/[0-9b-z].*-drafts\\.pdf", - "^/wg/[a-z0-9-]+/deps/svg/", - # This bad url occurs in an uploaded html agenda: - r"/site/ietfdhcwg/_/rsrc/1311005436000/system/app/css/overlay.css\?cb=simple100%250150goog-ws-left", - r"/dir/tsvdir/reviews/", - r"draft-touch-msword-template-v2\.0", - + r"^/meeting/[0-9]+/agenda/[0-9b-z].*-drafts\\.pdf", + r"^/wg/[a-z0-9-]+/deps/svg/", + # Skip other bad urls + r"^/dir/tsvdir/reviews/", + r"^/ipr/\d{,3}/history/", # Skip most html conversions, not worth the time - "^/doc/html/draft-[0-9ac-z]", - "^/doc/html/draft-b[0-9b-z]", - "^/doc/pdf/draft-[0-9ac-z]", - "^/doc/pdf/draft-b[0-9b-z]", - "^/doc/html/charter-.*", - "^/doc/html/status-.*", - "^/doc/html/rfc.*", - # These will always 404, but include only those not excluded above -# r"^/doc/html/charter-ietf-cicm", -# r"^/doc/html/charter-ietf-dcon", -# r"^/doc/html/charter-ietf-fun", -# r"^/doc/html/charter-ietf-multrans", -# r"^/doc/html/charter-ietf-sdn", -# r"^/doc/html/charter-ietf-woes", -# r"^/doc/html/draft-allan-mpls-loadbal-06", -# r"^/doc/html/draft-allocchio-mail11-00", -# r"^/doc/html/draft-almquist-leak-01", -# r"^/doc/html/draft-almquist-nexthop-01", -# r"^/doc/html/draft-armitage-ion-mars-scsp-06", - r"^/doc/html/draft-balakrishnan-cm-03", - r"^/doc/html/draft-ballardie-cbt-02", -# r"^/doc/html/draft-bellovin-ipng-shape-of-bits-00", -# r"^/doc/html/draft-bellovin-itrace-04", -# r"^/doc/html/draft-bhaskar-pim-ss-04", -# r"^/doc/html/draft-bhattach-diot-pimso-04", -# r"^/doc/html/draft-bierman-rmonmib-apmcaps-04", -# r"^/doc/html/draft-blaze-ipsp-trustmgt-04", -# r"^/doc/html/draft-blumenthal-snmpv2a-community-00", -# r"^/doc/html/draft-borenstein-agc-spec-00", -# r"^/doc/html/draft-borenstein-kidcode-00", -# r"^/doc/html/draft-borenstein-mailcap-00", -# r"^/doc/html/draft-borenstein-pgp-mime-00", -# r"^/doc/html/draft-brockhaus-lamps-cmp-updates-00", -# r"^/doc/html/draft-brockhaus-lamps-cmp-updates-03", -# r"^/doc/html/draft-brockhaus-lamps-lightweight-cmp-profile-00", -# r"^/doc/html/draft-brockhaus-lamps-lightweight-cmp-profile-03", -# r"^/doc/html/draft-brown-supplpkgs-04", -# r"^/doc/html/draft-brownlee-acct-arch-report-03", -# r"^/doc/html/draft-cain-igmp-00", -# r"^/doc/html/draft-calhoun-aaa-diameter-comp-01", -# r"^/doc/html/draft-calhoun-mobileip-fa-tokens-04", -# r"^/doc/html/draft-calhoun-mobileip-min-lat-handoff-02", -# r"^/doc/html/draft-callon-addflow-support-clnp-00", -# r"^/doc/html/draft-callon-routing-00", -# r"^/doc/html/draft-carpenter-ipng-nosi-00", -# r"^/doc/html/draft-carpenter-percep-00", -# r"^/doc/html/draft-casati-gtp-03", -# r"^/doc/html/draft-ceuppens-mpls-optical-04", -# r"^/doc/html/draft-chapin-clnp-ISO8473-00", -# r"^/doc/html/draft-cheng-modular-ikmp-00", -# r"^/doc/html/draft-cole-appm-01", -# r"^/doc/html/draft-conta-ipv6-icmp-igmp-00", -# r"^/doc/html/draft-coya-test-01", -# r"^/doc/html/draft-crocker-cidrd-myth-00", -# r"^/doc/html/draft-crocker-pci-00", -# r"^/doc/html/draft-crocker-stif-00", -# r"^/doc/html/draft-daigle-napstr-05", -# r"^/doc/html/draft-davie-intserv-compress-01", -# r"^/doc/html/draft-davie-tag-switching-atm-04", -# r"^/doc/html/draft-davin-qosrep-00", -# r"^/doc/html/draft-davin-rsvfms-00", -# r"^/doc/html/draft-dckmtr-proxy-00", -# r"^/doc/html/draft-doolan-tdp-spec-04", -# r"^/doc/html/draft-duerst-ruby-04", -# r"^/doc/html/draft-dusse-pem-message-00", -# r"^/doc/html/draft-dutt-rap-rsvp-proxy-01", -# r"^/doc/html/draft-eastlake-muse-05", -# r"^/doc/html/draft-elmalki-soliman-hmipv4v6-04", -# r"^/doc/html/draft-ema-vpim-simplev3-02", -# r"^/doc/html/draft-esaki-co-cl-ip-forw-atm-01", -# r"^/doc/html/draft-etf-marid-protocol-00", -# r"^/doc/html/draft-faltstrom-macmime-00", -# r"^/doc/html/draft-faltstrom-whois-05", -# r"^/doc/html/draft-fielding-http-spec-01", -# r"^/doc/html/draft-flick-interfaces-mib-00", -# r"^/doc/html/draft-flick-repeater-dev-mib-00", -# r"^/doc/html/draft-floyd-cc-alt", -# r"^/doc/html/draft-ford-bigten-bt-format-00", -# r"^/doc/html/draft-ford-sdrp-sipp16-format-00", -# r"^/doc/html/draft-freed-ftbp-00", -# r"^/doc/html/draft-freed-newenc-01", -# r"^/doc/html/draft-gellens-imapext-annotate-01", -# r"^/doc/html/draft-gharai-hdtv-video-04", -# r"^/doc/html/draft-glenn-layer-security-00", -# r"^/doc/html/draft-hares-idrp-familytree-00", -# r"^/doc/html/draft-harrington-control-mib-00", -# r"^/doc/html/draft-harrington-data-filter-mib-00", -# r"^/doc/html/draft-haskin-intra-route-server-00", -# r"^/doc/html/draft-helenius-ppp-subnet-04", -# r"^/doc/html/draft-hinden-ipng-addr-00", -# r"^/doc/html/draft-holbrook-ssm-04", -# r"^/doc/html/draft-hollenbeck-epp-tcp-02", -# r"^/doc/html/draft-houldsworth-ip6-nsap-use-00", -# r"^/doc/html/draft-houldsworth-sc6-hot-finland-00", -# r"^/doc/html/draft-huitema-shipworm-01", -# r"^/doc/html/draft-iab-liaisons-00", -# r"^/doc/html/draft-iab-mou2jtc1-03", -# r"^/doc/html/draft-iab-standards-processv3-00", -# r"^/doc/html/draft-ietf-aft-socks-md5-auth-00", -# r"^/doc/html/draft-ietf-bgpdepl-minutes-93feb-00", -# r"^/doc/html/draft-ietf-bmwg-overallperf-00", -# r"^/doc/html/draft-ietf-bridge-sr-obj-00", -# r"^/doc/html/draft-ietf-cat-altftp-00", -# r"^/doc/html/draft-ietf-cip-apisocket-00", -# r"^/doc/html/draft-ietf-cipso-ipsec-option-00", -# r"^/doc/html/draft-ietf-decnetiv-mib-implement-00", -# r"^/doc/html/draft-ietf-dhc-problem-stmt-00", -# r"^/doc/html/draft-ietf-dns-idpr-02", -# r"^/doc/html/draft-ietf-dns-ixfr-01", -# r"^/doc/html/draft-ietf-dnsind-dynDNS-arch-00", -# r"^/doc/html/draft-ietf-dnsind-dynDNS-impl-00", -# r"^/doc/html/draft-ietf-dtn-tcpclv4-00", -# r"^/doc/html/draft-ietf-dtn-tcpclv4-15", -# r"^/doc/html/draft-ietf-dtn-tcpclv4-18", -# r"^/doc/html/draft-ietf-dtn-tcpclv4-19", -# r"^/doc/html/draft-ietf-ethermib-objects-00", -# r"^/doc/html/draft-ietf-fax-tiff-f-reg-01", -# r"^/doc/html/draft-ietf-geopriv-dhcp-lo-option-01", -# r"^/doc/html/draft-ietf-html-charset-harmful-00", -# r"^/doc/html/draft-ietf-iafa-templates-00", -# r"^/doc/html/draft-ietf-idmr-mtree-00", -# r"^/doc/html/draft-ietf-idmr-pim-dense-spec-00", -# r"^/doc/html/draft-ietf-idmr-pim-dm-spec-08", -# r"^/doc/html/draft-ietf-idr-bgp-tcp-md5bad-01", -# r"^/doc/html/draft-ietf-idr-community-00", -# r"^/doc/html/draft-ietf-idr-rifs-00", -# r"^/doc/html/draft-ietf-ids-iwps-design-spec-01", -# r"^/doc/html/draft-ietf-ids-pilots-00", -# r"^/doc/html/draft-ietf-iesg-evolutionplan-00", -# r"^/doc/html/draft-ietf-iiir-html-01", -# r"^/doc/html/draft-ietf-iiir-http-00", -# r"^/doc/html/draft-ietf-ipae-new-ip-00", -# r"^/doc/html/draft-ietf-ipidrp-sip-01", -# r"^/doc/html/draft-ietf-iplpdn-multi-isdn-02", -# r"^/doc/html/draft-ietf-iplpdn-para-negotiation-02", -# r"^/doc/html/draft-ietf-iplpdn-shortcutrouting-02", -# r"^/doc/html/draft-ietf-iplpdn-simple-multi-01", -# r"^/doc/html/draft-ietf-ipp-indp-04", -# r"^/doc/html/draft-ietf-ipsec-ike-base-mode-03", -# r"^/doc/html/draft-ietf-ipsec-intragkm-03", -# r"^/doc/html/draft-ietf-ipsp-spsl-04", -# r"^/doc/html/draft-ietf-ipsra-pic-07", -# r"^/doc/html/draft-ietf-isis-atipx-00", -# r"^/doc/html/draft-ietf-isis-multilevel-routing-00", -# r"^/doc/html/draft-ietf-isis-nbma-00", -# r"^/doc/html/draft-ietf-isis-tcpip-01", -# r"^/doc/html/draft-ietf-isn-aup-01", -# r"^/doc/html/draft-ietf-lsma-scenarios-03", -# r"^/doc/html/draft-ietf-mailext-lang-char-00", -# r"^/doc/html/draft-ietf-mhsds-822dir-03", -# r"^/doc/html/draft-ietf-mhsds-convert-01", -# r"^/doc/html/draft-ietf-mhsds-mhsprofile-04", -# r"^/doc/html/draft-ietf-mhsds-mhsuse-03", -# r"^/doc/html/draft-ietf-mmusic-agree-00", -# r"^/doc/html/draft-ietf-mobileip-aaa-req-00", -# r"^/doc/html/draft-ietf-mobileip-addr-ext-00", -# r"^/doc/html/draft-ietf-mobileip-integrated-00", -# r"^/doc/html/draft-ietf-mobileip-mib-fa-01", -# r"^/doc/html/draft-ietf-mobileip-mib-ha-01", -# r"^/doc/html/draft-ietf-mobileip-mib-mn-01", -# r"^/doc/html/draft-ietf-mobileip-mib-sec-01", -# r"^/doc/html/draft-ietf-msi-api-03", -# r"^/doc/html/draft-ietf-nasreq-nasrequirements-01", -# r"^/doc/html/draft-ietf-netdata-implement-03", -# r"^/doc/html/draft-ietf-netdata-netdata-04", -# r"^/doc/html/draft-ietf-nimrod-dns-01", -# r"^/doc/html/draft-ietf-nisi-nicdoc-00", -# r"^/doc/html/draft-ietf-nisi-nics-00", -# r"^/doc/html/draft-ietf-nntp-news-01", -# r"^/doc/html/draft-ietf-oncrpc-remote-06", -# r"^/doc/html/draft-ietf-osids-dirtree-00", -# r"^/doc/html/draft-ietf-osids-dsanaming-02", -# r"^/doc/html/draft-ietf-osids-requirements-00", -# r"^/doc/html/draft-ietf-osids-simple-stack-00", -# r"^/doc/html/draft-ietf-osids-treestructure-00", -# r"^/doc/html/draft-ietf-osinsap-format-01", -# r"^/doc/html/draft-ietf-osix500-directories-01", -# r"^/doc/html/draft-ietf-ospf-extattr-00", -# r"^/doc/html/draft-ietf-ospf-ipv6-ext-00", -# r"^/doc/html/draft-ietf-ospf-pmp-if-00", -# r"^/doc/html/draft-ietf-otp-ver-03", -# r"^/doc/html/draft-ietf-pana-aaa-interworking-00", -# r"^/doc/html/draft-ietf-pem-notary-00", -# r"^/doc/html/draft-ietf-pim-ipv6-04", -# r"^/doc/html/draft-ietf-pim-simplekmp-02", -# r"^/doc/html/draft-ietf-pint-conf-02", -# r"^/doc/html/draft-ietf-pip-vector-00", -# r"^/doc/html/draft-ietf-poised-nomcomm-00", -# r"^/doc/html/draft-ietf-pppext-aha-auth-00", -# r"^/doc/html/draft-ietf-pppext-ipcp-network-04", -# r"^/doc/html/draft-ietf-pppext-kap-auth-00", -# r"^/doc/html/draft-ietf-pppext-kapv4-auth-00", -# r"^/doc/html/draft-ietf-ripv2-ripng-00", -# r"^/doc/html/draft-ietf-rmon-trap-00", -# r"^/doc/html/draft-ietf-rmonmib-rmon2hc-01", -# r"^/doc/html/draft-ietf-roamops-actng-08", -# r"^/doc/html/draft-ietf-rohc-rtp-rocco-performance-01", -# r"^/doc/html/draft-ietf-rohc-rtp-rocco-video-01", -# r"^/doc/html/draft-ietf-rolc-nhrp-mib-00", -# r"^/doc/html/draft-ietf-rreq-iprouters-04", -# r"^/doc/html/draft-ietf-rsvp-policy-ext-05", -# r"^/doc/html/draft-ietf-rsvp-state-compression-04", -# r"^/doc/html/draft-ietf-sdr-IPv6-pack-00", -# r"^/doc/html/draft-ietf-sdr-pl-00", -# r"^/doc/html/draft-ietf-sdr-route-construction-01", -# r"^/doc/html/draft-ietf-sdr-route-setup-00", -# r"^/doc/html/draft-ietf-sdr-speakers-attribute-00", -# r"^/doc/html/draft-ietf-sip-64bit-plan-00", -# r"^/doc/html/draft-ietf-sip-dnss-00", -# r"^/doc/html/draft-ietf-sip-ospf-00", -# r"^/doc/html/draft-ietf-sip-rip-01", -# r"^/doc/html/draft-ietf-sip-unicast-addr-00", -# r"^/doc/html/draft-ietf-sipp-auto-addr-00", -# r"^/doc/html/draft-ietf-sipp-bsd-api-02", -# r"^/doc/html/draft-ietf-sipp-dhcpopt-01", -# r"^/doc/html/draft-ietf-sipp-discovery-04", -# r"^/doc/html/draft-ietf-sipp-discovery-formats-00", -# r"^/doc/html/draft-ietf-sipp-dns-01", -# r"^/doc/html/draft-ietf-sipp-dns-ext-00", -# r"^/doc/html/draft-ietf-sipp-icmp-igmp-00", -# r"^/doc/html/draft-ietf-sipp-routing-addr-02", -# r"^/doc/html/draft-ietf-sipp-sst-overview-00", -# r"^/doc/html/draft-ietf-sipping-overload-design", -# r"^/doc/html/draft-ietf-smime-certdist-06", -# r"^/doc/html/draft-ietf-smtpext-pipeline-02", -# r"^/doc/html/draft-ietf-snmp-isdn-cisco-00", -# r"^/doc/html/draft-ietf-snmpsec-m2mv2-01", -# r"^/doc/html/draft-ietf-snmpsec-mibv2-00", -# r"^/doc/html/draft-ietf-snmpsec-protov2-01", -# r"^/doc/html/draft-ietf-snmpsec-tmv2-00", -# r"^/doc/html/draft-ietf-stjohns-ipso-00", -# r"^/doc/html/draft-ietf-svrloc-discovery-11", -# r"^/doc/html/draft-ietf-tcplw-extensions-00", -# r"^/doc/html/draft-ietf-tcplw-high-performance-01", -# r"^/doc/html/draft-ietf-telnet-authker-v5-01", -# r"^/doc/html/draft-ietf-telnet-compression-00", -# r"^/doc/html/draft-ietf-telnet-encryption-02", -# r"^/doc/html/draft-ietf-tewg-measure-07", -# r"^/doc/html/draft-ietf-thinosi-profile-00", -# r"^/doc/html/draft-ietf-tnfs-spec-03", -# r"^/doc/html/draft-ietf-ucp-connectivity-01", -# r"^/doc/html/draft-ietf-udlr-life-03", -# r"^/doc/html/draft-ietf-ufdl-spec-01", -# r"^/doc/html/draft-ietf-uri-roy-urn-urc-00", -# r"^/doc/html/draft-ietf-uri-urc-00", -# r"^/doc/html/draft-ietf-uri-urc-sgml-00", -# r"^/doc/html/draft-ietf-uri-urc-spec-00", -# r"^/doc/html/draft-ietf-uri-urc-trivial-00", -# r"^/doc/html/draft-ietf-uri-urn-issues-00", -# r"^/doc/html/draft-ietf-uri-urn-madsen-critique-00", -# r"^/doc/html/draft-ietf-uri-urn-res-descript-00", -# r"^/doc/html/draft-ietf-uri-urn-res-thoughts-00", -# r"^/doc/html/draft-ietf-uri-urn-syntax-00", -# r"^/doc/html/draft-ietf-uri-urn-x-dns-2-00", -# r"^/doc/html/draft-ietf-uri-urn2urc-00", -# r"^/doc/html/draft-ietf-uri-yaurn-00", -# r"^/doc/html/draft-ietf-userdoc2-fyi-biblio-00", -# r"^/doc/html/draft-ietf-uswg-fyi1-02", -# r"^/doc/html/draft-ietf-whip-reqs-summary-01", -# r"^/doc/html/draft-ietf-x400ops-admd-03", -# r"^/doc/html/draft-ietf-x400ops-dnsx400rout-02", -# r"^/doc/html/draft-ietf-x400ops-tbl-dist-00", -# r"^/doc/html/draft-ietf-x400ops-tbl-dist-part1-01", -# r"^/doc/html/draft-ietf-x400ops-tbl-dist-part2-01", -# r"^/doc/html/draft-ipsec-isakmp-mode-cfg-02", -# r"^/doc/html/draft-johnson-imhp-00", -# r"^/doc/html/draft-jseng-utf5-02", -# r"^/doc/html/draft-just-ldapv3-rescodes-03", -# r"^/doc/html/draft-karrenberg-proposal-00", -# r"^/doc/html/draft-kastenholz-loki-00", -# r"^/doc/html/draft-kempf-scope-rules-04", -# r"^/doc/html/draft-klyne-conneg-feature-match-03", -# r"^/doc/html/draft-koch-dnsind-local-compression-01", -# r"^/doc/html/draft-kzm-rap-sppi-04", -# r"^/doc/html/draft-kzm-snmpv2-adminv2-alt-00", -# r"^/doc/html/draft-kzm-snmpv2-coex-alt-00", -# r"^/doc/html/draft-kzm-snmpv2-conf-alt-00", -# r"^/doc/html/draft-kzm-snmpv2-intro-alt-00", -# r"^/doc/html/draft-kzm-snmpv2-mib-alt-00", -# r"^/doc/html/draft-kzm-snmpv2-smi-alt-00", -# r"^/doc/html/draft-kzm-snmpv2-usec-conf-alt-00", -# r"^/doc/html/draft-larson-bad-dns-res-01", -# r"^/doc/html/draft-lear-foglamps-03", -# r"^/doc/html/draft-leech-socks-protocol-v4-01", -# r"^/doc/html/draft-levi-snmp-mid-level-mgr-00", -# r"^/doc/html/draft-levi-snmp-script-language-00", -# r"^/doc/html/draft-levinson-sgml-02", -# r"^/doc/html/draft-li-bigten-addr-format-00", -# r"^/doc/html/draft-li-tap-ipv7-00", -# r"^/doc/html/draft-lloyd-ip6-iso-itu-reg-00", -# r"^/doc/html/draft-macker-mdp-framework-05", -# r"^/doc/html/draft-mahoney-snmpv2-features-00", -# r"^/doc/html/draft-mahoney-snmpv2-proto-alt-00", -# r"^/doc/html/draft-martensson-rocco-video-04", -# r"^/doc/html/draft-mccann-mobileip-sessionid-04", -# r"^/doc/html/draft-megginson-ldup-lcup-01", -# r"^/doc/html/draft-metzger-ah-sha-00", -# r"^/doc/html/draft-mpls-rsvpte-attributes-00", -# r"^/doc/html/draft-myers-imap-imsp-01", -# r"^/doc/html/draft-myers-imap-mbox-00", -# r"^/doc/html/draft-myers-smtp-mult-01", -# r"^/doc/html/draft-nelson-model-mailext-00", -# r"^/doc/html/draft-newman-imap-annotate-00", -# r"^/doc/html/draft-nguyen-bgp-ipv6-vpn-03", -# r"^/doc/html/draft-nordmark-ipv6-aaa-hooks-04", -# r"^/doc/html/draft-nyckelgard-isl-arch-04", -# r"^/doc/html/draft-ohta-address-allocation-01", -# r"^/doc/html/draft-ohta-dynamic-dns-00", -# r"^/doc/html/draft-ohta-ip-over-atm-02", -# r"^/doc/html/draft-ohta-mime-charset-names-00", -# r"^/doc/html/draft-ohta-shared-media-02", -# r"^/doc/html/draft-ohta-simple-dns-01", -# r"^/doc/html/draft-ohta-text-encoding-01", -# r"^/doc/html/draft-ohta-translation-instr-01", -# r"^/doc/html/draft-ooms-cl-multicast-03", -# r"^/doc/html/draft-ops-rfc2011-update-01", -# r"^/doc/html/draft-ouldbrahim-bgpvpn-auto-03", -# r"^/doc/html/draft-palme-autosub-06", -# r"^/doc/html/draft-pan-diffserv-mib-01", -# r"^/doc/html/draft-perkins-cnlp-support-00", -# r"^/doc/html/draft-perkins-homeaddr-dhcpopt-00", -# r"^/doc/html/draft-perkins-opaque-04", -# r"^/doc/html/draft-polk-slp-loc-auth-server-04", -# r"^/doc/html/draft-popp-cnrp-goals-01", -# r"^/doc/html/draft-pusateri-igmp-mib-00", -# r"^/doc/html/draft-pusateri-ipmulti-mib-00", -# r"^/doc/html/draft-reddy-opsawg-mud-tls-00", -# r"^/doc/html/draft-reddy-opsawg-mud-tls-03", -# r"^/doc/html/draft-reichmeyer-polterm-terminology-04", -# r"^/doc/html/draft-rekhter-arch-sipp16-addr-00", -# r"^/doc/html/draft-rekhter-bigten-addr-arch-00", -# r"^/doc/html/draft-rekhter-direct-provider-01", -# r"^/doc/html/draft-rekhter-idr-over-atm-00", -# r"^/doc/html/draft-rekhter-lsr-mobile-hosts-00", -# r"^/doc/html/draft-rekhter-select-providers-02", -# r"^/doc/html/draft-rekhter-sops-02", -# r"^/doc/html/draft-rekhter-stratum-aggregation-01", -# r"^/doc/html/draft-renwick-hippiarp-01", -# r"^/doc/html/draft-renwick-hippimib-01", -# r"^/doc/html/draft-rfced-info-corson-00", -# r"^/doc/html/draft-rfced-info-katsube-oops-00", -# r"^/doc/html/draft-rfced-info-perkins-05", -# r"^/doc/html/draft-rfced-info-pi-vs-pa-addrspac-00", -# r"^/doc/html/draft-rfced-info-senie-00", -# r"^/doc/html/draft-ronc-domain-phb-set-ldap-rep-04", -# r"^/doc/html/draft-ronc-domain-phb-set-specification-04", -# r"^/doc/html/draft-rose-limit-01", -# r"^/doc/html/draft-rose-smxp-spec-00", -# r"^/doc/html/draft-rosen-ppvpn-l2vpn-01", -# r"^/doc/html/draft-rosen-tag-stack-05", -# r"^/doc/html/draft-rosenberg-mmusic-sdp-offer-answer-01", -# r"^/doc/html/draft-rosenberg-sip-tunnels-01", -# r"^/doc/html/draft-salzr-ldap-repsig-01", -# r"^/doc/html/draft-sandick-pimsm-ssmrules-04", -# r"^/doc/html/draft-schroeppel-dnsind-ecc-04", -# r"^/doc/html/draft-simpson-exchanges-00", -# r"^/doc/html/draft-simpson-ipv6-deploy-00", -# r"^/doc/html/draft-simpson-ipv6-discovery-req-00", -# r"^/doc/html/draft-simpson-ipv6-hc-00", -# r"^/doc/html/draft-simpson-sipp-64-bit-plan-00", -# r"^/doc/html/draft-sinnreich-interdomain-sip-qos-osp-02", -# r"^/doc/html/draft-slutsman-aicd-02", -# r"^/doc/html/draft-speer-avt-layered-video-05", -# r"^/doc/html/draft-stein-green-commerce-model-00", -# r"^/doc/html/draft-svanbro-rohc-lower-layer-guidelines-04", -# r"^/doc/html/draft-templin-atn-aero-interface-00", -# r"^/doc/html/draft-templin-atn-aero-interface-21", -# r"^/doc/html/draft-teraoka-ipv6-mobility-sup-07", -# r"^/doc/html/draft-thayer-seccomp-04", -# r"^/doc/html/draft-traina-bgp-confed-00", -# r"^/doc/html/draft-treese-class-desc-00", -# r"^/doc/html/draft-vaudreuil-binaryheaders-01", -# r"^/doc/html/draft-vaudreuil-enum-e164dir-05", -# r"^/doc/html/draft-veizades-ipng-svrloc-00", -# r"^/doc/html/draft-villamizar-isis-omp-01", -# r"^/doc/html/draft-waldbusser-conventions-01", -# r"^/doc/html/draft-waldbusser-rmonmib-apm-04", -# r"^/doc/html/draft-waldbusser-ssecimpl-01", -# r"^/doc/html/draft-waters-snmpv1-sec-mech-00", -# r"^/doc/html/draft-weider-comindex-00", -# r"^/doc/html/draft-wijnen-snmpv2-snmpv2t-00", -# r"^/doc/html/draft-woundy-dhcpleasequery-04", -# r"^/doc/html/draft-wright-policy-mpls-04", -# r"^/doc/html/draft-yu-asn1-pitfalls-04", -# r"^/doc/html/draft-yu-rpd-00", -# r"^/doc/html/draft-zaccone-nat-rsip-gen-arch-02", -# r"^/doc/html/draft-zaccone-nat-transp-fram-02", -# r"^/doc/html/draft-zaccone-nat-transport-03", -# r"^/doc/html/status-change-icmpv6-dns-ipv6-to-internet-standard", - + r"^/doc/html/draft-[0-9ac-z]", + r"^/doc/html/draft-b[0-9b-z]", + r"^/doc/pdf/draft-[0-9ac-z]", + r"^/doc/pdf/draft-b[0-9b-z]", + r"^/doc/html/charter-.*", + r"^/doc/html/status-.*", + r"^/doc/html/rfc.*", r"^/static/coverage/", - r"^/meeting/6[0-4]/agenda", - r"^https?://www.ietf.org/", + r"^/meeting/\d{,2}/agenda", # no agendas < 100 ): if re.search(pattern, url): return True @@ -691,8 +290,16 @@ logfile = None if args.logfile: logfile = open(args.logfile, "w") +vnu = None + # --- Main --- +def do_exit(code): + if vnu: + vnu.terminate() + sys.exit(code) + + if __name__ == "__main__": if (args.user): # log in as user, to have the respective HTML generated by the templates @@ -702,7 +309,7 @@ if __name__ == "__main__": if (response.status_code != 200): log("Could not log in as %s, HTML response %d" % (args.user, response.status_code)) - sys.exit(1) + do_exit(1) # Run django system checks and checks from ietf.checks: error_list = django.core.checks.run_checks() @@ -718,10 +325,13 @@ if __name__ == "__main__": for entry in error_list: print(entry) + if args.validator_nu: + vnu = start_vnu_server(port=8887) + while urls: if args.random: # popitem() is documented to be random, but really isn't - url = random.choice(urls.keys()) + url = random.choice(list(urls.keys())) referrer = urls.pop(url) else: url, referrer = urls.popitem() @@ -746,10 +356,10 @@ if __name__ == "__main__": elapsed = datetime.datetime.now() - request_start except KeyboardInterrupt: log(" ... was fetching %s" % url) - sys.exit(1) + do_exit(1) except: elapsed = datetime.datetime.now() - request_start - tags = [ u"FAIL (from [ %s ])" % (",\n\t".join(get_referrers(url))) ] + tags = [ "FAIL (from [ %s ])" % (",\n\t".join(get_referrers(url))) ] log("%2d:%02d:%02d %7d %6d %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), 500, elapsed.total_seconds(), url, " ".join(tags))) log("=============") log(traceback.format_exc()) @@ -760,7 +370,7 @@ if __name__ == "__main__": if r.status_code in (301, 302): u = strip_url(r["Location"]) - if u not in visited and u not in urls: + if not url.startswith("/") and u not in visited and u not in urls: urls[u] = referrer # referrer is original referrer, not redirected url referrers[u] = referrer @@ -799,8 +409,9 @@ if __name__ == "__main__": log("=============") else: - tags.append(u"FAIL (from %s)" % (referrer, )) - errors += 1 + tags.append("FAIL (from {})".format(referrer)) + if not url.startswith("/person/"): # FIXME: those fail sometimes + errors += 1 if elapsed.total_seconds() > slow_threshold: tags.append("SLOW") @@ -816,7 +427,7 @@ if __name__ == "__main__": log("%2d:%02d:%02d %7d %6d %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), r.status_code, elapsed.total_seconds(), url, " ".join(tags))) if ((errors or warnings) and args.pedantic): - sys.exit(1) + do_exit(1) if logfile: logfile.close() @@ -824,7 +435,7 @@ if __name__ == "__main__": if errors > 0: sys.stderr.write("Found %s errors, grep output for FAIL for details\n" % errors) - sys.exit(1) + do_exit(1) else: sys.stderr.write("Found no errors.\n") if warnings > 0: diff --git a/bin/vnu.jar b/bin/vnu.jar index 661366f39..dde1acf66 100644 Binary files a/bin/vnu.jar and b/bin/vnu.jar differ diff --git a/dev/tests/Dockerfile b/dev/tests/Dockerfile index e7b562555..10d89a58f 100644 --- a/dev/tests/Dockerfile +++ b/dev/tests/Dockerfile @@ -23,6 +23,7 @@ RUN apt-get install -qy \ bash \ build-essential \ curl \ + default-jdk \ docker-ce-cli \ enscript \ gawk \ diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index 181195517..722df05bc 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -32,6 +32,7 @@ RUN apt-get install -qy \ bash \ build-essential \ curl \ + default-jdk \ docker-ce-cli \ enscript \ fish \ diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 76f73eaba..f162a6bbc 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -4,6 +4,7 @@ import datetime import re +import os from urllib.parse import urljoin from email.utils import parseaddr @@ -17,10 +18,13 @@ from django.utils.html import strip_tags from django.utils.encoding import force_text from django.utils.encoding import force_str # pyflakes:ignore force_str is used in the doctests from django.urls import reverse as urlreverse +from django.core.cache import cache +from django.core.validators import URLValidator +from django.core.exceptions import ValidationError import debug # pyflakes:ignore -from ietf.doc.models import BallotDocEvent +from ietf.doc.models import BallotDocEvent, DocAlias from ietf.doc.models import ConsensusDocEvent from ietf.utils.html import sanitize_fragment from ietf.utils import log @@ -184,49 +188,113 @@ def rfceditor_info_url(rfcnum : str): """Link to the RFC editor info page for an RFC""" return urljoin(settings.RFC_EDITOR_INFO_BASE_URL, f'rfc{rfcnum}') + +def doc_exists(name): + """Check whether a given document exists""" + def find_unique(n): + key = hash(n) + found = cache.get(key) + if not found: + exact = DocAlias.objects.filter(name=n).first() + found = exact.name if exact else "_" + cache.set(key, found) + return None if found == "_" else found + + # all documents exist when tests are running + if settings.SERVER_MODE == 'test': + # unless we are running test-crawl, which would otherwise 404 + if "DJANGO_URLIZE_IETF_DOCS_PRODUCTION" not in os.environ: + return True + + # chop away extension + extension_split = re.search(r"^(.+)\.(txt|ps|pdf)$", name) + if extension_split: + name = extension_split.group(1) + + if find_unique(name): + return True + + # check for embedded rev - this may be ambiguous, so don't + # chop it off if we don't find a match + rev_split = re.search("^(.+)-([0-9]{2,})$", name) + if rev_split: + name = rev_split.group(1) + if find_unique(name): + return True + + return False + + +def link_charter_doc_match1(match): + if not doc_exists(match[0]): + return match[0] + return f'{match[0]}' + + +def link_charter_doc_match2(match): + if not doc_exists(match[0]): + return match[0] + return f'{match[0]}' + + def link_non_charter_doc_match(match): - if len(match[3])==2 and match[3].isdigit(): + if not doc_exists(match[0]): + return match[0] + if len(match[3]) == 2 and match[3].isdigit(): return f'{match[0]}' else: return f'{match[0]}' -@register.filter(name='urlize_ietf_docs', is_safe=True, needs_autoescape=True) + +def link_other_doc_match(match): + # there may be whitespace in the match + doc = re.sub(r"\s+", "", match[0]) + if not doc_exists(doc): + return match[0] + return f'{match[1]}' + + +@register.filter(name="urlize_ietf_docs", is_safe=True, needs_autoescape=True) def urlize_ietf_docs(string, autoescape=None): """ Make occurrences of RFC NNNN and draft-foo-bar links to /doc/. """ if autoescape and not isinstance(string, SafeData): - string = escape(string) - exp1 = r"\b(charter-(?:[\d\w\.+]+-)*)(\d\d-\d\d)(\.txt)?\b" - exp2 = r"\b(charter-(?:[\d\w\.+]+-)*)(\d\d)(\.txt)?\b" + if "<" in string: + string = escape(string) + else: + string = mark_safe(string) + exp1 = r"\b(?{x[0]}', + link_charter_doc_match1, string, flags=re.IGNORECASE | re.ASCII, ) - elif re.search(exp2, string): + elif re.search(exp2, string): string = re.sub( exp2, - lambda x: f'{x[0]}', + link_charter_doc_match2, string, flags=re.IGNORECASE | re.ASCII, ) string = re.sub( - r"\b(?{x[1]}', + r"\b(?Edit' % (urlreverse(url_name, args=args, kwargs=kwargs))) + return mark_safe('Edit' % (urlreverse(url_name, args=args, kwargs=kwargs))) @register.filter def textify(text): @@ -765,3 +833,16 @@ def absurl(viewname, **kwargs): Uses settings.IDTRACKER_BASE_URL as the base. """ return urljoin(settings.IDTRACKER_BASE_URL, urlreverse(viewname, kwargs=kwargs)) + + +@register.filter +def is_valid_url(url): + """ + Check if the given URL is syntactically valid + """ + validate_url = URLValidator() + try: + validate_url(url) + except ValidationError: + return False + return True diff --git a/ietf/doc/templatetags/tests_ietf_filters.py b/ietf/doc/templatetags/tests_ietf_filters.py index cd363d113..81fccc01d 100644 --- a/ietf/doc/templatetags/tests_ietf_filters.py +++ b/ietf/doc/templatetags/tests_ietf_filters.py @@ -46,14 +46,22 @@ class IetfFiltersTests(TestCase): ), ( "draft-madanapalli-nd-over-802.16-problems", - 'draft-madanapalli-nd-over-802.16-problems' + 'draft-madanapalli-nd-over-802.16-problems' ), ( "draft-madanapalli-nd-over-802.16-problems-02.txt", - 'draft-madanapalli-nd-over-802.16-problems-02.txt' + 'draft-madanapalli-nd-over-802.16-problems-02.txt' + ), + ( + 'draft-ietf-some-names@ietf.org', + 'draft-ietf-some-names@ietf.org', + ), + ( + "http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf", + "http://ieee802.org/1/files/public/docs2015/cn-thaler-Qcn-draft-PAR.pdf" ) ] - + # Some edge cases scraped from existing old draft names for name in [ # "draft-odell-8+8", # This fails since + matches the right side of \b diff --git a/ietf/group/utils.py b/ietf/group/utils.py index 8d27507db..e6628ca41 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -23,6 +23,7 @@ from ietf.person.models import Email from ietf.review.utils import can_manage_review_requests_for_team from ietf.utils import log from ietf.utils.history import get_history_object_for, copy_many_to_many_for_history +from ietf.doc.templatetags.ietf_filters import is_valid_url from functools import reduce def save_group_in_history(group): @@ -208,7 +209,8 @@ def construct_group_menu_context(request, group, selected, group_type, others): entries.append(("Photos", urlreverse("ietf.group.views.group_photos", kwargs=kwargs))) entries.append(("Email expansions", urlreverse("ietf.group.views.email", kwargs=kwargs))) if group.list_archive.startswith("http:") or group.list_archive.startswith("https:") or group.list_archive.startswith("ftp:"): - entries.append((mark_safe("List archive »"), group.list_archive)) + if is_valid_url(group.list_archive): + entries.append((mark_safe("List archive »"), group.list_archive)) # actions diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index e4b0d7b37..3785cee4a 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -35,7 +35,7 @@ from ietf.utils.validators import ( validate_file_size, validate_mime_type, # need to insert empty option for use in ChoiceField # countries.insert(0, ('', '-'*9 )) -countries.insert(0, ('', '')) +countries.insert(0, ('', '-' * 9)) timezones.insert(0, ('', '-' * 9)) # ------------------------------------------------- @@ -827,4 +827,4 @@ def sessiondetailsformset_factory(min_num=1, max_num=3): min_num=min_num, max_num=max_num, extra=max_num, # only creates up to max_num total - ) \ No newline at end of file + ) diff --git a/ietf/secr/proceedings/forms.py b/ietf/secr/proceedings/forms.py index fcfbeea98..26dd419d6 100644 --- a/ietf/secr/proceedings/forms.py +++ b/ietf/secr/proceedings/forms.py @@ -22,7 +22,9 @@ VALID_BLUESHEET_EXTENSIONS = ('.pdf','.jpg','.jpeg') class RecordingForm(forms.Form): external_url = forms.URLField(label='Url') - session = forms.ModelChoiceField(queryset=Session.objects,empty_label='') + session = forms.ModelChoiceField(queryset=Session.objects) + session.widget.attrs['class'] = "select2-field" + session.widget.attrs['data-minimum-input-length'] = 0 def __init__(self, *args, **kwargs): self.meeting = kwargs.pop('meeting') diff --git a/ietf/secr/static/js/proceedings-recording.js b/ietf/secr/static/js/proceedings-recording.js deleted file mode 100644 index 64cf9503e..000000000 --- a/ietf/secr/static/js/proceedings-recording.js +++ /dev/null @@ -1,6 +0,0 @@ -/* proceedings-recordings.js - utility functions */ - - -$(document).ready(function() { - $('#id_session').select2({ placeholder: 'Type group acronym or part of session name', width: '450px' });; -}); diff --git a/ietf/secr/templates/proceedings/recording.html b/ietf/secr/templates/proceedings/recording.html index 6b9498379..906218f97 100755 --- a/ietf/secr/templates/proceedings/recording.html +++ b/ietf/secr/templates/proceedings/recording.html @@ -11,7 +11,6 @@ {% block extrahead %}{{ block.super }} - {% endblock %} {% block breadcrumbs %}{{ block.super }} @@ -119,4 +118,4 @@ {% block footer-extras %} {% include "includes/upload_footer.html" %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/static/js/agenda_timezone.js b/ietf/static/js/agenda_timezone.js index 34b3a61d7..187bea0e8 100644 --- a/ietf/static/js/agenda_timezone.js +++ b/ietf/static/js/agenda_timezone.js @@ -25,7 +25,7 @@ .utc(); item.end_ts = moment.unix(this.getAttribute("data-end-time")) .utc(); - if (this.hasAttribute("weekday")) { + if (this.hasAttribute("data-weekday")) { item.format = 2; } else { item.format = 1; diff --git a/ietf/templates/doc/ballot/send_ballot_comment.html b/ietf/templates/doc/ballot/send_ballot_comment.html index fdffbd997..8f87bf51a 100644 --- a/ietf/templates/doc/ballot/send_ballot_comment.html +++ b/ietf/templates/doc/ballot/send_ballot_comment.html @@ -32,8 +32,8 @@
{{ body|maybewordwrap }}+
Body
+{{ body|maybewordwrap }}
{{ p.discuss|linkify|urlize_ietf_docs }}+
{{ p.discuss|urlize_ietf_docs|linkify }}
{{ p.comment|linkify|urlize_ietf_docs }}+
{{ p.comment|urlize_ietf_docs|linkify }}
{{ p.discuss|linkify|urlize_ietf_docs }}+
{{ p.discuss|urlize_ietf_docs|linkify }}
{{ p.comment|linkify|urlize_ietf_docs }}+
{{ p.comment|urlize_ietf_docs|linkify }}
{{ content|maybewordwrap|linkify|urlize_ietf_docs }}+
{{ content|maybewordwrap|urlize_ietf_docs|linkify }}
{{ content|maybewordwrap|linkify|urlize_ietf_docs }}+
{{ content|maybewordwrap|urlize_ietf_docs|linkify }}
{{ content|maybewordwrap|linkify|urlize_ietf_docs }}+
{{ content|maybewordwrap|urlize_ietf_docs|linkify }}{% endif %} {% else %} Not available as plain text. @@ -136,4 +136,4 @@ {% block js %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/templates/doc/document_review.html b/ietf/templates/doc/document_review.html index 2e307ec6a..bf249ff53 100644 --- a/ietf/templates/doc/document_review.html +++ b/ietf/templates/doc/document_review.html @@ -47,7 +47,7 @@
{{ content|maybewordwrap|linkify|urlize_ietf_docs }}+
{{ content|maybewordwrap|urlize_ietf_docs|linkify }}
{{ content|maybewordwrap|linkify|urlize_ietf_docs }}+
{{ content|maybewordwrap|urlize_ietf_docs|linkify }}
{{ text|linkify|urlize_ietf_docs }}{% endif %} + {% if text %}
{{ text|urlize_ietf_docs|linkify }}{% endif %} {% if can_edit %}
@@ -32,4 +32,4 @@
{% endif %}
{% endfor %}
{% endfor %}
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/ietf/templates/doc/downref.html b/ietf/templates/doc/downref.html
index c25dc52b3..b82b5027c 100644
--- a/ietf/templates/doc/downref.html
+++ b/ietf/templates/doc/downref.html
@@ -18,7 +18,7 @@
- {{ area.description|linkify|urlize_ietf_docs|safe }}
+ {{ area.description|urlize_ietf_docs|linkify|safe }}
, no surrounding necessary: #}
- {{ group.charter_text|linkify|urlize_ietf_docs|linebreaks }}
+ {{ group.charter_text|urlize_ietf_docs|linkify|linebreaks }}
{% else %}
@@ -41,4 +41,4 @@
{% endblock %}
{% block js %}
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/ietf/templates/doc/drafts_in_iesg_process.html b/ietf/templates/doc/drafts_in_iesg_process.html
index 4fbb6e06a..528678118 100644
--- a/ietf/templates/doc/drafts_in_iesg_process.html
+++ b/ietf/templates/doc/drafts_in_iesg_process.html
@@ -60,7 +60,7 @@
{% endif %}
{% if doc.note %}
Referenced RFC
- Internet-Draft making the reference
+ Internet-Draft making the reference
- Note: {{ doc.note|linkify|urlize_ietf_docs|linebreaksbr }}
+ Note: {{ doc.note|urlize_ietf_docs|linkify|linebreaksbr }}
{% endif %}
diff --git a/ietf/templates/doc/review/request_info.html b/ietf/templates/doc/review/request_info.html
index 5a647baf8..cc253ee7b 100644
--- a/ietf/templates/doc/review/request_info.html
+++ b/ietf/templates/doc/review/request_info.html
@@ -19,7 +19,7 @@
@@ -314,7 +312,7 @@
- Requested rev.
+ Requested revision
{% if review_req.requested_rev %}
{{ review_req.requested_rev }}
@@ -299,12 +299,10 @@
- Reviewed rev.
+ Reviewed revision
-
- {{ assignment.reviewed_rev }}
-
+ {{ assignment.reviewed_rev }}
{% if assignment.reviewed_rev != review_req.doc.rev %}(document currently at {{ review_req.doc.rev }}){% endif %}
- Review result
+ Result
{{ assignment.result.name }}
@@ -326,7 +324,7 @@
- Review completed:
+ Completed
{{ assignment.completed_on|date:"Y-m-d" }}
@@ -343,4 +341,4 @@
Assign reviewer
-{% endif %}
\ No newline at end of file
+{% endif %}
diff --git a/ietf/templates/doc/shepherd_writeup.html b/ietf/templates/doc/shepherd_writeup.html
index 7c28c49dc..8cf99aff0 100644
--- a/ietf/templates/doc/shepherd_writeup.html
+++ b/ietf/templates/doc/shepherd_writeup.html
@@ -11,11 +11,11 @@
{{ doc.canonical_name }}-{{ doc.rev }}
- {{writeup|linkify|urlize_ietf_docs|wordwrap:"80"}}
+ {{writeup|maybewordwrap|urlize_ietf_docs|linkify}}
{% if can_edit %}
Edit
{% endif %}
Back
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/ietf/templates/group/active_areas.html b/ietf/templates/group/active_areas.html
index 38153c1c3..d3fe45f45 100644
--- a/ietf/templates/group/active_areas.html
+++ b/ietf/templates/group/active_areas.html
@@ -50,7 +50,7 @@
{% if area.description %}
{{ rpt.desc|default:"(none)"|linkify|urlize_ietf_docs }}
+ {{ rpt.desc|default:"(none)"|urlize_ietf_docs|linkify }}
{{ rpt.desc|default:"(none)"|linkify|urlize_ietf_docs }}
+ {{ rpt.desc|default:"(none)"|urlize_ietf_docs|linkify }}
{% endif %}{% endif %}
{% endfor %}
{% endif %}
@@ -309,7 +309,7 @@
{{ group.type.desc.title }}
{# the linebreaks filter adds
{% if requested_close or group.state_id == "conclude" %}Final{% endif %}
@@ -333,4 +333,4 @@
{% endif %}
{% endif %}
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/ietf/templates/group/group_about_status.html b/ietf/templates/group/group_about_status.html
index e9cc486cb..c2fe7e0ae 100644
--- a/ietf/templates/group/group_about_status.html
+++ b/ietf/templates/group/group_about_status.html
@@ -28,7 +28,7 @@
{% endif %}
-
{{ status_update.desc|default:"(none)"|linkify|urlize_ietf_docs }}
+ {{ status_update.desc|default:"(none)"|urlize_ietf_docs|linkify }}
{% if can_provide_status_update %}
Status update for {{ group.type.name }} {{ group.acronym }} at {{ meeting }}
- {{ status_update.desc|default:"(none)"|linkify|urlize_ietf_docs }}
+ {{ status_update.desc|default:"(none)"|urlize_ietf_docs|linkify }}
Back
{% endblock %}
\ No newline at end of file
diff --git a/ietf/templates/group/group_base.html b/ietf/templates/group/group_base.html
index ab2ddb354..9fd1e71f3 100644
--- a/ietf/templates/group/group_base.html
+++ b/ietf/templates/group/group_base.html
@@ -9,12 +9,12 @@
{% block content %}
{% origin %}
- {{ group.name }} ({{ group.acronym }})
{% if group.state_id == "dormant" or group.state_id == "conclude" %}
- Concluded {{ group.type.name }}
+ Concluded {{ group.type.name }}
{% endif %}
- {% if group.state_id == "replaced" %}Replaced {{ group.type.name }}{% endif %}
- {% if group.state_id == "proposed" %}Proposed {{ group.type.name }}{% endif %}
+ {% if group.state_id == "replaced" %}Replaced {{ group.type.name }}{% endif %}
+ {% if group.state_id == "proposed" %}Proposed {{ group.type.name }}{% endif %}
+ {{ group.name }} ({{ group.acronym }})