#!/usr/bin/env python # Copyright The IETF Trust 2013-2019, All Rights Reserved import os, sys, re, datetime, argparse, traceback, json, subprocess import html5lib import random # Set up import path to find our own Django basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) if not basedir in sys.path: sys.path.insert(0, basedir) # Parse args now, so we can use custom settings when importing django parser = argparse.ArgumentParser( description="""Perform a test crawl of the project. For each found URL, the HTTP response status is printed. If it's not OK/redirect, FAIL is printed - in case of errors, a stacktrace is also included.""") parser.add_argument('urls', metavar='URL', nargs='*', help='One or more URLs to start the crawl from') parser.add_argument('--urls', '-u', dest='url_file', help='File with URLs to start the crawl from') parser.add_argument('--slow', dest='slow_threshold', type=float, default=1.0, help='Responses taking longer than this (in seconds) results in SLOW being printed') parser.add_argument('--settings', help='Custom settings file') parser.add_argument('--logfile', help='Write to logfile') parser.add_argument('--user', help='Crawl logged in as this user', default=None) parser.add_argument('--no-follow', dest='follow', action='store_false', default=True, help='Do not follow URLs found in fetched pages, just check the given URLs') parser.add_argument('--validator-nu', dest='validator_nu', action='store_true', help='Use validator.nu instead of html5lib for HTML validation') parser.add_argument('--pedantic', action='store_true', help='Stop the crawl on the first HTML validation issue') parser.add_argument('--random', action='store_true', help='Crawl URLs randomly') parser.add_argument('--validate-all', dest='validate_all', action='store_true', default=False, help='Run html 5 validation on all pages, without skipping similar urls. ' '(The default is to only run validation on one of /foo/1/, /foo/2/, /foo/3/, etc.)') parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Be more verbose') args = parser.parse_args() # Import Django, call setup() os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_testcrawl") import django import django.test import django.core.checks from django.conf import settings django.setup() # This needs to come after we set up sys path to include the local django import debug # pyflakes:ignore # prevent memory from leaking when settings.DEBUG=True # from django.db import connection # class DontSaveQueries(list): # def append(self, x): # pass # connection.queries = DontSaveQueries() from ietf.name.models import DocTypeName from ietf.utils.html import unescape from ietf.utils.test_utils import unicontent # --- Constants --- MAX_URL_LENGTH = 500 # --- Functions --- def note(s): if args.verbose: sys.stderr.write(s) sys.stderr.write('\n') def strip_url(url): if url.startswith("http://testserver"): url = url[len("http://testserver"):] fragment_url = re.search("^(.+)#[a-z_.-]+$", url) if fragment_url: url = fragment_url.group(1) return url def extract_html_urls(content): for m in re.finditer(r'(<(?:(?:a|link) [^>]*href|(?:img|script) [^>]*src)=[\'"]([^"]+)[\'"][^>]*>)', content): if re.search(r'rel=["\']?nofollow["\']', m.group(1)): continue url = strip_url(m.group(2)) if len(url) > MAX_URL_LENGTH: continue # avoid infinite GET parameter appendages if not url.startswith("/"): continue if url.startswith("//"): continue yield unescape(url) def extract_tastypie_urls(content): VISIT_OBJECTS = False VISIT_NEXT = False data = json.loads(content) for item in data: if type(data[item]) is dict: if "list_endpoint" in data[item]: uri = data[item]["list_endpoint"] yield uri if VISIT_NEXT: if "meta" in data and "next" in data["meta"]: uri = data["meta"]["next"] if uri != None: yield uri if VISIT_OBJECTS: if "objects" in data: object_list = data["objects"] for i in range(len(object_list)): if "resource_uri" in object_list[i]: uri = object_list[i]["resource_uri"] yield uri def check_html_valid(url, response, args): global parser, validated_urls, doc_types, warnings key = url if not args.validate_all: # derive a key for urls like this by replacing primary keys key = re.sub("#.*$", "", key) key = re.sub("/.+@.+/", "/x@x.org/", key) key = re.sub("/[0-9.]+/", "/mmmm/", key) key = re.sub("/[0-9.]+/", "/nnnn/", key) key = re.sub("/ag/[a-z0-9-]+/", "/ag/foo/", key) key = re.sub("/area/[a-z0-9-]+/", "/area/foo/", key) key = re.sub("/bcp[0-9]+/", "/bcpnnn/", key) key = re.sub("/conflict-review-[a-z0-9-]+/", "/conflrev-foo/", key) key = re.sub("/dir/[a-z0-9-]+/", "/dir/foo/", key) key = re.sub("/draft-[a-z0-9-]+/", "/draft-foo/", key) key = re.sub("/group/[a-z0-9-]+/", "/group/foo/", key) key = re.sub("/ipr/search/.*", "/ipr/search/", key) key = re.sub("/meeting/[-0-9a-z]+/agenda/[0-9a-z]+/", "/meeting/nn/agenda/foo/", key) key = re.sub("/release/[0-9dev.]+/", "/release/n.n.n/", key) key = re.sub("/rfc[0-9]+/", "/rfcnnnn/", key) key = re.sub("/rg/[a-z0-9-]+/", "/rg/foo/", key) key = re.sub("/secr/srec/nnnn/[0-9a-z-]+/", "/secr/sreq/nn/bar/", key) key = re.sub("/state/[a-z0-9-]+/", "/state/foo/", key) key = re.sub("/state/[a-z0-9-]+/[a-z0-9-]+/", "/state/foo/bar/", key) key = re.sub("/status-change-[a-z0-9-]+/", "/statchg-foo/", key) key = re.sub("/std[0-9]+/", "/stdnnn/", key) key = re.sub("/submit/status/nnnn/[0-9a-f]+/", "/submit/status/nnnn/bar/", key) key = re.sub("/team/[a-z0-9-]+/", "/team/foo/", key) key = re.sub("/wg/[a-z0-9-]+/", "/wg/foo/", key) key = re.sub("\?.*$", "", key) for slug in doc_types: key = re.sub("/%s-.*/"%slug, "/%s-nnnn/"%slug, key) if not key in validated_urls: note('Validate: %-32s: %s' % (url[:32], key)) # These URLs have known issues, skip them until those are fixed for pattern in ( '/secr', 'admin/', '/doc/.*/edit/info/', 'rfc542$', 'rfc776$', 'draft-leroux-pce-pcecp-interarea-reqs', 'draft-fujiwara-dnsop-resolver-update', ): if re.search(pattern, url): validated_urls[key] = True log("%s blacklisted; skipping HTML validation" % url) return if hasattr(response, "content"): content = response.content else: content = response.streaming_content validated_urls[key] = True if args.validator_nu: v = subprocess.Popen(["java", "-jar", basedir + "/bin/vnu.jar", "--format", "json", "-"], stdin=subprocess.PIPE, stderr=subprocess.PIPE) for m in json.loads(v.communicate(content)[1])["messages"]: t = m["subType"] if m["type"] == "info" else m["type"] tags.append("\n%s\tLine %d: %s" % (t.upper(), m["lastLine"], m["message"])) tags.append("\n\t%s" % m["extract"].replace('\n', ' ')) tags.append("\n\t%s%s" % (" " * m["hiliteStart"], "^" * m["hiliteLength"])) # disregard some HTML issues that are (usually) due to invalid # database content if not re.search('Forbidden code point|Bad value|seamless|The first child|Duplicate ID|The first occurrence of ID', m["message"]): warnings += 1 else: try: parser.parse(content) except Exception as e: for err in parser.errors: pos, code, data = err tags.append(u"WARN invalid html at line, pos %s: %s" % (pos, e)) warnings += 1 def skip_extract_from(url): for pattern in ( r'^/doc/html/[a-z0-9-]+', r'^/meeting/[a-z0-9-]+/agenda/[a-z0-9-]+', r'^/static/coverage/', ): if re.search(pattern, url): return True return False def skip_url(url): for pattern in ( "^/community/[0-9]+/remove_document/", "^/community/personal/", # Skip most of the slow pdf composite generation urls and svg urls "^/meeting/[0-9]+/agenda/[0-9b-z].*-drafts\\.pdf", "^/wg/[a-z0-9-]+/deps/svg/", # This bad url occurs in an uploaded html agenda: r"/site/ietfdhcwg/_/rsrc/1311005436000/system/app/css/overlay.css\?cb=simple100%250150goog-ws-left", r"/dir/tsvdir/reviews/", r"draft-touch-msword-template-v2\.0", # These will always 404: r"^/doc/html/charter-ietf-cicm", r"^/doc/html/charter-ietf-dcon", r"^/doc/html/charter-ietf-fun", r"^/doc/html/charter-ietf-multrans", r"^/doc/html/charter-ietf-sdn", r"^/doc/html/charter-ietf-woes", r"^/doc/html/draft-floyd-cc-alt", r"^/doc/html/draft-ietf-sipping-overload-design", r"^/doc/html/status-change-icmpv6-dns-ipv6-to-internet-standard", r"^/static/coverage/", r"^/meeting/6[0-4]/agenda", r"^https?://www.ietf.org/", ): if re.search(pattern, url): return True return False def log(s): print(s) if logfile: if not type(s) is str: s = s.encode('utf-8') logfile.write(s) logfile.write('\n') def get_referrers(url): ref_list = [] while url in referrers: url = referrers[url] if url in ref_list: log("Circular referral list, discovered at %s" % url) break ref_list.append(url) return ref_list # --- GLobals --- slow_threshold = args.slow_threshold visited = set() urls = {} # url -> referrer referrers = {} initial_urls = [] initial_urls.extend(args.urls) if args.url_file: with open(args.url_file) as f: for line in f: line = line.partition("#")[0].strip() if line: initial_urls.append(line) if not initial_urls: initial_urls.append("/") initial_urls.append("/api/v1") for url in initial_urls: urls[url] = "[initial]" parser = html5lib.HTMLParser(strict=True) # initialise validated_urls with some patterns we don't want to check, # because they aren't under our control, such as uploaded group agendas. validated_urls = {'/meeting/nn/agenda/foo/': True, } doc_types = [ t.slug for t in DocTypeName.objects.all() ] errors = 0 warnings = 0 count = 0 start_time = datetime.datetime.now() client = django.test.Client(Accept='text/html,text/plain,application/json') logfile = None if args.logfile: logfile = open(args.logfile, "w") # --- Main --- if __name__ == "__main__": if (args.user): # log in as user, to have the respective HTML generated by the templates response = client.post('/accounts/login/', {'username': args.user, 'password': 'password'}, secure=True, follow=True) if (response.status_code != 200): log("Could not log in as %s, HTML response %d" % (args.user, response.status_code)) sys.exit(1) # Run django system checks and checks from ietf.checks: error_list = django.core.checks.run_checks() silenced = [] for i in range(len(error_list)): if error_list[i].id in settings.SILENCED_SYSTEM_CHECKS: silenced.append(i) silenced.sort(reverse=True) for i in silenced: del error_list[i] if error_list: print("") for entry in error_list: print(entry) while urls: if args.random: # popitem() is documented to be random, but really isn't url = random.choice(urls.keys()) referrer = urls.pop(url) else: url, referrer = urls.popitem() visited.add(url) if skip_url(url): continue timestamp = datetime.datetime.now() acc_time = (timestamp - start_time).total_seconds() acc_secs = (timestamp - start_time).total_seconds() hrs = acc_secs // (60*60) min = (acc_secs % (60*60)) // 60 sec = acc_secs % 60 try: request_start = datetime.datetime.now() if args.verbose: sys.stderr.write(url+'\n') r = client.get(url, secure=True) elapsed = datetime.datetime.now() - request_start except KeyboardInterrupt: log(" ... was fetching %s" % url) sys.exit(1) except: elapsed = datetime.datetime.now() - request_start tags = [ u"FAIL (from [ %s ])" % (",\n\t".join(get_referrers(url))) ] log("%2d:%02d:%02d %7d %6d %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), 500, elapsed.total_seconds(), url, " ".join(tags))) log("=============") log(traceback.format_exc()) log("=============") errors += 1 else: tags = [] if r.status_code in (301, 302): u = strip_url(r["Location"]) if u not in visited and u not in urls: urls[u] = referrer # referrer is original referrer, not redirected url referrers[u] = referrer elif r.status_code == 200: ctype = r["Content-Type"] if ";" in ctype: ctype = ctype[:ctype.index(";")] if ctype == "text/html": try: if args.follow and not skip_extract_from(url): for u in extract_html_urls(unicontent(r)): if u not in visited and u not in urls: urls[u] = url referrers[u] = url check_html_valid(url, r, args) except: log("error extracting HTML urls from %s" % url) log("=============") log(traceback.format_exc()) log("=============") elif ctype == "application/json": try: if args.follow: for u in extract_tastypie_urls(unicontent(r)): if u not in visited and u not in urls: urls[u] = url referrers[u] = url except: log("error extracting urls from %s" % url) log("=============") log(traceback.format_exc()) log("=============") else: tags.append(u"FAIL (from %s)" % (referrer, )) errors += 1 if elapsed.total_seconds() > slow_threshold: tags.append("SLOW") acc_time = (timestamp - start_time).total_seconds() acc_secs = (timestamp - start_time).total_seconds() hrs = acc_secs // (60*60) min = (acc_secs % (60*60)) // 60 sec = acc_secs % 60 if (len(visited) % 100) == 1: log("\nElapsed Visited Queue Code Time Url ... Notes") log("%2d:%02d:%02d %7d %6d %s %6.3fs %s %s" % (hrs,min,sec, len(visited), len(urls), r.status_code, elapsed.total_seconds(), url, " ".join(tags))) if ((errors or warnings) and args.pedantic): sys.exit(1) if logfile: logfile.close() sys.stderr.write("Output written to %s\n\n" % logfile.name) if errors > 0: sys.stderr.write("Found %s errors, grep output for FAIL for details\n" % errors) sys.exit(1) else: sys.stderr.write("Found no errors.\n") if warnings > 0: sys.stderr.write("Found %s warnings, grep output for WARN for details\n" % warnings) else: sys.stderr.write("Found no warnings.\n")