Add extra exception trap to test crawler to output URL under test for None responses

- Legacy-Id: 6056
2013-08-14 16:15:42 +00:00 · 2013-08-14 16:15:42 +00:00 · 0b1aabc64d
parent c182d4e5e9
commit 0b1aabc64d
1 changed files with 9 additions and 3 deletions
--- a/ietf/bin/test-crawl
+++ b/ietf/bin/test-crawl
@ -82,9 +82,15 @@ while urls:
                ctype = ctype[:ctype.index(";")]

            if ctype == "text/html":
-                for u in extract_html_urls(r.content):
-                    if u not in visited and u not in urls:
-                        urls[u] = url
+                try:
+                    for u in extract_html_urls(r.content):
+                        if u not in visited and u not in urls:
+                            urls[u] = url
+                except:
+                    print "error extracting HTML urls from", url
+                    print "============="
+                    print traceback.format_exc()
+                    print "============="
        else:
            tags.append(u"FAIL (from %s)" % referrer)