altf4arnold/datatracker
1
0
Fork 0
Code Issues Pull requests Actions 3 Packages Projects Releases Wiki Activity

feat: add ability to skip html validation to test crawler. (#4299)

Browse source
This commit is contained in:
Robert Sparks 2022-07-29 15:37:24 -05:00 committed by GitHub
parent 35106c10a4
commit f5df294574
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
bin/test-crawl
View file

@ -54,6 +54,7 @@ parser.add_argument('--validator-nu', dest='validator_nu', action='store_true',
parser.add_argument('--validate-all', dest='validate_all', action='store_true', default=False, parser.add_argument('--validate-all', dest='validate_all', action='store_true', default=False,
help='Run html 5 validation on all pages, without skipping similar urls. ' help='Run html 5 validation on all pages, without skipping similar urls. '
'(The default is to only run validation on one of /foo/1/, /foo/2/, /foo/3/, etc.)') '(The default is to only run validation on one of /foo/1/, /foo/2/, /foo/3/, etc.)')
parser.add_argument('--skip-html-validation', dest='skip_html_validation', action='store_true', help='Skip HTML validation.',default=False)
parser.add_argument('-v', '--verbose', action='store_true', default=False, parser.add_argument('-v', '--verbose', action='store_true', default=False,
help='Be more verbose') help='Be more verbose')
parser.add_argument('-x', '--exclude', action='append', default=[], help="Exclude URLs matching pattern") parser.add_argument('-x', '--exclude', action='append', default=[], help="Exclude URLs matching pattern")
@ -149,6 +150,9 @@ def extract_tastypie_urls(content):
yield uri yield uri
def check_html_valid(url, response, args): def check_html_valid(url, response, args):
if args.skip_html_validation:
return
global parser, validated_urls, doc_types, warnings global parser, validated_urls, doc_types, warnings
key = url key = url
if not args.validate_all: if not args.validate_all: