altf4arnold/czds_python_download
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: altf4arnold:11d2de4aef8651de432740a1a01682bff28e10a2
Branches Tags
altf4arnold:master
..
compare: altf4arnold:7b49a2fde2619c8c7de6915017088440a1d33f09
Branches Tags
altf4arnold:master

10 commits
11d2de4aef ... 7b49a2fde2

Author SHA1 Message Date
Dongmei Cao 7b49a2fde2
Merge pull request #6 from case/patch-1 
Support loading config via env var
 2020-09-25 09:19:24 -07:00
Dongmei Cao 2d777c6369
Merge pull request #8 from kidmose/feature/allow-defaulting-workdir 
Allow omitting `working.directory` in `config.json` acc. to README
 2020-09-25 09:18:02 -07:00
Dongmei Cao cbb6838c97
Merge pull request #10 from spjj/filename-fix 
correct handling of filename when not extracted from header
 2020-09-25 08:58:25 -07:00
John Jensen 5802e7f1a1 correct handling of filename when not extracted from header 2020-09-25 12:17:44 -03:00
Egon Kidmose 3ab555eeb6
Allow omitting working.directory in config.json acc. to README 
README suggests that `working.directory` can be omitted from
`config.json`, but doing this will cause a KeyNotFoundException.

The dictionary lookup has been modified to include a default value of
`'.'` which is according to README.
 2020-03-09 13:10:21 +01:00
Eric Case d2cc85b697
Support loading config via env var 
In addition to loading the config via a `config.json` file, this change allows it to be loaded via an environment variable called `CZDS_CONFIG`. This makes it straightforward to deploy to services like Heroku, which tend to use env vars for config. See https://12factor.net/ for details.
 2019-03-28 15:58:45 -07:00
dongmei-cao 6c082c8fe4
Merge pull request #3 from liheyuan/master 
Add stream flag to requests.get avoid Memory Error
 2019-02-19 12:51:34 -08:00
Coder4 b8f178ba25
Add stream flag to avoid Memory Error 
If not add stream flag, large file will be total load into memory, which cause MemoryError on small RAM server.
 2019-02-15 17:40:29 +08:00
Dongmei Cao b63c004c32 Change the URLs to production in the config.sample.json 2019-01-31 16:18:45 -08:00
Dongmei Cao 51644f09db Update README 2019-01-23 12:19:44 -08:00
4 changed files with 17 additions and 12 deletions
Show stats Expand all files Collapse all files

5
README.md
View file

@ -32,3 +32,8 @@ Contributing
------------ ------------
Contributions are welcome. Contributions are welcome.
Other
-----
Reference Implementation in Java: https://github.com/icann/czds-api-client-java

4
config.sample.json
View file

@ -1,7 +1,7 @@
{ {
"icann.account.username": "username@example.com", "icann.account.username": "username@example.com",
"icann.account.password": "Abcdef#12345678", "icann.account.password": "Abcdef#12345678",
"authentication.base.url": "https://account-api-test.icann.org", "authentication.base.url": "https://account-api.icann.org",
"czds.base.url": "https://czds-api-test.icann.org", "czds.base.url": "https://czds-api.icann.org",
"working.directory": "/where/zonefiles/will/be/saved" "working.directory": "/where/zonefiles/will/be/saved"
} }

2
do_http_get.py
View file

@ -6,6 +6,6 @@ def do_get(url, access_token):
'Accept': 'application/json', 'Accept': 'application/json',
'Authorization': 'Bearer {0}'.format(access_token)} 'Authorization': 'Bearer {0}'.format(access_token)}
response = requests.get(url, params=None, headers=bearer_headers) response = requests.get(url, params=None, headers=bearer_headers, stream=True)
return response return response

12
download.py
View file

@ -12,6 +12,10 @@ from do_http_get import do_get
############################################################################################################## ##############################################################################################################
try: try:
if 'CZDS_CONFIG' in os.environ:
config_data = os.environ['CZDS_CONFIG']
config = json.loads(config_data)
else:
config_file = open("config.json", "r") config_file = open("config.json", "r")
config = json.load(config_file) config = json.load(config_file)
config_file.close() config_file.close()
@ -26,7 +30,7 @@ authen_base_url = config['authentication.base.url']
czds_base_url = config['czds.base.url'] czds_base_url = config['czds.base.url']
# This is optional. Default to current directory # This is optional. Default to current directory
working_directory = config['working.directory'] working_directory = config.get('working.directory', '.') # Default to current directory
if not username: if not username:
sys.stderr.write("'icann.account.username' parameter not found in the config.json file\n") sys.stderr.write("'icann.account.username' parameter not found in the config.json file\n")
@ -44,10 +48,6 @@ if not czds_base_url:
sys.stderr.write("'czds.base.url' parameter not found in the config.json file\n") sys.stderr.write("'czds.base.url' parameter not found in the config.json file\n")
exit(1) exit(1)
if not working_directory:
# Default to current directory
working_directory = '.'
############################################################################################################## ##############################################################################################################
@ -109,7 +109,7 @@ def download_one_zone(url, output_directory):
if status_code == 200: if status_code == 200:
# Try to get the filename from the header # Try to get the filename from the header
_,option = cgi.parse_header(download_zone_response.headers['content-disposition']) _,option = cgi.parse_header(download_zone_response.headers['content-disposition'])
filename = option['filename'] filename = option.get('filename')
# If could get a filename from the header, then makeup one like [tld].txt.gz # If could get a filename from the header, then makeup one like [tld].txt.gz
if not filename: if not filename: