Skip to content

Commit

Permalink
Implement more efficient download code
Browse files Browse the repository at this point in the history
  • Loading branch information
mhucka committed Jan 22, 2019
1 parent 67b3992 commit 1e0ef1a
Showing 1 changed file with 3 additions and 4 deletions.
7 changes: 3 additions & 4 deletions eprints2bags/network.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from time import sleep
import shutil
import ssl
import urllib
from urllib import request
Expand Down Expand Up @@ -142,11 +143,9 @@ def addurl(text):
download(url, user, password, local_destination, recursing)
elif 200 <= code < 400:
# The following originally started out as the code here:
# https://stackoverflow.com/a/16696317/743730
# https://stackoverflow.com/a/39217788/743730
with open(local_destination, 'wb') as f:
for chunk in req.iter_content(chunk_size = 1024):
if chunk:
f.write(chunk)
shutil.copyfileobj(req.raw, f)
req.close()
elif code in [401, 402, 403, 407, 451, 511]:
raise AuthenticationFailure(addurl('Access is forbidden'))
Expand Down

0 comments on commit 1e0ef1a

Please sign in to comment.