Merge pull request #96 from OrdnanceSurvey/dev

version 1.2.6

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [1.2.6] - 2023/06/28
+
+### Features
+- Added check for chunk size when streaming data. Program should error if file download is incomplete [JEPooley]
+
+
+### Changed
+- Upgrade requests version in dependencies to 2.31.0 [gwionap]
+
 ## [1.2.5] - 2023/05/17
 
 ### Fixed

docs/index.rst

@@ -1,8 +1,3 @@
-.. osdatahub documentation master file, created by
-sphinx-quickstart on Tue Nov  2 16:09:27 2021.
-You can adapt this file completely to your liking, but it should at least
-contain the root `toctree` directive.
-
 Welcome to ``osdatahub's`` documentation!
 ==========================================
 

requirements.txt

@@ -1,6 +1,7 @@
 geojson~=3.0.1
-requests~=2.25.0
+requests~=2.31.0
 typeguard~=2.13.0
 shapely~=2.0.0
 tqdm~=4.65.0
-setuptools~=67.7.2
+setuptools~=67.7.2
+

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = osdatahub
-version = 1.2.5
+version = 1.2.6
 author = OS Rapid Prototyping
 author_email = [email protected]
 classifiers =
@@ -34,7 +34,7 @@ url = https://github.com/OrdnanceSurvey/osdatahub
 include_package_data = True
 install_requires =
     geojson~=3.0.1
-    requests~=2.25.0
+    requests~=2.31.0
     typeguard~=2.13.0
     shapely~=2.0.0
     tqdm~=4.65.0

src/osdatahub/DownloadsAPI/downloads_api.py

@@ -45,18 +45,25 @@ def download(self, output_dir: Union[str, Path], overwrite: bool = False, pbar:
 
         response = requests.get(self.url, stream=True, proxies=osdatahub.get_proxies())
         response.raise_for_status()
-        size = int(response.headers.get('content-length'))
-        chunk_size = 1024
+        expected_size = int(response.headers.get('content-length'))
+        current_size = 0
+        chunk_size = 1048576 # 1024 ** 2 -> 1MB
         if response.status_code == 200:
             with open(output_path, 'wb') as f:
                 if not pbar:
-                    pbar = tqdm(total=size, desc=self.file_name, unit="B", unit_scale=True, leave=True)
+                    pbar = tqdm(total=expected_size, desc=self.file_name, unit="B", unit_scale=True, leave=True)
                 for chunk in response.iter_content(chunk_size=chunk_size):
+                    current_size += len(chunk)
                     f.write(chunk)
                     f.flush()
                     pbar.update(chunk_size)
 
-        # pbar.write(f"Finished downloading {self.file_name} to {output_path}")
+        if expected_size != current_size:
+            deficit = expected_size - current_size
+            raise IOError(
+                f'incomplete read ({current_size} bytes read, {deficit} more expected)'
+            ) 
+        pbar.write(f"Finished downloading {self.file_name} to {output_path}")
         return output_path
 
 

src/osdatahub/__init__.py

@@ -9,7 +9,7 @@ def set_proxies(proxies):
 def get_proxies():
     return json.loads(os.environ["_OSDATAHUB_PROXIES"])
 
-__version__ = "1.2.5"
+__version__ = "1.2.6"
 
 from osdatahub.extent import Extent
 from osdatahub.FeaturesAPI import FeaturesAPI