Merge pull request #82 from OrdnanceSurvey/dev

Release v1.2.4

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [1.2.4] - 2023/04/20
+
+### Fixed
+
+- Removed requests wrapper for Downloads API as it breaks requests that stream the data (issue #79)
+- Improved pbar to include total number of files to be downloaded. Also removed the finished download writeout to preserve aesthetics
 
 ## [1.2.3] - 2023/04/12
 

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = osdatahub
-version = 1.2.3
+version = 1.2.4
 author = OS Rapid Prototyping
 author_email = [email protected]
 classifiers =

src/osdatahub/DownloadsAPI/downloads_api.py

@@ -43,7 +43,7 @@ def download(self, output_dir: Union[str, Path], overwrite: bool = False, pbar:
                             f"Skipping download...")
             return output_path
 
-        response = osdatahub.get(self.url, stream=True, proxies=osdatahub.get_proxies())
+        response = requests.get(self.url, stream=True, proxies=osdatahub.get_proxies())
         response.raise_for_status()
         size = int(response.headers.get('content-length'))
         chunk_size = 1024
@@ -56,7 +56,7 @@ def download(self, output_dir: Union[str, Path], overwrite: bool = False, pbar:
                     f.flush()
                     pbar.update(chunk_size)
 
-        pbar.write(f"Finished downloading {self.file_name} to {output_path}")
+        # pbar.write(f"Finished downloading {self.file_name} to {output_path}")
         return output_path
 
 
@@ -154,14 +154,14 @@ def _download(download_list: Union[list, _DownloadObj], output_dir: Union[str, P
                 processes = cpu_count()
             with ThreadPoolExecutor(max_workers=processes) as executor:
                 pbar = tqdm(total=sum([d.size for d in download_list]), unit="B", unit_scale=True, leave=True,
-                            desc=f"Downloading {len(download_list)} files from osdatahub")
+                            desc=f"Downloaded 0/{len(download_list)} files from osdatahub")
                 results = list([executor.submit(p.download, output_dir, overwrite, pbar) for p in download_list])
 
                 num_downloads_completed = 0
                 for _ in as_completed(results):
                     num_downloads_completed += 1
                     pbar.set_description(
-                        f"Downloading {len(download_list) - num_downloads_completed} files from osdatahub")
+                        f"Downloaded {num_downloads_completed}/{len(download_list)} files from osdatahub")
         else:
             # download single file
             d = download_list[0] if isinstance(download_list, list) else download_list

src/osdatahub/__init__.py

@@ -9,7 +9,7 @@ def set_proxies(proxies):
 def get_proxies():
     return json.loads(os.environ["_OSDATAHUB_PROXIES"])
 
-__version__ = "1.2.3"
+__version__ = "1.2.4"
 
 from osdatahub.extent import Extent
 from osdatahub.FeaturesAPI import FeaturesAPI

tests/test_downloads_api.py

@@ -1,5 +1,9 @@
 import os
+import tempfile
 import unittest.mock as mock
+import dotenv
+
+dotenv.load_dotenv()
 
 import pytest
 from osdatahub import OpenDataDownload, DataPackageDownload
@@ -70,6 +74,19 @@ def data_package(self):
         data_package = DataPackageDownload(key="test_key", product_id="test_id")
         yield data_package
 
+    @pytest.mark.skipif(API_KEY is None, reason="Test API key not available")
+    def test_download_pass(self):
+        # Arrange
+        product_package = DataPackageDownload(API_KEY, "98")
+        files_to_download = product_package.product_list("156")
+
+        # Act
+        with tempfile.TemporaryDirectory() as tmpdirname:
+            downloaded = product_package.download("156", tmpdirname)
+
+        # Assert
+        assert len(downloaded) == len(files_to_download["downloads"])
+
     def test_download_list_pass(self):
         # TODO: implement download_list_pass
         pass