Merge pull request #336 from Backblaze/fix-staging-integration-tests

Fix integration tests on non-production environments

.github/workflows/ci.yml

@@ -29,11 +29,11 @@ jobs:
         run: python -m pip install --upgrade nox pip setuptools
       - name: Run linters
         run: nox -vs lint
-      - name: Validate changelog
-        if: ${{ ! startsWith(github.ref, 'refs/heads/dependabot/') }}
-        uses: zattoo/changelog@v1
-        with:
-          token: ${{ github.token }}
+      #- name: Validate changelog
+      #- if: ${{ ! startsWith(github.ref, 'refs/heads/dependabot/') }}
+      #- uses: zattoo/changelog@v1
+      #- with:
+      #-   token: ${{ github.token }}
   build:
     needs: lint
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Infrastructure
+* Fix download integration tests on non-production environments
+* Add `B2_DEBUG_HTTP` env variable to enable network-level test debugging
+* Disable changelog validation temporarily
+
 ## [1.17.2] - 2022-06-24
 
 ### Fixed

b2sdk/file_version.py

@@ -9,6 +9,7 @@
 ######################################################################
 
 from typing import Dict, Optional, Union, Tuple, TYPE_CHECKING
+import re
 
 from .encryption.setting import EncryptionSetting, EncryptionSettingFactory
 from .replication.types import ReplicationStatus
@@ -47,6 +48,13 @@ class BaseFileVersion:
         'mod_time_millis',
         'replication_status',
     ]
+    _TYPE_MATCHER = re.compile('[a-z0-9]+_[a-z0-9]+_f([0-9]).*')
+    _FILE_TYPE = {
+        1: 'small',
+        2: 'large',
+        3: 'part',
+        4: 'tiny',
+    }
 
     def __init__(
         self,
@@ -179,6 +187,15 @@ def update_retention(
         )
         return self._clone(file_retention=file_retention)
 
+    def _type(self):
+        """
+        FOR TEST PURPOSES ONLY
+        not guaranteed to work for perpetuity (using undocumented server behavior)
+        """
+        m = self._TYPE_MATCHER.match(self.id_)
+        assert m, self.id_
+        return self._FILE_TYPE[int(m.group(1))]
+
 
 class FileVersion(BaseFileVersion):
     """

test/integration/base.py

@@ -9,6 +9,8 @@
 ######################################################################
 
 from typing import Optional
+import http.client
+import os
 import random
 import string
 
@@ -21,6 +23,11 @@
 
 
 class IntegrationTestBase:
+    @pytest.fixture(autouse=True)
+    def set_http_debug(self):
+        if os.environ.get('B2_DEBUG_HTTP'):
+            http.client.HTTPConnection.debuglevel = 1
+
     @pytest.fixture(autouse=True)
     def save_settings(self, dont_cleanup_old_buckets, b2_auth_data):
         type(self).dont_cleanup_old_buckets = dont_cleanup_old_buckets

test/integration/helpers.py

@@ -9,6 +9,7 @@
 ######################################################################
 
 from typing import Optional
+import os
 import random
 import string
 
@@ -29,5 +30,6 @@ def bucket_name_part(length):
 def authorize(b2_auth_data, api_config=DEFAULT_HTTP_API_CONFIG):
     info = InMemoryAccountInfo()
     b2_api = B2Api(info, api_config=api_config)
-    b2_api.authorize_account("production", *b2_auth_data)
+    realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production')
+    b2_api.authorize_account(realm, *b2_auth_data)
     return b2_api, info

test/integration/test_download.py

@@ -11,6 +11,8 @@
 import gzip
 import io
 import pathlib
+from pprint import pprint
+from typing import Optional
 from unittest import mock
 
 from b2sdk.v2 import *
@@ -41,42 +43,49 @@ def test_large_file(self):
             ):
 
                 # let's check that small file downloads fail with these settings
-                bucket.upload_bytes(b'0', 'a_single_zero')
+                zero = bucket.upload_bytes(b'0', 'a_single_zero')
                 with pytest.raises(ValueError) as exc_info:
                     with io.BytesIO() as io_:
                         bucket.download_file_by_name('a_single_zero').save(io_)
                 assert exc_info.value.args == ('no strategy suitable for download was found!',)
+
                 f = self._file_helper(bucket)
-                assert f.download_version.content_sha1_verified
+                if zero._type() != 'large':
+                    # if we are here, that's not the production server!
+                    assert f.download_version.content_sha1_verified  # large files don't have sha1, lets not check
 
-    def _file_helper(self, bucket, sha1_sum=None):
-        bytes_to_write = int(self.info.get_absolute_minimum_part_size()) * 2 + 1
+    def _file_helper(
+        self, bucket, sha1_sum=None, bytes_to_write: Optional[int] = None
+    ) -> DownloadVersion:
+        bytes_to_write = bytes_to_write or int(self.info.get_absolute_minimum_part_size()) * 2 + 1
         with TempDir() as temp_dir:
             temp_dir = pathlib.Path(temp_dir)
-            source_large_file = pathlib.Path(temp_dir) / 'source_large_file'
-            with open(source_large_file, 'wb') as large_file:
-                self.write_zeros(large_file, bytes_to_write)
+            source_small_file = pathlib.Path(temp_dir) / 'source_small_file'
+            with open(source_small_file, 'wb') as small_file:
+                self.write_zeros(small_file, bytes_to_write)
             bucket.upload_local_file(
-                source_large_file,
-                'large_file',
+                source_small_file,
+                'small_file',
                 sha1_sum=sha1_sum,
             )
-            target_large_file = pathlib.Path(temp_dir) / 'target_large_file'
+            target_small_file = pathlib.Path(temp_dir) / 'target_small_file'
 
-            f = bucket.download_file_by_name('large_file')
-            f.save_to(target_large_file)
-            assert hex_sha1_of_file(source_large_file) == hex_sha1_of_file(target_large_file)
+            f = bucket.download_file_by_name('small_file')
+            f.save_to(target_small_file)
+            assert hex_sha1_of_file(source_small_file) == hex_sha1_of_file(target_small_file)
         return f
 
     def test_small(self):
         bucket = self.create_bucket()
-        f = self._file_helper(bucket)
+        f = self._file_helper(bucket, bytes_to_write=1)
         assert f.download_version.content_sha1_verified
 
     def test_small_unverified(self):
         bucket = self.create_bucket()
-        f = self._file_helper(bucket, sha1_sum='do_not_verify')
-        assert not f.download_version.content_sha1_verified
+        f = self._file_helper(bucket, sha1_sum='do_not_verify', bytes_to_write=1)
+        if f.download_version.content_sha1_verified:
+            pprint(f.download_version._get_args_for_clone())
+            assert not f.download_version.content_sha1_verified
 
     def test_gzip(self):
         bucket = self.create_bucket()