From cb481978b3e136fe9877785d56663d94c5a11f41 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 23 Jun 2022 19:05:52 +0200 Subject: [PATCH 1/8] Fix download integration tests on non-production environments --- CHANGELOG.md | 3 +++ test/integration/helpers.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a9ae66a7..df1011ecb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Infrastructure +* Fix download integration tests on non-production environments + ## [1.17.2] - 2022-06-24 ### Fixed diff --git a/test/integration/helpers.py b/test/integration/helpers.py index 145e0f13c..60ea3539e 100644 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -9,6 +9,7 @@ ###################################################################### from typing import Optional +import os import random import string @@ -29,5 +30,6 @@ def bucket_name_part(length): def authorize(b2_auth_data, api_config=DEFAULT_HTTP_API_CONFIG): info = InMemoryAccountInfo() b2_api = B2Api(info, api_config=api_config) - b2_api.authorize_account("production", *b2_auth_data) + realm = os.environ.get('B2_TEST_ENVIRONMENT', 'production') + b2_api.authorize_account(realm, *b2_auth_data) return b2_api, info From c800241341a55d1b96c57dbd1c8ba04a660d8d34 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 23 Jun 2022 19:23:56 +0200 Subject: [PATCH 2/8] Enhanced debug for TestDownload.test_small_unverified --- test/integration/test_download.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/test_download.py b/test/integration/test_download.py index 8d1c85117..c89c62ca1 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -11,6 +11,7 @@ import gzip import io import pathlib +from pprint import pprint from unittest import mock from b2sdk.v2 import * @@ -76,7 +77,9 @@ def test_small(self): def test_small_unverified(self): bucket = self.create_bucket() f = self._file_helper(bucket, sha1_sum='do_not_verify') - assert not f.download_version.content_sha1_verified + if f.download_version.content_sha1_verified: + pprint(f.download_version._get_args_for_clone()) + assert not f.download_version.content_sha1_verified def test_gzip(self): bucket = self.create_bucket() From bbf176ad47e019a40a774a068208ade3c712b887 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 23 Jun 2022 20:01:49 +0200 Subject: [PATCH 3/8] Make test_download work on non-production realms --- test/integration/test_download.py | 33 +++++++++++++++++-------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/test/integration/test_download.py b/test/integration/test_download.py index c89c62ca1..330ef480f 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -42,41 +42,44 @@ def test_large_file(self): ): # let's check that small file downloads fail with these settings - bucket.upload_bytes(b'0', 'a_single_zero') + zero = bucket.upload_bytes(b'0', 'a_single_zero') with pytest.raises(ValueError) as exc_info: with io.BytesIO() as io_: bucket.download_file_by_name('a_single_zero').save(io_) assert exc_info.value.args == ('no strategy suitable for download was found!',) + f = self._file_helper(bucket) - assert f.download_version.content_sha1_verified + if zero._type() != 'large': + # if we are here, that's not the production server! + assert f.download_version.content_sha1_verified # large files don't have sha1, lets not check - def _file_helper(self, bucket, sha1_sum=None): - bytes_to_write = int(self.info.get_absolute_minimum_part_size()) * 2 + 1 + def _file_helper(self, bucket, sha1_sum=None, bytes_to_write=None): + bytes_to_write = bytes_to_write or int(self.info.get_absolute_minimum_part_size()) * 2 + 1 with TempDir() as temp_dir: temp_dir = pathlib.Path(temp_dir) - source_large_file = pathlib.Path(temp_dir) / 'source_large_file' - with open(source_large_file, 'wb') as large_file: - self.write_zeros(large_file, bytes_to_write) + source_small_file = pathlib.Path(temp_dir) / 'source_small_file' + with open(source_small_file, 'wb') as small_file: + self.write_zeros(small_file, bytes_to_write) bucket.upload_local_file( - source_large_file, - 'large_file', + source_small_file, + 'small_file', sha1_sum=sha1_sum, ) - target_large_file = pathlib.Path(temp_dir) / 'target_large_file' + target_small_file = pathlib.Path(temp_dir) / 'target_small_file' - f = bucket.download_file_by_name('large_file') - f.save_to(target_large_file) - assert hex_sha1_of_file(source_large_file) == hex_sha1_of_file(target_large_file) + f = bucket.download_file_by_name('small_file') + f.save_to(target_small_file) + assert hex_sha1_of_file(source_small_file) == hex_sha1_of_file(target_small_file) return f def test_small(self): bucket = self.create_bucket() - f = self._file_helper(bucket) + f = self._file_helper(bucket, bytes_to_write=1) assert f.download_version.content_sha1_verified def test_small_unverified(self): bucket = self.create_bucket() - f = self._file_helper(bucket, sha1_sum='do_not_verify') + f = self._file_helper(bucket, sha1_sum='do_not_verify', bytes_to_write=1) if f.download_version.content_sha1_verified: pprint(f.download_version._get_args_for_clone()) assert not f.download_version.content_sha1_verified From 52fc01738235839d32238da7bb0baefbf92df48e Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 23 Jun 2022 21:13:53 +0200 Subject: [PATCH 4/8] Add FileVersion._type --- b2sdk/file_version.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index d58fdff9d..73974e1e7 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -9,6 +9,7 @@ ###################################################################### from typing import Dict, Optional, Union, Tuple, TYPE_CHECKING +import re from .encryption.setting import EncryptionSetting, EncryptionSettingFactory from .replication.types import ReplicationStatus @@ -47,6 +48,13 @@ class BaseFileVersion: 'mod_time_millis', 'replication_status', ] + _TYPE_MATCHER = re.compile('[a-z0-9]+_[a-z0-9]+_f([0-9]).*') + _FILE_TYPE = { + 1: 'small', + 2: 'large', + 3: 'part', + 4: 'tiny', + } def __init__( self, @@ -179,6 +187,15 @@ def update_retention( ) return self._clone(file_retention=file_retention) + def _type(self): + """ + FOR TEST PURPOSES ONLY + not guaranteed to work for perpetuity (using undocumented server behavior) + """ + m = self._TYPE_MATCHER.match(self.id_) + assert m, self.id_ + return self._FILE_TYPE[int(m.group(1))] + class FileVersion(BaseFileVersion): """ From 7077ae25ea0af2d569cbe89683a70411a2683bed Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 23 Jun 2022 21:20:37 +0200 Subject: [PATCH 5/8] Add `B2_DEBUG_HTTP` env variable to enable network-level test debugging --- CHANGELOG.md | 1 + test/integration/base.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index df1011ecb..e9d85738a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Infrastructure * Fix download integration tests on non-production environments +* Add `B2_DEBUG_HTTP` env variable to enable network-level test debugging ## [1.17.2] - 2022-06-24 diff --git a/test/integration/base.py b/test/integration/base.py index 62f26d57e..2d95b18db 100644 --- a/test/integration/base.py +++ b/test/integration/base.py @@ -9,6 +9,8 @@ ###################################################################### from typing import Optional +import http.client +import os import random import string @@ -21,6 +23,11 @@ class IntegrationTestBase: + @pytest.fixture(autouse=True) + def set_http_debug(self): + if os.environ.get('B2_DEBUG_HTTP'): + http.client.HTTPConnection.debuglevel = 1 + @pytest.fixture(autouse=True) def save_settings(self, dont_cleanup_old_buckets, b2_auth_data): type(self).dont_cleanup_old_buckets = dont_cleanup_old_buckets From 31690c5f17d7baa3bf8089192673352f52cfb879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Polewicz?= Date: Thu, 14 Jul 2022 16:23:00 +0200 Subject: [PATCH 6/8] Apply suggestions from code review --- test/integration/test_download.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/test_download.py b/test/integration/test_download.py index 330ef480f..9dfb22bc9 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -12,6 +12,7 @@ import io import pathlib from pprint import pprint +from typing import Optional from unittest import mock from b2sdk.v2 import * @@ -53,7 +54,7 @@ def test_large_file(self): # if we are here, that's not the production server! assert f.download_version.content_sha1_verified # large files don't have sha1, lets not check - def _file_helper(self, bucket, sha1_sum=None, bytes_to_write=None): + def _file_helper(self, bucket, sha1_sum=None, bytes_to_write: Optional[int] = None) -> DownloadVersion: bytes_to_write = bytes_to_write or int(self.info.get_absolute_minimum_part_size()) * 2 + 1 with TempDir() as temp_dir: temp_dir = pathlib.Path(temp_dir) From d963963f46366a0a90cb80e1de9eb00a2a539de3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Polewicz?= Date: Thu, 14 Jul 2022 16:26:13 +0200 Subject: [PATCH 7/8] Update test/integration/test_download.py --- test/integration/test_download.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/test_download.py b/test/integration/test_download.py index 9dfb22bc9..d7f851e45 100644 --- a/test/integration/test_download.py +++ b/test/integration/test_download.py @@ -54,7 +54,9 @@ def test_large_file(self): # if we are here, that's not the production server! assert f.download_version.content_sha1_verified # large files don't have sha1, lets not check - def _file_helper(self, bucket, sha1_sum=None, bytes_to_write: Optional[int] = None) -> DownloadVersion: + def _file_helper( + self, bucket, sha1_sum=None, bytes_to_write: Optional[int] = None + ) -> DownloadVersion: bytes_to_write = bytes_to_write or int(self.info.get_absolute_minimum_part_size()) * 2 + 1 with TempDir() as temp_dir: temp_dir = pathlib.Path(temp_dir) From 6233c6609ea5ef1d122ba89f38f0b5a66ecaa834 Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 14 Jul 2022 17:50:26 +0200 Subject: [PATCH 8/8] Disable broken changelog linter temporarily --- .github/workflows/ci.yml | 10 +++++----- CHANGELOG.md | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6587aa321..5fb5d1c76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,11 +29,11 @@ jobs: run: python -m pip install --upgrade nox pip setuptools - name: Run linters run: nox -vs lint - - name: Validate changelog - if: ${{ ! startsWith(github.ref, 'refs/heads/dependabot/') }} - uses: zattoo/changelog@v1 - with: - token: ${{ github.token }} + #- name: Validate changelog + #- if: ${{ ! startsWith(github.ref, 'refs/heads/dependabot/') }} + #- uses: zattoo/changelog@v1 + #- with: + #- token: ${{ github.token }} build: needs: lint runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index e9d85738a..013e5c219 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Infrastructure * Fix download integration tests on non-production environments * Add `B2_DEBUG_HTTP` env variable to enable network-level test debugging +* Disable changelog validation temporarily ## [1.17.2] - 2022-06-24