From f0a16d2c5efc9c54b47900cbd15f15af72e95b93 Mon Sep 17 00:00:00 2001 From: tttc3 Date: Sun, 5 Mar 2023 13:57:43 +0000 Subject: [PATCH 1/9] Support SHA224, SHA384 and SHA512 source hashes. --- conda_build/metadata.py | 3 +++ conda_build/source.py | 24 ++++++++++++------- docs/source/resources/define-metadata.rst | 3 +++ .../metadata/source_url/meta.yaml | 3 +++ 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 99f1b423d9..26940c6b2f 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -433,7 +433,10 @@ def parse(data, config, path=None): "url": None, "md5": str, "sha1": None, + "sha224": None, "sha256": None, + "sha384": None, + "sha512": None, "path": str, "path_via_symlink": None, "git_url": str, diff --git a/conda_build/source.py b/conda_build/source.py index 572666ccdb..3d9b1ccedd 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -38,6 +38,9 @@ ext_re = re.compile(r"(.*?)(\.(?:tar\.)?[^.]+)$") +ACCEPTED_HASH_TYPES = ("md5", "sha1", "sha224", "sha256", "sha384", "sha512") + + def append_hash_to_fn(fn, hash_value): return ext_re.sub(fr"\1_{hash_value[:10]}\2", fn) @@ -54,16 +57,19 @@ def download_to_cache(cache_folder, recipe_path, source_dict, verbose=False): source_urls = [source_urls] unhashed_fn = fn = source_dict['fn'] if 'fn' in source_dict else basename(source_urls[0]) hash_added = False - for hash_type in ('md5', 'sha1', 'sha256'): + for hash_type in ACCEPTED_HASH_TYPES: if hash_type in source_dict: if source_dict[hash_type] in (None, ""): raise ValueError(f'Empty {hash_type} hash provided for {fn}') fn = append_hash_to_fn(fn, source_dict[hash_type]) hash_added = True - break else: - log.warn("No hash (md5, sha1, sha256) provided for {}. Source download forced. " - "Add hash to recipe to use source cache.".format(unhashed_fn)) + log.warn( + "No hash {} provided for {}. Source download forced. " + "Add hash to recipe to use source cache.".format( + ACCEPTED_HASH_TYPES, unhashed_fn + ) + ) path = join(cache_folder, fn) if isfile(path): if verbose: @@ -102,15 +108,17 @@ def download_to_cache(cache_folder, recipe_path, source_dict, verbose=False): raise RuntimeError("Could not download %s" % url) hashed = None - for tp in ('md5', 'sha1', 'sha256'): + for tp in ACCEPTED_HASH_TYPES: if tp in source_dict: expected_hash = source_dict[tp] hashed = hashsum_file(path, tp) if expected_hash != hashed: rm_rf(path) - raise RuntimeError("%s mismatch: '%s' != '%s'" % - (tp.upper(), hashed, expected_hash)) - break + raise RuntimeError( + "{} mismatch: '{}' != '{}'".format( + tp.upper(), hashed, expected_hash + ) + ) # this is really a fallback. If people don't provide the hash, we still need to prevent # collisions in our source cache, but the end user will get no benefit from the cache. diff --git a/docs/source/resources/define-metadata.rst b/docs/source/resources/define-metadata.rst index 59d9444e51..7f792e7c46 100644 --- a/docs/source/resources/define-metadata.rst +++ b/docs/source/resources/define-metadata.rst @@ -115,7 +115,10 @@ Source from tarball or zip archive url: https://pypi.python.org/packages/source/b/bsdiff4/bsdiff4-1.1.4.tar.gz md5: 29f6089290505fc1a852e176bd276c43 sha1: f0a2c9a30073449cfb7d171c57552f3109d93894 + sha224: ebf3e3b54353146ca21128ed6399739663a1256a223f438ed0223845 sha256: 5a022ff4c1d1de87232b1c70bde50afbb98212fd246be4a867d8737173cf1f8f + sha384: 23eee6ee2e5d1054780e331857589bfba098255a88ae4edd47102fce676694ce0f543dc5c0d27c51f77cc4546d4e74c0 + sha512: b968c7dc99132252a83b175a96ec75ec842edf9e2494db2c07b419e61a0b1cf6984e7c544452f9ab56aa8581caf966c0f6933fc22a071ccc4fbb5d22b363fe54 If an extracted archive contains only 1 folder at its top level, its contents will be moved 1 level up, so that the extracted package contents sit in the diff --git a/tests/test-recipes/metadata/source_url/meta.yaml b/tests/test-recipes/metadata/source_url/meta.yaml index e8fc55dad7..7056ca487b 100644 --- a/tests/test-recipes/metadata/source_url/meta.yaml +++ b/tests/test-recipes/metadata/source_url/meta.yaml @@ -7,7 +7,10 @@ source: url: https://github.com/conda/conda-build/archive/1.8.1.tar.gz md5: 0bf1f3598a659a0e8fb5ee6bbb3fd9fd sha1: c464a8995ad6bbf0480abd2883876cc9b4913fa7 + sha224: 96d76b37dcc8c28577ae5776e3aa3eb3b057af60983ce4005bbd3d61 sha256: f82b0bd5c809c9a7c7256c26364a0065e57732788b7a74c7ea2169135ed2f598 + sha384: 5b407afd0c41028bd2443b13a1e34092053452db9e1872eed427f9a6042626837d16f0268568e8e54a07d173a3c80e6b + sha512: 6bf7d22fc65c111402d22fd27bbcf7ea36eec03efded33ece74a54bf30f0cb87b6f31b789276e990dd6e9e41ac78739eac8c6b2954144096ccf0bb2ec1cc4fd7 requirements: build: From 46ba6699319d37f86798f645b4021d250ca82c0b Mon Sep 17 00:00:00 2001 From: tttc3 Date: Sun, 5 Mar 2023 14:23:22 +0000 Subject: [PATCH 2/9] Added news item. --- news/4793-support-sha224-sha384-and-sha512 | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 news/4793-support-sha224-sha384-and-sha512 diff --git a/news/4793-support-sha224-sha384-and-sha512 b/news/4793-support-sha224-sha384-and-sha512 new file mode 100644 index 0000000000..05870ce0a8 --- /dev/null +++ b/news/4793-support-sha224-sha384-and-sha512 @@ -0,0 +1,18 @@ +### Enhancements + +* Added SHA224, SHA384 and SHA512 support for validating downloaded sources. + +### Bug fixes + +* If multiple hashing algorithims are used to validated a downloaded source, a `RuntimeError` will now be thrown if **any** of those hashes are incorrect. This is stricter and more consistent than the previous behaviour. +### Deprecations + +* + +### Docs + +* + +### Other + +* From 61a72f719ba843e0c7be6a6de4794f41a229906b Mon Sep 17 00:00:00 2001 From: tttc3 Date: Sun, 5 Mar 2023 14:25:11 +0000 Subject: [PATCH 3/9] Fixed typos. --- news/4793-support-sha224-sha384-and-sha512 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/4793-support-sha224-sha384-and-sha512 b/news/4793-support-sha224-sha384-and-sha512 index 05870ce0a8..a54beafea0 100644 --- a/news/4793-support-sha224-sha384-and-sha512 +++ b/news/4793-support-sha224-sha384-and-sha512 @@ -4,7 +4,7 @@ ### Bug fixes -* If multiple hashing algorithims are used to validated a downloaded source, a `RuntimeError` will now be thrown if **any** of those hashes are incorrect. This is stricter and more consistent than the previous behaviour. +* If multiple hashing algorithms are used to validate a downloaded source, a `RuntimeError` will now be thrown if **any** of those hashes are incorrect. This is stricter and more consistent than the previous behaviour. ### Deprecations * From 8e328f4afac2b3190f82d6a6b7c690470eb39b4a Mon Sep 17 00:00:00 2001 From: T Coxon <97948946+tttc3@users.noreply.github.com> Date: Mon, 26 Feb 2024 18:59:50 +0000 Subject: [PATCH 4/9] Apply suggestions from code review Co-authored-by: Ken Odegard --- conda_build/source.py | 6 ++---- news/4793-support-sha224-sha384-and-sha512 | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/conda_build/source.py b/conda_build/source.py index 1bff2f7da9..e32f783ca0 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -71,8 +71,7 @@ def download_to_cache(cache_folder, recipe_path, source_dict, verbose=False): ) hash_added = False - for hash_type in ACCEPTED_HASH_TYPES: - if hash_type in source_dict: + for hash_type in set(source_dict).intersection(ACCEPTED_HASH_TYPES): if source_dict[hash_type] in (None, ""): raise ValueError(f"Empty {hash_type} hash provided for {fn}") fn = append_hash_to_fn(fn, source_dict[hash_type]) @@ -121,8 +120,7 @@ def download_to_cache(cache_folder, recipe_path, source_dict, verbose=False): hashed = None - for tp in ACCEPTED_HASH_TYPES: - if tp in source_dict: + for tp in set(source_dict).intersection(ACCEPTED_HASH_TYPES): expected_hash = source_dict[tp] hashed = hashsum_file(path, tp) if expected_hash != hashed: diff --git a/news/4793-support-sha224-sha384-and-sha512 b/news/4793-support-sha224-sha384-and-sha512 index a54beafea0..efc63160a9 100644 --- a/news/4793-support-sha224-sha384-and-sha512 +++ b/news/4793-support-sha224-sha384-and-sha512 @@ -1,10 +1,11 @@ ### Enhancements -* Added SHA224, SHA384 and SHA512 support for validating downloaded sources. +* Added SHA224, SHA384 and SHA512 support for validating downloaded sources. (#4793) ### Bug fixes -* If multiple hashing algorithms are used to validate a downloaded source, a `RuntimeError` will now be thrown if **any** of those hashes are incorrect. This is stricter and more consistent than the previous behaviour. +* Verify all source hashes when multiple are defined (not just the first one). (#4793) + ### Deprecations * From f84a66ad835cbb4a29a6a575af57ec8ad0ffb5b7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 21:03:52 +0000 Subject: [PATCH 5/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- conda_build/source.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/conda_build/source.py b/conda_build/source.py index e32f783ca0..78ed5f4636 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -72,10 +72,10 @@ def download_to_cache(cache_folder, recipe_path, source_dict, verbose=False): hash_added = False for hash_type in set(source_dict).intersection(ACCEPTED_HASH_TYPES): - if source_dict[hash_type] in (None, ""): - raise ValueError(f"Empty {hash_type} hash provided for {fn}") - fn = append_hash_to_fn(fn, source_dict[hash_type]) - hash_added = True + if source_dict[hash_type] in (None, ""): + raise ValueError(f"Empty {hash_type} hash provided for {fn}") + fn = append_hash_to_fn(fn, source_dict[hash_type]) + hash_added = True else: log.warn( f"No hash {ACCEPTED_HASH_TYPES} provided for {unhashed_fn}. Source download forced. " @@ -121,13 +121,13 @@ def download_to_cache(cache_folder, recipe_path, source_dict, verbose=False): hashed = None for tp in set(source_dict).intersection(ACCEPTED_HASH_TYPES): - expected_hash = source_dict[tp] - hashed = hashsum_file(path, tp) - if expected_hash != hashed: - rm_rf(path) - raise RuntimeError( - f"{tp.upper()} mismatch: '{hashed}' != '{expected_hash}'" - ) + expected_hash = source_dict[tp] + hashed = hashsum_file(path, tp) + if expected_hash != hashed: + rm_rf(path) + raise RuntimeError( + f"{tp.upper()} mismatch: '{hashed}' != '{expected_hash}'" + ) # this is really a fallback. If people don't provide the hash, we still need to prevent # collisions in our source cache, but the end user will get no benefit from the cache. From ba87925170ce3de86a161fd092d3d1880877cccb Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Mon, 26 Feb 2024 15:07:34 -0600 Subject: [PATCH 6/9] Update conda_build/source.py --- conda_build/source.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conda_build/source.py b/conda_build/source.py index 78ed5f4636..6b633c478e 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -120,13 +120,13 @@ def download_to_cache(cache_folder, recipe_path, source_dict, verbose=False): hashed = None - for tp in set(source_dict).intersection(ACCEPTED_HASH_TYPES): - expected_hash = source_dict[tp] - hashed = hashsum_file(path, tp) + for hash_type in set(source_dict).intersection(ACCEPTED_HASH_TYPES): + expected_hash = source_dict[hash_type] + hashed = hashsum_file(path, hash_type) if expected_hash != hashed: rm_rf(path) raise RuntimeError( - f"{tp.upper()} mismatch: '{hashed}' != '{expected_hash}'" + f"{hash_type.upper()} mismatch: '{hashed}' != '{expected_hash}'" ) # this is really a fallback. If people don't provide the hash, we still need to prevent From 2f0000866e41315d60df9fb964081d3e97fd5303 Mon Sep 17 00:00:00 2001 From: tttc3 <97948946+tttc3@users.noreply.github.com> Date: Sun, 21 Apr 2024 15:23:04 +0100 Subject: [PATCH 7/9] Fixup merge --- conda_build/source.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conda_build/source.py b/conda_build/source.py index 04945168e7..c2f6e8961a 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -122,14 +122,13 @@ def download_to_cache(cache_folder, recipe_path, source_dict, verbose=False): for hash_type in set(source_dict).intersection(ACCEPTED_HASH_TYPES): if hash_type in source_dict: expected_hash = source_dict[hash_type] - hashed = hashsum_file(path, hash_type) + hashed = compute_sum(path, hash_type) if expected_hash != hashed: rm_rf(path) raise RuntimeError( f"{hash_type.upper()} mismatch: '{hashed}' != '{expected_hash}'" ) - # this is really a fallback. If people don't provide the hash, we still need to prevent # collisions in our source cache, but the end user will get no benefit from the cache. if not hash_added: From 5340007d2ab2541e0f49f56d6900aaa232c265fb Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Tue, 26 Nov 2024 06:52:16 -0600 Subject: [PATCH 8/9] Update conda_build/source.py Co-authored-by: jaimergp --- conda_build/source.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_build/source.py b/conda_build/source.py index 5040897b1e..2f5c68d640 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -75,6 +75,7 @@ def download_to_cache(cache_folder, recipe_path, source_dict, verbose=False): raise ValueError(f"Empty {hash_type} hash provided for {fn}") fn = append_hash_to_fn(fn, source_dict[hash_type]) hash_added = True + break else: log.warning( f"No hash {ACCEPTED_HASH_TYPES} provided for {unhashed_fn}. Source download forced. " From f6e498190650510f0481f835a5c3697840e4559e Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Tue, 26 Nov 2024 06:52:27 -0600 Subject: [PATCH 9/9] Update conda_build/source.py Co-authored-by: jaimergp --- conda_build/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/source.py b/conda_build/source.py index 2f5c68d640..936f4fa771 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -70,7 +70,7 @@ def download_to_cache(cache_folder, recipe_path, source_dict, verbose=False): ) hash_added = False - for hash_type in set(source_dict).intersection(ACCEPTED_HASH_TYPES): + for hash_type in sorted(set(source_dict).intersection(ACCEPTED_HASH_TYPES)): if source_dict[hash_type] in (None, ""): raise ValueError(f"Empty {hash_type} hash provided for {fn}") fn = append_hash_to_fn(fn, source_dict[hash_type])