Remove drpm test and fix borked CI (pulp#3323)

* Skip test that uses DRPM fixture, for now. It's not worth maintaining the fixture, and in the future we ought to be
able to do this test without it.
* Made metadata downloading util agnostic to compression type
* Ditch all usages of http_get and http_get_headers for requests
* Fix CI after working directory adjustment in the template

.github/workflows/scripts/post_before_script.sh

@@ -6,7 +6,7 @@ if [[ "$TEST" == "upgrade" ]]; then
     exit
 fi
 
-cmd_stdin_prefix bash -c "cat > /var/lib/pulp/scripts/sign-metadata.sh" < "$GITHUB_WORKSPACE"/pulp_rpm/tests/functional/sign-metadata.sh
+cmd_stdin_prefix bash -c "cat > /var/lib/pulp/scripts/sign-metadata.sh" < pulp_rpm/tests/functional/sign-metadata.sh
 
 curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-KEY-fixture-signing | cmd_stdin_prefix su pulp -c "cat > /tmp/GPG-KEY-fixture-signing"
 curl -L https://github.com/pulp/pulp-fixtures/raw/master/common/GPG-PRIVATE-KEY-fixture-signing | cmd_stdin_prefix su pulp -c "gpg --import"

functest_requirements.txt

@@ -1,8 +1,10 @@
+django  # TODO: test_sync.py has a dependency on date parsing functions
 git+https://github.com/pulp/pulp-smash.git#egg=pulp-smash
 productmd>=1.25
 pytest
 dictdiffer
 xmltodict
 pyyaml
 lxml
-django  # TODO: test_sync.py has a dependency on date parsing functions
+pyzstd
+requests

pulp_rpm/tests/functional/api/test_character_encoding.py

@@ -1,7 +1,9 @@
 """Tests for Pulp's characters encoding."""
-import pytest
 import uuid
 
+import pytest
+import requests
+
 from pulpcore.tests.functional.utils import PulpTaskError
 from pulp_rpm.tests.functional.constants import (
     RPM_WITH_NON_ASCII_NAME,
@@ -21,11 +23,11 @@
 
 
 def test_upload_non_ascii(
-    tmp_path, artifacts_api_client, http_get, rpm_package_api, monitor_task, delete_orphans_pre
+    tmp_path, artifacts_api_client, rpm_package_api, monitor_task, delete_orphans_pre
 ):
     """Test whether one can upload an RPM with non-ascii metadata."""
     temp_file = tmp_path / str(uuid.uuid4())
-    temp_file.write_bytes(http_get(RPM_WITH_NON_ASCII_URL))
+    temp_file.write_bytes(requests.get(RPM_WITH_NON_ASCII_URL).content)
     artifact = artifacts_api_client.create(temp_file)
     response = rpm_package_api.create(
         artifact=artifact.pulp_href,
@@ -36,11 +38,11 @@ def test_upload_non_ascii(
 
 
 def test_upload_non_utf8(
-    tmp_path, artifacts_api_client, http_get, rpm_package_api, monitor_task, delete_orphans_pre
+    tmp_path, artifacts_api_client, rpm_package_api, monitor_task, delete_orphans_pre
 ):
     """Test whether an exception is raised when non-utf-8 is uploaded."""
     temp_file = tmp_path / str(uuid.uuid4())
-    temp_file.write_bytes(http_get(RPM_WITH_NON_UTF_8_URL))
+    temp_file.write_bytes(requests.get(RPM_WITH_NON_UTF_8_URL).content)
     artifact = artifacts_api_client.create(temp_file)
     with pytest.raises(PulpTaskError) as ctx:
         response = rpm_package_api.create(

pulp_rpm/tests/functional/api/test_consume_content.py

@@ -1,14 +1,15 @@
 """Verify whether package manager, yum/dnf, can consume content from Pulp."""
-import pytest
 import subprocess
 import itertools
 
+import pytest
+import requests
+
 from pulp_rpm.tests.functional.constants import (
     # REPO_WITH_XML_BASE_URL,
     RPM_UNSIGNED_FIXTURE_URL,
 )
 
-
 dnf_installed = subprocess.run(("which", "dnf")).returncode == 0
 
 
@@ -189,7 +190,6 @@ def test_config_dot_repo(
     has_signing_service,
     rpm_metadata_signing_service,
     create_distribution,
-    http_get,
 ):
     """Test if the generated config.repo has the right content."""
     if has_signing_service and rpm_metadata_signing_service is None:
@@ -198,7 +198,7 @@ def test_config_dot_repo(
     distribution = create_distribution(
         gpgcheck=gpgcheck, repo_gpgcheck=repo_gpgcheck, has_signing_service=has_signing_service
     )
-    content = http_get(f"{distribution.base_url}config.repo").decode("utf-8")
+    content = requests.get(f"{distribution.base_url}config.repo").text
 
     assert f"[{distribution.name}]\n" in content
     assert f"baseurl={distribution.base_url}\n" in content

pulp_rpm/tests/functional/api/test_crud_content_unit.py

@@ -38,14 +38,7 @@
 
 
 class ContentUnitTestCase(PulpTestCase):
-    """CRUD content unit.
-
-    This test targets the following issues:
-
-    * `Pulp #2872 <https://pulp.plan.io/issues/2872>`_
-    * `Pulp #3445 <https://pulp.plan.io/issues/3445>`_
-    * `Pulp Smash #870 <https://github.com/pulp/pulp-smash/issues/870>`_
-    """
+    """CRUD content unit."""
 
     @classmethod
     def setUpClass(cls):

pulp_rpm/tests/functional/api/test_download_content.py

@@ -1,9 +1,11 @@
 """Tests that verify download of content served by Pulp."""
-import pytest
 import hashlib
 from random import choice
 from urllib.parse import urljoin
 
+import pytest
+import requests
+
 from pulp_rpm.tests.functional.constants import RPM_UNSIGNED_FIXTURE_URL
 from pulp_rpm.tests.functional.utils import (
     get_package_repo_path,
@@ -18,7 +20,6 @@ def test_all(
     rpm_publication_api,
     rpm_distribution_factory,
     download_content_unit,
-    http_get,
     gen_object_with_cleanup,
 ):
     """Verify whether content served by pulp can be downloaded.
@@ -40,11 +41,6 @@ def test_all(
     2. Select a random content unit in the distribution. Download that
        content unit from Pulp, and verify that the content unit has the
        same checksum when fetched directly from Pulp-Fixtures.
-
-    This test targets the following issues:
-
-    * `Pulp #2895 <https://pulp.plan.io/issues/2895>`_
-    * `Pulp Smash #872 <https://github.com/pulp/pulp-smash/issues/872>`_
     """
     # Sync a Repository
     repo = rpm_unsigned_repo_immediate
@@ -61,7 +57,7 @@ def test_all(
     package_paths = [p.location_href for p in packages.results]
     unit_path = choice(package_paths)
     fixture_hash = hashlib.sha256(
-        http_get(urljoin(RPM_UNSIGNED_FIXTURE_URL, unit_path))
+        requests.get(urljoin(RPM_UNSIGNED_FIXTURE_URL, unit_path)).content
     ).hexdigest()
 
     # …and Pulp.

pulp_rpm/tests/functional/api/test_fips_workflow.py

@@ -1,8 +1,10 @@
 """Tests that create/sync/distribute/publish MANY rpm plugin repositories."""
-import pytest
 import os
 import re
 
+import pytest
+import requests
+
 
 @pytest.fixture
 def cdn_certs_and_keys():
@@ -178,7 +180,6 @@ def test_fips_workflow(
     rpm_publication_factory,
     rpm_distribution_factory,
     cdn_certs_and_keys,
-    http_get,
 ):
     # Convert a url into a name-string
     name = _name_from_url(url)
@@ -208,5 +209,5 @@ def test_fips_workflow(
     assert distribution is not None
 
     # Test we can access the index of the distribution
-    response = http_get(distribution.base_url)
+    response = requests.get(distribution.base_url)
     assert response is not None

pulp_rpm/tests/functional/api/test_publish.py

@@ -6,6 +6,7 @@
 from lxml import etree
 from django.conf import settings
 from xml.etree import ElementTree
+import requests
 
 import xmltodict
 import dictdiffer
@@ -26,7 +27,7 @@
     RPM_UNSIGNED_FIXTURE_URL,
     SRPM_UNSIGNED_FIXTURE_URL,
 )
-from pulp_rpm.tests.functional.utils import gen_rpm_remote, read_xml_gz
+from pulp_rpm.tests.functional.utils import gen_rpm_remote, download_and_decompress_file
 from pulp_rpm.tests.functional.utils import set_up_module as setUpModule  # noqa:F401
 
 from pulpcore.client.pulp_rpm import RpmRepositorySyncURL, RpmRpmPublication
@@ -40,7 +41,6 @@ def test_sqlite_metadata(
         self,
         with_sqlite,
         rpm_unsigned_repo_on_demand,
-        http_get,
         rpm_publication_api,
         gen_object_with_cleanup,
         rpm_distribution_api,
@@ -66,7 +66,7 @@ def test_sqlite_metadata(
         distribution = gen_object_with_cleanup(rpm_distribution_api, body)
 
         repomd = ElementTree.fromstring(
-            http_get(os.path.join(distribution.base_url, "repodata/repomd.xml"))
+            requests.get(os.path.join(distribution.base_url, "repodata/repomd.xml")).text
         )
 
         data_xpath = "{{{}}}data".format(RPM_NAMESPACES["metadata/repo"])
@@ -80,7 +80,7 @@ def test_sqlite_metadata(
             for db_elem in sqlite_files:
                 location_xpath = "{{{}}}location".format(RPM_NAMESPACES["metadata/repo"])
                 db_href = db_elem.find(location_xpath).get("href")
-                http_get(os.path.join(distribution.base_url, db_href))
+                assert requests.get(os.path.join(distribution.base_url, db_href)).status_code == 200
         else:
             assert 0 == len(sqlite_files)
 
@@ -154,7 +154,6 @@ def test_validate_no_checksum_tag(
         rpm_unsigned_repo_immediate,
         rpm_publication_api,
         gen_object_with_cleanup,
-        http_get,
         rpm_distribution_api,
         monitor_task,
     ):
@@ -173,12 +172,13 @@ def test_validate_no_checksum_tag(
 
         # 2. check the tag 'sum' is not present in updateinfo.xml
         repomd = ElementTree.fromstring(
-            http_get(os.path.join(distribution.base_url, "repodata/repomd.xml"))
+            requests.get(os.path.join(distribution.base_url, "repodata/repomd.xml")).text
         )
 
         update_xml_url = self._get_updateinfo_xml_path(repomd)
-        update_xml_content = http_get(os.path.join(distribution.base_url, update_xml_url))
-        update_xml = read_xml_gz(update_xml_content)
+        update_xml = download_and_decompress_file(
+            os.path.join(distribution.base_url, update_xml_url)
+        )
         update_info_content = ElementTree.fromstring(update_xml)
 
         tags = {elem.tag for elem in update_info_content.iter()}
@@ -259,7 +259,6 @@ def test_complex_repo_core_metadata(
     init_and_sync,
     rpm_publication_api,
     gen_object_with_cleanup,
-    http_get,
     rpm_distribution_api,
     monitor_task,
     delete_orphans_pre,
@@ -286,11 +285,11 @@ def test_complex_repo_core_metadata(
 
     # Download and parse the metadata.
     original_repomd = ElementTree.fromstring(
-        http_get(os.path.join(repo_url, "repodata/repomd.xml"))
+        requests.get(os.path.join(repo_url, "repodata/repomd.xml")).text
     )
 
     reproduced_repomd = ElementTree.fromstring(
-        http_get(os.path.join(distribution.base_url, "repodata/repomd.xml"))
+        requests.get(os.path.join(distribution.base_url, "repodata/repomd.xml")).text
     )
 
     def get_metadata_content(base_url, repomd_elem, meta_type):
@@ -317,7 +316,7 @@ def get_metadata_content(base_url, repomd_elem, meta_type):
         xpath = "{{{}}}location".format(RPM_NAMESPACES["metadata/repo"])
         location_href = data_elems[0].find(xpath).get("href")
 
-        return read_xml_gz(http_get(os.path.join(base_url, location_href)))
+        return download_and_decompress_file(os.path.join(base_url, location_href))
 
     # Convert the metadata into a more workable form and then compare.
     for metadata_file in ["primary", "filelists", "other"]:
@@ -444,7 +443,6 @@ def _compare_xml_metadata_file(original_metadata_text, generated_metadata_text,
 def test_distribution_tree_metadata_publish(
     mirror,
     gen_object_with_cleanup,
-    http_get,
     rpm_repository_api,
     rpm_rpmremote_api,
     rpm_distribution_api,
@@ -476,17 +474,17 @@ def test_distribution_tree_metadata_publish(
     distribution = gen_object_with_cleanup(rpm_distribution_api, body)
 
     # 4. Download and parse the metadata.
-    original_treeinfo = http_get(os.path.join(RPM_KICKSTART_FIXTURE_URL, ".treeinfo"))
-    generated_treeinfo = http_get(os.path.join(distribution.base_url, ".treeinfo"))
+    original_treeinfo = requests.get(os.path.join(RPM_KICKSTART_FIXTURE_URL, ".treeinfo")).text
+    generated_treeinfo = requests.get(os.path.join(distribution.base_url, ".treeinfo")).text
 
     config = ConfigParser()
     config.optionxform = str  # by default it will cast keys to lower case
-    config.read_string(original_treeinfo.decode("utf-8"))
+    config.read_string(original_treeinfo)
     original_treeinfo = config._sections
 
     config = ConfigParser()
     config.optionxform = str  # by default it will cast keys to lower case
-    config.read_string(generated_treeinfo.decode("utf-8"))
+    config.read_string(generated_treeinfo)
     generated_treeinfo = config._sections
 
     # 5, 6. Re-arrange the metadata so that it can be compared, and do the comparison.
@@ -527,13 +525,12 @@ def test_distribution_tree_metadata_publish(
             continue
 
         checksum_type, checksum = checksum.split(":")
-        http_get(os.path.join(distribution.base_url, path))
+        assert requests.get(os.path.join(distribution.base_url, path)).status_code == 200
 
 
 @pytest.fixture
 def get_checksum_types(
     init_and_sync,
-    http_get,
     rpm_publication_api,
     gen_object_with_cleanup,
     rpm_distribution_api,
@@ -564,7 +561,7 @@ def _get_checksum_types(**kwargs):
         distribution = gen_object_with_cleanup(rpm_distribution_api, body)
 
         repomd = ElementTree.fromstring(
-            http_get(os.path.join(distribution.base_url, "repodata/repomd.xml"))
+            requests.get(os.path.join(distribution.base_url, "repodata/repomd.xml")).text
         )
 
         data_xpath = "{{{}}}data".format(RPM_NAMESPACES["metadata/repo"])
@@ -580,7 +577,7 @@ def _get_checksum_types(**kwargs):
                 location_xpath = "{{{}}}location".format(RPM_NAMESPACES["metadata/repo"])
                 primary_href = data_elem.find(location_xpath).get("href")
                 primary = ElementTree.fromstring(
-                    read_xml_gz(http_get(os.path.join(distribution.base_url, primary_href)))
+                    download_and_decompress_file(os.path.join(distribution.base_url, primary_href))
                 )
                 package_checksum_xpath = "{{{}}}checksum".format(RPM_NAMESPACES["metadata/common"])
                 package_xpath = "{{{}}}package".format(RPM_NAMESPACES["metadata/common"])
@@ -718,11 +715,11 @@ def test_immediate_specified_metadata_and_package_checksum_type(get_checksum_typ
 
 
 @pytest.mark.parallel
-def test_directory_layout_distribute_with_modules(generate_distribution, http_get):
+def test_directory_layout_distribute_with_modules(generate_distribution):
     """Ensure no more files or folders are present when distribute repository with modules."""
     distribution = generate_distribution(RPM_MODULAR_FIXTURE_URL)
     parser = etree.XMLParser(recover=True)
-    repository = etree.fromstring(http_get(distribution), parser=parser)
+    repository = etree.fromstring(requests.get(distribution).text, parser=parser)
     # Get links from repository HTML
     # Each link is an item (file or directory) in repository root
     repository_root_items = []
@@ -740,11 +737,11 @@ def test_directory_layout_distribute_with_modules(generate_distribution, http_ge
 
 
 @pytest.mark.parallel
-def test_directory_layout_distribute_with_treeinfo(generate_distribution, http_get):
+def test_directory_layout_distribute_with_treeinfo(generate_distribution):
     """Ensure no more files or folders are present when distribute repository with treeinfo."""
     distribution = generate_distribution(RPM_KICKSTART_FIXTURE_URL)
     parser = etree.XMLParser(recover=True)
-    repository = etree.fromstring(http_get(distribution), parser=parser)
+    repository = etree.fromstring(requests.get(distribution).text, parser=parser)
     # Get links from repository HTML
     # Each link is an item (file or directory) in repository root
     repository_root_items = []

pulp_rpm/tests/functional/api/test_sync.py

@@ -19,7 +19,6 @@
 
 from pulp_rpm.tests.functional.constants import (
     AMAZON_MIRROR,
-    DRPM_UNSIGNED_FIXTURE_URL,
     CENTOS7_OPSTOOLS_URL,
     PULP_TYPE_ADVISORY,
     PULP_TYPE_MODULEMD,
@@ -858,6 +857,9 @@ def test_complete_mirror_with_external_location_href_fails(init_and_sync):
     assert "features which are incompatible with 'mirror' sync" in error_description
 
 
+# We can restore this test when we are able to generate repositories on-demand. We just need to
+# create a "prestodelta" entry in the repomd.xml, we need not have it actually be a valid one.
+@pytest.mark.skip("No DRPM fixture repo.")
 @pytest.mark.parallel
 def test_complete_mirror_with_delta_metadata_fails(init_and_sync):
     """
@@ -866,7 +868,8 @@ def test_complete_mirror_with_delta_metadata_fails(init_and_sync):
     Otherwise we would be mirroring the metadata without mirroring the DRPM packages.
     """
     with pytest.raises(PulpTaskError) as exc:
-        init_and_sync(url=DRPM_UNSIGNED_FIXTURE_URL, sync_policy="mirror_complete")
+        pass
+        # init_and_sync(url=DRPM_UNSIGNED_FIXTURE_URL, sync_policy="mirror_complete")
 
     error_description = exc.value.task.to_dict()["error"]["description"]
     assert "features which are incompatible with 'mirror' sync" in error_description