Merge branch 'main' into gh_pr_target

.github/workflows/auto_updates.yml

@@ -35,7 +35,7 @@ jobs:
           git_commit_gpgsign: true
 
       - name: Install poetry
-        run: pipx install poetry==1.7.0
+        run: pipx install poetry==1.7.1
 
       - name: Configure poetry
         run: poetry config virtualenvs.in-project true
@@ -84,7 +84,7 @@ jobs:
 
       - name: Create Pull Request
         if: ${{ steps.commit.outcome == 'success' }}
-        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
         with:
           # This PAT is for the `phylum-bot` account and only has the `public_repo` scope to limit privileges.
           github-token: ${{ secrets.GH_RELEASE_PAT }}

.github/workflows/preview.yml

@@ -37,7 +37,7 @@ jobs:
           fetch-depth: 0
 
       - name: Install poetry
-        run: pipx install poetry==1.7.0
+        run: pipx install poetry==1.7.1
 
       - name: Configure poetry
         run: |

.github/workflows/release.yml

@@ -65,7 +65,7 @@ jobs:
           git_tag_gpgsign: true
 
       - name: Install poetry
-        run: pipx install poetry==1.7.0
+        run: pipx install poetry==1.7.1
 
       - name: Configure poetry
         run: |

.github/workflows/test.yml

@@ -27,7 +27,7 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Install poetry
-        run: pipx install poetry==1.7.0
+        run: pipx install poetry==1.7.1
 
       - name: Configure poetry
         run: poetry config virtualenvs.in-project true
@@ -70,7 +70,7 @@ jobs:
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Install poetry
-        run: pipx install poetry==1.7.0
+        run: pipx install poetry==1.7.1
 
       - name: Configure poetry
         run: poetry config virtualenvs.in-project true
@@ -112,7 +112,7 @@ jobs:
 
       - name: Install poetry
         if: ${{ matrix.build == 'wheel' }}
-        run: pipx install poetry==1.7.0
+        run: pipx install poetry==1.7.1
 
       - name: Configure poetry
         if: ${{ matrix.build == 'wheel' }}

.pre-commit-config.yaml

@@ -28,12 +28,12 @@ repos:
       - id: black
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 3d440bd6c67653f33f1c89ff9d6d450685ef9bfc  # frozen: v0.1.5
+    rev: 75d1925b523dd109cc200d1b29142ff36ec785ad  # frozen: v0.1.6
     hooks:
       - id: ruff
 
   - repo: https://github.com/dosisod/refurb
-    rev: 63209fc1735ef2497dd9c00774ba72a23bb1cdf9  # frozen: v1.23.0
+    rev: a25b5d6087bba1509f96654c583efcd7796452cd  # frozen: v1.24.0
     hooks:
       - id: refurb
 
@@ -52,7 +52,7 @@ repos:
     # https://python-poetry.org/docs/pre-commit-hooks/#why-does-pre-commit-autoupdate-not-update-to-the-latest-version
     # NOTE: This `rev` should be manually updated whenever `poetry` is updated. The `rev` can be found by checking out
     #       the `poetry` repository and running the command: `git show-ref --tags <tag_name>`
-    rev: ceeb1892a1f7456a65cd847ad36d740a577dee5e  # frozen: 1.7.0
+    rev: 7696e93872a965261ea154b5e423c44e8317fd38  # frozen: 1.7.1
     hooks:
       - id: poetry-check
         args: [--lock]

CHANGELOG.md

@@ -10,6 +10,13 @@ However, some entries may be manually edited, where it helps for clarity and und
 
 <!--next-version-placeholder-->
 
+## v0.39.0 (2023-11-27)
+
+### Feature
+
+* Set repository URL for CI environments ([#355](https://github.com/phylum-dev/phylum-ci/issues/355)) ([`28cf1a9`](https://github.com/phylum-dev/phylum-ci/commit/28cf1a9dd687f1912943e70775692e8babea223b))
+* Ensure remote `HEAD` set for `CINone` implementation ([#351](https://github.com/phylum-dev/phylum-ci/issues/351)) ([`e303919`](https://github.com/phylum-dev/phylum-ci/commit/e30391908d9a50f0d68bd8cc9c717f51f22d74a0))
+
 ## v0.38.0 (2023-11-09)
 
 ### Feature

Dockerfile

@@ -84,7 +84,7 @@ ENV PHYLUM_VENV="/opt/venv"
 ENV PHYLUM_VENV_PIP="${PHYLUM_VENV}/bin/pip"
 ENV PIP_NO_COMPILE=1
 ENV PIP_DISABLE_PIP_VERSION_CHECK=1
-ENV POETRY_VERSION="1.7.0"
+ENV POETRY_VERSION="1.7.1"
 
 WORKDIR ${APP_PATH}
 

Dockerfile.slim

@@ -92,7 +92,7 @@ ENV PHYLUM_VENV="/opt/venv"
 ENV PHYLUM_VENV_PIP="${PHYLUM_VENV}/bin/pip"
 ENV PIP_NO_COMPILE=1
 ENV PIP_DISABLE_PIP_VERSION_CHECK=1
-ENV POETRY_VERSION="1.7.0"
+ENV POETRY_VERSION="1.7.1"
 
 WORKDIR ${APP_PATH}
 

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "phylum"
-version = "0.38.0"
+version = "0.39.0"
 description = "Utilities for integrating Phylum into CI pipelines (and beyond)"
 license = "GPL-3.0-or-later"
 authors = ["Phylum, Inc. <[email protected]>"]

src/phylum/ci/ci_azure.py

@@ -269,6 +269,26 @@ def phylum_comment_exists(self) -> bool:
             return bool(get_most_recent_phylum_comment_github(comments_url, self.github_token))
         return False
 
+    @property
+    def repo_url(self) -> Optional[str]:
+        """Get the repository URL for reference in Phylum project metadata."""
+        # Ref: https://learn.microsoft.com/azure/devops/pipelines/build/variables
+        if self.triggering_repo == "TfsGit":
+            # SYSTEM_TEAMPROJECT provides the name that corresponds to SYSTEM_TEAMPROJECTID.
+            # Even though the ID will never change while the name might, the name is used for better human consumption.
+            if team_project_id := os.getenv("SYSTEM_TEAMPROJECT") is None:
+                LOG.debug("`SYSTEM_TEAMPROJECT` missing. Can't get repository URL.")
+            if instance := os.getenv("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI") is None:
+                LOG.debug("`SYSTEM_TEAMFOUNDATIONCOLLECTIONURI` missing. Can't get repository URL.")
+            if team_project_id is None or instance is None:
+                return None
+            # This format is correct regardless of the pipeline context
+            return f"{instance}{team_project_id}"
+        if self.triggering_repo == "GitHub":
+            # BUILD_REPOSITORY_URI will have the correct format regardless of the pipeline context
+            return os.getenv("BUILD_REPOSITORY_URI")
+        return None
+
     def post_output(self) -> None:
         """Post the output of the analysis.
 

src/phylum/ci/ci_base.py

@@ -122,18 +122,8 @@ def _find_potential_depfiles(self) -> None:
             msg = "Phylum `find-lockable-files` command failed"
             raise PhylumCalledProcessError(err, msg) from err
         lockable_files: dict = json.loads(result)
-        self._potential_manifests = list(starmap(LockfileEntry, lockable_files.get("manifests", [])))
-        self._potential_lockfiles = list(starmap(LockfileEntry, lockable_files.get("lockfiles", [])))
-
-    @property
-    def potential_manifests(self) -> LockfileEntries:
-        """Get all the potential manifests at the current directory or below."""
-        return self._potential_manifests
-
-    @property
-    def potential_lockfiles(self) -> LockfileEntries:
-        """Get all the potential lockfiles at the current directory or below."""
-        return self._potential_lockfiles
+        self._potential_manifests: LockfileEntries = list(starmap(LockfileEntry, lockable_files.get("manifests", [])))
+        self._potential_lockfiles: LockfileEntries = list(starmap(LockfileEntry, lockable_files.get("lockfiles", [])))
 
     @cached_property
     def depfiles(self) -> Depfiles:
@@ -207,17 +197,17 @@ def _filter_depfiles(self, provided_depfiles: LockfileEntries) -> Depfiles:
                 continue
 
             # Classify the file as a manifest or lockfile
-            if provided_depfile in self.potential_manifests and provided_depfile in self.potential_lockfiles:
+            if provided_depfile in self._potential_manifests and provided_depfile in self._potential_lockfiles:
                 msg = f"""\
                     Provided dependency file [code]{provided_depfile!r}[/] is a [b]lockifest[/].
                     It will be treated as a [b]manifest[/].
                     For more info, see: https://docs.phylum.io/docs/lockfile_generation"""
                 LOG.warning(textwrap.dedent(msg), extra=MARKUP)
                 depfile = Depfile(provided_depfile, self.cli_path, DepfileType.LOCKIFEST)
-            elif provided_depfile in self.potential_manifests:
+            elif provided_depfile in self._potential_manifests:
                 LOG.info("Provided dependency file [code]%r[/] is a [b]manifest[/]", provided_depfile, extra=MARKUP)
                 depfile = Depfile(provided_depfile, self.cli_path, DepfileType.MANIFEST)
-            elif provided_depfile in self.potential_lockfiles:
+            elif provided_depfile in self._potential_lockfiles:
                 LOG.info("Provided dependency file [code]%r[/] is a [b]lockfile[/]", provided_depfile, extra=MARKUP)
                 depfile = Depfile(provided_depfile, self.cli_path, DepfileType.LOCKFILE)
             else:
@@ -362,6 +352,19 @@ def phylum_label(self) -> str:
         """
         raise NotImplementedError
 
+    @property
+    @abstractmethod
+    def repo_url(self) -> Optional[str]:
+        """Get the repository URL for reference in Phylum project metadata.
+
+        The value should only exist for implementations that make use of a web hosted CI environment. The local use
+        cases should not set this value. The expected format of the repository URL is the "URL of the web UI for the
+        project root." This is not the URL to clone the repository.
+
+        `None` should be returned when the repository URL can't be found, shouldn't be set, or there is an error.
+        """
+        raise NotImplementedError
+
     @property
     def depfile_hash_object(self) -> str:
         """Get the dependency file hash object of the first changed dependency file and return it.
@@ -462,10 +465,14 @@ def _ensure_project_exists(self) -> None:
         file that already exists. Continue on without error when the specified project already exists.
         """
         LOG.info("Attempting to create a Phylum project with the name: %s ...", self.phylum_project)
-        cmd = [str(self.cli_path), "project", "create", self.phylum_project]
+        cmd = [str(self.cli_path), "project", "create"]
         if self.phylum_group:
             LOG.debug("Using Phylum group: %s", self.phylum_group)
-            cmd = [str(self.cli_path), "project", "create", "--group", self.phylum_group, self.phylum_project]
+            cmd.extend(["--group", self.phylum_group])
+        if self.repo_url:
+            LOG.debug("Using repository URL: %s", self.repo_url)
+            cmd.extend(["--repository-url", self.repo_url])
+        cmd.append(self.phylum_project)
         if not Path.cwd().joinpath(".git").is_dir():
             LOG.warning("Attempting to create a Phylum project outside the top level of a `git` repository")
         try:
@@ -477,6 +484,7 @@ def _ensure_project_exists(self) -> None:
             cli_exit_code_project_already_exists = 14
             if err.returncode == cli_exit_code_project_already_exists:
                 LOG.info("Project %s already exists. Continuing with it ...", self.phylum_project)
+                self._set_repo_url()
                 return
             msg = """\
                 There was a problem creating the project.
@@ -487,6 +495,73 @@ def _ensure_project_exists(self) -> None:
         if self._project_file_already_existed:
             LOG.warning("Overwrote previous `.phylum_project` file found at: %s", self._phylum_project_file)
 
+    def _set_repo_url(self) -> None:
+        """Set the repository URL for the project.
+
+        The value is meant to be the Web UI URL for where the project is hosted.
+        It should not be set or changed if a value already exists.
+        """
+        if self.repo_url is None:
+            LOG.debug("Repository URL not available to set")
+            return
+
+        # This block can be simplified after https://github.com/phylum-dev/cli/issues/1300
+        LOG.info("Checking project for existing repository URL ...")
+        LOG.debug("Finding ID for Phylum project: %s", self.phylum_project)
+        cmd = [str(self.cli_path), "project", "list", "--json"]
+        if self.phylum_group:
+            LOG.debug("Using Phylum group: %s", self.phylum_group)
+            cmd.extend(["--group", self.phylum_group])
+        try:
+            projects_list = subprocess.run(cmd, check=True, capture_output=True, text=True).stdout.strip()  # noqa: S603
+        except subprocess.CalledProcessError as err:
+            pprint_subprocess_error(err)
+            msg = """\
+                Phylum project listing failed. Skipping repository URL check. Use CLI
+                to manually set it: https://docs.phylum.io/docs/phylum_project_update"""
+            LOG.warning(textwrap.dedent(msg))
+            return
+        projects: list[dict] = json.loads(projects_list)
+        project = next(filter(lambda project: project.get("name") == self.phylum_project, projects), {})
+        project_id = project.get("id")
+        repo_url = project.get("repository_url")
+        if not project_id:
+            msg = """\
+                Could not find the project ID. Skipping repository URL check. Use CLI
+                to manually set it: https://docs.phylum.io/docs/phylum_project_update"""
+            LOG.warning(textwrap.dedent(msg))
+            return
+        LOG.debug("Found project ID: %s", project_id)
+
+        if repo_url is not None:
+            LOG.info("Repository URL already set: %s", repo_url)
+            if repo_url != self.repo_url:
+                msg = f"""\
+                    Repository URL differs from what would be set! Keeping existing value.
+                    Existing: {repo_url}
+                    Proposed: {self.repo_url}
+                    Use CLI to override: https://docs.phylum.io/docs/phylum_project_update"""
+                LOG.warning(textwrap.dedent(msg))
+                return
+            LOG.info("Repository URL matches what would be set. Nothing to do.")
+            return
+
+        LOG.info("Repository URL not set. Setting it to: %s", self.repo_url)
+        cmd = [str(self.cli_path), "project", "update", "--project-id", project_id, "--repository-url", self.repo_url]
+        if self.phylum_group:
+            cmd.extend(["--group", self.phylum_group])
+        LOG.debug("Using command: %s", shlex.join(cmd))
+        try:
+            subprocess.run(cmd, check=True, capture_output=True, text=True)  # noqa: S603
+        except subprocess.CalledProcessError as err:
+            pprint_subprocess_error(err)
+            msg = """\
+                Phylum project update failed. Skipping repository URL check. Use CLI
+                to manually set it: https://docs.phylum.io/docs/phylum_project_update"""
+            LOG.warning(textwrap.dedent(msg))
+            return
+        LOG.info("Repository URL successfully set")
+
     def post_output(self) -> None:
         """Post the output of the analysis as markdown rendered for output to the terminal/logs.
 

src/phylum/ci/ci_bitbucket.py

@@ -211,6 +211,14 @@ def phylum_comment_exists(self) -> bool:
         """Predicate for detecting whether a Phylum-generated comment exists."""
         return bool(self.most_recent_phylum_comment)
 
+    @property
+    def repo_url(self) -> Optional[str]:
+        """Get the repository URL for reference in Phylum project metadata."""
+        # This is the "URL for the origin", which uses `HTTP:` instead of
+        # `HTTPS:`, even though the value redirects to the `HTTPS:` site.
+        # Ref: https://support.atlassian.com/bitbucket-cloud/docs/variables-and-secrets/
+        return os.getenv("BITBUCKET_GIT_HTTP_ORIGIN")
+
     def post_output(self) -> None:
         """Post the output of the analysis.
 

src/phylum/ci/ci_github.py

@@ -1,7 +1,7 @@
 """Define an implementation for the GitHub Actions platform.
 
 GitHub References:
-  * https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
+  * https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
   * https://docs.github.com/en/actions/security-guides/automatic-token-authentication
   * https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token
   * https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes
@@ -176,6 +176,18 @@ def phylum_comment_exists(self) -> bool:
         """Predicate for detecting whether a Phylum-generated comment exists."""
         return bool(get_most_recent_phylum_comment_github(self.comments_url, self.github_token))
 
+    @property
+    def repo_url(self) -> Optional[str]:
+        """Get the repository URL for reference in Phylum project metadata."""
+        # Ref: https://docs.github.com/actions/learn-github-actions/variables#default-environment-variables
+        if server_url := os.getenv("GITHUB_SERVER_URL") is None:
+            LOG.debug("`GITHUB_SERVER_URL` missing. Can't get repository URL.")
+        if repo := os.getenv("GITHUB_REPOSITORY") is None:
+            LOG.debug("`GITHUB_REPOSITORY` missing. Can't get repository URL.")
+        if server_url is None or repo is None:
+            return None
+        return f"{server_url}/{repo}"
+
     def post_output(self) -> None:
         """Post the output of the analysis.
 

src/phylum/ci/ci_gitlab.py

@@ -174,6 +174,12 @@ def phylum_comment_exists(self) -> bool:
         """Predicate for detecting whether a Phylum-generated note (comment) exists."""
         return bool(self.most_recent_phylum_note)
 
+    @property
+    def repo_url(self) -> Optional[str]:
+        """Get the repository URL for reference in Phylum project metadata."""
+        # Ref: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
+        return os.getenv("CI_PROJECT_URL")
+
     def post_output(self) -> None:
         """Post the output of the analysis.
 

src/phylum/ci/ci_none.py

@@ -97,3 +97,9 @@ def phylum_comment_exists(self) -> bool:
         """Predicate for detecting whether a Phylum-generated comment exists."""
         # There are no historical comments in this implementation
         return False
+
+    @property
+    def repo_url(self) -> Optional[str]:
+        """Get the repository URL for reference in Phylum project metadata."""
+        # There is no repository URL in this implementation
+        return None

src/phylum/ci/ci_precommit.py

@@ -129,3 +129,9 @@ def phylum_comment_exists(self) -> bool:
         """Predicate for detecting whether a Phylum-generated comment exists."""
         # There are no historical comments in this implementation
         return False
+
+    @property
+    def repo_url(self) -> Optional[str]:
+        """Get the repository URL for reference in Phylum project metadata."""
+        # There is no repository URL in this implementation
+        return None