From 3051504ce5970700e6ce4625e100721ff4142a4a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 8 Jun 2021 16:53:52 +0100 Subject: [PATCH 1/3] OPS: Add conventional commits pre-commit check skip_ci_tests --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 25f8fa5..9a165f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -59,3 +59,9 @@ repos: - '^refactor/([a-z][a-z0-9]*)(-[a-z0-9]+)*$' - '^review/([a-z][a-z0-9]*)(-[a-z0-9]+)*$' - '^release/(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$' + + - repo: https://github.com/octue/pre-commit-hooks + rev: 0.0.2 + hooks: + - id: check-commit-message-is-conventional + stages: [commit-msg] From e8084162c73cbefd1163a837142c6cd10b59b3c0 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 8 Jun 2021 17:18:21 +0100 Subject: [PATCH 2/3] OPS: Update version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 207fcae..9378083 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name="twined", - version="0.0.20", + version="0.0.21", py_modules=[], install_requires=["jsonschema ~= 3.2.0", "python-dotenv"], url="https://www.github.com/octue/twined", From 5bc70249a4c721fecb503f7847023096358cf535 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 14 Jun 2021 19:15:39 +0100 Subject: [PATCH 3/3] OPS: Enable automatic semantic deployment --- .github/workflows/python-ci.yml | 31 +-- .github/workflows/release.yml | 4 +- .../scripts/check-semantic-version.py | 40 ++++ .../scripts/check-version-consistency.py | 68 ------- .../scripts/compile-release-notes.py | 176 ++++++++++++++++++ .github/workflows/update-pull-request.yml | 33 ++++ 6 files changed, 270 insertions(+), 82 deletions(-) create mode 100644 .github/workflows/scripts/check-semantic-version.py delete mode 100644 .github/workflows/scripts/check-version-consistency.py create mode 100644 .github/workflows/scripts/compile-release-notes.py create mode 100644 .github/workflows/update-pull-request.yml diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 5176f2c..32d1824 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -10,22 +10,30 @@ on: [push] jobs: - check-version-consistency: + check-semantic-version: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + # Set fetch-depth to 0 to fetch all tags (necessary for git-mkver to determine the correct semantic version). + fetch-depth: 0 - uses: actions/setup-python@v2 - - run: python .github/workflows/scripts/check-version-consistency.py + - name: Install git-mkver + run: | + curl -L https://github.com/idc101/git-mkver/releases/download/v1.2.1/git-mkver-linux-amd64-1.2.1.tar.gz \ + | tar xvz \ + && sudo mv git-mkver /usr/local/bin + - name: Check version + run: python .github/workflows/scripts/check-semantic-version.py - tests: - runs-on: ubuntu-latest + run-tests: if: "!contains(github.event.head_commit.message, 'skip_ci_tests')" + runs-on: ubuntu-latest env: USING_COVERAGE: '3.8' strategy: matrix: python: [3.8] - os: [ubuntu-latest, windows-latest, macos-latest] steps: - name: Checkout Repository uses: actions/checkout@v2 @@ -33,10 +41,10 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} - - name: Install Tox and any other packages + - name: Install tox run: pip install tox - - name: Run Tox - run: tox -e py + - name: Run tests + run: tox - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: @@ -45,13 +53,13 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} publish: - if: contains(github.ref, 'main') || contains(github.ref, 'release/') + if: "!contains(github.event.head_commit.message, 'skip_ci_publish')" runs-on: ubuntu-latest - needs: tests + needs: [check-semantic-version, run-tests] steps: - name: Checkout Repository uses: actions/checkout@v2 - - name: Setup Python + - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.8 @@ -60,7 +68,6 @@ jobs: python3 -m pip install --upgrade setuptools wheel python3 setup.py sdist bdist_wheel - name: Test package is publishable with PyPI test server - if: contains(github.ref, 'release/') uses: pypa/gh-action-pypi-publish@master with: user: __token__ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0d07e42..62538ba 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Release the package on merge of release/x.y.z into main +name: Release the package on merge into main # Only trigger when a pull request into main branch is closed. on: @@ -10,7 +10,7 @@ on: jobs: release: # This job will only run if the PR has been merged (and not closed without merging). - if: github.event.pull_request.merged == true && startsWith( github.head_ref, 'release/' ) + if: "github.event.pull_request.merged == true && !contains(github.event.pull_request.head.message, 'skip_ci_release')" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/scripts/check-semantic-version.py b/.github/workflows/scripts/check-semantic-version.py new file mode 100644 index 0000000..06192ab --- /dev/null +++ b/.github/workflows/scripts/check-semantic-version.py @@ -0,0 +1,40 @@ +import os +import subprocess +import sys + + +PACKAGE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) + + +RED = '\033[0;31m' +GREEN = "\033[0;32m" +NO_COLOUR = '\033[0m' + + +def get_setup_version(): + process = subprocess.run(["python", "setup.py", "--version"], capture_output=True) + return process.stdout.strip().decode("utf8") + + +def get_expected_semantic_version(): + process = subprocess.run(["git-mkver", "next"], capture_output=True) + return process.stdout.strip().decode("utf8") + + +if __name__ == "__main__": + os.chdir(PACKAGE_ROOT) + setup_version = get_setup_version() + expected_semantic_version = get_expected_semantic_version() + + if setup_version != expected_semantic_version: + print( + f"{RED}VERSION FAILED CHECKS:{NO_COLOUR} The version stated in 'setup.py' ({setup_version}) is different " + f"from the expected semantic version ({expected_semantic_version})." + ) + sys.exit(1) + + print( + f"{GREEN}VERSION PASSED CHECKS:{NO_COLOUR} The version stated in 'setup.py' is the same as the expected " + f"semantic version: {expected_semantic_version}." + ) + sys.exit(0) diff --git a/.github/workflows/scripts/check-version-consistency.py b/.github/workflows/scripts/check-version-consistency.py deleted file mode 100644 index 7ab9453..0000000 --- a/.github/workflows/scripts/check-version-consistency.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import subprocess -import sys - - -PACKAGE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) - - -class NotAReleaseBranchException(Exception): - pass - - -class InvalidBranchNameFormat(Exception): - pass - - -def get_setup_version(): - process = subprocess.run(["python", "setup.py", "--version"], capture_output=True) - return process.stdout.strip().decode("utf8") - - -def get_branch_name(): - process = subprocess.run(["git", "branch", "--show-current"], capture_output=True) - return process.stdout.strip().decode("utf8") - - -def release_branch_version_matches_setup_version(setup_version, full_branch_name): - """ Check if the package version stated in setup.py matches the semantic version 'x.y.z' included in the branch name - of the format 'release/x.y.z'. - - :param str setup_version: - :param str full_branch_name: - :raise NotAVersionBranchException: - :return bool: - """ - if not full_branch_name.startswith("release"): - raise NotAReleaseBranchException(f"The branch is not a release branch: {full_branch_name!r}.") - - try: - branch_type, branch_name = full_branch_name.split("/") - except ValueError: - raise InvalidBranchNameFormat( - f"The branch name must be in the form 'branch_type/branch_name'; received {full_branch_name!r}" - ) - - return branch_name == setup_version - - -if __name__ == "__main__": - - os.chdir(PACKAGE_ROOT) - setup_version = get_setup_version() - full_branch_name = get_branch_name() - - try: - if release_branch_version_matches_setup_version(setup_version, full_branch_name): - print(f"Release branch name matches setup.py version: {setup_version!r}.") - sys.exit(0) - - print( - f"Release branch name does not match setup.py version: branch is {full_branch_name!r} but setup.py version " - f"is {setup_version!r}." - ) - sys.exit(1) - - except NotAReleaseBranchException as e: - print(e.args[0]) - sys.exit(0) diff --git a/.github/workflows/scripts/compile-release-notes.py b/.github/workflows/scripts/compile-release-notes.py new file mode 100644 index 0000000..68c918c --- /dev/null +++ b/.github/workflows/scripts/compile-release-notes.py @@ -0,0 +1,176 @@ +import re +import subprocess +import sys + + +LAST_RELEASE = "LAST_RELEASE" +LAST_PULL_REQUEST = "LAST_PULL_REQUEST" + +SEMANTIC_VERSION_PATTERN = r"tag: (\d+\.\d+\.\d+)" +PULL_REQUEST_INDICATOR = "Merge pull request #" + +COMMIT_CODES_TO_HEADINGS_MAPPING = { + "FEA": "### New features", + "ENH": "### Enhancements", + "FIX": "### Fixes", + "OPS": "### Operations", + "DEP": "### Dependencies", + "REF": "### Refactoring", + "TST": "### Testing", + "MRG": "### Other", + "REV": "### Reversions", + "CHO": "### Chores", + "WIP": "### Other", + "DOC": "### Other", + "STY": "### Other", +} + +AUTO_GENERATION_START_INDICATOR = "" +AUTO_GENERATION_END_INDICATOR = "" + + +class ReleaseNoteCompiler: + """A release/pull request notes compiler. The notes are pulled together from Conventional Commit messages, stopping + at the specified stop point. The stop point can either be the last merged pull request in the branch or the last + semantically-versioned release tagged in the branch. If previous notes are provided, only the text between the + comment lines `` and `` will be replaced - + anything outside of this will appear in the new release notes. + + :param str header: + :param str|None previous_notes: + :param str list_item_symbol: + :param dict|None commit_codes_to_headings_mapping: + :param str stop_point: + :return None: + """ + + def __init__( + self, + stop_point, + previous_notes=None, + header="## Contents", + list_item_symbol="- [x] ", + commit_codes_to_headings_mapping=None, + ): + if stop_point.upper() not in {LAST_RELEASE, LAST_PULL_REQUEST}: + raise ValueError(f"`stop_point` must be one of {LAST_RELEASE, LAST_PULL_REQUEST!r}; received {stop_point!r}.") + + self.stop_point = stop_point.upper() + self.previous_notes = previous_notes + self.header = header + self.list_item_symbol = list_item_symbol + self.commit_codes_to_headings_mapping = commit_codes_to_headings_mapping or COMMIT_CODES_TO_HEADINGS_MAPPING + + def compile_release_notes(self): + """Compile the release or pull request notes into a multiline string, sorting the commit messages into headed + sections according to their commit codes and the commit-codes-to-headings mapping. + + :return str: + """ + git_log = subprocess.run(["git", "log", "--pretty=format:%s|%d"], capture_output=True).stdout.strip().decode() + parsed_commits, unparsed_commits = self._parse_commit_messages(git_log) + categorised_commit_messages = self._categorise_commit_messages(parsed_commits, unparsed_commits) + autogenerated_release_notes = self._build_release_notes(categorised_commit_messages) + + if not self.previous_notes: + return autogenerated_release_notes + + previous_notes_lines = self.previous_notes.splitlines() + + for i, line in enumerate(previous_notes_lines): + if AUTO_GENERATION_START_INDICATOR in line: + autogeneration_start_index = i + + elif AUTO_GENERATION_END_INDICATOR in line: + autogeneration_end_index = i + 1 + + try: + return "\n".join( + ( + *previous_notes_lines[:autogeneration_start_index], + autogenerated_release_notes, + *previous_notes_lines[autogeneration_end_index:] + ) + ) + + except NameError: + return autogenerated_release_notes + + def _parse_commit_messages(self, formatted_oneline_git_log): + """Parse commit messages from the git log (formatted using `--pretty=format:%s|%d`) until the stop point is + reached. The parsed commit messages are returned separately to any that fail to parse. + + :param str formatted_oneline_git_log: + :return list(tuple), list(str): + """ + parsed_commits = [] + unparsed_commits = [] + + for commit in formatted_oneline_git_log.splitlines(): + split_commit = commit.split("|") + + if len(split_commit) == 2: + message, decoration = split_commit + + try: + code, message = message.split(":") + except ValueError: + unparsed_commits.append(message.strip()) + continue + + if self.stop_point == LAST_RELEASE: + if "tag" in decoration: + if re.compile(SEMANTIC_VERSION_PATTERN).search(decoration): + break + + if self.stop_point == LAST_PULL_REQUEST: + if PULL_REQUEST_INDICATOR in message: + break + + parsed_commits.append((code.strip(), message.strip(), decoration.strip())) + + return parsed_commits, unparsed_commits + + def _categorise_commit_messages(self, parsed_commits, unparsed_commits): + """Categorise the commit messages into headed sections. Unparsed commits are put under an "uncategorised" + header. + + :param iter(tuple)) parsed_commits: + :param iter(str) unparsed_commits: + :return dict: + """ + release_notes = {heading: [] for heading in self.commit_codes_to_headings_mapping.values()} + + for code, message, _ in parsed_commits: + try: + release_notes[self.commit_codes_to_headings_mapping[code]].append(message) + except KeyError: + release_notes["### Other"].append(message) + + release_notes["### Uncategorised!"] = unparsed_commits + return release_notes + + def _build_release_notes(self, categorised_commit_messages): + """Build the the categorised commit messages into a single multi-line string ready to be used as formatted + release notes. + + :param dict categorised_commit_messages: + :return str: + """ + release_notes_for_printing = f"{AUTO_GENERATION_START_INDICATOR}\n{self.header}\n\n" + + for heading, notes in categorised_commit_messages.items(): + + if len(notes) == 0: + continue + + note_lines = "\n".join(self.list_item_symbol + note for note in notes) + release_notes_for_printing += f"{heading}\n{note_lines}\n\n" + + release_notes_for_printing += AUTO_GENERATION_END_INDICATOR + return release_notes_for_printing + + +if __name__ == "__main__": + release_notes = ReleaseNoteCompiler(*sys.argv[1:]).compile_release_notes() + print(release_notes) diff --git a/.github/workflows/update-pull-request.yml b/.github/workflows/update-pull-request.yml new file mode 100644 index 0000000..2b28cf2 --- /dev/null +++ b/.github/workflows/update-pull-request.yml @@ -0,0 +1,33 @@ +# This workflow updates the pull request description with an auto-generated section containing the categorised commit +# message headers of the commits since the last pull request merged into main. The auto generated section is enveloped +# between two comments: "" and "". Anything +# outside these in the description is left untouched. Auto-generated updates can be skipped for a commit if +# "skip_ci_update_description" is in the commit message body. + +name: update-pull-request + +# Only trigger for pull requests into main branch. +on: + pull_request: + branches: + - main + +jobs: + description: + if: "!contains(github.event.pull_request.head.message, 'skip_ci_update_description')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # Set fetch-depth to 0 to fetch all commit history (necessary for compiling pull request description). + fetch-depth: 0 + - name: Compile new pull request description + run: | + echo 'PULL_REQUEST_NOTES<> $GITHUB_ENV + echo "$(python .github/workflows/scripts/compile-release-notes.py LAST_PULL_REQUEST '${{ github.event.pull_request.body }}')" >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + - name: Update pull request body + uses: riskledger/update-pr-description@v2 + with: + body: ${{ env.PULL_REQUEST_NOTES }} + token: ${{ secrets.GITHUB_TOKEN }}