From fbb0514749279695d0f0c46e2096e93522a271fc Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Mon, 13 Jan 2025 20:32:39 +0000 Subject: [PATCH 1/6] feat: improvements to release.yml --- .github/workflows/release.yml | 217 ++++++++++++++-------------------- 1 file changed, 92 insertions(+), 125 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 588cacf5..ee6eebf3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,31 +1,18 @@ -name: Release +name: Automatic Release Creation on: - schedule: - # Run every day at 9:00 UTC - - cron: '0 9 * * *' - # Allow manual trigger for testing workflow_dispatch: jobs: - prepare: + detect-last-release: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} last_release: ${{ steps.last-release.outputs.hash }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Find package directories - id: set-matrix - run: | - # Find all package.json and pyproject.toml files, excluding root - DIRS=$(git ls-tree -r HEAD --name-only | grep -E "package.json|pyproject.toml" | xargs dirname | grep -v "^.$" | jq -R -s -c 'split("\n")[:-1]') - echo "matrix=${DIRS}" >> $GITHUB_OUTPUT - echo "Found directories: ${DIRS}" - - name: Get last release hash id: last-release run: | @@ -33,142 +20,122 @@ jobs: echo "hash=${HASH}" >> $GITHUB_OUTPUT echo "Using last release hash: ${HASH}" - release: - needs: prepare + create-tag-name: runs-on: ubuntu-latest - environment: release - strategy: - matrix: - directory: ${{ fromJson(needs.prepare.outputs.matrix) }} - fail-fast: false - permissions: - contents: write - packages: write + outputs: + tag_name: ${{ steps.last-release.outputs.tag}} + steps: + - name: Get last release hash + id: last-release + run: | + DATE=$(date +%Y.%m.%d) + echo "tag=v${DATE}" >> $GITHUB_OUTPUT + echo "Using tag: v${DATE}" + detect-packages: + needs: [detect-last-release] + runs-on: ubuntu-latest + outputs: + packages: ${{ steps.find-packages.outputs.packages }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: astral-sh/setup-uv@v5 - - - name: Setup Node.js - if: endsWith(matrix.directory, '/package.json') - uses: actions/setup-node@v4 - with: - node-version: '18' - registry-url: 'https://registry.npmjs.org' + - name: Install uv + uses: astral-sh/setup-uv@v5 - - name: Setup Python - if: endsWith(matrix.directory, '/pyproject.toml') - run: uv python install - - - name: Release package - id: release - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }} + - name: Find packages + id: find-packages + working-directory: src run: | - # Create unique hash for this directory - dir_hash=$(echo "${{ matrix.directory }}" | sha256sum | awk '{print $1}') + cat << 'EOF' > find_packages.py + import json + import os + import subprocess + from itertools import chain + from pathlib import Path - # Run git diff first to show changes - echo "Changes since last release:" - git diff --name-only "${{ needs.prepare.outputs.last_release }}" -- "${{ matrix.directory }}" || true + packages = [] - # Run the release - output=$(uv run --script scripts/release.py "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" 2>&1) - exit_code=$? + print("Starting package detection...") + print(f"Using LAST_RELEASE: {os.environ['LAST_RELEASE']}") - echo "Release output (exit code: $exit_code):" - echo "$output" + # Find all directories containing package.json or pyproject.toml + paths = chain(Path('.').glob('*/package.json'), Path('.').glob('*/pyproject.toml')) + for path in paths: + print(f"\nChecking path: {path}") + # Check for changes in .py or .ts files + # Run git diff from the specific directory + cmd = ['git', 'diff', '--name-only', f'{os.environ["LAST_RELEASE"]}..HEAD', '--', '.'] + result = subprocess.run(cmd, capture_output=True, text=True, cwd=path.parent) - # Extract package info if successful - if [ $exit_code -eq 0 ]; then - pkg_info=$(echo "$output" | grep -o -E "[a-zA-Z0-9\-]+@[0-9]+\.[0-9]+\.[0-9]+" || true) - else - echo "Release failed" - exit 1 - fi + # Check if any .py or .ts files were changed + changed_files = result.stdout.strip().split('\n') + print(f"Changed files found: {changed_files}") - if [ ! -z "$pkg_info" ]; then - echo "Released package: $pkg_info" + has_changes = any(f.endswith(('.py', '.ts')) for f in changed_files if f) + if has_changes: + print(f"Adding package: {path.parent}") + packages.append(str(path.parent)) - # Create outputs directory - mkdir -p ./outputs + print(f"\nFinal packages list: {packages}") - # Save both package info and full changes - echo "$pkg_info" > "./outputs/${dir_hash}_info" - echo "dir_hash=${dir_hash}" >> $GITHUB_OUTPUT + # Write output + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f"packages={json.dumps(packages)}\n") + EOF - # Log what we're saving - echo "Saved package info to ./outputs/${dir_hash}_info:" - cat "./outputs/${dir_hash}_info" - else - echo "No release needed for this package" - fi - - - name: Set artifact name - if: steps.release.outputs.dir_hash - id: artifact - run: | - # Replace forward slashes with dashes - SAFE_DIR=$(echo "${{ matrix.directory }}" | tr '/' '-') - echo "name=release-outputs-${SAFE_DIR}" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v4 - if: steps.release.outputs.dir_hash - with: - name: ${{ steps.artifact.outputs.name }} - path: ./outputs/${{ steps.release.outputs.dir_hash }}* + LAST_RELEASE=${{ needs.detect-last-release.outputs.last_release }} uv run --script --python 3.12 find_packages.py create-tag: - needs: [prepare, release] + needs: [detect-packages, create-tag-name] runs-on: ubuntu-latest + environment: release permissions: contents: write steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - pattern: release-outputs-src-* - merge-multiple: true - path: outputs - - - name: Create tag and release + - name: Create tag env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - if [ -d outputs ]; then - # Collect package info - find outputs -name "*_info" -exec cat {} \; > packages.txt - - if [ -s packages.txt ]; then - DATE=$(date +%Y.%m.%d) - echo "Creating tag v${DATE}" - - # Generate comprehensive release notes - { - echo "# Release ${DATE}" - echo "" - echo "## Updated Packages" - while IFS= read -r line; do - echo "- $line" - done < packages.txt - } > notes.md - - # Create and push tag - git tag -a "v${DATE}" -m "Release ${DATE}" - git push origin "v${DATE}" - - # Create GitHub release - gh release create "v${DATE}" \ - --title "Release ${DATE}" \ - --notes-file notes.md - else - echo "No packages need release" - fi + # Configure git + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + + # Get packages array + PACKAGES='${{ needs.detect-packages.outputs.packages }}' + + if [ "$(echo "$PACKAGES" | jq 'length')" -gt 0 ]; then + # Create and push tag + git tag -a "${{ needs.create-tag-name.outputs.tag_name }}" -m "Release ${{ needs.create-tag-name.outputs.tag_name }}" + git push origin "${{ needs.create-tag-name.outputs.tag_name }}" + else + echo "No packages need release" + fi + + - name: Create release + env: + GH_TOKEN: ${{ secrets.RELEASE_TOKEN }} + run: | + PACKAGES='${{ needs.detect-packages.outputs.packages }}' + + if [ "$(echo "$PACKAGES" | jq 'length')" -gt 0 ]; then + # Generate comprehensive release notes + { + echo "# Release ${{ needs.create-tag-name.outputs.tag_name }}" + echo "" + echo "## Updated Packages" + echo "$PACKAGES" | jq -r '.[]' | while read -r package; do + echo "- $package" + done + } > notes.md + # Create GitHub release + gh release create "${{ needs.create-tag-name.outputs.tag_name }}" \ + --title "Release ${{ needs.create-tag-name.outputs.tag_name }}" \ + --notes-file notes.md else - echo "No release artifacts found" - fi \ No newline at end of file + echo "No packages need release" + fi From 85e303aab801f2ec49944f946f6cd634295d9501 Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Tue, 14 Jan 2025 00:35:12 +0000 Subject: [PATCH 2/6] feat: remove release check --- .github/workflows/release-check.yml | 159 ---------------------------- 1 file changed, 159 deletions(-) delete mode 100644 .github/workflows/release-check.yml diff --git a/.github/workflows/release-check.yml b/.github/workflows/release-check.yml deleted file mode 100644 index 63d7dfa0..00000000 --- a/.github/workflows/release-check.yml +++ /dev/null @@ -1,159 +0,0 @@ -name: Release Check - -on: - # Allow manual trigger for testing - workflow_dispatch: - -jobs: - prepare: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - last_release: ${{ steps.last-release.outputs.hash }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Find package directories - id: set-matrix - run: | - # Find all package.json and pyproject.toml files, excluding root - DIRS=$(git ls-tree -r HEAD --name-only | grep -E "package.json|pyproject.toml" | xargs dirname | grep -v "^.$" | jq -R -s -c 'split("\n")[:-1]') - echo "matrix=${DIRS}" >> $GITHUB_OUTPUT - echo "Found directories: ${DIRS}" - - - name: Get last release hash - id: last-release - run: | - HASH=$(git rev-list --tags --max-count=1 || echo "HEAD~1") - echo "hash=${HASH}" >> $GITHUB_OUTPUT - echo "Using last release hash: ${HASH}" - - check-release: - needs: prepare - runs-on: ubuntu-latest - strategy: - matrix: - directory: ${{ fromJson(needs.prepare.outputs.matrix) }} - fail-fast: false - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: astral-sh/setup-uv@v5 - - - name: Setup Node.js - if: endsWith(matrix.directory, '/package.json') - uses: actions/setup-node@v4 - with: - node-version: '18' - - - name: Setup Python - if: endsWith(matrix.directory, '/pyproject.toml') - run: uv python install - - - name: Check release - id: check - run: | - # Create unique hash for this directory - dir_hash=$(echo "${{ matrix.directory }}" | sha256sum | awk '{print $1}') - - # Run release check script with verbose output - echo "Running release check against last release: ${{ needs.prepare.outputs.last_release }}" - - # Run git diff first to show changes - echo "Changes since last release:" - git diff --name-only "${{ needs.prepare.outputs.last_release }}" -- "${{ matrix.directory }}" || true - - # Run the release check - output=$(uv run --script scripts/release.py --dry-run "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" 2>&1) - exit_code=$? - - echo "Release check output (exit code: $exit_code):" - echo "$output" - - # Extract package info if successful - if [ $exit_code -eq 0 ]; then - pkg_info=$(echo "$output" | grep -o -E "[a-zA-Z0-9\-]+@[0-9]+\.[0-9]+\.[0-9]+" || true) - else - echo "Release check failed" - exit 1 - fi - - if [ ! -z "$pkg_info" ]; then - echo "Found package that needs release: $pkg_info" - - # Create outputs directory - mkdir -p ./outputs - - # Save both package info and full changes - echo "$pkg_info" > "./outputs/${dir_hash}_info" - echo "dir_hash=${dir_hash}" >> $GITHUB_OUTPUT - - # Log what we're saving - echo "Saved package info to ./outputs/${dir_hash}_info:" - cat "./outputs/${dir_hash}_info" - else - echo "No release needed for this package" - fi - - - name: Set artifact name - if: steps.check.outputs.dir_hash - id: artifact - run: | - # Replace forward slashes with dashes - SAFE_DIR=$(echo "${{ matrix.directory }}" | tr '/' '-') - echo "name=release-outputs-${SAFE_DIR}" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v4 - if: steps.check.outputs.dir_hash - with: - name: ${{ steps.artifact.outputs.name }} - path: ./outputs/${{ steps.check.outputs.dir_hash }}* - - check-tag: - needs: [prepare, check-release] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/download-artifact@v4 - with: - pattern: release-outputs-src-* - merge-multiple: true - path: outputs - - - name: Simulate tag creation - run: | - if [ -d outputs ]; then - # Collect package info - find outputs -name "*_info" -exec cat {} \; > packages.txt - - if [ -s packages.txt ]; then - DATE=$(date +%Y.%m.%d) - echo "🔍 Dry run: Would create tag v${DATE} if this was a real release" - - # Generate comprehensive release notes - { - echo "# Release ${DATE}" - echo "" - echo "## Updated Packages" - while IFS= read -r line; do - echo "- $line" - done < packages.txt - } > notes.md - - echo "🔍 Would create release with following notes:" - cat notes.md - - echo "🔍 Would create tag v${DATE} with the above release notes" - echo "🔍 Would create GitHub release from tag v${DATE}" - else - echo "No packages need release" - fi - else - echo "No release artifacts found" - fi From a156ab635b4e10e64f6ce273c211b425bc3956e4 Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Tue, 14 Jan 2025 00:38:44 +0000 Subject: [PATCH 3/6] feat: only run create-tag job when packages are detected --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee6eebf3..b9cec3de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -90,6 +90,7 @@ jobs: create-tag: needs: [detect-packages, create-tag-name] + if: fromJson(needs.detect-packages.outputs.packages)[0] != null runs-on: ubuntu-latest environment: release permissions: From 9edf9fcaf043a7ef73cb8027742f3814407fe7aa Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Tue, 14 Jan 2025 00:39:59 +0000 Subject: [PATCH 4/6] feat: update package versions before creating release tag --- .github/workflows/release.yml | 76 ++++++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b9cec3de..2bd97e67 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -98,7 +98,15 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Create tag + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Update package versions and create tag env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -106,16 +114,66 @@ jobs: git config --global user.name "GitHub Actions" git config --global user.email "actions@github.com" - # Get packages array + # Get packages array and version PACKAGES='${{ needs.detect-packages.outputs.packages }}' + VERSION="${{ needs.create-tag-name.outputs.tag_name }}" + VERSION_NO_V="${VERSION#v}" # Remove 'v' prefix for package versions - if [ "$(echo "$PACKAGES" | jq 'length')" -gt 0 ]; then - # Create and push tag - git tag -a "${{ needs.create-tag-name.outputs.tag_name }}" -m "Release ${{ needs.create-tag-name.outputs.tag_name }}" - git push origin "${{ needs.create-tag-name.outputs.tag_name }}" - else - echo "No packages need release" - fi + # Create version update script + cat << 'EOF' > update_versions.py + import json + import os + import sys + import toml + from pathlib import Path + + def update_package_json(path, version): + with open(path) as f: + data = json.load(f) + data['version'] = version + with open(path, 'w') as f: + json.dump(data, f, indent=2) + f.write('\n') + + def update_pyproject_toml(path, version): + data = toml.load(path) + if 'project' in data: + data['project']['version'] = version + elif 'tool' in data and 'poetry' in data['tool']: + data['tool']['poetry']['version'] = version + with open(path, 'w') as f: + toml.dump(data, f) + + packages = json.loads(os.environ['PACKAGES']) + version = os.environ['VERSION'] + + for package_dir in packages: + package_dir = Path('src') / package_dir + + # Update package.json if it exists + package_json = package_dir / 'package.json' + if package_json.exists(): + update_package_json(package_json, version) + + # Update pyproject.toml if it exists + pyproject_toml = package_dir / 'pyproject.toml' + if pyproject_toml.exists(): + update_pyproject_toml(pyproject_toml, version) + EOF + + # Install toml package for Python + uv pip install toml + + # Update versions + PACKAGES="$PACKAGES" VERSION="$VERSION_NO_V" uv run update_versions.py + + # Commit version updates + git add src/*/package.json src/*/pyproject.toml + git commit -m "chore: update package versions to $VERSION" + + # Create and push tag + git tag -a "$VERSION" -m "Release $VERSION" + git push origin HEAD "$VERSION" - name: Create release env: From 8e944369b76dfb2bb8b47eec863cbdda49fef5b1 Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Tue, 14 Jan 2025 01:18:25 +0000 Subject: [PATCH 5/6] improve release workflow more --- .github/workflows/release.yml | 207 +++++++++------------------------- scripts/release.py | 201 +++++++++++++++++---------------- 2 files changed, 159 insertions(+), 249 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2bd97e67..b25ca834 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,10 +4,11 @@ on: workflow_dispatch: jobs: - detect-last-release: + create-metadata: runs-on: ubuntu-latest outputs: - last_release: ${{ steps.last-release.outputs.hash }} + hash: ${{ steps.last-release.outputs.hash }} + version: ${{ steps.create-version.outputs.version}} steps: - uses: actions/checkout@v4 with: @@ -20,23 +21,33 @@ jobs: echo "hash=${HASH}" >> $GITHUB_OUTPUT echo "Using last release hash: ${HASH}" - create-tag-name: - runs-on: ubuntu-latest - outputs: - tag_name: ${{ steps.last-release.outputs.tag}} - steps: - - name: Get last release hash - id: last-release + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Create version name + id: create-version + run: | + VERSION=$(uv run --script scripts/release.py generate-version) + echo "version $VERSION" + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Create notes run: | - DATE=$(date +%Y.%m.%d) - echo "tag=v${DATE}" >> $GITHUB_OUTPUT - echo "Using tag: v${DATE}" + HASH="${{ steps.last-release.outputs.hash }}" + uv run --script scripts/release.py generate-notes --directory src/ $HASH > RELEASE_NOTES.md + cat RELEASE_NOTES.md + + - name: Release notes + uses: actions/upload-artifact@v4 + with: + name: release-notes + path: RELEASE_NOTES.md - detect-packages: - needs: [detect-last-release] + update-packages: + needs: [create-metadata] runs-on: ubuntu-latest outputs: - packages: ${{ steps.find-packages.outputs.packages }} + changes_made: ${{ steps.commit.outputs.changes_made }} steps: - uses: actions/checkout@v4 with: @@ -45,52 +56,33 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 - - name: Find packages - id: find-packages - working-directory: src + - name: Update packages run: | - cat << 'EOF' > find_packages.py - import json - import os - import subprocess - from itertools import chain - from pathlib import Path + HASH="${{ needs.create-metadata.outputs.hash }}" + uv run --script scripts/release.py update-packages --directory src/ $HASH - packages = [] - - print("Starting package detection...") - print(f"Using LAST_RELEASE: {os.environ['LAST_RELEASE']}") - - # Find all directories containing package.json or pyproject.toml - paths = chain(Path('.').glob('*/package.json'), Path('.').glob('*/pyproject.toml')) - for path in paths: - print(f"\nChecking path: {path}") - # Check for changes in .py or .ts files - # Run git diff from the specific directory - cmd = ['git', 'diff', '--name-only', f'{os.environ["LAST_RELEASE"]}..HEAD', '--', '.'] - result = subprocess.run(cmd, capture_output=True, text=True, cwd=path.parent) - - # Check if any .py or .ts files were changed - changed_files = result.stdout.strip().split('\n') - print(f"Changed files found: {changed_files}") - - has_changes = any(f.endswith(('.py', '.ts')) for f in changed_files if f) - if has_changes: - print(f"Adding package: {path.parent}") - packages.append(str(path.parent)) - - print(f"\nFinal packages list: {packages}") - - # Write output - with open(os.environ['GITHUB_OUTPUT'], 'a') as f: - f.write(f"packages={json.dumps(packages)}\n") - EOF + - name: Configure git + run: | + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" - LAST_RELEASE=${{ needs.detect-last-release.outputs.last_release }} uv run --script --python 3.12 find_packages.py + - name: Commit changes + id: commit + run: | + VERSION="${{ needs.create-metadata.outputs.version }}" + git add -u + if git diff-index --quiet HEAD; then + echo "changes_made=false" >> $GITHUB_OUTPUT + else + git commit -m 'Automatic update of packages' + git tag -a "$VERSION" -m "Release $VERSION" + git push origin "$VERSION" + echo "changes_made=true" >> $GITHUB_OUTPUT + fi - create-tag: - needs: [detect-packages, create-tag-name] - if: fromJson(needs.detect-packages.outputs.packages)[0] != null + create-release: + needs: [update-packages, create-metadata] + if: needs.update-packages.outputs.changes_made == 'true' runs-on: ubuntu-latest environment: release permissions: @@ -98,103 +90,16 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install Node.js - uses: actions/setup-node@v4 + - name: Download release notes + uses: actions/download-artifact@v4 with: - node-version: '20' - - - name: Update package versions and create tag - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Configure git - git config --global user.name "GitHub Actions" - git config --global user.email "actions@github.com" - - # Get packages array and version - PACKAGES='${{ needs.detect-packages.outputs.packages }}' - VERSION="${{ needs.create-tag-name.outputs.tag_name }}" - VERSION_NO_V="${VERSION#v}" # Remove 'v' prefix for package versions - - # Create version update script - cat << 'EOF' > update_versions.py - import json - import os - import sys - import toml - from pathlib import Path - - def update_package_json(path, version): - with open(path) as f: - data = json.load(f) - data['version'] = version - with open(path, 'w') as f: - json.dump(data, f, indent=2) - f.write('\n') - - def update_pyproject_toml(path, version): - data = toml.load(path) - if 'project' in data: - data['project']['version'] = version - elif 'tool' in data and 'poetry' in data['tool']: - data['tool']['poetry']['version'] = version - with open(path, 'w') as f: - toml.dump(data, f) - - packages = json.loads(os.environ['PACKAGES']) - version = os.environ['VERSION'] - - for package_dir in packages: - package_dir = Path('src') / package_dir - - # Update package.json if it exists - package_json = package_dir / 'package.json' - if package_json.exists(): - update_package_json(package_json, version) - - # Update pyproject.toml if it exists - pyproject_toml = package_dir / 'pyproject.toml' - if pyproject_toml.exists(): - update_pyproject_toml(pyproject_toml, version) - EOF - - # Install toml package for Python - uv pip install toml - - # Update versions - PACKAGES="$PACKAGES" VERSION="$VERSION_NO_V" uv run update_versions.py - - # Commit version updates - git add src/*/package.json src/*/pyproject.toml - git commit -m "chore: update package versions to $VERSION" - - # Create and push tag - git tag -a "$VERSION" -m "Release $VERSION" - git push origin HEAD "$VERSION" + name: release-notes - name: Create release env: GH_TOKEN: ${{ secrets.RELEASE_TOKEN }} run: | - PACKAGES='${{ needs.detect-packages.outputs.packages }}' - - if [ "$(echo "$PACKAGES" | jq 'length')" -gt 0 ]; then - # Generate comprehensive release notes - { - echo "# Release ${{ needs.create-tag-name.outputs.tag_name }}" - echo "" - echo "## Updated Packages" - echo "$PACKAGES" | jq -r '.[]' | while read -r package; do - echo "- $package" - done - } > notes.md - # Create GitHub release - gh release create "${{ needs.create-tag-name.outputs.tag_name }}" \ - --title "Release ${{ needs.create-tag-name.outputs.tag_name }}" \ - --notes-file notes.md - else - echo "No packages need release" - fi + VERSION="${{ needs.create-metadata.outputs.version }}" + gh release create "$VERSION" \ + --title "Release $VERSION" \ + --notes-file RELEASE_NOTES.md diff --git a/scripts/release.py b/scripts/release.py index ace528eb..4b84c0af 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -1,6 +1,6 @@ #!/usr/bin/env uv run --script # /// script -# requires-python = ">=3.11" +# requires-python = ">=3.12" # dependencies = [ # "click>=8.1.8", # "tomlkit>=0.13.2" @@ -14,8 +14,8 @@ import tomlkit import datetime import subprocess -from enum import Enum -from typing import Any, NewType +from dataclasses import dataclass +from typing import Any, Iterator, NewType, Protocol Version = NewType("Version", str) @@ -51,25 +51,58 @@ def convert( GIT_HASH = GitHashParamType() -class PackageType(Enum): - NPM = 1 - PYPI = 2 +class Package(Protocol): + path: Path - @classmethod - def from_path(cls, directory: Path) -> "PackageType": - if (directory / "package.json").exists(): - return cls.NPM - elif (directory / "pyproject.toml").exists(): - return cls.PYPI - else: - raise Exception("No package.json or pyproject.toml found") + def package_name(self) -> str: ... + def update_version(self, version: Version) -> None: ... -def get_changes(path: Path, git_hash: str) -> bool: + +@dataclass +class NpmPackage: + path: Path + + def package_name(self) -> str: + with open(self.path / "package.json", "r") as f: + return json.load(f)["name"] + + def update_version(self, version: Version): + with open(self.path / "package.json", "r+") as f: + data = json.load(f) + data["version"] = version + f.seek(0) + json.dump(data, f, indent=2) + f.truncate() + + +@dataclass +class PyPiPackage: + path: Path + + def package_name(self) -> str: + with open(self.path / "pyproject.toml") as f: + toml_data = tomlkit.parse(f.read()) + name = toml_data.get("project", {}).get("name") + if not name: + raise Exception("No name in pyproject.toml project section") + return str(name) + + def update_version(self, version: Version): + # Update version in pyproject.toml + with open(self.path / "pyproject.toml") as f: + data = tomlkit.parse(f.read()) + data["project"]["version"] = version + + with open(self.path / "pyproject.toml", "w") as f: + f.write(tomlkit.dumps(data)) + + +def has_changes(path: Path, git_hash: GitHash) -> bool: """Check if any files changed between current state and git hash""" try: output = subprocess.run( - ["git", "diff", "--name-only", git_hash, "--", path], + ["git", "diff", "--name-only", git_hash, "--", "."], cwd=path, check=True, capture_output=True, @@ -77,105 +110,77 @@ def get_changes(path: Path, git_hash: str) -> bool: ) changed_files = [Path(f) for f in output.stdout.splitlines()] - relevant_files = [f for f in changed_files if f.suffix in ['.py', '.ts']] + relevant_files = [f for f in changed_files if f.suffix in [".py", ".ts"]] return len(relevant_files) >= 1 except subprocess.CalledProcessError: return False -def get_package_name(path: Path, pkg_type: PackageType) -> str: - """Get package name from package.json or pyproject.toml""" - match pkg_type: - case PackageType.NPM: - with open(path / "package.json", "rb") as f: - return json.load(f)["name"] - case PackageType.PYPI: - with open(path / "pyproject.toml") as f: - toml_data = tomlkit.parse(f.read()) - name = toml_data.get("project", {}).get("name") - if not name: - raise Exception("No name in pyproject.toml project section") - return str(name) - - -def generate_version() -> Version: +def gen_version() -> Version: """Generate version based on current date""" now = datetime.datetime.now() return Version(f"{now.year}.{now.month}.{now.day}") -def publish_package( - path: Path, pkg_type: PackageType, version: Version, dry_run: bool = False -): - """Publish package based on type""" - try: - match pkg_type: - case PackageType.NPM: - # Update version in package.json - with open(path / "package.json", "rb+") as f: - data = json.load(f) - data["version"] = version - f.seek(0) - json.dump(data, f, indent=2) - f.truncate() - - if not dry_run: - # Publish to npm - subprocess.run(["npm", "publish"], cwd=path, check=True) - case PackageType.PYPI: - # Update version in pyproject.toml - with open(path / "pyproject.toml") as f: - data = tomlkit.parse(f.read()) - data["project"]["version"] = version - - with open(path / "pyproject.toml", "w") as f: - f.write(tomlkit.dumps(data)) - - if not dry_run: - # Build and publish to PyPI - subprocess.run(["uv", "build"], cwd=path, check=True) - subprocess.run( - ["uv", "publish"], - cwd=path, - check=True, - ) - except Exception as e: - raise Exception(f"Failed to publish: {e}") from e - - -@click.command() -@click.argument("directory", type=click.Path(exists=True, path_type=Path)) +def find_changed_packages(directory: Path, git_hash: GitHash) -> Iterator[Package]: + for path in directory.glob("*/package.json"): + if has_changes(path.parent, git_hash): + yield NpmPackage(path.parent) + for path in directory.glob("*/pyproject.toml"): + if has_changes(path.parent, git_hash): + yield PyPiPackage(path.parent) + + +@click.group() +def cli(): + pass + + +@cli.command("update-packages") +@click.option( + "--directory", type=click.Path(exists=True, path_type=Path), default=Path.cwd() +) @click.argument("git_hash", type=GIT_HASH) +def update_packages(directory: Path, git_hash: GitHash) -> int: + # Detect package type + path = directory.resolve(strict=True) + version = gen_version() + + for package in find_changed_packages(path, git_hash): + name = package.package_name() + package.update_version(version) + + click.echo(f"{name}@{version}") + + return 0 + + +@cli.command("generate-notes") @click.option( - "--dry-run", is_flag=True, help="Update version numbers but don't publish" + "--directory", type=click.Path(exists=True, path_type=Path), default=Path.cwd() ) -def main(directory: Path, git_hash: GitHash, dry_run: bool) -> int: - """Release package if changes detected""" +@click.argument("git_hash", type=GIT_HASH) +def generate_notes(directory: Path, git_hash: GitHash) -> int: # Detect package type - try: - path = directory.resolve(strict=True) - pkg_type = PackageType.from_path(path) - except Exception as e: - return 1 + path = directory.resolve(strict=True) + version = gen_version() - # Check for changes - if not get_changes(path, git_hash): - return 0 + click.echo(f"# Release : v{version}") + click.echo("") + click.echo("## Updated packages") + for package in find_changed_packages(path, git_hash): + name = package.package_name() + click.echo(f"- {name}@{version}") + + return 0 - try: - # Generate version and publish - version = generate_version() - name = get_package_name(path, pkg_type) - publish_package(path, pkg_type, version, dry_run) - if not dry_run: - click.echo(f"{name}@{version}") - else: - click.echo(f"Dry run: Would have published {name}@{version}") - return 0 - except Exception as e: - return 1 +@cli.command("generate-version") +def generate_version() -> int: + # Detect package type + click.echo(gen_version()) + return 0 if __name__ == "__main__": - sys.exit(main()) + sys.exit(cli()) From 4f3dc110651e4f094c64a321ffb947840bcfe47e Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Tue, 14 Jan 2025 02:22:40 +0000 Subject: [PATCH 6/6] new release workflow --- .github/workflows/release.yml | 111 +++++++++++++++++++++++++++++++++- scripts/release.py | 24 ++++++++ 2 files changed, 133 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b25ca834..4f687118 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,6 +2,8 @@ name: Automatic Release Creation on: workflow_dispatch: + schedule: + - cron: '0 10 * * *' jobs: create-metadata: @@ -9,6 +11,8 @@ jobs: outputs: hash: ${{ steps.last-release.outputs.hash }} version: ${{ steps.create-version.outputs.version}} + npm_packages: ${{ steps.create-npm-packages.outputs.npm_packages}} + pypi_packages: ${{ steps.create-pypi-packages.outputs.pypi_packages}} steps: - uses: actions/checkout@v4 with: @@ -43,8 +47,25 @@ jobs: name: release-notes path: RELEASE_NOTES.md + - name: Create python matrix + id: create-pypi-packages + run: | + HASH="${{ steps.last-release.outputs.hash }}" + PYPI=$(uv run --script scripts/release.py generate-matrix --pypi --directory src $HASH) + echo "pypi_packages $PYPI" + echo "pypi_packages=$PYPI" >> $GITHUB_OUTPUT + + - name: Create npm matrix + id: create-npm-packages + run: | + HASH="${{ steps.last-release.outputs.hash }}" + NPM=$(uv run --script scripts/release.py generate-matrix --npm --directory src $HASH) + echo "npm_packages $NPM" + echo "npm_packages=$NPM" >> $GITHUB_OUTPUT + update-packages: needs: [create-metadata] + if: ${{ needs.create-metadata.outputs.npm_packages != '[]' || needs.create-metadata.outputs.pypi_packages != '[]' }} runs-on: ubuntu-latest outputs: changes_made: ${{ steps.commit.outputs.changes_made }} @@ -80,8 +101,94 @@ jobs: echo "changes_made=true" >> $GITHUB_OUTPUT fi - create-release: + publish-pypi: + needs: [update-packages, create-metadata] + strategy: + fail-fast: false + matrix: + package: ${{ fromJson(needs.create-metadata.outputs.pypi_packages) }} + name: Build ${{ matrix.package }} + environment: release + permissions: + id-token: write # Required for trusted publishing + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.create-metadata.outputs.version }} + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: "src/${{ matrix.package }}/.python-version" + + - name: Install dependencies + working-directory: src/${{ matrix.package }} + run: uv sync --frozen --all-extras --dev + + - name: Run pyright + working-directory: src/${{ matrix.package }} + run: uv run --frozen pyright + + - name: Build package + working-directory: src/${{ matrix.package }} + run: uv build + + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: src/${{ matrix.package }}/dist + + publish-npm: needs: [update-packages, create-metadata] + strategy: + fail-fast: false + matrix: + package: ${{ fromJson(needs.create-metadata.outputs.npm_packages) }} + name: Build ${{ matrix.package }} + environment: release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.create-metadata.outputs.version }} + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + registry-url: 'https://registry.npmjs.org' + + - name: Install dependencies + working-directory: src/${{ matrix.package }} + run: npm ci + + - name: Check if version exists on npm + working-directory: src/${{ matrix.package }} + run: | + VERSION=$(jq -r .version package.json) + if npm view --json | jq --arg version "$VERSION" '[.[]][0].versions | contains([$version])'; then + echo "Version $VERSION already exists on npm" + exit 1 + fi + echo "Version $VERSION is new, proceeding with publish" + + - name: Build package + working-directory: src/${{ matrix.package }} + run: npm run build + + - name: Publish package + working-directory: src/${{ matrix.package }} + run: | + npm publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + create-release: + needs: [update-packages, create-metadata, publish-pypi, publish-npm] if: needs.update-packages.outputs.changes_made == 'true' runs-on: ubuntu-latest environment: release @@ -97,7 +204,7 @@ jobs: - name: Create release env: - GH_TOKEN: ${{ secrets.RELEASE_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN}} run: | VERSION="${{ needs.create-metadata.outputs.version }}" gh release create "$VERSION" \ diff --git a/scripts/release.py b/scripts/release.py index 4b84c0af..05d76c0a 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -182,5 +182,29 @@ def generate_version() -> int: return 0 +@cli.command("generate-matrix") +@click.option( + "--directory", type=click.Path(exists=True, path_type=Path), default=Path.cwd() +) +@click.option("--npm", is_flag=True, default=False) +@click.option("--pypi", is_flag=True, default=False) +@click.argument("git_hash", type=GIT_HASH) +def generate_matrix(directory: Path, git_hash: GitHash, pypi: bool, npm: bool) -> int: + # Detect package type + path = directory.resolve(strict=True) + version = gen_version() + + changes = [] + for package in find_changed_packages(path, git_hash): + pkg = package.path.relative_to(path) + if npm and isinstance(package, NpmPackage): + changes.append(str(pkg)) + if pypi and isinstance(package, PyPiPackage): + changes.append(str(pkg)) + + click.echo(json.dumps(changes)) + return 0 + + if __name__ == "__main__": sys.exit(cli())