Galaxy Training Network news #1915
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # Pull news from the Galaxy Training Network's RSS feed and add them as | |
| # Galaxy Hub posts. | |
| name: Galaxy Training Network news | |
| env: | |
| base: master | |
| branch: gtn-news | |
| github_api_version: "2022-11-28" | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: 0 * * * * | |
| jobs: | |
| collect: | |
| name: Collect news from the Galaxy Training Network | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install feedparser. | |
| run: pip install feedparser~=6.0 | |
| - name: Parse GTN RSS feed. | |
| id: parse | |
| shell: python -u {0} | |
| run: | | |
| import json | |
| import os | |
| from datetime import datetime, timedelta, timezone | |
| import feedparser | |
| feedparser.sanitizer._HTMLSanitizer.acceptable_elements.remove("a") | |
| feedparser.sanitizer._HTMLSanitizer.acceptable_elements.remove("img") | |
| feed = feedparser.parse("https://training.galaxyproject.org/training-material/feed.xml") | |
| posts = {} | |
| oldest = datetime.now().astimezone() | |
| for entry in feed.entries: | |
| feed_title = f"[GTN news] {entry['title']}" | |
| feed_link = entry["link"] | |
| feed_blurb = entry["summary"] | |
| feed_published = entry["published"].split("T")[0] | |
| feed_name = feed_link.split("/")[-1].replace(".html", "") | |
| feed_author = entry["author"] | |
| posts[f"{feed_published}-{feed_name}"] = { | |
| "title": feed_title, | |
| "link": feed_link, | |
| "blurb": feed_blurb, | |
| "published": feed_published, | |
| "name": feed_name, | |
| "author": feed_author, | |
| } | |
| feed_published_dt = datetime.fromisoformat(entry["published"]) | |
| oldest = feed_published_dt if feed_published_dt < oldest else oldest | |
| oldest = (oldest - timedelta(days=1)).astimezone(timezone.utc).date() | |
| # reverse ordering (sort by ascending date) | |
| posts = { | |
| post: posts[post] | |
| for post in tuple(posts)[::-1] | |
| } | |
| with open(os.environ["GITHUB_OUTPUT"], 'w') as file: | |
| print("posts=%s" % json.dumps(posts), file=file) | |
| print( | |
| "matrix=%s" % f"{json.dumps({'name': list(posts)})}", | |
| file=file | |
| ) | |
| # Not strictly necessary but produces a workflow with less jobs. | |
| - name: Filter out already published posts. | |
| id: filter | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| readarray -t all_posts <<< $(jq -r 'keys_unsorted | .[]' <<< '${{ steps.parse.outputs.posts }}') | |
| posts=() | |
| set +e | |
| for post in "${all_posts[@]}"; do | |
| # check in the base branch | |
| gh api \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: ${{ env.github_api_version }}" \ | |
| /repos/${{ github.repository }}/contents/content/news/"$post"/index.md?ref=${{ env.base }} | |
| test $? -ne 0 || continue | |
| # check in the target branch | |
| gh api \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: ${{ env.github_api_version }}" \ | |
| /repos/${{ github.repository }}/contents/content/news/"$post"/index.md?ref=${{ env.branch }} | |
| test $? -ne 0 || continue | |
| posts+=("$post") | |
| done | |
| set -e | |
| json=$(printf '%s\n' "${posts[@]}" | jq -R . | jq -s --compact-output .) | |
| matrix=$(jq -n --compact-output --argjson v "$json" '{"name": $v}') | |
| echo matrix="$matrix" >> $GITHUB_OUTPUT | |
| outputs: | |
| posts: ${{ steps.parse.outputs.posts }} | |
| matrix: ${{ steps.filter.outputs.matrix }} | |
| commit: | |
| name: Create commits | |
| runs-on: ubuntu-latest | |
| needs: collect | |
| if: ${{ needs.collect.outputs.matrix != '{"name":[""]}' }} | |
| strategy: | |
| max-parallel: 1 | |
| fail-fast: false | |
| matrix: ${{fromJson(needs.collect.outputs.matrix)}} | |
| steps: | |
| - name: Create the Galaxy Hub post. | |
| shell: python -u {0} | |
| run: | | |
| import json | |
| import textwrap | |
| post = json.loads(""" | |
| ${{ needs.collect.outputs.posts }} | |
| """)["${{ matrix.name }}"] | |
| post = textwrap.dedent(f""" | |
| --- | |
| subsites: [all] | |
| main_subsite: global | |
| date: "{post['published']}" | |
| tags: [training, gtn-news] | |
| title: "{post['title']}" | |
| authors: "{post['author']}" | |
| external_url: '{post["link"]}' | |
| --- | |
| {post["blurb"]} | |
| """)[1:] | |
| with open("index.md", "w") as file: | |
| file.write(post) | |
| - name: Create the PR branch. | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Check if the PR branch exists. | |
| set +e | |
| gh api \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: ${{ env.github_api_version }}" \ | |
| /repos/${{ github.repository }}/git/ref/heads/${{ env.branch }} | |
| test $? -ne 0 || exit 0 | |
| set -e | |
| # Get commit sha of the tip of the base branch. | |
| sha=$(\ | |
| gh api \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "X-GitHub-Api-Version: ${{ env.github_api_version }}" \ | |
| /repos/${{ github.repository }}/git/ref/heads/${{ env.base }} \ | |
| | jq -r '.object.sha' \ | |
| ) | |
| # Create the PR branch. | |
| gh api \ | |
| --method POST \ | |
| -H "Accept: application/vnd.github.v3+json" \ | |
| -H "X-GitHub-Api-Version: ${{ env.github_api_version }}" \ | |
| /repos/${{ github.repository }}/git/refs \ | |
| -f ref='refs/heads/${{ env.branch }}' \ | |
| -f sha=$sha | |
| - name: Commit the Galaxy Hub post. | |
| run: | | |
| base64=$(base64 --wrap=0 < index.md) | |
| commit='{ | |
| "message": "Post ${{ matrix.name }} from GTN", | |
| "author": { | |
| "name": "GitHub", | |
| "email": "[email protected]" | |
| }, | |
| "branch": "${{ env.branch }}", | |
| "content": "'$base64'" | |
| }' | |
| echo $commit > data.json | |
| curl -L \ | |
| -X PUT \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ github.token }}" \ | |
| -H "X-GitHub-Api-Version: ${{ env.github_api_version }}" \ | |
| https://api.github.com/repos/${{ github.repository }}/contents/content/news/${{ matrix.name }}/index.md \ | |
| -d @data.json | |
| pull_request: | |
| name: Create pull request | |
| runs-on: ubuntu-latest | |
| needs: commit | |
| if: always() | |
| steps: | |
| - name: Create a pull request. | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Check if an open pull request already exists. | |
| exists=$(gh pr list \ | |
| --repo ${{ github.repository }} \ | |
| --head ${{ env.branch }} \ | |
| --state open \ | |
| --limit 1 \ | |
| --json number \ | |
| | jq '.[].number' | wc -l) | |
| test $exists -ne 1 || exit 0 | |
| set +e | |
| gh pr create \ | |
| --repo ${{ github.repository }} \ | |
| --base "${{ env.base }}" \ | |
| --head "${{ env.branch }}" \ | |
| --title "Publish latest news from the Galaxy Training Network" \ | |
| --body "Publish the latest news from the Galaxy Training Network." | |
| true |