Skip to content

Commit

Permalink
refactor(framework:skip) Refactor docker build matrix (#4224)
Browse files Browse the repository at this point in the history
Signed-off-by: Robert Steiner <[email protected]>
  • Loading branch information
Robert-Steiner authored Nov 13, 2024
1 parent 34b2bd6 commit 1308b8d
Show file tree
Hide file tree
Showing 4 changed files with 357 additions and 190 deletions.
33 changes: 16 additions & 17 deletions .github/workflows/docker-build-main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
outputs:
pip-version: ${{ steps.versions.outputs.pip-version }}
setuptools-version: ${{ steps.versions.outputs.setuptools-version }}
flwr-version-ref: ${{ steps.versions.outputs.flwr-version-ref }}
matrix: ${{ steps.versions.outputs.matrix }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

Expand All @@ -25,21 +25,26 @@ jobs:
run: |
echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
echo "flwr-version-ref=git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}" >> "$GITHUB_OUTPUT"
FLWR_VERSION_REF="git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}"
python dev/build-docker-image-matrix.py --flwr-version "${FLWR_VERSION_REF}" --matrix unstable > matrix.json
echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
build-docker-base-images:
name: Build base images
if: github.repository == 'adap/flower'
uses: ./.github/workflows/_docker-build.yml
needs: parameters
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.parameters.outputs.matrix).base }}
with:
namespace-repository: flwr/base
file-dir: src/docker/base/ubuntu
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: |
PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
FLWR_VERSION_REF=${{ needs.parameters.outputs.flwr-version-ref }}
tags: unstable
${{ matrix.images.build_args_encoded }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
Expand All @@ -48,21 +53,15 @@ jobs:
name: Build binary images
if: github.repository == 'adap/flower'
uses: ./.github/workflows/_docker-build.yml
needs: build-docker-base-images
needs: [parameters, build-docker-base-images]
strategy:
fail-fast: false
matrix:
images: [
{ repository: "flwr/superlink", file_dir: "src/docker/superlink" },
{ repository: "flwr/supernode", file_dir: "src/docker/supernode" },
{ repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
{ repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
]
matrix: ${{ fromJson(needs.parameters.outputs.matrix).binary }}
with:
namespace-repository: ${{ matrix.images.repository }}
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: BASE_IMAGE=unstable
tags: unstable
build-args: BASE_IMAGE=${{ matrix.images.base_image }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
8 changes: 4 additions & 4 deletions .github/workflows/framework-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ jobs:
- id: matrix
run: |
python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" > matrix.json
python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" --matrix stable > matrix.json
echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
build-base-images:
Expand All @@ -88,8 +88,8 @@ jobs:
build-args: |
PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
${{ matrix.images.build_args }}
tags: ${{ matrix.images.tag }}
${{ matrix.images.build_args_encoded }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
Expand All @@ -106,7 +106,7 @@ jobs:
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: BASE_IMAGE=${{ matrix.images.base_image }}
tags: ${{ matrix.images.tags }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
40 changes: 17 additions & 23 deletions .github/workflows/release-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,10 @@ jobs:
name: Relase nightly on PyPI
if: github.repository == 'adap/flower'
outputs:
name: ${{ steps.release.outputs.name }}
version: ${{ steps.release.outputs.version }}
skip: ${{ steps.release.outputs.skip }}
pip-version: ${{ steps.release.outputs.pip-version }}
setuptools-version: ${{ steps.release.outputs.setuptools-version }}
matrix: ${{ steps.release.outputs.matrix }}
steps:
- uses: actions/checkout@v4
- name: Bootstrap
Expand All @@ -33,27 +32,30 @@ jobs:
echo "skip=true" >> $GITHUB_OUTPUT
fi
echo "name=$(poetry version | awk {'print $1'})" >> $GITHUB_OUTPUT
echo "version=$(poetry version -s)" >> $GITHUB_OUTPUT
echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
NAME=$(poetry version | awk {'print $1'})
VERSION=$(poetry version -s)
python dev/build-docker-image-matrix.py --flwr-version "${VERSION}" --matrix nightly --flwr-package "${NAME}" > matrix.json
echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
build-docker-base-images:
name: Build nightly base images
if: github.repository == 'adap/flower' && needs.release-nightly.outputs.skip != 'true'
uses: ./.github/workflows/_docker-build.yml
needs: release-nightly
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.release-nightly.outputs.matrix).base }}
with:
namespace-repository: flwr/base
file-dir: src/docker/base/ubuntu
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: |
PIP_VERSION=${{ needs.release-nightly.outputs.pip-version }}
SETUPTOOLS_VERSION=${{ needs.release-nightly.outputs.setuptools-version }}
FLWR_VERSION=${{ needs.release-nightly.outputs.version }}
FLWR_PACKAGE=${{ needs.release-nightly.outputs.name }}
tags: |
${{ needs.release-nightly.outputs.version }}
nightly
${{ matrix.images.build_args_encoded }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
Expand All @@ -65,20 +67,12 @@ jobs:
needs: [release-nightly, build-docker-base-images]
strategy:
fail-fast: false
matrix:
images: [
{ repository: "flwr/superlink", file_dir: "src/docker/superlink" },
{ repository: "flwr/supernode", file_dir: "src/docker/supernode" },
{ repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
{ repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
]
matrix: ${{ fromJson(needs.release-nightly.outputs.matrix).binary }}
with:
namespace-repository: ${{ matrix.images.repository }}
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: BASE_IMAGE=${{ needs.release-nightly.outputs.version }}
tags: |
${{ needs.release-nightly.outputs.version }}
nightly
build-args: BASE_IMAGE=${{ matrix.images.base_image }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
Loading

0 comments on commit 1308b8d

Please sign in to comment.