Example for Custom Metrics calculation during Federated Learning #6277
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| FLWR_TELEMETRY_ENABLED: 0 | |
| jobs: | |
| wheel: | |
| runs-on: ubuntu-22.04 | |
| name: Build, test and upload wheel | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Bootstrap | |
| uses: ./.github/actions/bootstrap | |
| - name: Install dependencies (mandatory only) | |
| run: python -m poetry install | |
| - name: Build wheel | |
| run: ./dev/build.sh | |
| - name: Test wheel | |
| run: ./dev/test-wheel.sh | |
| - name: Upload wheel | |
| if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }} | |
| id: upload | |
| env: | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: | | |
| cd ./dist | |
| echo "WHL_PATH=$(ls *.whl)" >> "$GITHUB_OUTPUT" | |
| sha_short=$(git rev-parse --short HEAD) | |
| echo "SHORT_SHA=$sha_short" >> "$GITHUB_OUTPUT" | |
| [ -z "${{ github.head_ref }}" ] && dir="${{ github.ref_name }}" || dir="pr/${{ github.head_ref }}" | |
| echo "DIR=$dir" >> "$GITHUB_OUTPUT" | |
| aws s3 cp --content-disposition "attachment" --cache-control "no-cache" ./ s3://artifact.flower.dev/py/$dir/$sha_short --recursive | |
| outputs: | |
| whl_path: ${{ steps.upload.outputs.WHL_PATH }} | |
| short_sha: ${{ steps.upload.outputs.SHORT_SHA }} | |
| dir: ${{ steps.upload.outputs.DIR }} | |
| frameworks: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 10 | |
| needs: wheel | |
| # Using approach described here: | |
| # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs | |
| strategy: | |
| matrix: | |
| include: | |
| - directory: bare | |
| - directory: bare-https | |
| - directory: jax | |
| - directory: pytorch | |
| dataset: | | |
| from torchvision.datasets import CIFAR10 | |
| CIFAR10('./data', download=True) | |
| - directory: tensorflow | |
| dataset: | | |
| import tensorflow as tf | |
| tf.keras.datasets.cifar10.load_data() | |
| - directory: tabnet | |
| dataset: | | |
| import tensorflow_datasets as tfds | |
| tfds.load(name='iris', split=tfds.Split.TRAIN) | |
| - directory: opacus | |
| dataset: | | |
| from torchvision.datasets import CIFAR10 | |
| CIFAR10('./data', download=True) | |
| - directory: pytorch-lightning | |
| dataset: | | |
| from torchvision.datasets import MNIST | |
| MNIST('./data', download=True) | |
| - directory: mxnet | |
| dataset: | | |
| import mxnet as mx | |
| mx.test_utils.get_mnist() | |
| - directory: scikit-learn | |
| dataset: | | |
| import openml | |
| openml.datasets.get_dataset(554) | |
| - directory: fastai | |
| dataset: | | |
| from fastai.vision.all import untar_data, URLs | |
| untar_data(URLs.MNIST) | |
| - directory: pandas | |
| dataset: | | |
| from pathlib import Path | |
| from sklearn.datasets import load_iris | |
| Path('data').mkdir(exist_ok=True) | |
| load_iris(as_frame=True)['data'].to_csv('./data/client.csv') | |
| name: Framework / ${{matrix.directory}} | |
| defaults: | |
| run: | |
| working-directory: e2e/${{ matrix.directory }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Bootstrap | |
| uses: ./.github/actions/bootstrap | |
| with: | |
| python-version: 3.8 | |
| - name: Install dependencies | |
| run: python -m poetry install | |
| - name: Install Flower wheel from artifact store | |
| if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }} | |
| run: | | |
| python -m pip install https://artifact.flower.dev/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }} | |
| - name: Download dataset | |
| if: ${{ matrix.dataset }} | |
| run: python -c "${{ matrix.dataset }}" | |
| - name: Run edge client test | |
| run: ./../test.sh "${{ matrix.directory }}" | |
| - name: Run virtual client test | |
| run: python simulation.py | |
| - name: Run driver test | |
| run: ./../test_driver.sh "${{ matrix.directory }}" | |
| strategies: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 10 | |
| needs: wheel | |
| strategy: | |
| matrix: | |
| strat: ["FedMedian", "FedTrimmedAvg", "QFedAvg", "FaultTolerantFedAvg", "FedAvgM", "FedAdam", "FedAdagrad", "FedYogi"] | |
| name: Strategy / ${{ matrix.strat }} | |
| defaults: | |
| run: | |
| working-directory: e2e/strategies | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Bootstrap | |
| uses: ./.github/actions/bootstrap | |
| - name: Install dependencies | |
| run: | | |
| python -m poetry install | |
| - name: Install Flower wheel from artifact store | |
| if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }} | |
| run: | | |
| python -m pip install https://artifact.flower.dev/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }} | |
| - name: Cache Datasets | |
| uses: actions/cache@v3 | |
| with: | |
| path: "~/.keras" | |
| key: keras-datasets | |
| - name: Download Datasets | |
| run: | | |
| python -c "import tensorflow as tf; tf.keras.datasets.mnist.load_data()" | |
| - name: Test strategies | |
| run: | | |
| python test.py "${{ matrix.strat }}" |