Skip to content

Commit

Permalink
Merge branch 'main' into add-fds-e2e-tests-to-ci
Browse files Browse the repository at this point in the history
  • Loading branch information
tanertopal authored Nov 14, 2023
2 parents 50726ef + 4f3bb4f commit 6748217
Show file tree
Hide file tree
Showing 714 changed files with 77,728 additions and 2,812 deletions.
4 changes: 2 additions & 2 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ RUN apt install -y curl wget gnupg python3 python-is-python3 python3-pip git \
build-essential tmux vim

RUN python -m pip install \
pip==23.1.2 \
setuptools==68.0.0 \
pip==23.3.1 \
setuptools==68.2.2 \
poetry==1.5.1

USER $USERNAME
Expand Down
4 changes: 2 additions & 2 deletions .github/actions/bootstrap/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ inputs:
default: 3.8
pip-version:
description: "Version of pip to be installed using pip"
default: 23.1.2
default: 23.3.1
setuptools-version:
description: "Version of setuptools to be installed using pip"
default: 68.0.0
default: 68.2.2
poetry-version:
description: "Version of poetry to be installed using pip"
default: 1.5.1
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/android-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ jobs:
run:
working-directory: src/kotlin
name: Release build and publish
if: github.repository == 'adap/flower'
runs-on: ubuntu-latest
steps:
- name: Check out code
Expand Down
96 changes: 96 additions & 0 deletions .github/workflows/cpp.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
name: C++ SDK

on:
push:
branches: ['main']
paths: ['src/cc/flwr/**']
pull_request:
branches: ['main']
paths: ['src/cc/flwr/**']

jobs:
build_and_test:
name: Build and test
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- name: Bootstrap
uses: ./.github/actions/bootstrap

- name: Cache restore SDK build
uses: actions/cache/restore@v3
with:
path: build/
key: ${{ runner.os }}-sdk-build

- name: Cache restore example build
uses: actions/cache/restore@v3
with:
path: examples/quickstart-cpp/build/
key: ${{ runner.os }}-example-build

- name: Install Dependencies
run: |
sudo apt-get update
sudo apt-get install -y clang-format cmake g++ clang-tidy cppcheck
- name: Check source Formatting
run: |
find src/cc/flwr/src -name '*.cc' | xargs clang-format -i
git diff --exit-code
- name: Check header Formatting
run: |
find src/cc/flwr/include -name '*.h' -not -path "src/cc/flwr/include/flwr/*" | xargs clang-format -i
git diff --exit-code
- name: Build
run: |
mkdir -p build
cd build
cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ../src/cc/flwr
make
- name: Run clang-tidy
run: |
cd build
find ../src/cc/flwr/src -name '*.cc' | xargs clang-tidy
- name: Run cppcheck
run: |
cd build
cppcheck --enable=all -I../src/cc/flwr/include ../src/cc/flwr/src
- name: End-to-end test
run: |
cd examples/quickstart-cpp
cmake -DUSE_LOCAL_FLWR=ON -S . -B build
cmake --build build
pip install ../..
timeout 2m python server.py &
pid=$!
sleep 3
build/flwr_client 0 127.0.0.1:8080 &
sleep 3
build/flwr_client 1 127.0.0.1:8080 &
wait $pid
res=$?
if [[ "$res" = "0" ]];
then echo "Training worked correctly";
else echo "Training had an issue" && exit 1;
fi
- name: Cache save SDK build
uses: actions/cache/save@v3
if: github.ref_name == 'main'
with:
path: build/
key: ${{ runner.os }}-sdk-build

- name: Cache save example build
uses: actions/cache/save@v3
if: github.ref_name == 'main'
with:
path: examples/quickstart-cpp/build/
key: ${{ runner.os }}-example-build
4 changes: 4 additions & 0 deletions .github/workflows/datasets.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,13 @@ on:
push:
branches:
- main
paths:
- "datasets/**"
pull_request:
branches:
- main
paths:
- "datasets/**"

concurrency:
group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}
Expand Down
11 changes: 9 additions & 2 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,22 @@ jobs:
name: Build and deploy
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Bootstrap
uses: ./.github/actions/bootstrap
- name: Install pandoc
run: sudo apt install pandoc
- name: Install dependencies (mandatory only)
- name: Install Flower dependencies (mandatory only)
run: python -m poetry install --extras "simulation"
- name: Install Flower Datasets
run: |
cd datasets
python -m poetry install
- name: Build docs
run: ./dev/build-docs.sh
- name: Deploy docs
if: github.ref == 'refs/heads/main'
if: ${{ github.ref == 'refs/heads/main' && github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork }}
env:
AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
Expand All @@ -39,3 +45,4 @@ jobs:
aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./doc/build/html/ s3://flower.dev/docs/framework
aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./baselines/doc/build/html/ s3://flower.dev/docs/baselines
aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./examples/doc/build/html/ s3://flower.dev/docs/examples
aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./datasets/doc/build/html/ s3://flower.dev/docs/datasets
43 changes: 43 additions & 0 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,43 @@ env:
FLWR_TELEMETRY_ENABLED: 0

jobs:
wheel:
runs-on: ubuntu-22.04
name: Build, test and upload wheel
steps:
- uses: actions/checkout@v4
- name: Bootstrap
uses: ./.github/actions/bootstrap
- name: Install dependencies (mandatory only)
run: python -m poetry install
- name: Build wheel
run: ./dev/build.sh
- name: Test wheel
run: ./dev/test-wheel.sh
- name: Upload wheel
if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork }}
id: upload
env:
AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }}
run: |
cd ./dist
echo "WHL_PATH=$(ls *.whl)" >> "$GITHUB_OUTPUT"
sha_short=$(git rev-parse --short HEAD)
echo "SHORT_SHA=$sha_short" >> "$GITHUB_OUTPUT"
[ -z "${{ github.head_ref }}" ] && dir="${{ github.ref_name }}" || dir="pr/${{ github.head_ref }}"
echo "DIR=$dir" >> "$GITHUB_OUTPUT"
aws s3 cp --content-disposition "attachment" --cache-control "no-cache" ./ s3://artifact.flower.dev/py/$dir/$sha_short --recursive
outputs:
whl_path: ${{ steps.upload.outputs.WHL_PATH }}
short_sha: ${{ steps.upload.outputs.SHORT_SHA }}
dir: ${{ steps.upload.outputs.DIR }}

frameworks:
runs-on: ubuntu-22.04
timeout-minutes: 10
needs: wheel
# Using approach described here:
# https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
strategy:
Expand Down Expand Up @@ -89,6 +123,10 @@ jobs:
python-version: 3.8
- name: Install dependencies
run: python -m poetry install
- name: Install Flower wheel from artifact store
if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork }}
run: |
python -m pip install https://artifact.flower.dev/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
- name: Download dataset
if: ${{ matrix.dataset }}
run: python -c "${{ matrix.dataset }}"
Expand All @@ -102,6 +140,7 @@ jobs:
strategies:
runs-on: ubuntu-22.04
timeout-minutes: 10
needs: wheel
strategy:
matrix:
strat: ["FedMedian", "FedTrimmedAvg", "QFedAvg", "FaultTolerantFedAvg", "FedAvgM", "FedAdam", "FedAdagrad", "FedYogi"]
Expand All @@ -119,6 +158,10 @@ jobs:
- name: Install dependencies
run: |
python -m poetry install
- name: Install Flower wheel from artifact store
if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork }}
run: |
python -m pip install https://artifact.flower.dev/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
- name: Cache Datasets
uses: actions/cache@v3
with:
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/flower-swift_sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ concurrency:
jobs:
build:
runs-on: ubuntu-latest
if: github.repository == 'adap/flower'
steps:
- uses: actions/checkout@v4
- name: Pushes src/swift to flower-swift repository
Expand Down
63 changes: 63 additions & 0 deletions .github/workflows/framework-release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
name: Release Framework

on:
push:
tags:
- "v*.*.*"

jobs:
publish:
if: ${{ github.repository == 'adap/flower' }}
name: Publish draft
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Wait for wheel to be built
uses: lewagon/[email protected]
with:
ref: ${{ github.ref }}
check-name: 'Build, test and upload wheel'
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 10
- name: Download wheel
run: |
tag_name=$(echo "${GITHUB_REF_NAME}" | cut -c2-)
echo "TAG_NAME=$tag_name" >> "$GITHUB_ENV"
wheel_name="flwr-${tag_name}-py3-none-any.whl"
echo "WHEEL_NAME=$wheel_name" >> "$GITHUB_ENV"
tar_name="flwr-${tag_name}.tar.gz"
echo "TAR_NAME=$tar_name" >> "$GITHUB_ENV"
wheel_url="https://artifact.flower.dev/py/main/${GITHUB_SHA::7}/${wheel_name}"
tar_url="https://artifact.flower.dev/py/main/${GITHUB_SHA::7}/${tar_name}"
curl $wheel_url --output $wheel_name
curl $tar_url --output $tar_name
- name: Upload wheel
env:
AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }}
run: |
aws s3 cp --content-disposition "attachment" --cache-control "no-cache" ./${{ env.WHEEL_NAME }} s3://artifact.flower.dev/py/release/v${{ env.TAG_NAME }}/${{ env.WHEEL_NAME }}
aws s3 cp --content-disposition "attachment" --cache-control "no-cache" ./${{ env.TAR_NAME }} s3://artifact.flower.dev/py/release/v${{ env.TAG_NAME }}/${{ env.TAR_NAME }}
- name: Generate body
run: |
./dev/get-latest-changelog.sh > body.md
cat body.md
- name: Release
uses: softprops/action-gh-release@de2c0eb
with:
body_path: ./body.md
draft: true
name: Flower ${{ env.TAG_NAME }}
files: |
${{ env.WHEEL_NAME }}
${{ env.TAR_NAME }}
1 change: 1 addition & 0 deletions .github/workflows/release-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ jobs:
release_nightly:
runs-on: ubuntu-22.04
name: Nightly
if: github.repository == 'adap/flower'
steps:
- uses: actions/checkout@v4
- name: Bootstrap
Expand Down
26 changes: 0 additions & 26 deletions .github/workflows/release.yml

This file was deleted.

8 changes: 4 additions & 4 deletions .github/workflows/swift.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
name: Test
runs-on: macos-latest
steps:
- uses: fwal/setup-swift@cdcbe8e35d3801acd82d7109285e6ab22c9212c2
- uses: fwal/setup-swift@f51889efb55dccf13be0ee727e3d6c89a096fb4c
with:
swift-version: 5
- uses: actions/checkout@v4
Expand All @@ -31,7 +31,7 @@ jobs:
runs-on: macos-latest
name: Build docs
steps:
- uses: fwal/setup-swift@cdcbe8e35d3801acd82d7109285e6ab22c9212c2
- uses: fwal/setup-swift@f51889efb55dccf13be0ee727e3d6c89a096fb4c
with:
swift-version: 5
- uses: actions/checkout@v4
Expand All @@ -40,11 +40,11 @@ jobs:

deploy_docs:
needs: "build_docs"
if: github.ref == 'refs/heads/main'
if: ${{ github.ref == 'refs/heads/main' && github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork }}
runs-on: macos-latest
name: Deploy docs
steps:
- uses: fwal/setup-swift@cdcbe8e35d3801acd82d7109285e6ab22c9212c2
- uses: fwal/setup-swift@f51889efb55dccf13be0ee727e3d6c89a096fb4c
with:
swift-version: 5
- uses: actions/checkout@v4
Expand Down
Loading

0 comments on commit 6748217

Please sign in to comment.