Merge branch 'main' into fds-sklearn-integration-tests

adap · Oct 12, 2023 · 927fba3 · 927fba3
2 parents 835b219 + 3d75c12
commit 927fba3
Show file tree

Hide file tree

Showing 405 changed files with 7,710 additions and 1,446 deletions.
diff --git a/.github/workflows/android-release.yml b/.github/workflows/android-release.yml
@@ -15,6 +15,7 @@ jobs:
         run:
             working-directory: src/kotlin
     name: Release build and publish
+    if: github.repository == 'adap/flower'
     runs-on: ubuntu-latest
     steps:
       - name: Check out code

diff --git a/.github/workflows/datasets.yml b/.github/workflows/datasets.yml
@@ -0,0 +1,46 @@
+name: Datasets
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "datasets/**"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "datasets/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    working-directory: datasets
+
+jobs:
+  test_core:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        # Latest version which comes cached in the host image can be found here:
+        # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md#python
+        # In case of a mismatch, the job has to download Python to install it.
+        # Note: Due to a bug in actions/setup-python we have to put 3.10 in
+        # qoutes as it will otherwise will assume 3.1
+        python: [3.8, 3.9, '3.10']
+
+    name: Python ${{ matrix.python }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Bootstrap
+        uses: ./.github/actions/bootstrap
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install dependencies (mandatory only)
+        run: python -m poetry install --all-extras
+      - name: Test (formatting + unit tests)
+        run: ./dev/test.sh
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -21,16 +21,22 @@ jobs:
     name: Build and deploy
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Bootstrap
         uses: ./.github/actions/bootstrap
       - name: Install pandoc
         run: sudo apt install pandoc
-      - name: Install dependencies (mandatory only)
+      - name: Install Flower dependencies (mandatory only)
         run: python -m poetry install --extras "simulation"
+      - name: Install Flower Datasets
+        run: |
+          cd datasets
+          python -m poetry install
       - name: Build docs
         run: ./dev/build-docs.sh
       - name: Deploy docs
-        if: github.ref == 'refs/heads/main'
+        if: ${{ github.ref == 'refs/heads/main' && github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork }}
         env:
           AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -39,3 +45,4 @@ jobs:
           aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./doc/build/html/ s3://flower.dev/docs/framework
           aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./baselines/doc/build/html/ s3://flower.dev/docs/baselines
           aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./examples/doc/build/html/ s3://flower.dev/docs/examples
+          aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./datasets/doc/build/html/ s3://flower.dev/docs/datasets
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -16,9 +16,43 @@ env:
   FLWR_TELEMETRY_ENABLED: 0
 
 jobs:
+  wheel:
+    runs-on: ubuntu-22.04
+    name: Build, test and upload wheel
+    steps:
+      - uses: actions/checkout@v4
+      - name: Bootstrap
+        uses: ./.github/actions/bootstrap
+      - name: Install dependencies (mandatory only)
+        run: python -m poetry install
+      - name: Build wheel
+        run: ./dev/build.sh
+      - name: Test wheel
+        run: ./dev/test-wheel.sh
+      - name: Upload wheel
+        if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork }}
+        id: upload
+        env:
+          AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }}
+        run: |
+          cd ./dist
+          echo "WHL_PATH=$(ls *.whl)" >> "$GITHUB_OUTPUT"
+          sha_short=$(git rev-parse --short HEAD)
+          echo "SHORT_SHA=$sha_short" >> "$GITHUB_OUTPUT"
+          [ -z "${{ github.head_ref }}" ] && dir="${{ github.ref_name }}" || dir="pr/${{ github.head_ref }}"
+          echo "DIR=$dir" >> "$GITHUB_OUTPUT"
+          aws s3 cp --content-disposition "attachment" --cache-control "no-cache" ./ s3://artifact.flower.dev/py/$dir/$sha_short --recursive
+    outputs:
+      whl_path: ${{ steps.upload.outputs.WHL_PATH }}
+      short_sha: ${{ steps.upload.outputs.SHORT_SHA }}
+      dir: ${{ steps.upload.outputs.DIR }}
+
   frameworks:
     runs-on: ubuntu-22.04
     timeout-minutes: 10
+    needs: wheel
     # Using approach described here:
     # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
     strategy:
@@ -89,6 +123,10 @@ jobs:
           python-version: 3.8
       - name: Install dependencies
         run: python -m poetry install
+      - name: Install Flower wheel from artifact store
+        if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork }}
+        run: |
+          python -m pip install https://artifact.flower.dev/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
       - name: Download dataset
         if: ${{ matrix.dataset }}
         run: python -c "${{ matrix.dataset }}"
@@ -102,6 +140,7 @@ jobs:
   strategies:
     runs-on: ubuntu-22.04
     timeout-minutes: 10
+    needs: wheel
     strategy:
       matrix:
         strat: ["FedMedian", "FedTrimmedAvg", "QFedAvg", "FaultTolerantFedAvg", "FedAvgM", "FedAdam", "FedAdagrad", "FedYogi"]
@@ -119,6 +158,10 @@ jobs:
       - name: Install dependencies
         run: |
           python -m poetry install
+      - name: Install Flower wheel from artifact store
+        if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork }}
+        run: |
+          python -m pip install https://artifact.flower.dev/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
       - name: Cache Datasets
         uses: actions/cache@v3
         with:

diff --git a/.github/workflows/flower-swift_sync.yml b/.github/workflows/flower-swift_sync.yml
@@ -12,6 +12,7 @@ concurrency:
 jobs:
   build:
     runs-on: ubuntu-latest
+    if: github.repository == 'adap/flower'
     steps:
       - uses: actions/checkout@v4
       - name: Pushes src/swift to flower-swift repository

diff --git a/.github/workflows/framework-release.yml b/.github/workflows/framework-release.yml
@@ -0,0 +1,63 @@
+name: Release Framework
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+jobs:
+  publish:
+    if: ${{ github.repository == 'adap/flower' }}
+    name: Publish draft
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Wait for wheel to be built
+        uses: lewagon/[email protected]
+        with:
+          ref: ${{ github.ref }}
+          check-name: 'Build, test and upload wheel'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 10
+      - name: Download wheel
+        run: |
+          tag_name=$(echo "${GITHUB_REF_NAME}" | cut -c2-)
+          echo "TAG_NAME=$tag_name" >> "$GITHUB_ENV"
+          
+          wheel_name="flwr-${tag_name}-py3-none-any.whl"
+          echo "WHEEL_NAME=$wheel_name" >> "$GITHUB_ENV"
+          
+          tar_name="flwr-${tag_name}.tar.gz"
+          echo "TAR_NAME=$tar_name" >> "$GITHUB_ENV"
+
+          wheel_url="https://artifact.flower.dev/py/main/${GITHUB_SHA::7}/${wheel_name}"
+          tar_url="https://artifact.flower.dev/py/main/${GITHUB_SHA::7}/${tar_name}"
+          
+          curl $wheel_url --output $wheel_name
+          curl $tar_url --output $tar_name
+      - name: Upload wheel
+        env:
+          AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }}
+        run: |
+          aws s3 cp --content-disposition "attachment" --cache-control "no-cache" ./${{ env.WHEEL_NAME }} s3://artifact.flower.dev/py/release/v${{ env.TAG_NAME }}/${{ env.WHEEL_NAME }}
+          aws s3 cp --content-disposition "attachment" --cache-control "no-cache" ./${{ env.TAR_NAME }} s3://artifact.flower.dev/py/release/v${{ env.TAG_NAME }}/${{ env.TAR_NAME }}
+          
+      - name: Generate body
+        run: |
+          ./dev/get-latest-changelog.sh > body.md
+          cat body.md
+          
+      - name: Release
+        uses: softprops/action-gh-release@de2c0eb
+        with:
+          body_path: ./body.md
+          draft: true
+          name: Flower ${{ env.TAG_NAME }}
+          files: |
+            ${{ env.WHEEL_NAME }}
+            ${{ env.TAR_NAME }}
diff --git a/.github/workflows/release.yml → .github/workflows/release-nightly.yml b/.github/workflows/release.yml → .github/workflows/release-nightly.yml
@@ -1,20 +1,17 @@
-name: Release
+name: Release nightly
 
 on:
   schedule:
     - cron: "0 23 * * *"
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
 env:
   FLWR_TELEMETRY_ENABLED: 0
 
 jobs:
-  nightly_release:
+  release_nightly:
     runs-on: ubuntu-22.04
     name: Nightly
+    if: github.repository == 'adap/flower'
     steps:
       - uses: actions/checkout@v4
       - name: Bootstrap

diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml
@@ -40,7 +40,7 @@ jobs:
 
   deploy_docs:
     needs: "build_docs"
-    if: github.ref == 'refs/heads/main'
+    if: ${{ github.ref == 'refs/heads/main' && github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork }}
     runs-on: macos-latest
     name: Deploy docs 
     steps:

diff --git a/.gitignore b/.gitignore
@@ -11,7 +11,7 @@ node_modules
 
 # Flower Examples
 examples/**/dataset/**
-examples/**/dev/*.sh
+#examples/**/dev/*.sh
 
 # Flower Baselines
 baselines/datasets/leaf

diff --git a/baselines/doc/source/conf.py b/baselines/doc/source/conf.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Adap GmbH. All Rights Reserved.
+# Copyright 2020 Flower Labs GmbH. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,7 +32,7 @@
 # -- Project information -----------------------------------------------------
 
 project = "Flower"
-copyright = "2022 Adap GmbH"
+copyright = "2022 Flower Labs GmbH"
 author = "The Flower Authors"
 
 # The full version, including alpha/beta/rc tags