Merge branch 'develop' into dependabot/pip/protobuf-gte-4.22-and-lt-6…

….0.0

.github/dependabot.yml

@@ -5,3 +5,22 @@ updates:
     schedule:
       interval: "daily"
     open-pull-requests-limit: 3
+  # Don't update these directories.
+  - package-ecosystem: pip
+    directory: /tests
+    schedule:
+      interval: "daily"
+    ignore:
+      - dependency-name: "*"
+  - package-ecosystem: pip
+    directory: /openfl-workspace
+    schedule:
+      interval: "daily"
+    ignore:
+      - dependency-name: "*"
+  - package-ecosystem: pip
+    directory: /openfl-tutorials/interactive_api
+    schedule:
+      interval: "daily"
+    ignore:
+      - dependency-name: "*"

.github/workflows/docker-bench-security.yml

@@ -0,0 +1,56 @@
+name: Docker Bench for Security
+
+on:
+  pull_request:
+    branches: [ develop ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.8"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install .
+    - name: Clean Docker System
+      run: |
+        docker image prune -a -f
+        docker system prune -a -f
+    - name: Clone Docker Bench Security Repo
+      run: git clone https://github.com/docker/docker-bench-security.git
+    - name: Build Docker Bench Security Image
+      run: |
+        cd docker-bench-security
+        docker build --no-cache -t docker-bench-security .
+    - name: Create results directory
+      run: mkdir -p results
+    - name: Run Docker Bench for Security
+      run: |
+        docker run --rm --net host --pid host --userns host --cap-add audit_control \
+          -e DOCKER_CONTENT_TRUST=0 \
+          -v /etc:/etc:ro \
+          -v /lib/systemd/system:/lib/systemd/system:ro \
+          -v /usr/bin/containerd:/usr/bin/containerd:ro \
+          -v /usr/bin/runc:/usr/bin/runc:ro \
+          -v /usr/lib/systemd:/usr/lib/systemd:ro \
+          -v /var/lib:/var/lib:ro \
+          -v /var/run/docker.sock:/var/run/docker.sock:ro \
+          -v "$(pwd)/results:/results" \
+          --label docker_bench_security \
+          docker-bench-security | tee results/docker_bench_security_report.txt
+
+    - name: Upload Security Bench Report
+      uses: actions/upload-artifact@v3
+      with:
+        name: docker_bench_security-report
+        path: results/docker_bench_security_report.txt

.github/workflows/gandlf.yml

@@ -29,7 +29,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install typer==0.11.1
-        pip install torch==2.1.0+cpu torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
+        pip install torch==2.3.1+cpu torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
         pip install .
     - name: Install GaNDLF
       run: |

.github/workflows/lint.yml

@@ -24,6 +24,6 @@ jobs:
     - name: Install linters
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements-linters.txt 
+        pip install -r linters-requirements.txt 
     - name: Lint using built-in script
-      run: bash shell/lint.sh
+      run: bash shell/lint.sh

.github/workflows/pytest_coverage.yml

@@ -29,7 +29,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pytest coverage
-        pip install -r requirements-test.txt
+        pip install -r test-requirements.txt
         pip install .
     - name: Test with pytest and report code coverage
       run: |

.github/workflows/scans.yml

@@ -28,3 +28,9 @@ jobs:
     - name: Hadolint Dockerfile Scan
       run: |
         docker run -v ${PWD}/openfl-docker:/openfl-docker --rm -i hadolint/hadolint hadolint -t error /openfl-docker/Dockerfile.base
+        docker run -v ${PWD}/openfl-docker:/openfl-docker --rm -i hadolint/hadolint hadolint -t error -f json /openfl-docker/Dockerfile.base > hadolint_output.json
+    - name: Upload Hadolint JSON Report
+      uses: actions/upload-artifact@v3
+      with:
+        name: hadolint-report
+        path: hadolint_output.json

.github/workflows/trivy.yml

@@ -20,7 +20,7 @@ jobs:
           docker build --pull -t docker.io/securefederatedai/openfl:${{ github.sha }} -f openfl-docker/Dockerfile.base .
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@0.24.0
         with:
           image-ref: 'docker.io/securefederatedai/openfl:${{ github.sha }}'
           format: 'sarif'
@@ -30,3 +30,61 @@ jobs:
         uses: github/codeql-action/upload-sarif@v2
         with:
           sarif_file: 'trivy-results.sarif'
+
+      - name: Install Trivy
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.55.0
+  
+      - name: Run Trivy code vulnerability scanner (JSON Output)
+        run: |
+          trivy --quiet fs --format json --output trivy-code-results.json --ignore-unfixed --vuln-type os,library --severity CRITICAL,HIGH,MEDIUM,LOW .
+
+      - name: Upload Code Vulnerability Scan Results
+        uses: actions/upload-artifact@v3
+        with:
+          name: trivy-code-report-json
+          path: trivy-code-results.json
+
+      - name: Run Trivy vulnerability scanner for Docker image (JSON Output)
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: 'docker.io/securefederatedai/openfl:${{ github.sha }}'
+          format: 'json'
+          output: 'trivy-docker-results.json'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH,MEDIUM,LOW'
+
+      - name: Upload Docker Vulnerability Scan
+        uses: actions/upload-artifact@v3
+        with:
+          name: trivy-docker-report-json
+          path: trivy-docker-results.json
+
+      - name: Run Trivy code vulnerability scanner (SPDX-JSON Output)
+        run: |
+          trivy --quiet fs --format spdx-json --output trivy-code-spdx-results.json --ignore-unfixed --vuln-type os,library --severity CRITICAL,HIGH,MEDIUM,LOW .
+  
+      - name: Upload Code Vulnerability Scan Results
+        uses: actions/upload-artifact@v3
+        with:
+          name: trivy-code-spdx-report-json
+          path: trivy-code-spdx-results.json
+
+      - name: Run Trivy vulnerability scanner for Docker image (SPDX-JSON Output)
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: 'docker.io/securefederatedai/openfl:${{ github.sha }}'
+          format: 'spdx-json'
+          output: 'trivy-docker-spdx-results.json'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH,MEDIUM,LOW'
+
+      - name: Upload Docker Vulnerability Scan
+        uses: actions/upload-artifact@v3
+        with:
+          name: trivy-docker-spdx-report-json
+          path: trivy-docker-spdx-results.json

.github/workflows/ubuntu.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements-linters.txt 
+        pip install -r linters-requirements.txt 
         pip install .
     - name: Lint with flake8
       run: |
@@ -38,15 +38,15 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pytest coverage
-        pip install -r requirements-test.txt
+        pip install -r test-requirements.txt
         pip install .
     - name: Test with pytest and report code coverage
       run: |
         coverage run -m pytest -rA
         coverage report
   
   interactive-kvasir: # from interactive-kvasir.yml
-    needs: [lint, pytest-coverage, dockerization]
+    needs: [lint, pytest-coverage]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
@@ -66,7 +66,7 @@ jobs:
         python -m tests.github.interactive_api_director.experiments.pytorch_kvasir_unet.run
   
   cli:
-    needs: [lint, pytest-coverage, dockerization, interactive-kvasir]
+    needs: [lint, pytest-coverage, interactive-kvasir]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -80,4 +80,4 @@ jobs:
           pip install .
       - name: Test TaskRunner API
         run: |
-          python -m tests.github.test_hello_federation --template keras_cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3 --save-model output_model
+          python -m tests.github.test_hello_federation --template keras_cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3 --save-model output_model

.github/workflows/windows.yml

@@ -20,7 +20,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pytest coverage
-        pip install -r requirements-test.txt
+        pip install -r test-requirements.txt
         pip install .
     - name: Test with pytest and report code coverage
       run: |

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+---
+minimum_pre_commit_version: 3.3.3
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: end-of-file-fixer
+        files: \.py$
+      - id: requirements-txt-fixer
+      - id: check-merge-conflict
+      - id: check-case-conflict
+      - id: check-json
+        files: \.json$
+      - id: check-yaml
+        files: \.yaml$
+      - id: debug-statements
+      - id: pretty-format-json
+        args:
+          - --autofix
+        files: \.json$
+      - id: trailing-whitespace
+        files: \.py$
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.0
+    hooks:
+      - id: autoflake
+        args:
+          - --in-place
+          - --remove-unused-variables
+          - --recursive
+          - --ignore-pass-statements

.readthedocs.yaml

@@ -32,4 +32,6 @@ sphinx:
 # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
 python:
   install:
-    - requirements: docs/requirements-docs.txt
+    - method: pip
+      path: .
+    - requirements: docs/docs-requirements.txt

CONTRIBUTING.md

@@ -1,7 +1,7 @@
 # Contributing to OpenFL
 
 We welcome contributions from the community. There are several ways to contribute:
-* Improvements in [documentation](https://openfl.readthedocs.io/en/latest/install.html).
+* Improvements in [documentation](https://openfl.readthedocs.io/en/latest/).
 * Contributing to OpenFL's code-base: via bug-fixes or feature additions.
 * Answering questions on our [discussions page](https://github.com/securefederatedai/openfl/discussions).
 * Participating in our [roadmap](https://github.com/securefederatedai/openfl/blob/develop/ROADMAP.md) discussions.
@@ -55,11 +55,12 @@ git clone https://github.com/YOUR_GITHUB_USERNAME/openfl.git
 cd openfl
 pip install -U pip setuptools wheel
 pip install .
+pip install -r requirements-linters.txt
 ```
 
 ## Code style
 
-OpenFL uses [black](https://black.readthedocs.io/en/stable/) and [isort](https://pycqa.github.io/isort/) to format the code.
+OpenFL uses [black](https://black.readthedocs.io/en/stable/), [isort](https://pycqa.github.io/isort/) and [precommit](https://pre-commit.com/) to format the code.
 
 Run the following command at the **root** directory of the repo to format your code.
 
@@ -75,4 +76,4 @@ A **class** or a **function** docstring may contain:
 * A one-line description of the class/function.
 * Paragraph(s) of detailed information.
 * Optional `Examples` section.
-* `Args` section for arguments under `__init__()`.
+* `Args` section for arguments under `__init__()`.

docs/README.md

@@ -3,10 +3,16 @@
 We use sphinx to generate the documentation for this project.
 The documentation project has been initialized properly and we basically just need to update the actual content.
 
+Install the openfl package
+
+```sh
+pip install .
+```
+
 Install requirements for building documentation:
 
 ```sh
-pip install -r requirements-docs.txt
+pip install -r docs-requirements.txt
 ```
 
 

docs/about/releases.md

@@ -1,5 +1,31 @@
 # Releases
 
+## 1.6
+[Full Release Notes](https://github.com/securefederatedai/openfl/releases/tag/v1.6)
+
+### New Features and APIs: 
+- **Federated LLM fine-tuning**: 
+     - [**Horovod**](https://github.com/securefederatedai/openfl/tree/develop/openfl-workspace/torch_llm_horovod): Use horovod to efficiently train LLMs across multiple private clusters  
+     - **Neuralchat-7b fine-tuning**: Learn how to fine-tune [neuralchat-7b](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/experimental/LLM/neuralchat) using the Intel® Extension for Transformers and the workflow interface. 
+
+- **Workflow API enhancements**: Introducing an experimental [Workspace Export](https://github.com/securefederatedai/openfl/blob/develop/openfl-tutorials/experimental/Workflow_Interface_1001_Workspace_Creation_from_JupyterNotebook.ipynb) feature that can be used to transform a Workflow API-based FL experiment into the TaskRunner API format for running in a distributed deployment. There is also groundwork laid for a future FederatedRuntime implementation for Workflow API, in addition to the currently supported LocalRuntime.  
+- **Federated Evaluation**: Federated evaluation allows for the assessment of ML models in a federated learning system by validating the model's performance locally on decentralized collaborator nodes, and then aggregating these metrics to gauge overall effectiveness, without compromising data privacy and security. FE is now officially supported by OpenFL, including [example tutorials](https://openfl.readthedocs.io/en/latest/about/features_index/fed_eval.html) on how to use this new feature (via TaskRunner API). 
+
+- **Expanded AI Accelerator Support**:  Intel® Data Center GPU Max Series support via the Intel® Extension for PyTorch, including examples for training on datasets such as [MNIST](https://github.com/securefederatedai/openfl/blob/develop/openfl-tutorials/experimental/Workflow_Interface_104_MNIST_XPU.ipynb) (via Workflow API) and [TinyImageNet](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/interactive_api/PyTorch_TinyImageNet_XPU) (via Interactive API) 
+
+- **Improved straggler collaborator handling**: Improvements and bug fixes to aggregator’s fault-tolerance when collaborators stop responding or drop out of a federation. Introducing a cut-off timer-based policy and enabling other policies to be plugged-in. This capability is particularly relevant for large or geo-distributed federations. 
+
+- **fx CLI Improvements**: We have separated the CLI commands are separated for creating a collaborator from the certificate request generation, for offering improved control of the participant setup workflow. 
+
+### Highlighting Community Research
+
+- **Mitigating backdoor attacks in FL** - [Workflow API example](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/experimental/CrowdGuard) contributed by @perieger (TU Darmstadt) demonstrates how [Crowdguard](https://github.com/TRUST-TUDa/crowdguard) can be used to leverage clients' feedback on individual models, analyze the behavior of neurons in hidden layers, and eliminate poisoned models through an iterative pruning scheme.
+
+### Enhanced Documentation: 
+
+- **Quickstart Tutorial**: An updated [TaskRunner API quickstart](https://openfl.readthedocs.io/en/latest/get_started/quickstart.html#quick-start) tutorial makes it easier for newcomers to get up and running with built-in workspaces or custom code. 
+- **FL Plan Description Documentation**: There is now a detailed [FL plan description](https://openfl.readthedocs.io/en/latest/about/features_index/taskrunner.html#federated-learning-plan-fl-plan-settings) in the to our documentation to help users better understand and configure the federated learning process.
+
 ## 1.5.1
 [Full Release Notes](https://github.com/securefederatedai/openfl/releases/tag/v1.5.1)
 

docs/conf.py

@@ -43,6 +43,9 @@
     'sphinx.ext.autosummary',
     'recommonmark'
 ]
+autodoc_default_options = {
+    'imported-members': True,
+}
 autosummary_generate = True  # Turn on sphinx.ext.autosummary
 
 source_suffix = ['.rst', '.md']

docs/contributing_guidelines/contributing.md

@@ -5,7 +5,7 @@ We welcome contributions from the community. We believe that anyone can bring so
 
 We accept various contributions from documentation improvement and bug fixing to major features proposals and [roadmap](https://github.com/intel/openfl/blob/develop/ROADMAP.md) suggestions.
 
-Documentation improvement: review our [documentation](https://openfl.readthedocs.io/en/latest/install.html) and let us know if something is not clear or not relevant. 
+Documentation improvement: review our [documentation](https://openfl.readthedocs.io/en/latest) and let us know if something is not clear or not relevant. 
 Propose your own formulations or even write new section explaining something that you know how works, but do not see in the documentation. 
 Propose it through GitHub [issues](https://github.com/intel/openfl/issues/new/choose) or [Discussions](https://github.com/intel/openfl/discussions).
 
@@ -99,4 +99,4 @@ Then you just add a line to every git commit message:
 Use your real name (sorry, no pseudonyms or anonymous contributions.)
 
 If you set your `user.name` and `user.email` git configs, you can sign your
-commit automatically with `git commit -s`.
+commit automatically with `git commit -s`.