Merge branch 'securefederatedai:develop' into exp_agg_workflow_privat…

…e_attrs_fix
securefederatedai · Oct 15, 2024 · abdd735 · abdd735
2 parents 58b5d16 + 74497de
commit abdd735
Show file tree

Hide file tree

Showing 106 changed files with 921 additions and 684 deletions.
diff --git a/.github/workflows/docker-bench-security.yml b/.github/workflows/docker-bench-security.yml
@@ -0,0 +1,56 @@
+name: Docker Bench for Security
+
+on:
+  pull_request:
+    branches: [ develop ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.8"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install .
+    - name: Clean Docker System
+      run: |
+        docker image prune -a -f
+        docker system prune -a -f
+    - name: Clone Docker Bench Security Repo
+      run: git clone https://github.com/docker/docker-bench-security.git
+    - name: Build Docker Bench Security Image
+      run: |
+        cd docker-bench-security
+        docker build --no-cache -t docker-bench-security .
+    - name: Create results directory
+      run: mkdir -p results
+    - name: Run Docker Bench for Security
+      run: |
+        docker run --rm --net host --pid host --userns host --cap-add audit_control \
+          -e DOCKER_CONTENT_TRUST=0 \
+          -v /etc:/etc:ro \
+          -v /lib/systemd/system:/lib/systemd/system:ro \
+          -v /usr/bin/containerd:/usr/bin/containerd:ro \
+          -v /usr/bin/runc:/usr/bin/runc:ro \
+          -v /usr/lib/systemd:/usr/lib/systemd:ro \
+          -v /var/lib:/var/lib:ro \
+          -v /var/run/docker.sock:/var/run/docker.sock:ro \
+          -v "$(pwd)/results:/results" \
+          --label docker_bench_security \
+          docker-bench-security | tee results/docker_bench_security_report.txt
+
+    - name: Upload Security Bench Report
+      uses: actions/upload-artifact@v3
+      with:
+        name: docker_bench_security-report
+        path: results/docker_bench_security_report.txt
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -26,4 +26,4 @@ jobs:
         python -m pip install --upgrade pip
         pip install -r linters-requirements.txt 
     - name: Lint using built-in script
-      run: bash shell/lint.sh
+      run: bash shell/lint.sh
diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml
@@ -30,17 +30,61 @@ jobs:
         uses: github/codeql-action/upload-sarif@v2
         with:
           sarif_file: 'trivy-results.sarif'
+
+      - name: Install Trivy
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.55.0
+  
+      - name: Run Trivy code vulnerability scanner (JSON Output)
+        run: |
+          trivy --quiet fs --format json --output trivy-code-results.json --ignore-unfixed --vuln-type os,library --severity CRITICAL,HIGH,MEDIUM,LOW .
+
+      - name: Upload Code Vulnerability Scan Results
+        uses: actions/upload-artifact@v3
+        with:
+          name: trivy-code-report-json
+          path: trivy-code-results.json
 
-      - name: Run Trivy vulnerability scanner
+      - name: Run Trivy vulnerability scanner for Docker image (JSON Output)
         uses: aquasecurity/[email protected]
         with:
           image-ref: 'docker.io/securefederatedai/openfl:${{ github.sha }}'
-          format: 'table'
+          format: 'json'
+          output: 'trivy-docker-results.json'
           exit-code: '1'
           ignore-unfixed: true
           vuln-type: 'os,library'
           severity: 'CRITICAL,HIGH,MEDIUM,LOW'
 
-      - name: Log completion message
+      - name: Upload Docker Vulnerability Scan
+        uses: actions/upload-artifact@v3
+        with:
+          name: trivy-docker-report-json
+          path: trivy-docker-results.json
+
+      - name: Run Trivy code vulnerability scanner (SPDX-JSON Output)
         run: |
-          echo "Trivy scan completed. Review the output above for vulnerabilities."
+          trivy --quiet fs --format spdx-json --output trivy-code-spdx-results.json --ignore-unfixed --vuln-type os,library --severity CRITICAL,HIGH,MEDIUM,LOW .
+  
+      - name: Upload Code Vulnerability Scan Results
+        uses: actions/upload-artifact@v3
+        with:
+          name: trivy-code-spdx-report-json
+          path: trivy-code-spdx-results.json
+
+      - name: Run Trivy vulnerability scanner for Docker image (SPDX-JSON Output)
+        uses: aquasecurity/[email protected]
+        with:
+          image-ref: 'docker.io/securefederatedai/openfl:${{ github.sha }}'
+          format: 'spdx-json'
+          output: 'trivy-docker-spdx-results.json'
+          exit-code: '1'
+          ignore-unfixed: true
+          vuln-type: 'os,library'
+          severity: 'CRITICAL,HIGH,MEDIUM,LOW'
+
+      - name: Upload Docker Vulnerability Scan
+        uses: actions/upload-artifact@v3
+        with:
+          name: trivy-docker-spdx-report-json
+          path: trivy-docker-spdx-results.json
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,31 @@
+---
+minimum_pre_commit_version: 3.3.3
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: end-of-file-fixer
+        files: \.py$
+      - id: requirements-txt-fixer
+      - id: check-merge-conflict
+      - id: check-case-conflict
+      - id: check-json
+        files: \.json$
+      - id: check-yaml
+        files: \.yaml$
+      - id: debug-statements
+      - id: pretty-format-json
+        args:
+          - --autofix
+        files: \.json$
+      - id: trailing-whitespace
+        files: \.py$
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.0
+    hooks:
+      - id: autoflake
+        args:
+          - --in-place
+          - --remove-unused-variables
+          - --recursive
+          - --ignore-pass-statements
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -55,11 +55,12 @@ git clone https://github.com/YOUR_GITHUB_USERNAME/openfl.git
 cd openfl
 pip install -U pip setuptools wheel
 pip install .
+pip install -r requirements-linters.txt
 ```
 
 ## Code style
 
-OpenFL uses [black](https://black.readthedocs.io/en/stable/) and [isort](https://pycqa.github.io/isort/) to format the code.
+OpenFL uses [black](https://black.readthedocs.io/en/stable/), [isort](https://pycqa.github.io/isort/) and [precommit](https://pre-commit.com/) to format the code.
 
 Run the following command at the **root** directory of the repo to format your code.
 

diff --git a/docs/about/releases.md b/docs/about/releases.md
@@ -1,5 +1,31 @@
 # Releases
 
+## 1.6
+[Full Release Notes](https://github.com/securefederatedai/openfl/releases/tag/v1.6)
+
+### New Features and APIs: 
+- **Federated LLM fine-tuning**: 
+     - [**Horovod**](https://github.com/securefederatedai/openfl/tree/develop/openfl-workspace/torch_llm_horovod): Use horovod to efficiently train LLMs across multiple private clusters  
+     - **Neuralchat-7b fine-tuning**: Learn how to fine-tune [neuralchat-7b](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/experimental/LLM/neuralchat) using the Intel® Extension for Transformers and the workflow interface. 
+
+- **Workflow API enhancements**: Introducing an experimental [Workspace Export](https://github.com/securefederatedai/openfl/blob/develop/openfl-tutorials/experimental/Workflow_Interface_1001_Workspace_Creation_from_JupyterNotebook.ipynb) feature that can be used to transform a Workflow API-based FL experiment into the TaskRunner API format for running in a distributed deployment. There is also groundwork laid for a future FederatedRuntime implementation for Workflow API, in addition to the currently supported LocalRuntime.  
+- **Federated Evaluation**: Federated evaluation allows for the assessment of ML models in a federated learning system by validating the model's performance locally on decentralized collaborator nodes, and then aggregating these metrics to gauge overall effectiveness, without compromising data privacy and security. FE is now officially supported by OpenFL, including [example tutorials](https://openfl.readthedocs.io/en/latest/about/features_index/fed_eval.html) on how to use this new feature (via TaskRunner API). 
+
+- **Expanded AI Accelerator Support**:  Intel® Data Center GPU Max Series support via the Intel® Extension for PyTorch, including examples for training on datasets such as [MNIST](https://github.com/securefederatedai/openfl/blob/develop/openfl-tutorials/experimental/Workflow_Interface_104_MNIST_XPU.ipynb) (via Workflow API) and [TinyImageNet](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/interactive_api/PyTorch_TinyImageNet_XPU) (via Interactive API) 
+
+- **Improved straggler collaborator handling**: Improvements and bug fixes to aggregator’s fault-tolerance when collaborators stop responding or drop out of a federation. Introducing a cut-off timer-based policy and enabling other policies to be plugged-in. This capability is particularly relevant for large or geo-distributed federations. 
+
+- **fx CLI Improvements**: We have separated the CLI commands are separated for creating a collaborator from the certificate request generation, for offering improved control of the participant setup workflow. 
+
+### Highlighting Community Research
+
+- **Mitigating backdoor attacks in FL** - [Workflow API example](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/experimental/CrowdGuard) contributed by @perieger (TU Darmstadt) demonstrates how [Crowdguard](https://github.com/TRUST-TUDa/crowdguard) can be used to leverage clients' feedback on individual models, analyze the behavior of neurons in hidden layers, and eliminate poisoned models through an iterative pruning scheme.
+
+### Enhanced Documentation: 
+
+- **Quickstart Tutorial**: An updated [TaskRunner API quickstart](https://openfl.readthedocs.io/en/latest/get_started/quickstart.html#quick-start) tutorial makes it easier for newcomers to get up and running with built-in workspaces or custom code. 
+- **FL Plan Description Documentation**: There is now a detailed [FL plan description](https://openfl.readthedocs.io/en/latest/about/features_index/taskrunner.html#federated-learning-plan-fl-plan-settings) in the to our documentation to help users better understand and configure the federated learning process.
+
 ## 1.5.1
 [Full Release Notes](https://github.com/securefederatedai/openfl/releases/tag/v1.5.1)
 

diff --git a/docs/docs-requirements.txt b/docs/docs-requirements.txt
@@ -1,10 +1,10 @@
 # Copyright (C) 2020-2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
-sphinx-rtd-theme
+pygments>=2.7.4 # not directly required, pinned by Snyk to avoid a vulnerability
+recommonmark
+sphinx>=3.0.4 # not directly required, pinned by Snyk to avoid a vulnerability
+sphinx-copybutton
 sphinx-prompt 
+sphinx-rtd-theme
 sphinx_substitution_extensions 
-sphinx-copybutton
 sphinxcontrib-mermaid
-pygments>=2.7.4 # not directly required, pinned by Snyk to avoid a vulnerability
-sphinx>=3.0.4 # not directly required, pinned by Snyk to avoid a vulnerability
-recommonmark