Merge branch 'develop' into add_task_runner_example

securefederatedai · Dec 11, 2024 · 022edcb · 022edcb
2 parents 2c731c1 + 7e742dd
commit 022edcb
Show file tree

Hide file tree

Showing 36 changed files with 350 additions and 205 deletions.
diff --git a/.github/workflows/docker-bench-security.yml b/.github/workflows/docker-bench-security.yml
@@ -16,10 +16,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/double_ws_export.yml b/.github/workflows/double_ws_export.yml
@@ -23,10 +23,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/experimental_workflow_tests.yml b/.github/workflows/experimental_workflow_tests.yml
@@ -10,6 +10,10 @@ on:
 permissions:
   contents: read
 
+env:
+  # A workaround for long FQDN names provided by GitHub actions.
+  FQDN: "localhost"
+
 jobs:
   build:
     if: (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'workflow_interface'))
@@ -18,10 +22,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.yml
@@ -19,10 +19,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -19,10 +19,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install linters
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/pki.yml b/.github/workflows/pki.yml
@@ -23,10 +23,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -40,10 +40,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/pytest_coverage.yml b/.github/workflows/pytest_coverage.yml
@@ -24,10 +24,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/straggler-handling.yml b/.github/workflows/straggler-handling.yml
@@ -26,10 +26,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/task_runner_e2e.yml b/.github/workflows/task_runner_e2e.yml
@@ -39,7 +39,7 @@ jobs:
         # There are open issues for some of the models, so excluding them for now:
         # model_name: [ "torch_cnn_mnist", "keras_cnn_mnist", "torch_cnn_histology" ]
         model_name: ["torch_cnn_mnist", "keras_cnn_mnist"]
-        python_version: ["3.8", "3.9", "3.10"]
+        python_version: ["3.9", "3.10", "3.11"]
       fail-fast: false # do not immediately fail if one of the combinations fail
 
     env:
@@ -230,3 +230,70 @@ jobs:
         with:
           name: tr_no_client_auth_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
           path: result.tar
+
+  test_memory_logs:
+    name: tr_tls_memory_logs
+    runs-on: ubuntu-22.04
+    timeout-minutes: 15
+    strategy:
+      matrix:
+        # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10
+        # If required, this can be extended to other models and python versions
+        model_name: ["torch_cnn_mnist"]
+        python_version: ["3.10"]
+      fail-fast: false # do not immediately fail if one of the combinations fail
+
+    env:
+      MODEL_NAME: ${{ matrix.model_name }}
+      PYTHON_VERSION: ${{ matrix.python_version }}
+
+    steps:
+      - name: Checkout OpenFL repository
+        id: checkout_openfl
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 2 # needed for detecting changes
+          submodules: "true"
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python
+        id: setup_python
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        id: install_dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+          pip install -r test-requirements.txt
+
+      - name: Run Task Runner E2E tests without TLS
+        id: run_tests
+        run: |
+          python -m pytest -s tests/end_to_end/test_suites/memory_logs_tests.py \
+          --model_name ${{ env.MODEL_NAME }} --num_rounds ${{ env.NUM_ROUNDS }} \
+          --num_collaborators ${{ env.NUM_COLLABORATORS }} --log_memory_usage
+          echo "Task runner memory logs test run completed"
+
+      - name: Print test summary
+        id: print_test_summary
+        if: ${{ always() }}
+        run: |
+          export PYTHONPATH="$PYTHONPATH:."
+          python tests/end_to_end/utils/summary_helper.py
+          echo "Test summary printed"
+
+      - name: Tar files
+        id: tar_files
+        if: ${{ always() }}
+        run: tar -cvf result.tar results
+
+      - name: Upload Artifacts
+        id: upload_artifacts
+        uses: actions/upload-artifact@v4
+        if: ${{ always() }}
+        with:
+          name: tr_tls_memory_logs_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
+          path: result.tar
diff --git a/.github/workflows/taskrunner.yml b/.github/workflows/taskrunner.yml
@@ -21,7 +21,7 @@ jobs:
     strategy:
       matrix:
        os: ['ubuntu-latest', 'windows-latest']
-       python-version: ['3.8','3.9','3.10','3.11']
+       python-version: ["3.9", "3.10", "3.11"]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 15
 

diff --git a/.github/workflows/taskrunner_eden_pipeline.yml b/.github/workflows/taskrunner_eden_pipeline.yml
@@ -22,10 +22,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml
@@ -17,10 +17,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/tr_docker_native.yml b/.github/workflows/tr_docker_native.yml
@@ -17,10 +17,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -13,16 +13,15 @@ env:
 
 jobs:
   pytest-coverage: # from pytest_coverage.yml
-    needs: lint
     runs-on: ubuntu-latest
     timeout-minutes: 15
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -35,16 +34,16 @@ jobs:
         coverage report
   
   cli:
-    needs: [lint, pytest-coverage]
+    needs: [pytest-coverage]
     runs-on: ubuntu-latest
     timeout-minutes: 15
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3
         uses: actions/setup-python@v3
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -17,10 +17,10 @@ jobs:
     timeout-minutes: 15
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.8
+    - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.8"
+        python-version: "3.9"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -38,10 +38,10 @@ jobs:
     timeout-minutes: 15
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3
         uses: actions/setup-python@v3
         with:
-          python-version: "3.8"
+          python-version: "3.9"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -81,7 +81,7 @@ pipeline {
                         SNYK_ALLOW_LONG_PROJECT_NAME = true
                         SNYK_USE_MULTI_PROC = true
                         SNYK_DEBUG = true
-                        SNYK_PYTHON_VERSION = '3.8'
+                        SNYK_PYTHON_VERSION = '3.9'
 
                         BANDIT_SOURCE_PATH = 'openfl/ openfl-workspace/ openfl-tutorials/'
                         BANDIT_SEVERITY_LEVEL = 'high'
@@ -114,7 +114,7 @@ pipeline {
                 stage('Build Package') {
                     agent {
                         docker {
-                            image 'python:3.8'
+                            image 'python:3.9'
                         }
                     }
                     steps {

diff --git a/README.md b/README.md
@@ -2,7 +2,8 @@
   <img src="https://github.com/securefederatedai/artwork/blob/main/PNG/OpenFL%20Logo%20-%20color.png?raw=true">
 </div>
 
-[![PyPI - Python Version](https://img.shields.io/badge/python-3.7%20%7C%203.8%20%7C%203.9%20%7C%203.10-blue)](https://pypi.org/project/openfl/)
+
+[![PyPI - Python Version](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11-blue)](https://pypi.org/project/openfl/)
 [![Ubuntu CI status](https://github.com/securefederatedai/openfl/actions/workflows/ubuntu.yml/badge.svg)](https://github.com/securefederatedai/openfl/actions/workflows/ubuntu.yml)
 [![Windows CI status](https://github.com/securefederatedai/openfl/actions/workflows/windows.yml/badge.svg)](https://github.com/securefederatedai/openfl/actions/workflows/windows.yml)
 [![Documentation Status](https://readthedocs.org/projects/openfl/badge/?version=latest)](https://openfl.readthedocs.io/en/latest/?badge=latest)

diff --git a/docs/about/features_index/taskrunner.rst b/docs/about/features_index/taskrunner.rst
@@ -147,7 +147,7 @@ Bare Metal Approach
 STEP 1: Create a Workspace
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-1.	Start a Python 3.8 (>=3.6, <3.11) virtual environment and confirm |productName| is available.
+1.	Start a Python 3.9 (>=3.9, <3.12) virtual environment and confirm |productName| is available.
 
 	.. code-block:: console
 

diff --git a/docs/developer_guide/running_the_federation.notebook.rst b/docs/developer_guide/running_the_federation.notebook.rst
@@ -17,7 +17,7 @@ You will start a Jupyter\* \  lab server and receive a URL you can use to access
 Start the Tutorials
 ===================
 
-1. Start a Python\* \  3.8 (>=3.6, <3.9) virtual environment and confirm |productName| is available.
+1. Start a Python\* \  3.9 (>=3.9, <3.12) virtual environment and confirm |productName| is available.
 
     .. code-block:: python
 

diff --git a/docs/developer_guide/running_the_federation_with_gandlf.rst b/docs/developer_guide/running_the_federation_with_gandlf.rst
@@ -114,7 +114,7 @@ However, continue with the following procedure for details in creating a federat
 STEP 1: Install GaNDLF prerequisites and Create a Workspace
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-1.	Start a Python 3.8 (>=3.7, <3.11) virtual environment and confirm |productName| is available.
+1.	Start a Python 3.9 (>=3.9, <3.12) virtual environment and confirm |productName| is available.
 
 	.. code-block:: python
 

diff --git a/docs/get_started/installation.rst b/docs/get_started/installation.rst
@@ -19,7 +19,7 @@ Install the Package
 Follow this procedure to prepare the environment and install the |productName| package.
 Perform this procedure on every node in the federation.
 
-1. Install a Python 3.8 (>=3.6, <3.9) virtual environment using venv.
+1. Install a Python 3.9 (>=3.9, <3.12) virtual environment using venv.
 
  See the `Venv installation guide <https://docs.python.org/3/library/venv.html>`_ for details.
 

diff --git a/openfl-tutorials/__init__.py b/openfl-tutorials/__init__.py
diff --git a/openfl-tutorials/experimental/workflow/workflow_interface_requirements.txt b/openfl-tutorials/experimental/workflow/workflow_interface_requirements.txt
@@ -1,10 +1,10 @@
-astor==0.8.1
 chardet
 charset-normalizer
 dill==0.3.6
 matplotlib>=2.0.0
 metaflow==2.7.15
 nbdev==2.3.12
+nbformat==5.10.4
 ray==2.9.2
 torch
 torchvision
diff --git a/openfl-workspace/keras_cnn_mnist/requirements.txt b/openfl-workspace/keras_cnn_mnist/requirements.txt
@@ -1 +1,2 @@
+numpy==1.23.5
 tensorflow==2.13