Bring python versions and and callbacks into 1.7 release (#1228)

* Bump Python version to 3.10 - 3.12 (#1213) * update python version to 3.12 Signed-off-by: yes <[email protected]> * dummy commit Signed-off-by: yes <[email protected]> * dummy commit Signed-off-by: yes <[email protected]> * update python version to 3.12 Signed-off-by: yes <[email protected]> * dummy commit Signed-off-by: yes <[email protected]> * dummy commit Signed-off-by: yes <[email protected]> * removed files Signed-off-by: yes <[email protected]> * reverted doc change Signed-off-by: yes <[email protected]> * added missing requirements for Workflow Interface Tests Signed-off-by: yes <[email protected]> * added tensorboard Signed-off-by: yes <[email protected]> --------- Signed-off-by: yes <[email protected]> * Introduce `callbacks` API (#1195) * Get rid of kwargs Signed-off-by: Shah, Karan <[email protected]> * Use module-level logger Signed-off-by: Shah, Karan <[email protected]> * Reduce keras verbosity Signed-off-by: Shah, Karan <[email protected]> * Remove all log_metric and log_memory_usage traces; add callback hooks Signed-off-by: Shah, Karan <[email protected]> * Add `openfl.callbacks` module Signed-off-by: Shah, Karan <[email protected]> * Include round_num for task callbacks Signed-off-by: Shah, Karan <[email protected]> * Add tensordb to callbacks Signed-off-by: Shah, Karan <[email protected]> * No round_num on task callbacks Signed-off-by: Shah, Karan <[email protected]> * Remove task boundary callbacks Signed-off-by: Shah, Karan <[email protected]> * Remove tb/model_ckpt. Add memory_profiler Signed-off-by: Shah, Karan <[email protected]> * Restore psutil and tbX Signed-off-by: Shah, Karan <[email protected]> * Format code Signed-off-by: Shah, Karan <[email protected]> * Define default callbacks Signed-off-by: Shah, Karan <[email protected]> * Add write_logs for bwd compat Signed-off-by: Shah, Karan <[email protected]> * Add log_metric_callback for bwd compat Signed-off-by: Shah, Karan <[email protected]> * Migrate to module-level logger for collaborator Signed-off-by: Shah, Karan <[email protected]> * Review comments Signed-off-by: Shah, Karan <[email protected]> * Add metric_writer Signed-off-by: Shah, Karan <[email protected]> * Add collaborator side metric logging Signed-off-by: Shah, Karan <[email protected]> * Make log dirs on exp begin Signed-off-by: Shah, Karan <[email protected]> * Do not print use_tls Signed-off-by: Shah, Karan <[email protected]> * Assume reportable metric to be a scalar Signed-off-by: Shah, Karan <[email protected]> * Add aggregator side callbacks Signed-off-by: Shah, Karan <[email protected]> * do_task test returns mock dict Signed-off-by: Shah, Karan <[email protected]> * Consistency changes Signed-off-by: Shah, Karan <[email protected]> * Add documentation hooks Signed-off-by: Shah, Karan <[email protected]> * Update docstring Signed-off-by: Shah, Karan <[email protected]> * Update docs hook Signed-off-by: Shah, Karan <[email protected]> * Remove all traces of log_metric_callback and write_metric Signed-off-by: Shah, Karan <[email protected]> * Do on_round_begin if not time_to_quit Signed-off-by: Shah, Karan <[email protected]> --------- Signed-off-by: Shah, Karan <[email protected]> --------- Signed-off-by: yes <[email protected]> Signed-off-by: Shah, Karan <[email protected]> Co-authored-by: Shailesh Tanwar <[email protected]> Co-authored-by: Karan Shah <[email protected]>
securefederatedai · Dec 23, 2024 · 6b69126 · 6b69126
1 parent 0c4be45
commit 6b69126
Show file tree

Hide file tree

Showing 59 changed files with 540 additions and 508 deletions.
diff --git a/.github/workflows/docker-bench-security.yml b/.github/workflows/docker-bench-security.yml
@@ -19,7 +19,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/double_ws_export.yml b/.github/workflows/double_ws_export.yml
@@ -26,7 +26,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/experimental_workflow_tests.yml b/.github/workflows/experimental_workflow_tests.yml
@@ -25,7 +25,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.yml
@@ -22,7 +22,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -22,7 +22,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install linters
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/pki.yml b/.github/workflows/pki.yml
@@ -26,7 +26,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -43,7 +43,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/pytest_coverage.yml b/.github/workflows/pytest_coverage.yml
@@ -27,7 +27,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/straggler-handling.yml b/.github/workflows/straggler-handling.yml
@@ -29,7 +29,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml
@@ -37,7 +37,7 @@ jobs:
         # There are open issues for some of the models, so excluding them for now:
         # model_name: [ "torch_cnn_mnist", "keras_cnn_mnist", "torch_cnn_histology" ]
         model_name: ["torch_cnn_mnist", "keras_cnn_mnist"]
-        python_version: ["3.9", "3.10", "3.11"]
+        python_version: ["3.10", "3.11", "3.12"]
       fail-fast: false # do not immediately fail if one of the combinations fail
 
     env:

diff --git a/.github/workflows/taskrunner.yml b/.github/workflows/taskrunner.yml
@@ -21,7 +21,7 @@ jobs:
     strategy:
       matrix:
        os: ['ubuntu-latest', 'windows-latest']
-       python-version: ["3.9", "3.10", "3.11"]
+       python-version: ["3.10", "3.11", "3.12"]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 15
 

diff --git a/.github/workflows/taskrunner_eden_pipeline.yml b/.github/workflows/taskrunner_eden_pipeline.yml
@@ -25,7 +25,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml
@@ -20,7 +20,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/tr_docker_native.yml b/.github/workflows/tr_docker_native.yml
@@ -20,7 +20,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -21,7 +21,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -43,7 +43,7 @@ jobs:
       - name: Set up Python 3
         uses: actions/setup-python@v3
         with:
-          python-version: "3.9"
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -20,7 +20,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -41,7 +41,7 @@ jobs:
       - name: Set up Python 3
         uses: actions/setup-python@v3
         with:
-          python-version: "3.9"
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -81,7 +81,7 @@ pipeline {
                         SNYK_ALLOW_LONG_PROJECT_NAME = true
                         SNYK_USE_MULTI_PROC = true
                         SNYK_DEBUG = true
-                        SNYK_PYTHON_VERSION = '3.9'
+                        SNYK_PYTHON_VERSION = '3.10'
 
                         BANDIT_SOURCE_PATH = 'openfl/ openfl-workspace/ openfl-tutorials/'
                         BANDIT_SEVERITY_LEVEL = 'high'
@@ -114,7 +114,7 @@ pipeline {
                 stage('Build Package') {
                     agent {
                         docker {
-                            image 'python:3.9'
+                            image 'python:3.10'
                         }
                     }
                     steps {

diff --git a/docs/about/features_index/taskrunner.rst b/docs/about/features_index/taskrunner.rst
@@ -142,7 +142,7 @@ Bare Metal Approach
 STEP 1: Create a Workspace
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-1.	Start a Python 3.9 (>=3.9, <3.12) virtual environment and confirm OpenFL is available.
+1.	Start a Python 3.10 (>=3.10, <3.13) virtual environment and confirm OpenFL is available.
 
 	.. code-block:: shell
 

diff --git a/docs/developer_guide/running_the_federation.notebook.rst b/docs/developer_guide/running_the_federation.notebook.rst
@@ -17,7 +17,7 @@ You will start a Jupyter\* \  lab server and receive a URL you can use to access
 Start the Tutorials
 ===================
 
-1. Start a Python\* \  3.9 (>=3.9, <3.12) virtual environment and confirm OpenFL is available.
+1. Start a Python\* \  3.10 (>=3.10, <3.13) virtual environment and confirm OpenFL is available.
 
     .. code-block:: python
 

diff --git a/docs/developer_guide/running_the_federation_with_gandlf.rst b/docs/developer_guide/running_the_federation_with_gandlf.rst
@@ -113,7 +113,7 @@ However, continue with the following procedure for details in creating a federat
 STEP 1: Install GaNDLF prerequisites and Create a Workspace
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-1.	Start a Python 3.9 (>=3.9, <3.12) virtual environment and confirm OpenFL is available.
+1.	Start a Python 3.10 (>=3.10, <3.13) virtual environment and confirm OpenFL is available.
 
 	.. code-block:: python
 

diff --git a/docs/openfl.callbacks.rst b/docs/openfl.callbacks.rst
@@ -0,0 +1,16 @@
+``openfl.callbacks`` module
+===========================
+
+.. currentmodule:: openfl.callbacks
+
+.. automodule:: openfl.callbacks
+
+.. autosummary::
+   :toctree: _autosummary
+   :recursive:
+
+   Callback
+   CallbackList
+   LambdaCallback
+   MetricWriter
+   MemoryProfiler
diff --git a/docs/openfl.rst b/docs/openfl.rst
@@ -1,6 +1,6 @@
 .. currentmodule:: openfl
 
-Public API: ``openfl`` package
+API Reference: ``openfl``
 ===========================
 
 Subpackages
@@ -10,6 +10,7 @@ Subpackages
     :maxdepth: 1
 
     openfl.component
+    openfl.callbacks
     openfl.cryptography
     openfl.experimental
     openfl.databases

diff --git a/openfl-tutorials/experimental/workflow/404_Keras_MNIST_with_FedProx.ipynb b/openfl-tutorials/experimental/workflow/404_Keras_MNIST_with_FedProx.ipynb
@@ -51,7 +51,7 @@
    "id": "4dbb89b6",
    "metadata": {},
    "source": [
-    "First we start by installing the necessary dependencies for the workflow interface. Note that this tuorial uses Keras 3, make sure you use python 3.9 or higher."
+    "First we start by installing the necessary dependencies for the workflow interface. Note that this tuorial uses Keras 3, make sure you use python 3.10 or higher."
    ]
   },
   {

diff --git a/...orkspace/experimental/workflow/AggregatorBasedWorkflow/101_torch_cnn_mnist/plan/plan.yaml b/...orkspace/experimental/workflow/AggregatorBasedWorkflow/101_torch_cnn_mnist/plan/plan.yaml
@@ -6,8 +6,6 @@ aggregator :
   template : openfl.experimental.workflow.component.Aggregator
   settings :
     rounds_to_train     : 1
-    log_metric_callback :
-      template : src.utils.write_metric
 
 
 collaborator :

diff --git a/...-workspace/experimental/workflow/AggregatorBasedWorkflow/101_torch_cnn_mnist/src/utils.py b/...-workspace/experimental/workflow/AggregatorBasedWorkflow/101_torch_cnn_mnist/src/utils.py
diff --git a/...ce/experimental/workflow/AggregatorBasedWorkflow/102_aggregator_validation/plan/plan.yaml b/...ce/experimental/workflow/AggregatorBasedWorkflow/102_aggregator_validation/plan/plan.yaml
@@ -6,8 +6,6 @@ aggregator :
   template : openfl.experimental.workflow.component.Aggregator
   settings :
     rounds_to_train     : 1
-    log_metric_callback :
-      template : src.utils.write_metric
 
 
 collaborator :

diff --git a/...pace/experimental/workflow/AggregatorBasedWorkflow/102_aggregator_validation/src/utils.py b/...pace/experimental/workflow/AggregatorBasedWorkflow/102_aggregator_validation/src/utils.py
diff --git a/...fl-workspace/experimental/workflow/AggregatorBasedWorkflow/104_keras_mnist/plan/plan.yaml b/...fl-workspace/experimental/workflow/AggregatorBasedWorkflow/104_keras_mnist/plan/plan.yaml
@@ -6,8 +6,6 @@ aggregator :
   template : openfl.experimental.workflow.component.Aggregator
   settings :
     rounds_to_train     : 1
-    log_metric_callback :
-      template : src.utils.write_metric
 
 
 collaborator :

diff --git a/openfl-workspace/experimental/workflow/AggregatorBasedWorkflow/104_keras_mnist/src/utils.py b/openfl-workspace/experimental/workflow/AggregatorBasedWorkflow/104_keras_mnist/src/utils.py
diff --git a/...rimental/workflow/AggregatorBasedWorkflow/301_torch_cnn_mnist_watermarking/plan/plan.yaml b/...rimental/workflow/AggregatorBasedWorkflow/301_torch_cnn_mnist_watermarking/plan/plan.yaml
@@ -6,8 +6,6 @@ aggregator :
   template : openfl.experimental.workflow.component.Aggregator
   settings :
     rounds_to_train     : 1
-    log_metric_callback :
-      template : src.utils.write_metric
 
 
 collaborator :

diff --git a/...perimental/workflow/AggregatorBasedWorkflow/301_torch_cnn_mnist_watermarking/src/utils.py b/...perimental/workflow/AggregatorBasedWorkflow/301_torch_cnn_mnist_watermarking/src/utils.py
diff --git a/...orkflow/AggregatorBasedWorkflow/501_pytorch_tinyimagenet_transfer_learning/plan/plan.yaml b/...orkflow/AggregatorBasedWorkflow/501_pytorch_tinyimagenet_transfer_learning/plan/plan.yaml
@@ -6,8 +6,6 @@ aggregator :
   template : openfl.experimental.workflow.component.Aggregator
   settings :
     rounds_to_train     : 1
-    log_metric_callback :
-      template : src.utils.write_metric
 
 
 collaborator :