From 0f2132a6fde997d2ac389d03bfc972af70e2613f Mon Sep 17 00:00:00 2001 From: Patrick Foley Date: Fri, 8 Sep 2023 14:35:21 -0700 Subject: [PATCH 1/7] Initial scans commit for bandit, hadolint, trivy Signed-off-by: Patrick Foley --- .github/workflows/scans.yml | 34 ++++++++++++++++++++++++++++++++++ .github/workflows/trivy.yml | 29 +++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 .github/workflows/scans.yml create mode 100644 .github/workflows/trivy.yml diff --git a/.github/workflows/scans.yml b/.github/workflows/scans.yml new file mode 100644 index 0000000000..7550d9b9c5 --- /dev/null +++ b/.github/workflows/scans.yml @@ -0,0 +1,34 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Lint with Flake8 + +on: + pull_request: + branches: [ develop ] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install bandit + pip install . + - name: Bandit Scan + run: | + bandit -r openfl + - name: Hadolint Dockerfile Scan + run: | + docker run --rm -i hadolint/hadolint < openfl-docker/Dockerfile.base diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml new file mode 100644 index 0000000000..9037d79139 --- /dev/null +++ b/.github/workflows/trivy.yml @@ -0,0 +1,29 @@ +name: build +on: + push: + branches: + - main + pull_request: +jobs: + build: + name: Build + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Build an image from Dockerfile + run: | + docker build -t docker.io/securefederatedai/openfl:${{ github.sha }} openfl-docker/Dockerfile.base + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: 'docker.io/securefederatedai/openfl:${{ github.sha }}' + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'trivy-results.sarif' From 0a0419eda77ca638c73c08dd198a620944c136e4 Mon Sep 17 00:00:00 2001 From: Patrick Foley Date: Fri, 8 Sep 2023 15:26:52 -0700 Subject: [PATCH 2/7] Address bandit scan results Signed-off-by: Patrick Foley --- .github/workflows/scans.yml | 6 +----- openfl/experimental/utilities/metaflow_utils.py | 6 ++++-- openfl/interface/tutorial.py | 2 +- openfl/interface/workspace.py | 6 +++--- openfl/plugins/interface_serializer/dill_serializer.py | 4 ++-- openfl/transport/grpc/aggregator_server.py | 2 +- openfl/utilities/ca/ca.py | 4 ++-- openfl/utilities/ca/downloader.py | 3 ++- openfl/utilities/workspace.py | 2 +- 9 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/workflows/scans.yml b/.github/workflows/scans.yml index 7550d9b9c5..e406f2a2d0 100644 --- a/.github/workflows/scans.yml +++ b/.github/workflows/scans.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Lint with Flake8 +name: Security Scans on: pull_request: @@ -24,11 +24,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install bandit pip install . - - name: Bandit Scan - run: | - bandit -r openfl - name: Hadolint Dockerfile Scan run: | docker run --rm -i hadolint/hadolint < openfl-docker/Dockerfile.base diff --git a/openfl/experimental/utilities/metaflow_utils.py b/openfl/experimental/utilities/metaflow_utils.py index 9e26d73409..b5d0b62e81 100644 --- a/openfl/experimental/utilities/metaflow_utils.py +++ b/openfl/experimental/utilities/metaflow_utils.py @@ -25,7 +25,8 @@ from metaflow.task import MetaDatum import fcntl import hashlib -from dill.source import getsource +from dill.source import getsource #nosec +# getsource only used to determine structure of FlowGraph from typing import TYPE_CHECKING if TYPE_CHECKING: from openfl.experimental.interface import FLSpec @@ -62,7 +63,8 @@ def __init__(self, name): self.name = name def __enter__(self): - lock_id = hashlib.new('md5', self.name.encode("utf8"), usedforsecurity=False).hexdigest() + lock_id = hashlib.new('md5', self.name.encode("utf8"), usedforsecurity=False).hexdigest() #nosec + # MD5sum used for concurrency purposes, not security self.fp = open(f"/tmp/.lock-{lock_id}.lck", "wb") fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX) diff --git a/openfl/interface/tutorial.py b/openfl/interface/tutorial.py index 634e9b107f..c8bdf030c7 100644 --- a/openfl/interface/tutorial.py +++ b/openfl/interface/tutorial.py @@ -30,7 +30,7 @@ def start(ip, port): """Start the Jupyter Lab from the tutorials directory.""" from os import environ from os import sep - from subprocess import check_call + from subprocess import check_call #nosec from sys import executable from openfl.interface.cli_helper import TUTORIALS diff --git a/openfl/interface/workspace.py b/openfl/interface/workspace.py index e28e6534f5..6ef525f550 100644 --- a/openfl/interface/workspace.py +++ b/openfl/interface/workspace.py @@ -3,7 +3,7 @@ """Workspace module.""" import os -import subprocess +import subprocess #nosec import sys from pathlib import Path from typing import Tuple, Union @@ -86,7 +86,7 @@ def create_(prefix, template): def create(prefix, template): """Create federated learning workspace.""" from os.path import isfile - from subprocess import check_call + from subprocess import check_call #nosec from sys import executable from openfl.interface.cli_helper import print_tree @@ -196,7 +196,7 @@ def import_(archive): from os.path import basename from os.path import isfile from shutil import unpack_archive - from subprocess import check_call + from subprocess import check_call #nosec from sys import executable archive = Path(archive).absolute() diff --git a/openfl/plugins/interface_serializer/dill_serializer.py b/openfl/plugins/interface_serializer/dill_serializer.py index 5b3aaaa42e..abad5a29e1 100644 --- a/openfl/plugins/interface_serializer/dill_serializer.py +++ b/openfl/plugins/interface_serializer/dill_serializer.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Dill serializer plugin.""" -import dill +import dill #nosec from .serializer_interface import Serializer @@ -24,4 +24,4 @@ def serialize(object_, filename): def restore_object(filename): """Load and deserialize an object.""" with open(filename, 'rb') as f: - return dill.load(f) + return dill.load(f) #nosec diff --git a/openfl/transport/grpc/aggregator_server.py b/openfl/transport/grpc/aggregator_server.py index fa71d98c9d..27ab1848c5 100644 --- a/openfl/transport/grpc/aggregator_server.py +++ b/openfl/transport/grpc/aggregator_server.py @@ -83,7 +83,7 @@ def validate_collaborator(self, request, context): if not self.aggregator.valid_collaborator_cn_and_id( common_name, collaborator_common_name): # Random delay in authentication failures - sleep(5 * random()) + sleep(5 * random()) #nosec context.abort( StatusCode.UNAUTHENTICATED, f'Invalid collaborator. CN: |{common_name}| ' diff --git a/openfl/utilities/ca/ca.py b/openfl/utilities/ca/ca.py index f5dace0be7..1b4da69e30 100644 --- a/openfl/utilities/ca/ca.py +++ b/openfl/utilities/ca/ca.py @@ -9,11 +9,11 @@ import sys import shutil import signal -import subprocess +import subprocess #nosec import time from logging import getLogger from pathlib import Path -from subprocess import check_call +from subprocess import check_call #nosec from click import confirm diff --git a/openfl/utilities/ca/downloader.py b/openfl/utilities/ca/downloader.py index 10b9465b79..2c518acba4 100644 --- a/openfl/utilities/ca/downloader.py +++ b/openfl/utilities/ca/downloader.py @@ -67,5 +67,6 @@ def _download(url, prefix, confirmation): if confirmation: confirm('CA binaries will be downloaded now', default=True, abort=True) name = url.split('/')[-1] - urllib.request.urlretrieve(url, f'{prefix}/{name}') + # nosec: private function definition with static urls + urllib.request.urlretrieve(url, f'{prefix}/{name}') #nosec shutil.unpack_archive(f'{prefix}/{name}', f'{prefix}/step') diff --git a/openfl/utilities/workspace.py b/openfl/utilities/workspace.py index bf392e0259..a12e51779c 100644 --- a/openfl/utilities/workspace.py +++ b/openfl/utilities/workspace.py @@ -10,7 +10,7 @@ import time from contextlib import contextmanager from pathlib import Path -from subprocess import check_call +from subprocess import check_call #nosec from sys import executable from typing import Optional from typing import Tuple From b0a166d4c94218c9a52379d76e6b862caeaf2dca Mon Sep 17 00:00:00 2001 From: Patrick Foley Date: Fri, 8 Sep 2023 15:38:53 -0700 Subject: [PATCH 3/7] Fix Trivy action Signed-off-by: Patrick Foley --- .github/workflows/trivy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 9037d79139..8618bdae16 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -14,7 +14,7 @@ jobs: - name: Build an image from Dockerfile run: | - docker build -t docker.io/securefederatedai/openfl:${{ github.sha }} openfl-docker/Dockerfile.base + docker build -t docker.io/securefederatedai/openfl:${{ github.sha }} -f openfl-docker/Dockerfile.base . - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master From 7fdc27bda2039d210a5d0c10dde372040a22255f Mon Sep 17 00:00:00 2001 From: Patrick Foley Date: Tue, 26 Sep 2023 15:37:43 -0700 Subject: [PATCH 4/7] Fix linting Signed-off-by: Patrick Foley --- openfl/experimental/utilities/metaflow_utils.py | 5 +++-- openfl/interface/tutorial.py | 2 +- openfl/interface/workspace.py | 6 +++--- openfl/plugins/interface_serializer/dill_serializer.py | 4 ++-- openfl/transport/grpc/aggregator_server.py | 2 +- openfl/utilities/ca/ca.py | 4 ++-- openfl/utilities/ca/downloader.py | 2 +- openfl/utilities/workspace.py | 2 +- 8 files changed, 14 insertions(+), 13 deletions(-) diff --git a/openfl/experimental/utilities/metaflow_utils.py b/openfl/experimental/utilities/metaflow_utils.py index b5d0b62e81..9dccca1487 100644 --- a/openfl/experimental/utilities/metaflow_utils.py +++ b/openfl/experimental/utilities/metaflow_utils.py @@ -25,7 +25,7 @@ from metaflow.task import MetaDatum import fcntl import hashlib -from dill.source import getsource #nosec +from dill.source import getsource # nosec # getsource only used to determine structure of FlowGraph from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -63,7 +63,8 @@ def __init__(self, name): self.name = name def __enter__(self): - lock_id = hashlib.new('md5', self.name.encode("utf8"), usedforsecurity=False).hexdigest() #nosec + lock_id = hashlib.new('md5', self.name.encode("utf8"), + usedforsecurity=False).hexdigest() # nosec # MD5sum used for concurrency purposes, not security self.fp = open(f"/tmp/.lock-{lock_id}.lck", "wb") fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX) diff --git a/openfl/interface/tutorial.py b/openfl/interface/tutorial.py index c8bdf030c7..85a0cb3f66 100644 --- a/openfl/interface/tutorial.py +++ b/openfl/interface/tutorial.py @@ -30,7 +30,7 @@ def start(ip, port): """Start the Jupyter Lab from the tutorials directory.""" from os import environ from os import sep - from subprocess import check_call #nosec + from subprocess import check_call # nosec from sys import executable from openfl.interface.cli_helper import TUTORIALS diff --git a/openfl/interface/workspace.py b/openfl/interface/workspace.py index 6ef525f550..1ee747ca7b 100644 --- a/openfl/interface/workspace.py +++ b/openfl/interface/workspace.py @@ -3,7 +3,7 @@ """Workspace module.""" import os -import subprocess #nosec +import subprocess # nosec import sys from pathlib import Path from typing import Tuple, Union @@ -86,7 +86,7 @@ def create_(prefix, template): def create(prefix, template): """Create federated learning workspace.""" from os.path import isfile - from subprocess import check_call #nosec + from subprocess import check_call # nosec from sys import executable from openfl.interface.cli_helper import print_tree @@ -196,7 +196,7 @@ def import_(archive): from os.path import basename from os.path import isfile from shutil import unpack_archive - from subprocess import check_call #nosec + from subprocess import check_call # nosec from sys import executable archive = Path(archive).absolute() diff --git a/openfl/plugins/interface_serializer/dill_serializer.py b/openfl/plugins/interface_serializer/dill_serializer.py index abad5a29e1..f4bb9ffd58 100644 --- a/openfl/plugins/interface_serializer/dill_serializer.py +++ b/openfl/plugins/interface_serializer/dill_serializer.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Dill serializer plugin.""" -import dill #nosec +import dill # nosec from .serializer_interface import Serializer @@ -24,4 +24,4 @@ def serialize(object_, filename): def restore_object(filename): """Load and deserialize an object.""" with open(filename, 'rb') as f: - return dill.load(f) #nosec + return dill.load(f) # nosec diff --git a/openfl/transport/grpc/aggregator_server.py b/openfl/transport/grpc/aggregator_server.py index 27ab1848c5..39fde16445 100644 --- a/openfl/transport/grpc/aggregator_server.py +++ b/openfl/transport/grpc/aggregator_server.py @@ -83,7 +83,7 @@ def validate_collaborator(self, request, context): if not self.aggregator.valid_collaborator_cn_and_id( common_name, collaborator_common_name): # Random delay in authentication failures - sleep(5 * random()) #nosec + sleep(5 * random()) # nosec context.abort( StatusCode.UNAUTHENTICATED, f'Invalid collaborator. CN: |{common_name}| ' diff --git a/openfl/utilities/ca/ca.py b/openfl/utilities/ca/ca.py index 1b4da69e30..1ff88c5742 100644 --- a/openfl/utilities/ca/ca.py +++ b/openfl/utilities/ca/ca.py @@ -9,11 +9,11 @@ import sys import shutil import signal -import subprocess #nosec +import subprocess # nosec import time from logging import getLogger from pathlib import Path -from subprocess import check_call #nosec +from subprocess import check_call # nosec from click import confirm diff --git a/openfl/utilities/ca/downloader.py b/openfl/utilities/ca/downloader.py index 2c518acba4..9331d1fd4a 100644 --- a/openfl/utilities/ca/downloader.py +++ b/openfl/utilities/ca/downloader.py @@ -68,5 +68,5 @@ def _download(url, prefix, confirmation): confirm('CA binaries will be downloaded now', default=True, abort=True) name = url.split('/')[-1] # nosec: private function definition with static urls - urllib.request.urlretrieve(url, f'{prefix}/{name}') #nosec + urllib.request.urlretrieve(url, f'{prefix}/{name}') # nosec shutil.unpack_archive(f'{prefix}/{name}', f'{prefix}/step') diff --git a/openfl/utilities/workspace.py b/openfl/utilities/workspace.py index a12e51779c..c1561b726e 100644 --- a/openfl/utilities/workspace.py +++ b/openfl/utilities/workspace.py @@ -10,7 +10,7 @@ import time from contextlib import contextmanager from pathlib import Path -from subprocess import check_call #nosec +from subprocess import check_call # nosec from sys import executable from typing import Optional from typing import Tuple From be7bb4c5c26272d4f43dc76183f0f49b89e8d306 Mon Sep 17 00:00:00 2001 From: Patrick Foley Date: Fri, 8 Sep 2023 16:26:13 -0700 Subject: [PATCH 5/7] Add Coverity Badge Signed-off-by: Patrick Foley --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0db6182bb2..ea4b780fc2 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,12 @@ [](https://join.slack.com/t/openfl/shared_invite/zt-ovzbohvn-T5fApk05~YS_iZhjJ5yaTw) [![License](https://img.shields.io/badge/License-Apache%202.0-brightgreen.svg)](https://opensource.org/licenses/Apache-2.0) [![Citation](https://img.shields.io/badge/cite-citation-brightgreen)](https://arxiv.org/abs/2105.06413) -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/intel/openfl/blob/develop/openfl-tutorials/interactive_api/numpy_linear_regression/workspace/SingleNotebook.ipynb) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/6599/badge)](https://bestpractices.coreinfrastructure.org/projects/6599) + + Coverity Scan Build Status + +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/intel/openfl/blob/develop/openfl-tutorials/interactive_api/numpy_linear_regression/workspace/SingleNotebook.ipynb) OpenFL is a Python 3 framework for Federated Learning. OpenFL is designed to be a _flexible_, _extensible_ and _easily learnable_ tool for data scientists. OpenFL is hosted by The Linux Foundation, aims to be community-driven, and welcomes contributions back to the project. From 098c82616c08489c92d11d65dfaaff872d7d0788 Mon Sep 17 00:00:00 2001 From: Patrick Foley Date: Fri, 29 Sep 2023 11:18:56 -0700 Subject: [PATCH 6/7] Update Hadolint threshold to flag errors only Signed-off-by: Patrick Foley --- .github/workflows/scans.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scans.yml b/.github/workflows/scans.yml index e406f2a2d0..e648130449 100644 --- a/.github/workflows/scans.yml +++ b/.github/workflows/scans.yml @@ -27,4 +27,4 @@ jobs: pip install . - name: Hadolint Dockerfile Scan run: | - docker run --rm -i hadolint/hadolint < openfl-docker/Dockerfile.base + docker run -v ${PWD}/openfl-docker:/openfl-docker --rm -i hadolint/hadolint hadolint -t warning /openfl-docker/Dockerfile.base From 1da72602686106b55a84cf9ab2c94c4626c6766a Mon Sep 17 00:00:00 2001 From: Patrick Foley Date: Fri, 29 Sep 2023 13:46:54 -0700 Subject: [PATCH 7/7] Update Hadolint threshold to flag errors only Signed-off-by: Patrick Foley --- .github/workflows/scans.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scans.yml b/.github/workflows/scans.yml index e648130449..93d0d75425 100644 --- a/.github/workflows/scans.yml +++ b/.github/workflows/scans.yml @@ -27,4 +27,4 @@ jobs: pip install . - name: Hadolint Dockerfile Scan run: | - docker run -v ${PWD}/openfl-docker:/openfl-docker --rm -i hadolint/hadolint hadolint -t warning /openfl-docker/Dockerfile.base + docker run -v ${PWD}/openfl-docker:/openfl-docker --rm -i hadolint/hadolint hadolint -t error /openfl-docker/Dockerfile.base