Use al2 as base image for fence

.github/workflows/ci.yaml

@@ -10,7 +10,7 @@ jobs:
   Security:
     name: Security Pipeline
     uses: uc-cdis/.github/.github/workflows/securitypipeline.yaml@master
-    secrets: inherit
+    secrets: inherit  # pragma: allowlist secret
 
   UnitTest:
     name: Python Unit Test with Postgres
@@ -19,13 +19,30 @@ jobs:
        python-version: '3.9'
        test-script: 'tests/ci_commands_script.sh'
        run-coveralls: true
-  ci:
+
+  BuildImageAndPush:
+    name: Build Image and Push
+    needs: Security
+    # https://github.com/uc-cdis/.github/blob/master/.github/workflows/image_build_push.yaml
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
+  # I did not test that this works
+  BuildMcryptImageAndPush:
     name: Build Image and Push
-    # TODO Uncomment after PXP-9212
-    # needs: Security
+    needs: Security
+    # https://github.com/uc-cdis/.github/blob/master/.github/workflows/image_build_push.yaml
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     secrets:
       ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
       ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+      # we can either use the repo name or
+#      OVERRIDE_REPO_NAME: fenceMcrypt
+      OVERRIDE_TAG_NAME: "mcrypt_$(echo ${GITHUB_REF#refs/*/} | tr / _)"
+      DOCKERFILE_LOCATION: "./DockerfileMcrypt"

.gitignore

@@ -102,7 +102,6 @@ ENV/
 .mypy_cache/
 
 # jwt keys
-keys
 tests/resources/keys/*.pem
 
 .DS_Store

.secrets.baseline

@@ -115,13 +115,13 @@
     }
   ],
   "results": {
-    ".github/workflows/ci.yaml": [
+    ".github/workflows/buildpipeline.yaml": [
       {
         "type": "Secret Keyword",
-        "filename": ".github/workflows/ci.yaml",
+        "filename": ".github/workflows/buildpipeline.yaml",
         "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0",
         "is_verified": false,
-        "line_number": 13
+        "line_number": 17
       }
     ],
     "deployment/scripts/postgresql/postgresql_init.sql": [
@@ -210,22 +210,13 @@
         "line_number": 137
       }
     ],
-    "fence/resources/storage/storageclient/cleversafe.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "fence/resources/storage/storageclient/cleversafe.py",
-        "hashed_secret": "7cb6efb98ba5972a9b5090dc2e517fe14d12cb04",
-        "is_verified": false,
-        "line_number": 274
-      }
-    ],
     "fence/utils.py": [
       {
         "type": "Secret Keyword",
         "filename": "fence/utils.py",
         "hashed_secret": "8318df9ecda039deac9868adf1944a29a95c7114",
         "is_verified": false,
-        "line_number": 129
+        "line_number": 128
       }
     ],
     "migrations/versions/a04a70296688_non_unique_client_name.py": [
@@ -268,14 +259,14 @@
         "filename": "tests/conftest.py",
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 1570
+        "line_number": 1556
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "tests/conftest.py",
         "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108",
         "is_verified": false,
-        "line_number": 1594
+        "line_number": 1579
       }
     ],
     "tests/credentials/google/test_credentials.py": [
@@ -394,24 +385,6 @@
         "line_number": 300
       }
     ],
-    "tests/storageclient/storage_client_mock.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/storageclient/storage_client_mock.py",
-        "hashed_secret": "37bbea9557f9efd1eeadb25dda9ab6514f08fde9",
-        "is_verified": false,
-        "line_number": 158
-      }
-    ],
-    "tests/storageclient/test_cleversafe_api_client.py": [
-      {
-        "type": "Secret Keyword",
-        "filename": "tests/storageclient/test_cleversafe_api_client.py",
-        "hashed_secret": "f683c485d521c2e45830146dd570111770baea29",
-        "is_verified": false,
-        "line_number": 130
-      }
-    ],
     "tests/test-fence-config.yaml": [
       {
         "type": "Basic Auth Credentials",
@@ -422,5 +395,5 @@
       }
     ]
   },
-  "generated_at": "2024-08-22T19:43:39Z"
+  "generated_at": "2023-10-20T20:37:17Z"
 }

Dockerfile

@@ -1,56 +1,49 @@
-# To run: docker run --rm -d -v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml --name=fence -p 80:80 fence
-# To check running container do: docker exec -it fence /bin/bash
+# To build: docker build -t fence:latest .
+# To run interactive:
+#   docker run -v ~/.gen3/fence/fence-config.yaml:/var/www/fence/fence-config.yaml -v ./keys/:/fence/keys/ fence:latest
+# To check running container do: docker exec -it CONTAINER bash
 
-FROM quay.io/cdis/python:python3.9-buster-2.0.0
+ARG AZLINUX_BASE_VERSION=feat_python-nginx
+
+# ------ Base stage ------
+FROM quay.io/cdis/python-nginx-al:${AZLINUX_BASE_VERSION} AS base
+
+# Comment this in, and comment out the line above, if quay is down
+# FROM 707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/python-nginx-al:${AZLINUX_BASE_VERSION} as base
 
 ENV appname=fence
 
-RUN pip install --upgrade pip
-RUN pip install --upgrade poetry
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends curl bash git \
-    && apt-get -y install vim \
-    libmcrypt4 mcrypt \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/
-
-RUN mkdir -p /var/www/$appname \
-    && mkdir -p /var/www/.cache/Python-Eggs/ \
-    && mkdir /run/nginx/ \
-    && ln -sf /dev/stdout /var/log/nginx/access.log \
-    && ln -sf /dev/stderr /var/log/nginx/error.log \
-    && chown nginx -R /var/www/.cache/Python-Eggs/ \
-    && chown nginx /var/www/$appname
-
-# aws cli v2 - needed for storing files in s3 during usersync k8s job
-RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
-    && unzip awscliv2.zip \
-    && ./aws/install \
-    && /bin/rm -rf awscliv2.zip ./aws
-
-WORKDIR /$appname
-
-# copy ONLY poetry artifact, install the dependencies but not fence
-# this will make sure than the dependencies is cached
-COPY poetry.lock pyproject.toml /$appname/
-RUN poetry config virtualenvs.create false \
-    && poetry install -vv --no-root --no-dev --no-interaction \
-    && poetry show -v
-
-# copy source code ONLY after installing dependencies
-COPY . /$appname
-COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
-COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py
-COPY clear_prometheus_multiproc /$appname/clear_prometheus_multiproc
-
-# install fence
-RUN poetry config virtualenvs.create false \
-    && poetry install -vv --no-dev --no-interaction \
-    && poetry show -v
-
-RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \
-    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py
-
-WORKDIR /var/www/$appname
-
-CMD ["sh","-c","bash /fence/dockerrun.bash && /dockerrun.sh"]
+WORKDIR /${appname}
+
+RUN chown -R gen3:gen3 /${appname}
+
+# ------ Builder stage ------
+FROM base AS builder
+
+# Install just the deps without the code as it's own step to avoid redoing this on code changes
+COPY poetry.lock pyproject.toml /${appname}/
+RUN poetry install -vv --only main --no-interaction
+
+# Move app files into working directory
+COPY --chown=gen3:gen3 . /$appname
+COPY --chown=gen3:gen3 ./deployment/wsgi/wsgi.py /$appname/wsgi.py
+
+# Do the install again incase the app itself needs install
+RUN poetry install -vv --only main --no-interaction
+
+ENV PATH="$(poetry env info --path)/bin:$PATH"
+
+# Setup version info
+RUN git config --global --add safe.directory /${appname} && COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" > /$appname/version_data.py \
+    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >> /$appname/version_data.py
+
+# install tar
+# RUN yum install tar -y
+# do we need to untar jwt-keys?
+
+# ------ Final stage ------
+FROM base
+
+COPY --chown=gen3:gen3 --from=builder /$appname /$appname
+
+CMD ["poetry", "run", "gunicorn", "-c", "deployment/wsgi/gunicorn.conf.py"]

DockerfileMcrypt

@@ -0,0 +1 @@
+# old Dockerfile

deployment/wsgi/gunicorn.conf.py

@@ -0,0 +1,9 @@
+wsgi_app = "deployment.wsgi.wsgi:application"
+bind = "0.0.0.0:8000"
+workers = 1
+preload_app = True
+user = "gen3"
+group = "gen3"
+timeout = 300
+keepalive = 2
+keepalive_timeout = 5

dockerrun.bash

@@ -4,7 +4,7 @@
 # Update certificate authority index -
 # environment may have mounted more authorities
 #
-update-ca-certificates
+# update-ca-certificates
 #
 # Kubernetes may mount jwt-keys as a tar ball
 #
@@ -15,6 +15,9 @@ if [ -f /fence/jwt-keys.tar ]; then
     if [ -d jwt-keys ]; then
       mkdir -p keys
       mv jwt-keys/* keys/
+      rm -rf /fence/keys/key/
     fi
   )
 fi
+nginx
+gunicorn -c /fence/deployment/wsgi/gunicorn.conf.py

docs/additional_documentation/fence_create.md

@@ -53,7 +53,7 @@ curl --request POST https://FENCE_URL/oauth2/token?grant_type=client_credentials
 
 The optional `--expires-in` parameter allows specifying the number of *days* until this client expires. The recommendation is to rotate credentials with the `client_credentials` grant at least once a year (see [Rotate client credentials](#rotate-client-credentials) section).
 
-NOTE: In Gen3, you can grant specific access to a client the same way you would to a user. See the [user.yaml guide](https://github.com/uc-cdis/fence/blob/master/docs/user.yaml_guide.md) for more details.
+NOTE: In Gen3, you can grant specific access to a client the same way you would to a user. See the [user.yaml guide](https://github.com/uc-cdis/fence/blob/master/docs/additional_documentation/user.yaml_guide.md) for more details.
 
 NOTE: Client credentials tokens are not linked to a user (the claims contain no `sub` or `context.user.name` like other tokens). Some Gen3 endpoints that assume the token is linked to a user, or whose logic require there being a user, do not support them. For an example of how to adapt an endpoint to support client credentials tokens, see [here](https://github.com/uc-cdis/requestor/commit/a5078fae27fa258ac78045cf2bb89cb2104f53cf). For an example of how to explicitly reject client credentials tokens, see [here](https://github.com/uc-cdis/requestor/commit/0f4974c25343d2185c7cdb48dcdeb58f97800672).
 

docs/additional_documentation/user.yaml_guide.md

@@ -16,7 +16,7 @@
 
 The `user.yaml` file is one way to get authorization information into Gen3. It is ingested via [Fence's `usersync` script](usersync.md). The format of this file is tightly coupled with the notions of resource, role and policy as defined by Gen3's policy engine, [Arborist](https://github.com/uc-cdis/arborist#arborist).
 
-For Gen3 Data Commons that do not use Arborist or that use the Google Data Access method of [Google Service Account Registration](https://github.com/uc-cdis/fence/blob/master/docs/google_architecture.md#google-account-linking-and-service-account-registration), refer to the [Deprecated format](#deprecated-format) section.
+For Gen3 Data Commons that do not use Arborist or that use the Google Data Access method of [Google Service Account Registration](https://github.com/uc-cdis/fence/blob/master/docs/additional_documentation/google_architecture.md#google-account-linking-and-service-account-registration), refer to the [Deprecated format](#deprecated-format) section.
 
 In a fully deployed Gen3 Commons using [Cloud Automation](https://github.com/uc-cdis/cloud-automation), the `user.yaml` file is usually hosted in S3 and configured via the `global.useryaml_s3path` setting of the Gen3 Data Commons manifest:
 ```