diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9d7ab6592..96b8db5d5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,7 +10,7 @@ jobs: Security: name: Security Pipeline uses: uc-cdis/.github/.github/workflows/securitypipeline.yaml@master - secrets: inherit + secrets: inherit # pragma: allowlist secret UnitTest: name: Python Unit Test with Postgres @@ -19,13 +19,30 @@ jobs: python-version: '3.9' test-script: 'tests/ci_commands_script.sh' run-coveralls: true - ci: + + BuildImageAndPush: + name: Build Image and Push + needs: Security + # https://github.com/uc-cdis/.github/blob/master/.github/workflows/image_build_push.yaml + uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + secrets: + ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} + ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + + # I did not test that this works + BuildMcryptImageAndPush: name: Build Image and Push - # TODO Uncomment after PXP-9212 - # needs: Security + needs: Security + # https://github.com/uc-cdis/.github/blob/master/.github/workflows/image_build_push.yaml uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master secrets: ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + # we can either use the repo name or +# OVERRIDE_REPO_NAME: fenceMcrypt + OVERRIDE_TAG_NAME: "mcrypt_$(echo ${GITHUB_REF#refs/*/} | tr / _)" + DOCKERFILE_LOCATION: "./DockerfileMcrypt" diff --git a/.gitignore b/.gitignore index 4a76c3a3e..c760e0ad2 100644 --- a/.gitignore +++ b/.gitignore @@ -102,7 +102,6 @@ ENV/ .mypy_cache/ # jwt keys -keys tests/resources/keys/*.pem .DS_Store diff --git a/.secrets.baseline b/.secrets.baseline index 82a2c6b2a..469ed8bfc 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -115,13 +115,13 @@ } ], "results": { - ".github/workflows/ci.yaml": [ + ".github/workflows/buildpipeline.yaml": [ { "type": "Secret Keyword", - "filename": ".github/workflows/ci.yaml", + "filename": ".github/workflows/buildpipeline.yaml", "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0", "is_verified": false, - "line_number": 13 + "line_number": 17 } ], "deployment/scripts/postgresql/postgresql_init.sql": [ @@ -210,22 +210,13 @@ "line_number": 137 } ], - "fence/resources/storage/storageclient/cleversafe.py": [ - { - "type": "Secret Keyword", - "filename": "fence/resources/storage/storageclient/cleversafe.py", - "hashed_secret": "7cb6efb98ba5972a9b5090dc2e517fe14d12cb04", - "is_verified": false, - "line_number": 274 - } - ], "fence/utils.py": [ { "type": "Secret Keyword", "filename": "fence/utils.py", "hashed_secret": "8318df9ecda039deac9868adf1944a29a95c7114", "is_verified": false, - "line_number": 129 + "line_number": 128 } ], "migrations/versions/a04a70296688_non_unique_client_name.py": [ @@ -268,14 +259,14 @@ "filename": "tests/conftest.py", "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", "is_verified": false, - "line_number": 1570 + "line_number": 1556 }, { "type": "Base64 High Entropy String", "filename": "tests/conftest.py", "hashed_secret": "227dea087477346785aefd575f91dd13ab86c108", "is_verified": false, - "line_number": 1594 + "line_number": 1579 } ], "tests/credentials/google/test_credentials.py": [ @@ -394,24 +385,6 @@ "line_number": 300 } ], - "tests/storageclient/storage_client_mock.py": [ - { - "type": "Secret Keyword", - "filename": "tests/storageclient/storage_client_mock.py", - "hashed_secret": "37bbea9557f9efd1eeadb25dda9ab6514f08fde9", - "is_verified": false, - "line_number": 158 - } - ], - "tests/storageclient/test_cleversafe_api_client.py": [ - { - "type": "Secret Keyword", - "filename": "tests/storageclient/test_cleversafe_api_client.py", - "hashed_secret": "f683c485d521c2e45830146dd570111770baea29", - "is_verified": false, - "line_number": 130 - } - ], "tests/test-fence-config.yaml": [ { "type": "Basic Auth Credentials", @@ -422,5 +395,5 @@ } ] }, - "generated_at": "2024-08-22T19:43:39Z" + "generated_at": "2023-10-20T20:37:17Z" } diff --git a/Dockerfile b/Dockerfile index f103e44f9..a2762ecce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,56 +1,49 @@ -# To run: docker run --rm -d -v /path/to/fence-config.yaml:/var/www/fence/fence-config.yaml --name=fence -p 80:80 fence -# To check running container do: docker exec -it fence /bin/bash +# To build: docker build -t fence:latest . +# To run interactive: +# docker run -v ~/.gen3/fence/fence-config.yaml:/var/www/fence/fence-config.yaml -v ./keys/:/fence/keys/ fence:latest +# To check running container do: docker exec -it CONTAINER bash -FROM quay.io/cdis/python:python3.9-buster-2.0.0 +ARG AZLINUX_BASE_VERSION=feat_python-nginx + +# ------ Base stage ------ +FROM quay.io/cdis/python-nginx-al:${AZLINUX_BASE_VERSION} AS base + +# Comment this in, and comment out the line above, if quay is down +# FROM 707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/python-nginx-al:${AZLINUX_BASE_VERSION} as base ENV appname=fence -RUN pip install --upgrade pip -RUN pip install --upgrade poetry -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl bash git \ - && apt-get -y install vim \ - libmcrypt4 mcrypt \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/ - -RUN mkdir -p /var/www/$appname \ - && mkdir -p /var/www/.cache/Python-Eggs/ \ - && mkdir /run/nginx/ \ - && ln -sf /dev/stdout /var/log/nginx/access.log \ - && ln -sf /dev/stderr /var/log/nginx/error.log \ - && chown nginx -R /var/www/.cache/Python-Eggs/ \ - && chown nginx /var/www/$appname - -# aws cli v2 - needed for storing files in s3 during usersync k8s job -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ - && unzip awscliv2.zip \ - && ./aws/install \ - && /bin/rm -rf awscliv2.zip ./aws - -WORKDIR /$appname - -# copy ONLY poetry artifact, install the dependencies but not fence -# this will make sure than the dependencies is cached -COPY poetry.lock pyproject.toml /$appname/ -RUN poetry config virtualenvs.create false \ - && poetry install -vv --no-root --no-dev --no-interaction \ - && poetry show -v - -# copy source code ONLY after installing dependencies -COPY . /$appname -COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini -COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py -COPY clear_prometheus_multiproc /$appname/clear_prometheus_multiproc - -# install fence -RUN poetry config virtualenvs.create false \ - && poetry install -vv --no-dev --no-interaction \ - && poetry show -v - -RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \ - && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py - -WORKDIR /var/www/$appname - -CMD ["sh","-c","bash /fence/dockerrun.bash && /dockerrun.sh"] +WORKDIR /${appname} + +RUN chown -R gen3:gen3 /${appname} + +# ------ Builder stage ------ +FROM base AS builder + +# Install just the deps without the code as it's own step to avoid redoing this on code changes +COPY poetry.lock pyproject.toml /${appname}/ +RUN poetry install -vv --only main --no-interaction + +# Move app files into working directory +COPY --chown=gen3:gen3 . /$appname +COPY --chown=gen3:gen3 ./deployment/wsgi/wsgi.py /$appname/wsgi.py + +# Do the install again incase the app itself needs install +RUN poetry install -vv --only main --no-interaction + +ENV PATH="$(poetry env info --path)/bin:$PATH" + +# Setup version info +RUN git config --global --add safe.directory /${appname} && COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" > /$appname/version_data.py \ + && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >> /$appname/version_data.py + +# install tar +# RUN yum install tar -y +# do we need to untar jwt-keys? + +# ------ Final stage ------ +FROM base + +COPY --chown=gen3:gen3 --from=builder /$appname /$appname + +CMD ["poetry", "run", "gunicorn", "-c", "deployment/wsgi/gunicorn.conf.py"] diff --git a/DockerfileMcrypt b/DockerfileMcrypt new file mode 100644 index 000000000..383c07bbf --- /dev/null +++ b/DockerfileMcrypt @@ -0,0 +1 @@ +# old Dockerfile diff --git a/deployment/fence.conf b/deployment/fence.conf deleted file mode 100644 index 03c5d6812..000000000 --- a/deployment/fence.conf +++ /dev/null @@ -1,19 +0,0 @@ - - WSGIDaemonProcess /fence processes=2 threads=4 python-path=/var/www/fence/:/fence/:/usr/bin/python - WSGIScriptAlias / /var/www/fence/wsgi.py - WSGIPassAuthorization On - - WSGIProcessGroup /fence - WSGIApplicationGroup %{GLOBAL} - Options +ExecCGI - Order deny,allow - Allow from all - - ErrorLog ${APACHE_LOG_DIR}/error.log - LogLevel info - LogFormat "%{X-Forwarded-For}i %l %{X-UserId}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" aws - LogFormat "{\"gen3log\": \"apache2\", \"date_access\": \"%t\", \"user_id\": \"%{X-UserId}i\", \"request_id\": \"%{X-ReqId}i\", \"session_id\": \"%{X-SessionId}i\", \"visitor_id\": \"%{X-VisitorId}i\", \"network_client_ip\": \"%{X-Forwarded-For}i\", \"network_bytes_write\": \"%b\", \"http_response_time\": \"%T\", \"http_status_code\": \"%>s\", \"http_request\": \"%U\", \"http_verb\": \"%m\", \"http_referer\": \"%{Referer}i\", \"http_useragent\": \"%{User-agent}i\", \"message\": \"%r\"}" json - SetEnvIf X-Forwarded-For "^..*" forwarded - #CustomLog ${APACHE_LOG_DIR}/access.log combined env=!forwarded - CustomLog ${APACHE_LOG_DIR}/access.log json - diff --git a/deployment/uwsgi/uwsgi.ini b/deployment/uwsgi/uwsgi.ini deleted file mode 100644 index 0ebedb7ce..000000000 --- a/deployment/uwsgi/uwsgi.ini +++ /dev/null @@ -1,37 +0,0 @@ -[uwsgi] -protocol = uwsgi -socket = /var/run/gen3/uwsgi.sock -buffer-size = 32768 -uid = nginx -gid = nginx -chown-socket = nginx:nginx -chmod-socket = 666 -master = true -harakiri-verbose = true -# No global HARAKIRI, using only user HARAKIRI, because export overwrites it -# Cannot overwrite global HARAKIRI with user's: https://git.io/fjYuD -# harakiri = 45 -http-timeout = 45 -socket-timeout = 45 -worker-reload-mercy = 45 -reload-mercy = 45 -mule-reload-mercy = 45 -disable-logging = true -wsgi-file=/fence/wsgi.py -plugins = python3 -vacuum = true -pythonpath = /var/www/fence/ -pythonpath = /fence/ -# poetry installs git dependencies at /usr/local/src -pythonpath = /usr/local/src/* - -# metrics setup -stats = 127.0.0.1:9191 -stats-http = true -env = prometheus_multiproc_dir=/var/tmp/uwsgi_flask_metrics -exec-asap = /fence/clear_prometheus_multiproc /var/tmp/uwsgi_flask_metrics - -# Initialize application in worker processes, not master. This prevents the -# workers from all trying to open the same database connections at startup. -lazy = true -lazy-apps = true diff --git a/deployment/wsgi/gunicorn.conf.py b/deployment/wsgi/gunicorn.conf.py new file mode 100644 index 000000000..64092ba3d --- /dev/null +++ b/deployment/wsgi/gunicorn.conf.py @@ -0,0 +1,9 @@ +wsgi_app = "deployment.wsgi.wsgi:application" +bind = "0.0.0.0:8000" +workers = 1 +preload_app = True +user = "gen3" +group = "gen3" +timeout = 300 +keepalive = 2 +keepalive_timeout = 5 diff --git a/deployment/uwsgi/wsgi.py b/deployment/wsgi/wsgi.py similarity index 100% rename from deployment/uwsgi/wsgi.py rename to deployment/wsgi/wsgi.py diff --git a/dockerrun.bash b/dockerrun.bash index ce7150731..9c9a67313 100755 --- a/dockerrun.bash +++ b/dockerrun.bash @@ -4,7 +4,7 @@ # Update certificate authority index - # environment may have mounted more authorities # -update-ca-certificates +# update-ca-certificates # # Kubernetes may mount jwt-keys as a tar ball # @@ -15,6 +15,9 @@ if [ -f /fence/jwt-keys.tar ]; then if [ -d jwt-keys ]; then mkdir -p keys mv jwt-keys/* keys/ + rm -rf /fence/keys/key/ fi ) fi +nginx +gunicorn -c /fence/deployment/wsgi/gunicorn.conf.py diff --git a/dockerrunshib.bash b/dockerrunshib.bash deleted file mode 100755 index 1ad35e65b..000000000 --- a/dockerrunshib.bash +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# -# Update certificate authority index - -# environment may have mounted more authorities -# -update-ca-certificates -# -# Kubernetes may mount jwt-keys as a tar ball -# -if [ -f /fence/jwt-keys.tar ]; then - ( - cd /fence - tar xvf jwt-keys.tar - if [ -d jwt-keys ]; then - mkdir -p keys - mv jwt-keys/* keys/ - fi - ) -fi -service shibd start -sed -i "s/ServerName SERVERNAME/ServerName https:\/\/$HOSTNAME/g" /etc/apache2/sites-available/fence.conf -rm -rf /var/run/apache2/apache2.pid -/usr/sbin/apache2ctl -D FOREGROUND diff --git a/docs/additional_documentation/fence_create.md b/docs/additional_documentation/fence_create.md index 398544f98..db2b735f3 100644 --- a/docs/additional_documentation/fence_create.md +++ b/docs/additional_documentation/fence_create.md @@ -53,7 +53,7 @@ curl --request POST https://FENCE_URL/oauth2/token?grant_type=client_credentials The optional `--expires-in` parameter allows specifying the number of *days* until this client expires. The recommendation is to rotate credentials with the `client_credentials` grant at least once a year (see [Rotate client credentials](#rotate-client-credentials) section). -NOTE: In Gen3, you can grant specific access to a client the same way you would to a user. See the [user.yaml guide](https://github.com/uc-cdis/fence/blob/master/docs/user.yaml_guide.md) for more details. +NOTE: In Gen3, you can grant specific access to a client the same way you would to a user. See the [user.yaml guide](https://github.com/uc-cdis/fence/blob/master/docs/additional_documentation/user.yaml_guide.md) for more details. NOTE: Client credentials tokens are not linked to a user (the claims contain no `sub` or `context.user.name` like other tokens). Some Gen3 endpoints that assume the token is linked to a user, or whose logic require there being a user, do not support them. For an example of how to adapt an endpoint to support client credentials tokens, see [here](https://github.com/uc-cdis/requestor/commit/a5078fae27fa258ac78045cf2bb89cb2104f53cf). For an example of how to explicitly reject client credentials tokens, see [here](https://github.com/uc-cdis/requestor/commit/0f4974c25343d2185c7cdb48dcdeb58f97800672). diff --git a/docs/additional_documentation/user.yaml_guide.md b/docs/additional_documentation/user.yaml_guide.md index 893d32045..77e9e2b1d 100644 --- a/docs/additional_documentation/user.yaml_guide.md +++ b/docs/additional_documentation/user.yaml_guide.md @@ -16,7 +16,7 @@ The `user.yaml` file is one way to get authorization information into Gen3. It is ingested via [Fence's `usersync` script](usersync.md). The format of this file is tightly coupled with the notions of resource, role and policy as defined by Gen3's policy engine, [Arborist](https://github.com/uc-cdis/arborist#arborist). -For Gen3 Data Commons that do not use Arborist or that use the Google Data Access method of [Google Service Account Registration](https://github.com/uc-cdis/fence/blob/master/docs/google_architecture.md#google-account-linking-and-service-account-registration), refer to the [Deprecated format](#deprecated-format) section. +For Gen3 Data Commons that do not use Arborist or that use the Google Data Access method of [Google Service Account Registration](https://github.com/uc-cdis/fence/blob/master/docs/additional_documentation/google_architecture.md#google-account-linking-and-service-account-registration), refer to the [Deprecated format](#deprecated-format) section. In a fully deployed Gen3 Commons using [Cloud Automation](https://github.com/uc-cdis/cloud-automation), the `user.yaml` file is usually hosted in S3 and configured via the `global.useryaml_s3path` setting of the Gen3 Data Commons manifest: ``` diff --git a/fence/jwt/keys.py b/fence/jwt/keys.py index d08e9d5f0..f423362fb 100644 --- a/fence/jwt/keys.py +++ b/fence/jwt/keys.py @@ -148,13 +148,26 @@ def from_directory(cls, keys_dir, naming_function=None): prv_filepath = os.path.join(keys_dir, "jwt_private_key.pem") if not os.path.isfile(pub_filepath): - raise EnvironmentError( - "missing public key file; expected file to exist: " + pub_filepath - ) + # Generate public key from private key + with open(prv_filepath, "r") as f: + private_key_file = f.read() + private_key = serialization.load_pem_private_key( + bytes(private_key_file, "utf-8"), + password=None, + backend=default_backend(), + ) + public_key = private_key.public_key() + public_key = public_key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + public_key = public_key.decode("utf-8") + with open(pub_filepath, "w") as f: + f.write(public_key) if not os.path.isfile(prv_filepath): raise EnvironmentError( - "missing public key file; expected file to exist: " + prv_filepath + "missing private key file; expected file to exist: " + prv_filepath ) with open(pub_filepath, "r") as f: diff --git a/poetry.lock b/poetry.lock index f28981dca..e74f5b3c5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1310,6 +1310,27 @@ files = [ docs = ["Sphinx", "furo"] test = ["objgraph", "psutil"] +[[package]] +name = "gunicorn" +version = "23.0.0" +description = "WSGI HTTP Server for UNIX" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d"}, + {file = "gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec"}, +] + +[package.dependencies] +packaging = "*" + +[package.extras] +eventlet = ["eventlet (>=0.24.1,!=0.36.0)"] +gevent = ["gevent (>=1.4.0)"] +setproctitle = ["setproctitle"] +testing = ["coverage", "eventlet", "gevent", "pytest", "pytest-cov"] +tornado = ["tornado (>=0.2)"] + [[package]] name = "h11" version = "0.14.0" @@ -2582,4 +2603,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0.0" -content-hash = "655105408255f1d68d57e3517f8d961b68d1389fe93aed82d1c1daed7042c444" +content-hash = "d9ada83a71b032e740eddfece93f2894892aba9645c90b70160139776f18a957" diff --git a/pyproject.toml b/pyproject.toml index 38ed3e908..96bcea822 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,8 @@ markupsafe = "^2.0.1" paramiko = ">=2.6.0" prometheus-client = "<1" -psycopg2 = "<3" +psycopg2-binary = "<3" +gunicorn = ">=21" PyJWT = "^2.4.0" python_dateutil = "^2.6.1" python-jose = "^2.0.2"