Merge branch 'master' into chore/adding_requestor_db_to_psql_fix_script

uc-cdis · Oct 3, 2022 · ecf6f42 · ecf6f42
2 parents ce8c7db + 0815359
commit ecf6f42
Show file tree

Hide file tree

Showing 242 changed files with 12,472 additions and 808 deletions.
diff --git a/.github/workflows/image_build_push.yaml b/.github/workflows/image_build_push.yaml
@@ -29,3 +29,14 @@ jobs:
       ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
       QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
       QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  awshelper:
+    name: AwsHelper Build and Push
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./Docker/awshelper/Dockerfile"
+      OVERRIDE_REPO_NAME: "awshelper"
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -1,9 +1,9 @@
 {
   "exclude": {
-    "files": "^.secrets.baseline$|^./.secrets.baseline$",
+    "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2022-03-29T17:29:39Z",
+  "generated_at": "2022-07-29T15:31:31Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -88,7 +88,7 @@
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 132,
+        "line_number": 138,
         "type": "Secret Keyword"
       }
     ],
@@ -101,6 +101,15 @@
         "type": "Secret Keyword"
       }
     ],
+    "Docker/Jenkins2/Dockerfile": [
+      {
+        "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 113,
+        "type": "Secret Keyword"
+      }
+    ],
     "Docker/sidecar/service.key": [
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
@@ -378,6 +387,15 @@
         "type": "Secret Keyword"
       }
     ],
+    "gen3/bin/kube-setup-dicom-server.sh": [
+      {
+        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 43,
+        "type": "Secret Keyword"
+      }
+    ],
     "gen3/bin/kube-setup-jenkins.sh": [
       {
         "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f",
@@ -532,14 +550,14 @@
         "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 550,
+        "line_number": 640,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "5b4b6c62d3d99d202f095c38c664eded8f640ce8",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 570,
+        "line_number": 660,
         "type": "Secret Keyword"
       }
     ],
@@ -722,7 +740,7 @@
         "hashed_secret": "52330dffa4d0795b4199a66428e54eca228e1661",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 7,
+        "line_number": 15,
         "type": "Secret Keyword"
       }
     ],
@@ -1179,6 +1197,93 @@
         "type": "JSON Web Token"
       }
     ],
+    "kube/services/superset/superset-deploy.yaml": [
+      {
+        "hashed_secret": "96e4aceb7cf284be363aa248a32a7cc89785a9f7",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 38,
+        "type": "Secret Keyword"
+      }
+    ],
+    "kube/services/superset/superset-redis.yaml": [
+      {
+        "hashed_secret": "4af3596275edcb7cd5cc6c3c38bc10479902a08f",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 165,
+        "type": "Secret Keyword"
+      },
+      {
+        "hashed_secret": "244f421f896bdcdd2784dccf4eaf7c8dfd5189b5",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 260,
+        "type": "Secret Keyword"
+      }
+    ],
+    "kube/services/superset/superset/superset-deploy.yaml": [
+      {
+        "hashed_secret": "96e4aceb7cf284be363aa248a32a7cc89785a9f7",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 38,
+        "type": "Secret Keyword"
+      }
+    ],
+    "kube/services/superset/superset/superset-redis.yaml": [
+      {
+        "hashed_secret": "4af3596275edcb7cd5cc6c3c38bc10479902a08f",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 169,
+        "type": "Secret Keyword"
+      },
+      {
+        "hashed_secret": "244f421f896bdcdd2784dccf4eaf7c8dfd5189b5",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 266,
+        "type": "Secret Keyword"
+      }
+    ],
+    "kube/services/superset/values.yaml": [
+      {
+        "hashed_secret": "6f803b24314c39062efe38d0c1da8c472f47eab3",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 54,
+        "type": "Secret Keyword"
+      },
+      {
+        "hashed_secret": "6eae3a5b062c6d0d79f070c26e6d62486b40cb46",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 86,
+        "type": "Secret Keyword"
+      },
+      {
+        "hashed_secret": "3eb416223e9e69e6bb8ee19793911ad1ad2027d8",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 212,
+        "type": "Secret Keyword"
+      },
+      {
+        "hashed_secret": "ff55435345834a3fe224936776c2aa15f6ed5358",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 396,
+        "type": "Secret Keyword"
+      },
+      {
+        "hashed_secret": "98a84a63e5633d17e3b27b69695f87aa7189e9dc",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 503,
+        "type": "Secret Keyword"
+      }
+    ],
     "package-lock.json": [
       {
         "hashed_secret": "c95b6bc99445e7ed9177040f5ef94d0cdb38fb21",
@@ -2130,12 +2235,21 @@
         "type": "Secret Keyword"
       }
     ],
+    "tf_files/aws/eks/sample.tfvars": [
+      {
+        "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
+        "is_secret": false,
+        "is_verified": false,
+        "line_number": 107,
+        "type": "Hex High Entropy String"
+      }
+    ],
     "tf_files/aws/eks/variables.tf": [
       {
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 135,
+        "line_number": 133,
         "type": "Hex High Entropy String"
       }
     ],
@@ -2307,15 +2421,6 @@
         "type": "Hex High Entropy String"
       }
     ],
-    "tf_files/aws/rds/sample.tfvars": [
-      {
-        "hashed_secret": "76c3c4836dee37d8d0642949f84092a9a24bbf46",
-        "is_secret": false,
-        "is_verified": false,
-        "line_number": 7,
-        "type": "Secret Keyword"
-      }
-    ],
     "tf_files/aws/slurm/README.md": [
       {
         "hashed_secret": "fd85d792fa56981cf6a8d2a5c0857c74af86e99d",

diff --git a/Docker/Jenkins-CI-Worker/Dockerfile b/Docker/Jenkins-CI-Worker/Dockerfile
@@ -1,4 +1,4 @@
-FROM jenkins/jnlp-slave:4.3-1
+FROM jenkins/jnlp-slave:4.9-1
 
 USER root
 

diff --git a/Docker/Jenkins-Worker/Dockerfile b/Docker/Jenkins-Worker/Dockerfile
@@ -8,7 +8,8 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils python python-setuptools python-dev python-pip python3 python3-pip build-essential libgit2-dev zip unzip less vim gettext-base
 RUN set -xe && python -m pip install awscli --upgrade && python -m pip install pytest --upgrade && python -m pip install PyYAML --upgrade && python -m pip install lxml --upgrade
 RUN set -xe && python3 -m pip install pytest --upgrade && python3 -m pip install PyYAML --upgrade
-RUN set -xe && python -m pip install yq --upgrade && python3 -m pip install yq --upgrade && python3 -m pip install pandas --upgrade
+RUN set -xe && python -m pip install yq --upgrade && python3 -m pip install yq --upgrade
+RUN set -xe && python3 -m pip install pandas --upgrade
 
 RUN apt-get update \
   && apt-get install -y lsb-release \
@@ -50,6 +51,11 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747
   && apt-get update \
   && apt-get install k6
 
+# install xk6-browser
+RUN cd /opt && wget --quiet https://github.com/grafana/xk6-browser/releases/download/v0.3.0/xk6-browser-v0.3.0-linux-amd64.tar.gz \
+  && tar -xvzf /opt/xk6-browser-v0.3.0-linux-amd64.tar.gz
+ENV PATH="/opt/xk6-browser-v0.3.0-linux-amd64:${PATH}"
+
 # install google tools
 RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" \
     && echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list \
@@ -99,9 +105,9 @@ RUN unzip /tmp/packer.zip -d /usr/local/bin; /bin/rm /tmp/packer.zip
 # add psql: https://www.postgresql.org/download/linux/debian/
 RUN DISTRO="$(lsb_release -c -s)"  \
       && echo "deb http://apt.postgresql.org/pub/repos/apt/ ${DISTRO}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
-      && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+      && wget --quiet --no-check-certificate -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
       && apt-get update \
-      && apt-get install -y postgresql-client-13 \
+      && apt-get install -y postgresql-client-13 libpq-dev \
       && rm -rf /var/lib/apt/lists/*
 
 # Copy sh script responsible for installing Python

diff --git a/Docker/Jenkins2/Dockerfile b/Docker/Jenkins2/Dockerfile
@@ -4,15 +4,122 @@ USER root
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-# install python and pip and aws cli
-RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils python python-setuptools python-dev python-pip python3 python3-pip build-essential zip unzip jq less vim gettext-base
-RUN set -xe && python -m pip install awscli --upgrade && python -m pip install pytest --upgrade && python -m pip install PyYAML --upgrade
-RUN set -xe && python3 -m pip install pytest --upgrade && python3 -m pip install PyYAML --upgrade
-RUN set -xe && python -m pip install yq --upgrade && python3 -m pip install yq --upgrade
+# install python
+RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils python python-setuptools python-dev python-pip python3 python3-pip python3-venv build-essential zip unzip jq less vim gettext-base
+
+RUN set -xe && apt-get update \
+  && apt-get install -y lsb-release \
+     apt-transport-https \
+     ca-certificates \
+     curl \
+     gnupg2 \
+     libffi-dev \
+     libssl-dev \
+     libcurl4-openssl-dev \
+     libncurses5-dev \
+     libncursesw5-dev \
+     libreadline-dev \
+     libsqlite3-dev \
+     libgdbm-dev \
+     libdb5.3-dev \
+     libbz2-dev \
+     libexpat1-dev \
+     liblzma-dev \
+     python-virtualenv \
+     lua5.3 \
+     r-base \
+     software-properties-common \
+     sudo \
+     tk-dev \
+     zlib1g-dev \
+     zsh \
+  && ln -s /usr/bin/lua5.3 /usr/local/bin/lua
+
+# install google tools
+RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" \
+    && echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list \
+    && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+    && apt-get update \
+    && apt-get install -y google-cloud-sdk \
+          google-cloud-sdk-cbt \
+          kubectl
+
+#
+# install docker tools:
+#  * https://docs.docker.com/install/linux/docker-ce/debian/#install-docker-ce-1
+#  * https://docs.docker.com/compose/install/#install-compose
+#
+RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
+    && add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/debian \
+   $(lsb_release -cs) \
+   stable" \
+   && apt-get update \
+   && apt-get install -y docker-ce \
+   && curl -L "https://github.com/docker/compose/releases/download/1.23.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose \
+   && chmod a+rx /usr/local/bin/docker-compose
+
+# install nodejs
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash -
+RUN apt-get update && apt-get install -y nodejs
+
+# add psql: https://www.postgresql.org/download/linux/debian/
+RUN DISTRO="$(lsb_release -c -s)"  \
+      && echo "deb http://apt.postgresql.org/pub/repos/apt/ ${DISTRO}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
+      && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+      && apt-get update \
+      && apt-get install -y postgresql-client-13 libpq-dev \
+      && rm -rf /var/lib/apt/lists/*
+
+# Copy sh script responsible for installing Python
+COPY install-python3.8.sh /root/tmp/install-python3.8.sh
+
+# Run the script responsible for installing Python 3.8.0 and link it to /usr/bin/python
+RUN chmod +x /root/tmp/install-python3.8.sh; sync && \
+	./root/tmp/install-python3.8.sh && \
+	rm -rf /root/tmp/install-python3.8.sh && \
+        unlink /usr/bin/python3 && \
+        ln -s /Python-3.8.0/python /usr/bin/python3
+
+# Fix shebang for lsb_release
+RUN sed -i 's/python3/python3.5/' /usr/bin/lsb_release && \
+    sed -i 's/python3/python3.5/' /usr/bin/add-apt-repository
+
+# install aws cli, poetry, pytest, etc.
+RUN set -xe && python3 -m pip install awscli --upgrade && python3 -m pip install pytest --upgrade && python3 -m pip install poetry && python3 -m pip install PyYAML --upgrade && python3 -m pip install lxml --upgrade && python3 -m pip install yq --upgrade
+
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3 -
+
+# install chrome (supports headless mode)
+RUN set -xe \
+    && curl -fsSL https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+    && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \
+    && apt-get update \
+    && apt-get install -y google-chrome-stable
+
+# install terraform
+RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.11.15/terraform_0.11.15_linux_amd64.zip \
+   && unzip /tmp/terraform.zip -d /usr/local/bin && /bin/rm /tmp/terraform.zip
+
+RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.12.31/terraform_0.12.31_linux_amd64.zip \
+   && unzip /tmp/terraform.zip -d /tmp && mv /tmp/terraform /usr/local/bin/terraform12 && /bin/rm /tmp/terraform.zip
+
+# install packer
+RUN curl -o /tmp/packer.zip https://releases.hashicorp.com/packer/1.5.1/packer_1.5.1_linux_amd64.zip
+RUN unzip /tmp/packer.zip -d /usr/local/bin; /bin/rm /tmp/packer.zip
+
+# update /etc/sudoers
+RUN sed 's/^%sudo/#%sudo/' /etc/sudoers > /etc/sudoers.bak \
+  && /bin/echo -e "\n%sudo    ALL=(ALL:ALL) NOPASSWD:ALL\n" >> /etc/sudoers.bak \
+  && cp /etc/sudoers.bak /etc/sudoers \
+  && usermod -G sudo jenkins
 
 # add our custom start script
 COPY jenkins2.sh /opt/cdis/bin/jenkins2.sh
 RUN chmod -R a+rx /opt/cdis
 ENTRYPOINT ["/sbin/tini", "--", "/opt/cdis/bin/jenkins2.sh"]
 
 USER jenkins
+
+RUN git config --global user.email jenkins \
+    && git config --global user.name jenkins