From 811f769604b7afd8c9acc6cd1376128931603232 Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Fri, 7 Jun 2024 10:52:10 -0400 Subject: [PATCH 01/22] [ALS-6100] All-in-one: Move PSAMA to its own Docker (#125) In order to move the pic-sure-auth-micro-app to its own container I did the following: - Update HTTPD networking to redirect PSAMA request to the PSAMA container. - Added a new `.env` file that is passed to PSAMA as part of the build. - Updated the configuration pipeline to set values in the PSAMA`.env` file. - Removed PSAMA from WildFly database configuration. - Added necessary docker commands to both the `start-picsure.sh` and `stop-picsure.sh`. --- .../config/httpd/httpd-vhosts-ssloffload.conf | 2 +- .../config/httpd/httpd-vhosts.conf | 2 +- .../config/httpd/picsureui_settings.json | 2 +- .../config/httpd/psamaui_settings.json | 2 +- initial-configuration/config/psama/.env | 28 +++++++++++++++++++ .../config/wildfly/standalone.xml | 25 ++--------------- initial-configuration/install-dependencies.sh | 2 +- .../config.xml | 10 +++++++ .../config.xml | 3 +- .../config.xml | 4 +-- .../PIC-SURE Auth Micro-App Build/config.xml | 17 ++--------- .../PIC-SURE Wildfly Image Build/config.xml | 2 +- initial-configuration/mysql-docker/setup.sh | 2 +- start-picsure.sh | 12 +++++++- stop-picsure.sh | 2 +- 15 files changed, 66 insertions(+), 49 deletions(-) create mode 100644 initial-configuration/config/psama/.env diff --git a/initial-configuration/config/httpd/httpd-vhosts-ssloffload.conf b/initial-configuration/config/httpd/httpd-vhosts-ssloffload.conf index 5f76eefd..76b8c35d 100644 --- a/initial-configuration/config/httpd/httpd-vhosts-ssloffload.conf +++ b/initial-configuration/config/httpd/httpd-vhosts-ssloffload.conf @@ -34,7 +34,7 @@ Listen 0.0.0.0:80 RewriteRule ^/picsure/(.*)$ "http://wildfly:8080/pic-sure-api-2/PICSURE/$1" [P] - RewriteRule ^/psama/(.*)$ "http://wildfly:8080/pic-sure-auth-services/auth/$1" [P] + RewriteRule ^/psama/(.*)$ "http://psama:8090/auth/$1" [P] RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-d diff --git a/initial-configuration/config/httpd/httpd-vhosts.conf b/initial-configuration/config/httpd/httpd-vhosts.conf index 9145565d..ef1241f4 100644 --- a/initial-configuration/config/httpd/httpd-vhosts.conf +++ b/initial-configuration/config/httpd/httpd-vhosts.conf @@ -88,7 +88,7 @@ ServerTokens Prod RewriteRule ^/picsure/(.*)$ "http://wildfly:8080/pic-sure-api-2/PICSURE/$1" [P] - RewriteRule ^/psama/(.*)$ "http://wildfly:8080/pic-sure-auth-services/auth/$1" [P] + RewriteRule ^/psama/(.*)$ "http://psama:8090/auth/$1" [P] RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-d diff --git a/initial-configuration/config/httpd/picsureui_settings.json b/initial-configuration/config/httpd/picsureui_settings.json index bc6655ef..6506b0a1 100644 --- a/initial-configuration/config/httpd/picsureui_settings.json +++ b/initial-configuration/config/httpd/picsureui_settings.json @@ -26,7 +26,7 @@ "customizeAuth0Login": true, "queryButtonLabel": "Export for analysis", "maxVariantCount": 10000, - "auth0domain":"__AUTH0_DOMAIN__", + "auth0domain":"__AUTH0_TENANT__", "client_id":"__PIC_SURE_CLIENT_ID__", "analyticsId": "__ANALYTICS_ID__", "tagManagerId": "__TAG_MANAGER_ID__" diff --git a/initial-configuration/config/httpd/psamaui_settings.json b/initial-configuration/config/httpd/psamaui_settings.json index 7d1e0a9c..2b96b8ec 100644 --- a/initial-configuration/config/httpd/psamaui_settings.json +++ b/initial-configuration/config/httpd/psamaui_settings.json @@ -6,7 +6,7 @@ "basePath" : "/psama", "uiPath": "", "customizeAuth0Login": true, - "auth0domain":"__AUTH0_DOMAIN__", + "auth0domain":"__AUTH0_TENANT__", "client_id":"__PIC_SURE_CLIENT_ID__" } diff --git a/initial-configuration/config/psama/.env b/initial-configuration/config/psama/.env new file mode 100644 index 00000000..34f938de --- /dev/null +++ b/initial-configuration/config/psama/.env @@ -0,0 +1,28 @@ +# Database Configuration +DATASOURCE_URL=jdbc:mysql://picsure-db:3306/auth?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&autoReconnectForPools=true&serverTimezone=UTC +DATASOURCE_USERNAME=auth +DATASOURCE_PASSWORD=__AUTH_MYSQL_PASSWORD__ + +# Mail Configuration +EMAIL_ADDRESS=__EMAIL_FROM_ADDR__ +EMAIL_PASSWORD=__EMAIL_PASSWORD__ + +# Application Properties +CLIENT_ID=__PIC_SURE_CLIENT_ID__ +CLIENT_SECRET=__PIC_SURE_CLIENT_SECRET__ +TOS_ENABLED=false +SYSTEM_NAME=PIC-SURE All-in-one +GRANT_EMAIL_SUBJECT=__ACCESS_GRANTED_EMAIL_SUBJECT__ +USER_ACTIVATION_REPLY_TO=__USER_ACTIVATION_REPLY_TO__ +ADMIN_USERS=__ADMIN_USERS__ +DENIED_EMAIL_ENABLED=false +STACK_SPECIFIC_APPLICATION_ID=__STACK_SPECIFIC_APPLICATION_ID__ + +# IDP Provider Configuration +IDP_PROVIDER=auth0 +IDP_PROVIDER_URI=https://__AUTH0_TENANT__.auth0.com/ +AUTH0_HOST=https://__AUTH0_TENANT__.auth0.com/ + +# Token Expiration Times +TOKEN_EXPIRATION_TIME=3600000 +LONG_TERM_TOKEN_EXPIRATION_TIME=2592000000 \ No newline at end of file diff --git a/initial-configuration/config/wildfly/standalone.xml b/initial-configuration/config/wildfly/standalone.xml index 9c32b3d9..e189ea68 100644 --- a/initial-configuration/config/wildfly/standalone.xml +++ b/initial-configuration/config/wildfly/standalone.xml @@ -158,26 +158,6 @@ sa - - jdbc:mysql://picsure-db:3306/auth?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&autoReconnectForPools=true&serverTimezone=UTC - mysql - - 2 - 10 - true - - - auth - __AUTH_MYSQL_PASSWORD__ - - - - SELECT 1 - true - false - - - jdbc:mysql://picsure-db:3306/picsure?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&autoReconnectForPools=true&serverTimezone=UTC mysql @@ -461,7 +441,8 @@ - + + @@ -475,7 +456,7 @@ - + diff --git a/initial-configuration/install-dependencies.sh b/initial-configuration/install-dependencies.sh index 7c3f3390..36f3511b 100755 --- a/initial-configuration/install-dependencies.sh +++ b/initial-configuration/install-dependencies.sh @@ -87,7 +87,7 @@ rm -f picsure.tmp echo "` < /dev/urandom tr -dc @^=+$*%_A-Z-a-z-0-9 | head -c${1:-24}`%4cA" > auth.tmp mysql -u root -e "grant all privileges on auth.* to 'auth'@'%' identified by '`cat auth.tmp`';flush privileges;"; -sed -i s/__AUTH_MYSQL_PASSWORD__/`cat auth.tmp`/g /usr/local/docker-config/wildfly/standalone.xml +sed -i s/__AUTH_MYSQL_PASSWORD__/`cat auth.tmp`/g /usr/local/docker-config/psama/.env rm -f auth.tmp echo "Building and installing Jenkins" diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Outbound Email Settings/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Outbound Email Settings/config.xml index ef75cad2..dca80b51 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Outbound Email Settings/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Outbound Email Settings/config.xml @@ -48,6 +48,16 @@ fi if [ -n "$EMAIL_FROM" ] && [ -n "$oldfrom" ] ; then sed -i "s/$oldfrom/$EMAIL_FROM/g" /usr/local/docker-config/wildfly/standalone.xml fi + +if [ -n "$OUTBOUND_EMAIL_USER_PASSWORD" ] && [ -n "$oldpassword" ] ; then + sed -i "s/$oldpassword/$OUTBOUND_EMAIL_USER_PASSWORD/g" /usr/local/docker-config/psama/.env +fi +if [ -n "$OUTBOUND_EMAIL_USER" ] && [ -n "$olduser" ] ; then + sed -i "s/$olduser/$OUTBOUND_EMAIL_USER/g" /usr/local/docker-config/psama/.env +fi +if [ -n "$EMAIL_FROM" ] && [ -n "$oldfrom" ] ; then + sed -i "s/$oldfrom/$EMAIL_FROM/g" /usr/local/docker-config/psama/.env +fi diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml index f7f391f7..33e73023 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml @@ -46,9 +46,10 @@ export appplication_uuid=`cat /usr/local/docker-config/httpd/picsureui_settings. export new_token_introspection_token=`java -jar generateJwt.jar secret.txt sub "PSAMA_APPLICATION|${appplication_uuid}" 365 day | grep -v "Generating"` export old_token_introspection_token=`cat /usr/local/docker-config/wildfly/standalone.xml | grep token_introspection_token | cut -d '=' -f 3 | sed 's/[\"/\>]//'g` - sed -i "s/$old_token_introspection_token/$new_token_introspection_token/g" /usr/local/docker-config/wildfly/standalone.xml +sed -i "s/$old_token_introspection_token/$new_token_introspection_token/g" /usr/local/docker-config/psama/.env + docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "update application set token='$new_token_introspection_token';" auth diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml index b78202ac..2fc0b5fd 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml @@ -65,7 +65,6 @@ echo "" flyway_auth_url=jdbc:mysql://$MYSQL_HOST_NAME:$MYSQL_PORT/auth?serverTimezone=UTC flyway_picsure_url=jdbc:mysql://$MYSQL_HOST_NAME:$MYSQL_PORT/picsure?serverTimezone=UTC - cd /usr/local/docker-config/flyway/auth sed -i '/flyway.url/d' ./flyway-auth.conf sed -i "1iflyway.url=$flyway_auth_url" ./flyway-auth.conf @@ -82,9 +81,9 @@ sed -i "1ihost=$MYSQL_HOST_NAME" ./sql.properties sed -i '/port/d' ./sql.properties sed -i "2iport=$MYSQL_PORT" ./sql.properties +sed -i 's/jdbc:mysql*.*auth/jdbc:mysql:\/\/'$MYSQL_HOST_NAME':'$MYSQL_PORT'\/auth/g' /usr/local/docker-config/psama/.env cd /usr/local/docker-config/wildfly -sed -i 's/jdbc:mysql*.*auth/jdbc:mysql:\/\/'$MYSQL_HOST_NAME':'$MYSQL_PORT'\/auth/g' /usr/local/docker-config/wildfly/standalone.xml sed -i 's/jdbc:mysql*.*picsure/jdbc:mysql:\/\/'$MYSQL_HOST_NAME':'$MYSQL_PORT'\/picsure/g' /usr/local/docker-config/wildfly/standalone.xml echo `grep "password" /usr/local/docker-config/flyway/auth/sql.properties | cut -d "=" -f2-` > airflow.tmp @@ -95,7 +94,6 @@ AIRFLOW_PASSWORD=`grep "password" /usr/local/docker-config/flyway/auth PICSURE_PASSWORD=`sed -n 's/<password>\(.*\)<\/password>/\1/p' picsure.tmp | xargs` AUTH_PASSWORD=`sed -n 's/<password>\(.*\)<\/password>/\1/p' auth.tmp | xargs` - docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on auth.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'picsure'@'%' identified by '$PICSURE_PASSWORD';flush privileges;" mysql diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml index ad717a3f..dc43fa06 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml @@ -44,14 +44,6 @@ false - - clean install -DskipTests - Maven Home - false - - - false - if [ -f "/usr/local/docker-config/setProxy.sh" ]; then @@ -61,13 +53,10 @@ fi GIT_BRANCH_SHORT=`echo ${GIT_BRANCH} | cut -d "/" -f 2` GIT_COMMIT_SHORT=`echo ${GIT_COMMIT} | cut -c1-7` -cd pic-sure-auth-services -docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$http_proxy --build-arg no_proxy="$no_proxy" \ +docker build -f ./pic-sure-auth-services/Dockerfile --build-arg http_proxy=$http_proxy --build-arg https_proxy=$http_proxy --build-arg no_proxy="$no_proxy" \ --build-arg HTTP_PROXY=$http_proxy --build-arg HTTPS_PROXY=$http_proxy --build-arg NO_PROXY="$no_proxy" \ - -t hms-dbmi/pic-sure-auth-microapp:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} . -docker tag hms-dbmi/pic-sure-auth-microapp:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} hms-dbmi/pic-sure-auth-microapp:LATEST -mkdir -p /usr/local/docker-config/wildfly/deployments/ -cp target/pic-sure-auth-services.war /usr/local/docker-config/wildfly/deployments/ + -t hms-dbmi/psama:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} . +docker tag hms-dbmi/psama:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} hms-dbmi/psama:LATEST diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Wildfly Image Build/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Wildfly Image Build/config.xml index e5bde8a7..53b1b148 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Wildfly Image Build/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Wildfly Image Build/config.xml @@ -46,7 +46,7 @@ fi docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$http_proxy --build-arg no_proxy="$no_proxy" \ --build-arg HTTP_PROXY=$http_proxy --build-arg HTTPS_PROXY=$http_proxy --build-arg NO_PROXY="$no_proxy" \ ---build-arg PIC_SURE_API_VERSION=LATEST --build-arg PIC_SURE_AUTH_VERSION=LATEST --build-arg PIC_SURE_PASSTHRU_RESOURCE_VERSION=LATEST \ +--build-arg PIC_SURE_API_VERSION=LATEST --build-arg PIC_SURE_PASSTHRU_RESOURCE_VERSION=LATEST \ -t hms-dbmi/pic-sure-wildfly:${pipeline_build_id} . docker tag hms-dbmi/pic-sure-wildfly:${pipeline_build_id} hms-dbmi/pic-sure-wildfly:LATEST diff --git a/initial-configuration/mysql-docker/setup.sh b/initial-configuration/mysql-docker/setup.sh index e06f169e..12ef66db 100755 --- a/initial-configuration/mysql-docker/setup.sh +++ b/initial-configuration/mysql-docker/setup.sh @@ -65,7 +65,7 @@ if [ -z "$(docker ps --format '{{.Names}}' | grep picsure-db)" ]; then echo "` < /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-24}`" > auth.tmp docker exec -t picsure-db mysql -u root -p`cat ../pass.tmp` -e "CREATE USER 'auth'@'%' IDENTIFIED BY '`cat auth.tmp`';"; docker exec -t picsure-db mysql -u root -p`cat ../pass.tmp` -e "GRANT ALL PRIVILEGES ON auth.* to 'auth'@'%';FLUSH PRIVILEGES;"; - sed_inplace s/__AUTH_MYSQL_PASSWORD__/`cat auth.tmp`/g $DOCKER_CONFIG_DIR/wildfly/standalone.xml + sed_inplace s/__AUTH_MYSQL_PASSWORD__/`cat auth.tmp`/g $DOCKER_CONFIG_DIR/psama/.env rm -f auth.tmp cd $CWD diff --git a/start-picsure.sh b/start-picsure.sh index 6d4c4d91..bbfbac0c 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -15,6 +15,7 @@ else export EXPORT_SIZE="0"; fi +export PSAMA_OPTS="-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true $PROXY_OPTS" export WILDFLY_JAVA_OPTS="-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true $PROXY_OPTS" export HPDS_OPTS="-XX:+UseParallelGC -XX:SurvivorRatio=250 -Xms1g -Xmx16g -DCACHE_SIZE=1500 -DSMALL_TASK_THREADS=1 -DLARGE_TASK_THREADS=1 -DSMALL_JOB_LIMIT=100 -DID_BATCH_SIZE=$EXPORT_SIZE -DALL_IDS_CONCEPT=NONE -DID_CUBE_NAME=NONE -Denable_file_sharing=true " export PICSURE_SETTINGS_VOLUME="-v $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json:/usr/local/apache2/htdocs/picsureui/settings/settings.json" @@ -29,7 +30,7 @@ export PROFILING_OPTS=" -Dcom.sun.management.jmxremote=true -Dcom.sun.management if [ -f $DOCKER_CONFIG_DIR/wildfly/application.truststore ]; then export TRUSTSTORE_VOLUME="-v $DOCKER_CONFIG_DIR/wildfly/application.truststore:/opt/jboss/wildfly/standalone/configuration/application.truststore" - export TRUSTSTORE_JAVA_OPTS="-Djavax.net.ssl.trustStore=/opt/jboss/wildfly/standalone/configuration/application.truststore -Djavax.net.ssl.trustStorePassword=password" + export TRUSTSTORE_JAVA_OPTS="-Djavax.net.ssl.trustStore=/opt/jboss/wildfly/standalone/configuration/application.truststore -Djavax.net.ssl.trustStorePassword=password" fi @@ -63,6 +64,15 @@ docker network connect selenium httpd docker exec httpd sed -i '/^#LoadModule proxy_wstunnel_module/s/^#//' conf/httpd.conf docker restart httpd +docker stop psama && docker rm psama +docker run --name=psama --restart always \ + --network=picsure \ + --env-file /usr/local/docker-config/psama/.env \ + $EMAIL_TEMPLATE_VOUME \ + $TRUSTSTORE_VOLUME \ + -e JAVA_OPTS="$PSAMA_OPTS $TRUSTSTORE_JAVA_OPTS" \ + -d hms-dbmi/psama:LATEST + docker stop wildfly && docker rm wildfly docker run --name=wildfly --restart always --network=picsure -u root \ -v /var/log/wildfly-docker-logs/:/opt/jboss/wildfly/standalone/log/ \ diff --git a/stop-picsure.sh b/stop-picsure.sh index b6b00cae..dd8b01e6 100755 --- a/stop-picsure.sh +++ b/stop-picsure.sh @@ -2,4 +2,4 @@ docker stop hpds && docker rm hpds docker stop httpd && docker rm httpd docker stop wildfly && docker rm wildfly - +docker stop psama && docker rm psama \ No newline at end of file From bb18eb8f705df650ac18ce282ad7b1d163e4da8f Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Fri, 12 Jul 2024 08:54:01 -0400 Subject: [PATCH 02/22] Update .env variables names and values (#128) The commit updates several .env variable names and values for a clearer and more accurate configuration. The changes include renaming CLIENT_ID and CLIENT_SECRET to inline with Application Properties. The flag to determine if secret is base64 is also added. In addition, a boolean flag is utilized to enable the use of Auth0 IDP Provider instead of setting the provider's name directly. --- initial-configuration/config/psama/.env | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/initial-configuration/config/psama/.env b/initial-configuration/config/psama/.env index 34f938de..12e613a8 100644 --- a/initial-configuration/config/psama/.env +++ b/initial-configuration/config/psama/.env @@ -8,8 +8,8 @@ EMAIL_ADDRESS=__EMAIL_FROM_ADDR__ EMAIL_PASSWORD=__EMAIL_PASSWORD__ # Application Properties -CLIENT_ID=__PIC_SURE_CLIENT_ID__ -CLIENT_SECRET=__PIC_SURE_CLIENT_SECRET__ +APPLICATION_CLIENT_SECRET=__PIC_SURE_CLIENT_SECRET__ +APPLICATION_CLIENT_SECRET_IS_BASE_64=false TOS_ENABLED=false SYSTEM_NAME=PIC-SURE All-in-one GRANT_EMAIL_SUBJECT=__ACCESS_GRANTED_EMAIL_SUBJECT__ @@ -19,7 +19,7 @@ DENIED_EMAIL_ENABLED=false STACK_SPECIFIC_APPLICATION_ID=__STACK_SPECIFIC_APPLICATION_ID__ # IDP Provider Configuration -IDP_PROVIDER=auth0 +AUTH0_IDP_PROVIDER_IS_ENABLED=true IDP_PROVIDER_URI=https://__AUTH0_TENANT__.auth0.com/ AUTH0_HOST=https://__AUTH0_TENANT__.auth0.com/ From 112eecb1e18639b4982942d768be0367328e842e Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Fri, 12 Jul 2024 09:25:12 -0400 Subject: [PATCH 03/22] [ALS-0000] removing docker config dir bugs --- start-picsure.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/start-picsure.sh b/start-picsure.sh index bbfbac0c..f5c4ed9f 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -1,5 +1,11 @@ #!/usr/bin/env bash +# A note to developers: if you use /usr/local/docker-config to refer to a place on the host file system +# 99 times out of 100 you are WRONG and you have just made a bug. Please: +# - Consider using $DOCKER_CONFIG_DIR instead +# - Challenge your own understanding of where files are located in docker and on the host file system and +# how that does or doesn't change the commands you run when inside Jenkins + if [ -f "$DOCKER_CONFIG_DIR/setProxy.sh" ]; then . $DOCKER_CONFIG_DIR/setProxy.sh fi @@ -19,7 +25,7 @@ export PSAMA_OPTS="-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m export WILDFLY_JAVA_OPTS="-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true $PROXY_OPTS" export HPDS_OPTS="-XX:+UseParallelGC -XX:SurvivorRatio=250 -Xms1g -Xmx16g -DCACHE_SIZE=1500 -DSMALL_TASK_THREADS=1 -DLARGE_TASK_THREADS=1 -DSMALL_JOB_LIMIT=100 -DID_BATCH_SIZE=$EXPORT_SIZE -DALL_IDS_CONCEPT=NONE -DID_CUBE_NAME=NONE -Denable_file_sharing=true " export PICSURE_SETTINGS_VOLUME="-v $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json:/usr/local/apache2/htdocs/picsureui/settings/settings.json" -export PICSURE_BANNER_VOLUME="-v /usr/local/docker-config/httpd/banner_config.json:/usr/local/apache2/htdocs/picsureui/settings/banner_config.json" +export PICSURE_BANNER_VOLUME="-v $DOCKER_CONFIG_DIR/httpd/banner_config.json:/usr/local/apache2/htdocs/picsureui/settings/banner_config.json" export PSAMA_SETTINGS_VOLUME="-v $DOCKER_CONFIG_DIR/httpd/psamaui_settings.json:/usr/local/apache2/htdocs/picsureui/psamaui/settings/settings.json" export EMAIL_TEMPLATE_VOUME="-v $DOCKER_CONFIG_DIR/wildfly/emailTemplates:/opt/jboss/wildfly/standalone/configuration/emailTemplates " @@ -67,7 +73,7 @@ docker restart httpd docker stop psama && docker rm psama docker run --name=psama --restart always \ --network=picsure \ - --env-file /usr/local/docker-config/psama/.env \ + --env-file $DOCKER_CONFIG_DIR/psama/.env \ $EMAIL_TEMPLATE_VOUME \ $TRUSTSTORE_VOLUME \ -e JAVA_OPTS="$PSAMA_OPTS $TRUSTSTORE_JAVA_OPTS" \ From 8d4476b6147c27013bd29a0f3dc7336b02927d0c Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Fri, 12 Jul 2024 12:51:44 -0400 Subject: [PATCH 04/22] Add Jenkins jobs for PIC-SURE Auth Micro-App (#130) This commit introduces three Jenkins jobs related to the PIC-SURE Auth Micro-App; one for building and deploying the application, one for uploading updated configurations, and one for downloading the current configuration file. --- .../Download PSAMA Configuration/config.xml | 38 ++++++++++ .../config.xml | 71 +++++++++++++++++++ .../Upload PSAMA Configuration/config.xml | 46 ++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Download PSAMA Configuration/config.xml create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build - Jenkinsfile/config.xml create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Upload PSAMA Configuration/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Download PSAMA Configuration/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Download PSAMA Configuration/config.xml new file mode 100644 index 00000000..ec1aa894 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Download PSAMA Configuration/config.xml @@ -0,0 +1,38 @@ + + + + + false + + + false + false + + + + true + false + false + false + + false + + + # Just get the file so a user can download it. +cp /usr/local/docker-config/psama/.env psama.env + + + + + + psama.env + false + false + false + true + true + false + + + + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build - Jenkinsfile/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build - Jenkinsfile/config.xml new file mode 100644 index 00000000..a37b2a8e --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build - Jenkinsfile/config.xml @@ -0,0 +1,71 @@ + + + + + + + + + REPOSITORY_NAME + DOCKER_REGISTRY + + + + + This Jenkins job will build and deploy the pic-sure-auth-micro-app. + false + + + false + false + + + + + DOCKER_REGISTRY + Docker registry URL (e.g., ECR URL) + hms-dbmi + false + + + REPOSITORY_NAME + Docker repository name + psama + false + + + pipeline_build_id + MANUAL_RUN + false + + + git_hash + */ALS-6103-Architectural-Changes-To-Support-Multiple-Auth-providers + false + + + + + + + 2 + + + https://github.com/hms-dbmi/pic-sure-auth-microapp.git + + + + + ${git_hash} + + + false + + + + jenkinsfile + false + + + false + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Upload PSAMA Configuration/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Upload PSAMA Configuration/config.xml new file mode 100644 index 00000000..c664e415 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Upload PSAMA Configuration/config.xml @@ -0,0 +1,46 @@ + + + + This job is used to upload an updated pic-sure-auth-micro-app (psama) configuration file. If you would like to download the current configuration file you can use the "Download PSAMA Configuration" Jenkins Job. + false + + + false + false + + + + + psama.env + The updated pic-sure-auth-micro-app configuration file. You can download the current configuration file using the "Download PSAMA Configuration". + + + + + + true + false + false + false + + false + + + # Replace psama's configuration file with the provided one. +cp psama.env /usr/local/docker-config/psama/.env + + + + + + psama.env + false + false + false + true + true + false + + + + \ No newline at end of file From 44053e008e79b98dfab6123d3ec7976ab7b9bbf4 Mon Sep 17 00:00:00 2001 From: indraniel Date: Wed, 24 Jul 2024 10:21:30 -0500 Subject: [PATCH 05/22] + set DOCKER_CONFIG_DIR environment variable - include a useful default value if the DOCKER_CONFIG_DIR environment variable isn't already set. --- start-picsure.sh | 2 ++ update-jenkins.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/start-picsure.sh b/start-picsure.sh index f5c4ed9f..2796f929 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -6,6 +6,8 @@ # - Challenge your own understanding of where files are located in docker and on the host file system and # how that does or doesn't change the commands you run when inside Jenkins +DOCKER_CONFIG_DIR="${DOCKER_CONFIG_DIR:-/usr/local/docker-config}" + if [ -f "$DOCKER_CONFIG_DIR/setProxy.sh" ]; then . $DOCKER_CONFIG_DIR/setProxy.sh fi diff --git a/update-jenkins.sh b/update-jenkins.sh index ed70f575..67f57d7d 100755 --- a/update-jenkins.sh +++ b/update-jenkins.sh @@ -5,6 +5,8 @@ git pull echo "Sometimes we have to update not just the Jenkins jobs, but also the docker image itself." echo "If you want to update that image. Rerun this command with the --rebuild flag added." +DOCKER_CONFIG_DIR="${DOCKER_CONFIG_DIR:-/usr/local/docker-config}" + if [ "$1" = "--rebuild" ]; then # Rebuild the docker image. This matches the initial dep script. The proxy args are generally empty, but you might # run into bugs if you have an http proxy, but don't set it somewhere clever like your bash profile From 3dcaadef9d042b0c2177771a6157fd3179bce36f Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Thu, 25 Jul 2024 07:29:45 -0400 Subject: [PATCH 06/22] Export -f doesn't work on all systems --- initial-configuration/install-dependencies-docker.sh | 1 - initial-configuration/mysql-docker/setup.sh | 7 +++++++ initial-configuration/pass.tmp | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 initial-configuration/pass.tmp diff --git a/initial-configuration/install-dependencies-docker.sh b/initial-configuration/install-dependencies-docker.sh index bf8b20a6..ac66304e 100755 --- a/initial-configuration/install-dependencies-docker.sh +++ b/initial-configuration/install-dependencies-docker.sh @@ -8,7 +8,6 @@ sed_inplace() { sed -i "$@" fi } -export -f sed_inplace CWD=$(pwd) # this makes tr work on OSX diff --git a/initial-configuration/mysql-docker/setup.sh b/initial-configuration/mysql-docker/setup.sh index 12ef66db..259088db 100755 --- a/initial-configuration/mysql-docker/setup.sh +++ b/initial-configuration/mysql-docker/setup.sh @@ -1,3 +1,10 @@ +sed_inplace() { + if [ "$(uname)" = "Darwin" ]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} if [ -z "$(docker ps --format '{{.Names}}' | grep picsure-db)" ]; then echo "Cleaning up old configs" rm -r "${DOCKER_CONFIG_DIR:?}"/* diff --git a/initial-configuration/pass.tmp b/initial-configuration/pass.tmp new file mode 100644 index 00000000..db557fe3 --- /dev/null +++ b/initial-configuration/pass.tmp @@ -0,0 +1 @@ +30Ik2r_^Y9CppDHHvmtbGzNf From 5c1d3a839163f0bb10974d9b2dabe50ba0759257 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Mon, 5 Aug 2024 12:55:25 -0400 Subject: [PATCH 07/22] [ALS-6725] Dictionary Devops - Build dict - Deploy dict - Add to c4u pipeline - Add new tab for Dictionary --- .../jenkins/jenkins-docker/config.xml | 23 +++++ .../jobs/Build Dictionary API/config.xml | 56 ++++++++++++ .../jobs/Deploy Dictionary API/config.xml | 63 +++++++++++++ .../jobs/PIC-SURE Pipeline/config.xml | 91 +++++++++++-------- 4 files changed, 197 insertions(+), 36 deletions(-) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Build Dictionary API/config.xml create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Deploy Dictionary API/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index 5345d196..da9faaae 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -153,6 +153,29 @@ false + + + Dictionary + false + false + + + + Build Dictionary API + Deploy Dictionary API + + + + + + + + + + + + false + Deployment 50000 diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Build Dictionary API/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Build Dictionary API/config.xml new file mode 100644 index 00000000..174a2963 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Build Dictionary API/config.xml @@ -0,0 +1,56 @@ + + + + Build the Dictionary API Container + false + + + + + pipeline_build_id + MANUAL_RUN + false + + + git_hash + origin/main + true + + + + + + 2 + + + https://github.com/hms-dbmi/picsure-dictionary.git + + + + + ${git_hash} + + + false + + + + true + false + false + false + + false + + + current_git_tag=$(git tag --points-at HEAD) +if [ -z "${current_git_tag}"]; then + current_git_tag=$(git log --pretty=format:'%h' -n 1) +fi +docker build . -t "avillach/dictionary-api:$current_git_tag" + + + + + + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Deploy Dictionary API/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Deploy Dictionary API/config.xml new file mode 100644 index 00000000..bbe9d588 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Deploy Dictionary API/config.xml @@ -0,0 +1,63 @@ + + + + Deploy Dictionary API Container + false + + + + + dictionary.env + + + pipeline_build_id + MANUAL_RUN + false + + + git_hash + origin/main + true + + + + + + 2 + + + https://github.com/hms-dbmi/picsure-dictionary.git + + + + + ${git_hash} + + + false + + + + true + false + false + false + + false + + + mkdir -p $DOCKER_CONFIG_DIR/dictionary +cp docker-compose.yml $DOCKER_CONFIG_DIR/dictionary/ + +if [ -f dictionary.env ]; then + cp dictionary.env $DOCKER_CONFIG_DIR/dictionary/.env +fi + +cd $DOCKER_CONFIG_DIR/dictionary/ +docker compose up -d + + + + + + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml index 30bbbd26..b7652bf4 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml @@ -1,8 +1,8 @@ - + - - + + @@ -12,16 +12,19 @@ false - + From a636af6897a438e74b66788eeba816f762c271ef Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Tue, 6 Aug 2024 10:54:16 -0400 Subject: [PATCH 08/22] [ALS-6725] Dictionary Start / Stop + .env - start changes - stop changes - upload env file --- .../jenkins/jenkins-docker/config.xml | 1 + .../Upload Dictionary .env File/config.xml | 30 +++++++++++++++++++ start-picsure.sh | 4 +++ stop-picsure.sh | 6 +++- 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Upload Dictionary .env File/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index da9faaae..18106b60 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -163,6 +163,7 @@ Build Dictionary API Deploy Dictionary API + Upload Dictionary .env File diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Upload Dictionary .env File/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Upload Dictionary .env File/config.xml new file mode 100644 index 00000000..edc1b056 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Upload Dictionary .env File/config.xml @@ -0,0 +1,30 @@ + + + + false + + + + + dictionary.env + + + + + + true + false + false + false + + false + + + mkdir -p /usr/local/docker-config/dictionary/ +cp dictionary.env /usr/local/docker-config/dictionary/.env + + + + + + \ No newline at end of file diff --git a/start-picsure.sh b/start-picsure.sh index 2796f929..7b87d3fc 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -96,3 +96,7 @@ docker run --name=wildfly --restart always --network=picsure -u root \ -v $DOCKER_CONFIG_DIR/wildfly/mysql-connector-java-5.1.49.jar:/opt/jboss/wildfly/modules/system/layers/base/com/sql/mysql/main/mysql-connector-java-5.1.49.jar \ -e JAVA_OPTS="$WILDFLY_JAVA_OPTS $TRUSTSTORE_JAVA_OPTS" \ -d hms-dbmi/pic-sure-wildfly:LATEST + +if test -d $DOCKER_CONFIG_DIR/dictionary then + docker compose -f $DOCKER_CONFIG_DIR/dictionary/docker-compose.yml --env-file $DOCKER_CONFIG_DIR/dictionary/.env up -d +fi diff --git a/stop-picsure.sh b/stop-picsure.sh index dd8b01e6..75fc3eb7 100755 --- a/stop-picsure.sh +++ b/stop-picsure.sh @@ -2,4 +2,8 @@ docker stop hpds && docker rm hpds docker stop httpd && docker rm httpd docker stop wildfly && docker rm wildfly -docker stop psama && docker rm psama \ No newline at end of file +docker stop psama && docker rm psama + +if test -d $DOCKER_CONFIG_DIR/dictionary then + docker compose -f $DOCKER_CONFIG_DIR/dictionary/docker-compose.yml --env-file $DOCKER_CONFIG_DIR/dictionary/.env down +fi \ No newline at end of file From b140991643951a49155de46c5b2457b4ed1a706f Mon Sep 17 00:00:00 2001 From: Samantha Date: Mon, 12 Aug 2024 14:38:36 -0400 Subject: [PATCH 09/22] [ALS-6923] Add visualization resource creation jenkins job (#137) --- .../jenkins/jenkins-docker/config.xml | 1 + .../config.xml | 84 +++++++++++++++++++ start-picsure.sh | 1 + 3 files changed, 86 insertions(+) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Visualization Build/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index 18106b60..326edfa2 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -139,6 +139,7 @@ Backup Jenkins Home Create PIC-SURE PassThrough Resource Create PIC-SURE Aggregate Resource + Create PIC-SURE Visualization Build Retrieve Build Spec diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Visualization Build/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Visualization Build/config.xml new file mode 100644 index 00000000..a2a4b982 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Visualization Build/config.xml @@ -0,0 +1,84 @@ + + + + + false + + + + + RESOURCE_NAME + Resource Name + PIC-SURE Visualization Resource + false + + + RESOURCE_PATH + The path to be used in wildfly deployments + pic-sure-visualization-resource + false + + + git_hash + */master + false + + + + + + 2 + + + https://github.com/hms-dbmi/pic-sure.git + + + + + ${git_hash} + + + false + + + + true + false + false + false + + false + + + # Copy WAR file from PIC-SURE-API Build resources +cd pic-sure-resources/pic-sure-visualization-resource +mkdir -p /usr/local/docker-config/wildfly/deployments +cp target/pic-sure-visualization-resource.war /usr/local/docker-config/wildfly/deployments/$RESOURCE_PATH.war + +# Make properties config file +export SQL="SELECT LOWER(CONCAT(SUBSTR(HEX(uuid), 1, 8), '-', SUBSTR(HEX(uuid), 9, 4), '-', SUBSTR(HEX(uuid), 13, 4), '-', SUBSTR(HEX(uuid), 17, 4), '-', SUBSTR(HEX(uuid), 21))) from picsure.resource where name = 'hpds'"; +HPDS_ID=$(docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); +RESOURCE_ID=`uuidgen -r` + +mkdir -p /usr/local/docker-config/wildfly/visualization/$RESOURCE_PATH +PROP_FILE="/usr/local/docker-config/wildfly/visualization/$RESOURCE_PATH/resource.properties" +cat > $PROP_FILE <<-END +target.origin.id=http://localhost:8080/pic-sure-api-2/PICSURE/ +visualization.resource.id=$RESOURCE_ID +auth.hpds.resource.id=$HPDS_ID +open.hpds.resource.id=$HPDS_ID +END + +# Insert into DB +RESOURCE_ID_HEX=`echo $RESOURCE_ID | awk '{ print toupper($0) }'|sed 's/-//g';` +export SQL="INSERT INTO resource (uuid, targetURL, resourceRSPath, description, name, token) \ + VALUES (unhex('$RESOURCE_ID_HEX'), NULL, 'http://wildfly:8080/$RESOURCE_PATH/pic-sure/visualization/', '$RESOURCE_DESC', '$RESOURCE_NAME', NULL);" + +# Run with config +docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure + + + + + + \ No newline at end of file diff --git a/start-picsure.sh b/start-picsure.sh index 7b87d3fc..2fdac158 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -88,6 +88,7 @@ docker run --name=wildfly --restart always --network=picsure -u root \ -v /var/log/wildfly-docker-os-logs/:/var/log/ \ -v $DOCKER_CONFIG_DIR/wildfly/passthru/:/opt/jboss/wildfly/standalone/configuration/passthru/ \ -v $DOCKER_CONFIG_DIR/wildfly/aggregate-data-sharing/:/opt/jboss/wildfly/standalone/configuration/aggregate-data-sharing/ \ + -v $DOCKER_CONFIG_DIR/wildfly/visualization/:/opt/jboss/wildfly/standalone/configuration/visualization/ \ -v $DOCKER_CONFIG_DIR/wildfly/deployments/:/opt/jboss/wildfly/standalone/deployments/ \ -v $DOCKER_CONFIG_DIR/wildfly/standalone.xml:/opt/jboss/wildfly/standalone/configuration/standalone.xml \ $TRUSTSTORE_VOLUME \ From 6349aa8b39c1ab3a5af1bc69b6f6251324022e17 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Sat, 10 Aug 2024 12:00:11 -0400 Subject: [PATCH 10/22] [CHORE] Clean up start jenkins script --- start-jenkins.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/start-jenkins.sh b/start-jenkins.sh index fafd5b4d..97671521 100755 --- a/start-jenkins.sh +++ b/start-jenkins.sh @@ -5,10 +5,6 @@ if [ -f $DOCKER_CONFIG_DIR/setProxy.sh ]; then . $DOCKER_CONFIG_DIR/setProxy.sh fi -if ! docker network inspect selenium > /dev/null 2>&1; then - docker network create selenium -fi - docker run -d \ -e http_proxy="$http_proxy" \ -e https_proxy="$https_proxy" \ @@ -23,8 +19,6 @@ docker run -d \ -v "$HOME"/.m2:/root/.m2 \ -v /etc/hosts:/etc/hosts \ -v /usr/local/pic-sure-services:/pic-sure-services \ - --env-file initial-configuration/mysql-docker/.env \ - --network selenium \ -p 8080:8080 --name jenkins pic-sure-jenkins:LATEST # These would normally be volume mounts, but mounting volumes in volumes is bad vibes From 1f79a3afabdd6e9757392279dd08e59277d6599f Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Mon, 12 Aug 2024 09:26:39 -0400 Subject: [PATCH 11/22] [ALS-6725] Add edit .env job --- .../jenkins/jenkins-docker/config.xml | 1 + .../jobs/Edit Dictionary .env/config.xml | 50 +++++++++++++++++++ start-picsure.sh | 6 --- 3 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Edit Dictionary .env/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index 326edfa2..8f0ef2eb 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -165,6 +165,7 @@ Build Dictionary API Deploy Dictionary API Upload Dictionary .env File + Edit Dictionary .env diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Edit Dictionary .env/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Edit Dictionary .env/config.xml new file mode 100644 index 00000000..e67dde95 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Edit Dictionary .env/config.xml @@ -0,0 +1,50 @@ + + + + + false + + + + + ENV_CHANGES + A space delimited list of key value pairs. Ex: +key1 value1 key2 value2 + false + + + + + + 2 + + + https://github.com/hms-dbmi/properties-editor.git + + + + + */master + + + false + + + + true + false + false + false + + false + + + docker build . -t avillachlab/properties-editor:LATEST +docker run --rm -v $DOCKER_CONFIG_DIR/dictionary/.env:/.env avillachlab/properties-editor:LATEST .env $ENV_CHANGES + + + + + + + \ No newline at end of file diff --git a/start-picsure.sh b/start-picsure.sh index 2fdac158..ab3d9f0d 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -12,11 +12,6 @@ if [ -f "$DOCKER_CONFIG_DIR/setProxy.sh" ]; then . $DOCKER_CONFIG_DIR/setProxy.sh fi -if ! docker network inspect selenium > /dev/null 2>&1; then - docker network create selenium -fi - - if [ -z "$(grep queryExportType $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json | grep DISABLED)" ]; then export EXPORT_SIZE="2000"; else @@ -68,7 +63,6 @@ docker run --name=httpd --restart always --network=picsure \ -p 80:80 \ -p 443:443 \ -d hms-dbmi/pic-sure-ui-overrides:LATEST -docker network connect selenium httpd docker exec httpd sed -i '/^#LoadModule proxy_wstunnel_module/s/^#//' conf/httpd.conf docker restart httpd From 772b6af5b87f2df7ad342c4f0c9a6ee53815ad10 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Tue, 13 Aug 2024 14:23:31 -0400 Subject: [PATCH 12/22] [ALS-6725] Create weighting utility --- .../jenkins/jenkins-docker/config.xml | 1 + .../Weigh Dictionary Search Fields/config.xml | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Weigh Dictionary Search Fields/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index 8f0ef2eb..1aa4fb5e 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -166,6 +166,7 @@ Deploy Dictionary API Upload Dictionary .env File Edit Dictionary .env + Weight Dictionary Search Fields diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Weigh Dictionary Search Fields/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Weigh Dictionary Search Fields/config.xml new file mode 100644 index 00000000..08a89c04 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Weigh Dictionary Search Fields/config.xml @@ -0,0 +1,59 @@ + + + + + false + + + + + weights.csv + EXAMPLE: +concept_node.DISPLAY,3 +concept_node.CONCEPT_PATH,2 +dataset.FULL_NAME,1 +dataset.DESCRIPTION,1 +concept_node_meta_str,1 + + + + + + 2 + + + https://github.com/hms-dbmi/picsure-dictionary/ + + + + + */weights + + + false + + + + true + false + false + false + + false + + + # These are bash commands, so use the internal location of the config volume +mkdir -p /usr/local/docker-config/dictionary-weights/ +cp /usr/local/docker-config/dictionary/.env /usr/local/docker-config/dictionary-weights/ +cp weights.csv /usr/local/docker-config/dictionary-weights/weights.csv + +cd dictionaryweights +# These are docker commands, so now we use the external location of the config volume +docker build . -t dictionary-weights +docker run --rm --env-file=/usr/local/docker-config/dictionary-weights/.env --network dictionary_dictionary -v $DOCKER_CONFIG_DIR/dictionary-weights/weights.csv:/weights.csv dictionary-weights + + + + + + \ No newline at end of file From 7b35d9310d7de791bf2b32cf7a8539588d0be6e3 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Wed, 14 Aug 2024 11:52:47 -0400 Subject: [PATCH 13/22] [CHORE] Unbreak start picsure --- start-picsure.sh | 2 +- stop-picsure.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/start-picsure.sh b/start-picsure.sh index ab3d9f0d..f56cccaa 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -92,6 +92,6 @@ docker run --name=wildfly --restart always --network=picsure -u root \ -e JAVA_OPTS="$WILDFLY_JAVA_OPTS $TRUSTSTORE_JAVA_OPTS" \ -d hms-dbmi/pic-sure-wildfly:LATEST -if test -d $DOCKER_CONFIG_DIR/dictionary then +if [ -d $DOCKER_CONFIG_DIR/dictionary ]; then docker compose -f $DOCKER_CONFIG_DIR/dictionary/docker-compose.yml --env-file $DOCKER_CONFIG_DIR/dictionary/.env up -d fi diff --git a/stop-picsure.sh b/stop-picsure.sh index 75fc3eb7..f448f321 100755 --- a/stop-picsure.sh +++ b/stop-picsure.sh @@ -4,6 +4,6 @@ docker stop httpd && docker rm httpd docker stop wildfly && docker rm wildfly docker stop psama && docker rm psama -if test -d $DOCKER_CONFIG_DIR/dictionary then +if [ -d $DOCKER_CONFIG_DIR/dictionary ]; then docker compose -f $DOCKER_CONFIG_DIR/dictionary/docker-compose.yml --env-file $DOCKER_CONFIG_DIR/dictionary/.env down fi \ No newline at end of file From d42f41a5f926910f41de4e7f67c5da552a6d623a Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Fri, 16 Aug 2024 10:05:31 -0400 Subject: [PATCH 14/22] Add defaults to build hashes in all places. --- .../jobs/Initial Configuration Pipeline/config.xml | 5 ++++- .../jobs/Passthrough Resource Pipeline/config.xml | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml index 0d7dde28..796aa8db 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml @@ -70,7 +70,10 @@ def retrieveBuildSpecId; def pipelineBuildId; -def build_hashes = {}; +def build_hashes = { + DICTIONARY: false + UPLOADER: false +}; pipeline { agent any stages { diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml index 364bb5b1..4ba346cb 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml @@ -58,8 +58,10 @@ def retrieveBuildSpecId; def pipelineBuildId; -def build_hashes = {}; - +def build_hashes = { + DICTIONARY: false + UPLOADER: false +}; def resourceURL = "https://${SERVERNAME}/picsure/" pipeline { From a7c1507775f97dc98336fa8000294ece9d296db1 Mon Sep 17 00:00:00 2001 From: Jeremy Nix Date: Tue, 20 Aug 2024 09:36:50 -0400 Subject: [PATCH 15/22] V2.5.0 changes (#144) * Copy global maven settings to be used in container build --- .../jobs/PIC-SURE Auth Micro-App Build/config.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml index dc43fa06..bab68826 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Auth Micro-App Build/config.xml @@ -50,7 +50,10 @@ if [ -f "/usr/local/docker-config/setProxy.sh" ]; then . /usr/local/docker-config/setProxy.sh fi -GIT_BRANCH_SHORT=`echo ${GIT_BRANCH} | cut -d "/" -f 2` +# Copy global maven settings to be used in container build +mkdir -p .m2 && cp /usr/local/docker-config/.m2/*.xml .m2/ 2</dev/null + +GIT_BRANCH_SHORT=`echo ${GIT_BRANCH} | cut -d "/" -f 2` GIT_COMMIT_SHORT=`echo ${GIT_COMMIT} | cut -c1-7` docker build -f ./pic-sure-auth-services/Dockerfile --build-arg http_proxy=$http_proxy --build-arg https_proxy=$http_proxy --build-arg no_proxy="$no_proxy" \ From 300bd7ba0fae7bf9ef2e66698a4e7171f5b999b7 Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Thu, 22 Aug 2024 15:32:50 -0400 Subject: [PATCH 16/22] [ALS-7065] All-In-One PSAMA initial configuration environment file (#139) * Add scripts for MySQL config and Auth0 integration Enhanced MySQL instance configuration with additional JDBC replacement commands in `config.xml` files. Introduced a new job for exporting Jenkins build configurations with archiving functionality. Updated the Auth0 integration script to include replacements for client secret, client ID, and tenant. * Update Docker MySQL volume paths to use configuration directory Changed all paths for the MySQL Docker volume from `/root/.my.cnf` to `$DOCKER_CONFIG_DIR/.my.cnf` across multiple Jenkins jobs. This improves the maintenance and flexibility of the project's configuration files. * Update MySQL configuration directory usage Changed references from $DOCKER_CONFIG_DIR to $MYSQL_CONFIG_DIR across multiple Jenkins job config files and setup scripts to better organize and separate MySQL-specific configurations. This improves clarity and management of MySQL configuration settings. * Add script to reset development environment This script stops running services, prunes Docker images, and clears configuration directories. It ensures no critical directories are set incorrectly and resets specific paths with proper permissions. * Update config.xml in Jenkins pipelines to fix array syntax Added missing commas in the build_hashes array for three Jenkins pipeline configurations. This ensures proper syntax and prevents potential issues during pipeline execution. * Update README for additional MySQL config directory parameter Adjusted the install command to include a MySQL config directory. Updated documentation to reflect the new `MYSQL_CONFIG_DIR` variable, clarifying its usage and importance. * Ignore and remove sensitive temp files Added 'pass.tmp' and 'initial-configuration/pass.tmp' to .gitignore to avoid accidental commits of sensitive files. Deleted the existing 'initial-configuration/pass.tmp' file from the repository. * Refactor MySQL connection settings in config.xml Removed unnecessary MySQL connection string replacement for 'picsure' in psama/.env and 'auth' in wildfly/standalone.xml. * Update MySQL config directory in Jenkins and install script Changed the MySQL configuration directory path from `/root/` to `/usr/local/docker-config/picsure-db/` in the Jenkins config file and the install dependencies script. --- .gitignore | 2 + README.md | 3 +- .../install-dependencies-docker.sh | 30 +++++++- initial-configuration/install-dependencies.sh | 1 + .../jenkins/jenkins-docker/config.xml | 4 +- .../Build and Deploy Microservice/config.xml | 6 +- .../Configure Auth0 Integration/config.xml | 3 + .../config.xml | 2 +- .../config.xml | 2 +- .../config.xml | 16 ++-- .../jobs/Create Admin User/config.xml | 6 +- .../config.xml | 4 +- .../config.xml | 2 +- .../config.xml | 2 +- .../config.xml | 12 +++ .../jobs/Create Test Users/config.xml | 6 +- .../jobs/Export builds/config.xml | 48 ++++++++++++ .../Initial Configuration Pipeline/config.xml | 6 +- .../jobs/PIC-SURE Pipeline/config.xml | 6 +- .../Passthrough Resource Pipeline/config.xml | 6 +- .../jobs/Remove Test Users/config.xml | 4 +- .../jobs/Update User Token/config.xml | 4 +- initial-configuration/mysql-docker/setup.sh | 13 ++-- initial-configuration/pass.tmp | 1 - reset_development_environment.sh | 73 +++++++++++++++++++ start-jenkins.sh | 3 +- start-picsure.sh | 14 +++- 27 files changed, 228 insertions(+), 51 deletions(-) create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Export builds/config.xml delete mode 100644 initial-configuration/pass.tmp create mode 100755 reset_development_environment.sh diff --git a/.gitignore b/.gitignore index d500923c..b75783ba 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .project *.iml initial-configuration/mysql-docker/.env +pass.tmp +initial-configuration/pass.tmp \ No newline at end of file diff --git a/README.md b/README.md index 1c137153..63e98caf 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ you exact instructions. If you're following the legacy install instructions, you `cd pic-sure-all-in-one/initial-configuration` Choose one of the following use cases: - *Fully dockerized install.* Our current happy path. -`sudo ./install-dependencies-docker.sh /path/to/desired/config/dir/ && source ~/.bashrc` +`sudo ./install-dependencies-docker.sh /path/to/desired/config/dir/ /path/to/desired/mysql/cnf/dir && source ~/.bashrc` - *Legacy install.* I know what I'm doing. `sudo ./install-dependencies.sh` - *Jenkins on https.* This is rare: ```shell @@ -131,6 +131,7 @@ Once you have logged into Jenkins and have set up your admin account, you need t system variables: - `DOCKER_CONFIG_DIR`: `/path/to/config/dir` This is the path you passed to `install-dependencies-docker` +- `MYSQL_CONFIG_DIR`: `/path/to/mysql/cnf/dir` This is the path you passed to `install-dependencies-docker` - `MYSQL_NETWORK`: `picsure` If you plan to switch to a remote database, this needs to be changed back to `host` 6. Run the Initial Configuration Pipeline job. diff --git a/initial-configuration/install-dependencies-docker.sh b/initial-configuration/install-dependencies-docker.sh index ac66304e..f7e4cf6b 100755 --- a/initial-configuration/install-dependencies-docker.sh +++ b/initial-configuration/install-dependencies-docker.sh @@ -1,4 +1,3 @@ - #!/usr/bin/env bash sed_inplace() { @@ -44,7 +43,28 @@ function set_docker_config_dir { echo 'alias picsure-db="docker exec -ti picsure-db bash -c '\''mysql -uroot -p\$MYSQL_ROOT_PASSWORD'\''"' >> "$rc_file" } +function set_mysql_config_dir() { + local mysql_config_dir=$1 + if [ -z "$mysql_config_dir" ]; then + mysql_config_dir="$DOCKER_CONFIG_DIR/picsure-db/" + fi + #Check if mysql_config_dir is a dir and exists + if [ ! -d "$mysql_config_dir" ]; then + echo "Creating dir $mysql_config_dir and setting MYSQL_CONFIG_DIR in $rc_file" + mkdir -p $mysql_config_dir + export MYSQL_CONFIG_DIR=$mysql_config_dir + echo "export MYSQL_CONFIG_DIR=$mysql_config_dir" >> "$rc_file" + else + echo "dir $mysql_config_dir exists, just setting MYSQL_CONFIG_DIR in $rc_file" + # If the config dir exists, we still want to clean up old settings for it + export MYSQL_CONFIG_DIR=$1 + grep 'MYSQL_CONFIG_DIR' "$rc_file" && sed_inplace '/MYSQL_CONFIG_DIR/d' "$rc_file" + echo "export MYSQL_CONFIG_DIR=$mysql_config_dir" >> "$rc_file" + fi +} + set_docker_config_dir "$1" +set_mysql_config_dir "$2" #-------------------------------------------------------------------------------------------------# # Docker Install # @@ -55,7 +75,7 @@ echo "Starting update" echo "Installing docker" if [ -n "$(command -v yum)" ] && [ -z "$(command -v docker)" ]; then echo "Yum detected. Assuming RHEL. Install commands will use yum" - set_docker_config_dir $1 "$HOME/.zshrc" + set_docker_config_dir $1 "$HOME/.zshrc" yum -y update # This repo can be removed after we move away from centos 7 I think yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo @@ -89,7 +109,7 @@ if [ -n "$(command -v apt-get)" ] && [ -z "$(command -v docker)" ]; then fi if [[ "$OSTYPE" =~ ^darwin ]]; then - echo "Darwin detected. Assuming macOS. Install commands will use brew." + echo "Darwin detected. Assuming macOS. Install commands will use brew." #check for brew if [ -z "$(command -v brew)" ]; then echo "Brew not detected. Please install brew and rerun this script." @@ -111,7 +131,7 @@ fi if [ -n "$(command -v apk)" ]; then echo "apk detected. Assuming alpine. Install commands will use apk" apk update && apk add --no-cache wget -fi +fi if [ -z "$(command -v docker)" ]; then echo "You dont have docker installed and we cant detect a supported package manager." @@ -171,11 +191,13 @@ export APP_ID=`uuidgen | tr '[:upper:]' '[:lower:]'` export APP_ID_HEX=`echo $APP_ID | awk '{ print toupper($0) }'|sed 's/-//g'` sed_inplace "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json sed_inplace "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" $DOCKER_CONFIG_DIR/wildfly/standalone.xml +sed_inplace "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" $DOCKER_CONFIG_DIR/psama/.env export RESOURCE_ID=`uuidgen | tr '[:upper:]' '[:lower:]'` export RESOURCE_ID_HEX=`echo $RESOURCE_ID | awk '{ print toupper($0) }'|sed 's/-//g'` sed_inplace "s/__STACK_SPECIFIC_RESOURCE_UUID__/$RESOURCE_ID/g" $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json + echo $APP_ID > $DOCKER_CONFIG_DIR/APP_ID_RAW echo $APP_ID_HEX > $DOCKER_CONFIG_DIR/APP_ID_HEX echo $RESOURCE_ID > $DOCKER_CONFIG_DIR/RESOURCE_ID_RAW diff --git a/initial-configuration/install-dependencies.sh b/initial-configuration/install-dependencies.sh index 36f3511b..d6743a0f 100755 --- a/initial-configuration/install-dependencies.sh +++ b/initial-configuration/install-dependencies.sh @@ -109,6 +109,7 @@ export APP_ID=`uuidgen -r` export APP_ID_HEX=`echo $APP_ID | awk '{ print toupper($0) }'|sed 's/-//g'` sed -i "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" /usr/local/docker-config/httpd/picsureui_settings.json sed -i "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" /usr/local/docker-config/wildfly/standalone.xml +sed -i "s/__STACK_SPECIFIC_APPLICATION_ID__/$APP_ID/g" /usr/local/docker-config/psama/.env export RESOURCE_ID=`uuidgen -r` export RESOURCE_ID_HEX=`echo $RESOURCE_ID | awk '{ print toupper($0) }'|sed 's/-//g'` diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index 1aa4fb5e..509de7a3 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -196,7 +196,7 @@ - 6 + 7 project_specific_override_repo __PROJECT_SPECIFIC_OVERRIDE_REPO__ release_control_branch @@ -209,6 +209,8 @@ /usr/local/docker-config/ MYSQL_NETWORK host + MYSQL_CONFIG_DIR + /usr/local/docker-config/picsure-db/ diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml index 5bf5ebc8..e99e67ac 100755 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml @@ -56,7 +56,7 @@ # Get the resource from the db if it exists export SQL="SELECT LOWER(CONCAT(SUBSTR(HEX(uuid), 1, 8), '-', SUBSTR(HEX(uuid), 9, 4), '-', SUBSTR(HEX(uuid), 13, 4), '-', SUBSTR(HEX(uuid), 17, 4), '-', SUBSTR(HEX(uuid), 21))) from picsure.resource where name = '$service_name'"; -export resource_uuid=$(docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); +export resource_uuid=$(docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); # Add the resource to the database if it doesn't already exist if [ -z "$resource_uuid" ]; then @@ -64,12 +64,12 @@ if [ -z "$resource_uuid" ]; then echo '' export SQL="INSERT IGNORE INTO picsure.resource (uuid, name, resourceRSPath, description) \ VALUES (UUID(), '$service_name', 'http://$service_name/', '$service_description')"; - docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure fi # Get the resource from the db export SQL="SELECT LOWER(CONCAT(SUBSTR(HEX(uuid), 1, 8), '-', SUBSTR(HEX(uuid), 9, 4), '-', SUBSTR(HEX(uuid), 13, 4), '-', SUBSTR(HEX(uuid), 17, 4), '-', SUBSTR(HEX(uuid), 21))) from picsure.resource where name = '$service_name'"; -export resource_uuid=$(docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); +export resource_uuid=$(docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); echo '' echo "Done adding to db. Using $resource_uuid as uuid"; diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Auth0 Integration/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Auth0 Integration/config.xml index e74b4840..3b087172 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Auth0 Integration/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Auth0 Integration/config.xml @@ -46,6 +46,9 @@ sed -i "s/$old_client_id/$AUTH0_CLIENT_ID/g" /usr/local/docker-config/ sed -i "s/$old_tenant/$AUTH0_TENANT/g" /usr/local/docker-config/httpd/picsureui_settings.json sed -i "s/$old_tenant/$AUTH0_TENANT/g" /usr/local/docker-config/wildfly/standalone.xml +sed -i "s/$old_client_secret/$AUTH0_CLIENT_SECRET/g" /usr/local/docker-config/psama/.env +sed -i "s/$old_client_id/$AUTH0_CLIENT_ID/g" /usr/local/docker-config/psama/.env +sed -i "s/$old_tenant/$AUTH0_TENANT/g" /usr/local/docker-config/psama/.env diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure JupyterHub Token Introspection Token/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure JupyterHub Token Introspection Token/config.xml index 43a64d10..c4f6a21d 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure JupyterHub Token Introspection Token/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure JupyterHub Token Introspection Token/config.xml @@ -50,7 +50,7 @@ export old_token_introspection_token=`cat /usr/local/docker-config/jupyterhub_co sed -i "s/$old_token_introspection_token/$new_token_introspection_token/g" /usr/local/docker-config/jupyterhub_config.py -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "update application set token='$new_token_introspection_token' where uuid=$application_id;" auth diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml index 33e73023..897e57a0 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure PIC-SURE Token Introspection Token/config.xml @@ -50,7 +50,7 @@ sed -i "s/$old_token_introspection_token/$new_token_introspection_token/g&q sed -i "s/$old_token_introspection_token/$new_token_introspection_token/g" /usr/local/docker-config/psama/.env -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "update application set token='$new_token_introspection_token';" auth diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml index 2fc0b5fd..88a45ed1 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Remote MySQL Instance/config.xml @@ -94,18 +94,18 @@ AIRFLOW_PASSWORD=`grep "password" /usr/local/docker-config/flyway/auth PICSURE_PASSWORD=`sed -n 's/<password>\(.*\)<\/password>/\1/p' picsure.tmp | xargs` AUTH_PASSWORD=`sed -n 's/<password>\(.*\)<\/password>/\1/p' auth.tmp | xargs` -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on auth.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'picsure'@'%' identified by '$PICSURE_PASSWORD';flush privileges;" mysql -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on auth.* to 'auth'@'%' identified by '$AUTH_PASSWORD';flush privileges;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on auth.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'airflow'@'%' identified by '$AIRFLOW_PASSWORD';flush privileges;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on picsure.* to 'picsure'@'%' identified by '$PICSURE_PASSWORD';flush privileges;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "grant all privileges on auth.* to 'auth'@'%' identified by '$AUTH_PASSWORD';flush privileges;" mysql if [ "$DROP_EXISTING_TABLES" = "TRUE"]; then - docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e &quot;drop database IF EXISTS auth;&quot; mysql - docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e &quot;drop database IF EXISTS picsure;&quot; mysql + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e &quot;drop database IF EXISTS auth;&quot; mysql + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e &quot;drop database IF EXISTS picsure;&quot; mysql fi -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "create database auth;" mysql -docker run -i -v /root/.my.cnf:/root/.my.cnf mysql mysql -e "create database picsure;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "create database auth;" mysql +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf mysql mysql -e "create database picsure;" mysql diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml index c49a9b59..dc684679 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml @@ -32,11 +32,11 @@ export USER_ID=`uuidgen -r` export USER_ID_HEX=`echo $USER_ID | awk '{ print toupper($0) }'|sed 's/-//g'` -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "INSERT INTO auth.user VALUES (unhex('$USER_ID_HEX'), null, '{\"email\":\"$EMAIL\"}', null, (select uuid from connection where label='$CONNECTION_LABEL'),'$EMAIL',0,null,1,null);" auth -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "INSERT INTO auth.user_role VALUES (unhex('$USER_ID_HEX'), unhex('002DC366B0D8420F998F885D0ED797FD'));" auth -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "INSERT INTO auth.user_role VALUES (unhex('$USER_ID_HEX'), unhex('797FD002DC366B0D8420F998F885D0ED'));" auth diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create Custom Login (IDP) Connection/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create Custom Login (IDP) Connection/config.xml index 819bc4cd..f2194667 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create Custom Login (IDP) Connection/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create Custom Login (IDP) Connection/config.xml @@ -41,11 +41,11 @@ CONNECTION_ID=`uuidgen -r` CONNECTION_ID_HEX=`echo $CONNECTION_ID | awk '{ print toupper($0) }'|sed 's/-//g';` -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "INSERT INTO auth.connection (uuid, label, id, subprefix, requiredFields) VALUES (unhex('$CONNECTION_ID_HEX'), '$CONNECTION_LABEL', '$ID', '$SUBPREFIX', '$requiredFields');" auth USER_MAPPING_ID=`uuidgen -r` USER_MAPPING_ID_HEX=`echo $USER_MAPPING_ID | awk '{ print toupper($0) }'|sed 's/-//g';` -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "INSERT INTO auth.userMetadataMapping (uuid, auth0MetadataJsonPath, connectionId, generalMetadataJsonPath) VALUES (unhex('$USER_MAPPING_ID_HEX'), '\$.email', unhex('$CONNECTION_ID_HEX'), '\$.email');" auth echo "Please now edit the connections.json file in the httpd container and add your IDP button there. For more instructions see: https://pic-sure.gitbook.io/pic-sure/" diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Aggregate Resource/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Aggregate Resource/config.xml index c830ed19..96a9041f 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Aggregate Resource/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE Aggregate Resource/config.xml @@ -93,7 +93,7 @@ RESOURCE_ID_HEX=`echo $RESOURCE_ID | awk '{ print toupper($0) }'|sed & export SQL="INSERT INTO resource (uuid, targetURL, resourceRSPath, description, name, token) \ VALUES (unhex('$RESOURCE_ID_HEX'), NULL, 'http://wildfly:8080/$RESOURCE_PATH/pic-sure/aggregate-data-sharing/', '$RESOURCE_DESC', '$RESOURCE_NAME', NULL);" -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE PassThrough Resource/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE PassThrough Resource/config.xml index 843bb22f..a0f7c80e 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE PassThrough Resource/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create PIC-SURE PassThrough Resource/config.xml @@ -102,7 +102,7 @@ RESOURCE_ID_HEX=`echo $RESOURCE_ID | awk '{ print toupper($0) }'|sed & export SQL="INSERT INTO resource (uuid, targetURL, resourceRSPath, description, name, token) \ VALUES (unhex('$RESOURCE_ID_HEX'), NULL, 'http://wildfly:8080/$RESOURCE_PATH/pic-sure/passthru/', '$RESOURCE_DESC', '$RESOURCE_NAME', NULL);" -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create Root Certs in TrustStore/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create Root Certs in TrustStore/config.xml index 0b69fc06..9b00ee85 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create Root Certs in TrustStore/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create Root Certs in TrustStore/config.xml @@ -13,6 +13,9 @@ + + rm -f /usr/local/docker-config/wildfly/application.truststore + curl https://letsencrypt.org/certs/isrgrootx1.der -o isrgrootx1.der curl https://letsencrypt.org/certs/lets-encrypt-r3.der -o lets-encrypt-r3.der @@ -20,6 +23,15 @@ keytool -import -keystore /usr/local/docker-config/wildfly/application.truststore -storepass password -noprompt -trustcacerts -alias letsencryptauthority1 -file isrgrootx1.der -storetype JKS keytool -import -keystore /usr/local/docker-config/wildfly/application.truststore -storepass password -noprompt -trustcacerts -alias letsencryptauthority2 -file lets-encrypt-r3.der -storetype JKS + + + rm -rf /usr/local/docker-config/psama/application.truststore + + curl https://letsencrypt.org/certs/isrgrootx1.der -o isrgrootx1.der + curl https://letsencrypt.org/certs/lets-encrypt-r3.der -o lets-encrypt-r3.der + + keytool -import -keystore /usr/local/docker-config/psama/application.truststore -storepass password -noprompt -trustcacerts -alias letsencryptauthority1 -file isrgrootx1.der -storetype JKS + keytool -import -keystore /usr/local/docker-config/psama/application.truststore -storepass password -noprompt -trustcacerts -alias letsencryptauthority2 -file lets-encrypt-r3.der -storetype JKS diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create Test Users/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create Test Users/config.xml index cc0e703f..9cf753e5 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create Test Users/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create Test Users/config.xml @@ -61,7 +61,7 @@ function run_sql_procedure() { local connection_id="$2" local role_name="$3" - docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "CALL CreateUserWithRole('$user_email', '$connection_id', '$role_name', '{\"email\": \"$user_email\"}');" auth } @@ -74,14 +74,14 @@ function update_user_token() { grep client_secret /usr/local/docker-config/wildfly/standalone.xml | cut -d '=' -f 3 | sed 's/[\"/\>]//g' > secret.txt # Get the user subject by email - USER_SUBJECT=$(docker run -i -v /root/.my.cnf:/root/.my.cnf --network=host mysql mysql -N -e \ + USER_SUBJECT=$(docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=host mysql mysql -N -e \ "SELECT subject FROM auth.user where email='$USERNAME';") # Create a new user token by subject and expiry user_token=$(java -jar target/generateJwt.jar secret.txt sub "${USER_SUBJECT}" ${DAYSUNTILEXPIRATION} day | grep -v "Generating") # Update user with the new token - docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "update auth.user set long_term_token='$user_token' where email='$USERNAME';" # Return the newly generated user token diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Export builds/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Export builds/config.xml new file mode 100644 index 00000000..688ff220 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Export builds/config.xml @@ -0,0 +1,48 @@ + + + + + false + + + false + false + + + + true + false + false + false + + false + + + #!/bin/bash + +# Jenkins jobs directory +jenkinsJobsDir="/var/jenkins_home/jobs" + +# Output tar file name +outputFileName="jenkins_jobs_backup.tar.gz" + +# Find all config.xml files and pass them to tar for archiving +find $jenkinsJobsDir -type f -name "config.xml" -print0 | tar -czvf $outputFileName --null -T - + +echo "Backup completed: $outputFileName" + + + + + + jenkins_jobs_backup.tar.gz + false + false + false + true + true + false + + + + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml index 796aa8db..d75c72f0 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml @@ -70,10 +70,10 @@ def retrieveBuildSpecId; def pipelineBuildId; -def build_hashes = { - DICTIONARY: false +def build_hashes = [ + DICTIONARY: false, UPLOADER: false -}; +]; pipeline { agent any stages { diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml index b7652bf4..e77dd38d 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml @@ -17,10 +17,10 @@ def retrieveBuildSpecId; def pipelineBuildId; -def build_hashes = { - DICTIONARY: false +def build_hashes = [ + DICTIONARY: false, UPLOADER: false -}; +]; pipeline { agent any stages { diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml index 4ba346cb..91e0842b 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Passthrough Resource Pipeline/config.xml @@ -58,10 +58,10 @@ def retrieveBuildSpecId; def pipelineBuildId; -def build_hashes = { - DICTIONARY: false +def build_hashes = [ + DICTIONARY: false, UPLOADER: false -}; +]; def resourceURL = "https://${SERVERNAME}/picsure/" pipeline { diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Remove Test Users/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Remove Test Users/config.xml index 3e02aaff..24478710 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Remove Test Users/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Remove Test Users/config.xml @@ -29,14 +29,14 @@ function delete_user_by_email() { # SQL command to remove user from assoc local remove_user_role="DELETE FROM auth.user_role WHERE user_id in (SELECT uuid FROM auth.user where email = '$user_email');" - docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "$remove_user_role" auth # SQL command to delete a user based on their email local remove_user="DELETE FROM auth.user WHERE email = '$user_email';" # Run the command using Docker and MySQL client - docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ + docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "$remove_user" auth } diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml index 4a0a9b74..424059ce 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml @@ -58,12 +58,12 @@ cd target grep client_secret /usr/local/docker-config/wildfly/standalone.xml | cut -d '=' -f 3 | sed 's/[\"/\>]//g' > secret.txt -export USER_SUBJECT=`docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -N -e \ +export USER_SUBJECT=`docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -N -e \ "SELECT subject FROM auth.user where email='$USERNAME'; "` export user_token=`java -jar generateJwt.jar secret.txt sub "${USER_SUBJECT}" ${DAYSUNTILEXPIRATION} day | grep -v "Generating"` -docker run -i -v /root/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ +docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "update auth.user set long_term_token='$user_token' where email='$USERNAME';" diff --git a/initial-configuration/mysql-docker/setup.sh b/initial-configuration/mysql-docker/setup.sh index 259088db..b2d9c336 100755 --- a/initial-configuration/mysql-docker/setup.sh +++ b/initial-configuration/mysql-docker/setup.sh @@ -9,23 +9,26 @@ if [ -z "$(docker ps --format '{{.Names}}' | grep picsure-db)" ]; then echo "Cleaning up old configs" rm -r "${DOCKER_CONFIG_DIR:?}"/* cp -r config/* "$DOCKER_CONFIG_DIR"/ + rm -f "$MYSQL_CONFIG_DIR"/.my.cnf echo "Starting mysql server" echo "$( < /dev/urandom tr -dc @^=+$*%_A-Z-a-z-0-9 | head -c${1:-24})" > pass.tmp rm -f mysql-docker/.env + # shellcheck disable=SC2129 echo "PICSURE_DB_ROOT_PASS=`cat pass.tmp`" >> mysql-docker/.env echo "PICSURE_DB_PASS=`cat pass.tmp`" >> mysql-docker/.env echo "PICSURE_DB_DATABASE=ignore" >> mysql-docker/.env echo "PICSURE_DB_USER=ignore" >> mysql-docker/.env + echo "DOCKER_CONFIG_DIR=$DOCKER_CONFIG_DIR" >> mysql-docker/.env echo "Configuring .my.cnf" # shellcheck disable=SC2129 - echo "[mysql]" >> "$HOME"/.my.cnf - echo "user=root" >> "$HOME"/.my.cnf - echo "password=\"$(cat pass.tmp)\"" >> "$HOME"/.my.cnf - echo "host=picsure-db" >> "$HOME"/.my.cnf - echo "port=3306" >> "$HOME"/.my.cnf + echo "[mysql]" >> "$MYSQL_CONFIG_DIR"/.my.cnf + echo "user=root" >> "$MYSQL_CONFIG_DIR"/.my.cnf + echo "password=\"$(cat pass.tmp)\"" >> "$MYSQL_CONFIG_DIR"/.my.cnf + echo "host=picsure-db" >> "$MYSQL_CONFIG_DIR"/.my.cnf + echo "port=3306" >> "$MYSQL_CONFIG_DIR"/.my.cnf cd mysql-docker docker compose up -d diff --git a/initial-configuration/pass.tmp b/initial-configuration/pass.tmp deleted file mode 100644 index db557fe3..00000000 --- a/initial-configuration/pass.tmp +++ /dev/null @@ -1 +0,0 @@ -30Ik2r_^Y9CppDHHvmtbGzNf diff --git a/reset_development_environment.sh b/reset_development_environment.sh new file mode 100755 index 00000000..e0b17cb5 --- /dev/null +++ b/reset_development_environment.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Check if DOCKER_CONFIG_DIR is set, if not, use default +if [ -z "$DOCKER_CONFIG_DIR" ]; then + echo "DOCKER_CONFIG_DIR is not set. Defaulting to /var/local/docker-config." + DOCKER_CONFIG_DIR="/var/local/docker-config" +else + echo "DOCKER_CONFIG_DIR is set to $DOCKER_CONFIG_DIR" +fi + +# Ensure DOCKER_CONFIG_DIR is not set to root "/" +if [ "$DOCKER_CONFIG_DIR" = "/" ]; then + echo "Error: DOCKER_CONFIG_DIR is set to root '/'. Aborting to prevent system damage." + exit 1 +fi + +#$MYSQL_CONFIG_DIR +if [ -z "$MYSQL_CONFIG_DIR" ]; then + echo "MYSQL_CONFIG_DIR is not set. Defaulting to $DOCKER_CONFIG_DIR." + MYSQL_CONFIG_DIR="$DOCKER_CONFIG_DIR" +else + echo "MYSQL_CONFIG_DIR is set to $MYSQL_CONFIG_DIR" +fi + +# Ensure DOCKER_CONFIG_DIR is not set to root "/" +if [ "$DOCKER_CONFIG_DIR" = "/" ]; then + echo "Error: DOCKER_CONFIG_DIR is set to root '/'. Aborting to prevent system damage." + exit 1 +fi + +# Step 1: Run stop-picsure.sh +echo "Stopping PIC-SURE..." +./stop-picsure.sh + +# Step 2: Run stop-jenkin.sh +echo "Stopping Jenkins..." +./stop-jenkin.sh + +# Step 3: Stop and remove the picsure-db container +echo "Stopping and removing PIC-SURE database container..." +docker stop picsure-db +docker rm picsure-db + +# Step 4: Run docker system prune -a +echo "Pruning Docker system and removing all images..." +docker system prune -a -f + +# Step 5: Clear the MYSQL_CONFIG_DIR +echo "Clearing the MySQL configuration directory..." +rm -rf "$MYSQL_CONFIG_DIR/*" + +# Step 6: Clear the DOCKER_CONFIG_DIR +echo "Clearing the Docker configuration directory..." +rm -rf "$DOCKER_CONFIG_DIR/*" + +# Step 7: Remove the jenkins_home directory and recreate necessary directories +echo "Removing and recreating Jenkins and log directories..." +sudo rm -rf /var/jenkins_home +sudo rm -rf /var/log/jenkins-docker-logs +sudo rm -rf /var/jenkins_home_bak + +sudo mkdir -p /var/log/jenkins-docker-logs +sudo mkdir -p /var/jenkins_home +sudo mkdir -p /var/jenkins_home_bak +sudo mkdir -p /var/log/httpd-docker-logs/ssl_mutex + +# Step 8: Set permissions for the directories +echo "Setting permissions for Jenkins and log directories..." +sudo chmod -R 777 /var/jenkins_home +sudo chmod -R 777 /var/jenkins_home_bak +sudo chmod -R 777 /var/log/httpd-docker-logs + +echo "All steps completed successfully." \ No newline at end of file diff --git a/start-jenkins.sh b/start-jenkins.sh index 97671521..b7cb8018 100755 --- a/start-jenkins.sh +++ b/start-jenkins.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash DOCKER_CONFIG_DIR="${DOCKER_CONFIG_DIR:-/usr/local/docker-config}" +MY_SQL_DIR="${MY_SQL_DIR:-/root/}" if [ -f $DOCKER_CONFIG_DIR/setProxy.sh ]; then . $DOCKER_CONFIG_DIR/setProxy.sh @@ -15,7 +16,7 @@ docker run -d \ -v /var/jenkins_home:/var/jenkins_home \ -v "$DOCKER_CONFIG_DIR":/usr/local/docker-config \ -v /var/run/docker.sock:/var/run/docker.sock \ - -v "$HOME"/.my.cnf:/root/.my.cnf \ + -v "$MYSQL_CONFIG_DIR"/.my.cnf:/root/.my.cnf \ -v "$HOME"/.m2:/root/.m2 \ -v /etc/hosts:/etc/hosts \ -v /usr/local/pic-sure-services:/pic-sure-services \ diff --git a/start-picsure.sh b/start-picsure.sh index f56cccaa..60fb1bca 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -34,8 +34,18 @@ export PROFILING_OPTS=" -Dcom.sun.management.jmxremote=true -Dcom.sun.management if [ -f $DOCKER_CONFIG_DIR/wildfly/application.truststore ]; then export TRUSTSTORE_VOLUME="-v $DOCKER_CONFIG_DIR/wildfly/application.truststore:/opt/jboss/wildfly/standalone/configuration/application.truststore" export TRUSTSTORE_JAVA_OPTS="-Djavax.net.ssl.trustStore=/opt/jboss/wildfly/standalone/configuration/application.truststore -Djavax.net.ssl.trustStorePassword=password" +else + echo "wildfly truststore not found" + exit 2 fi +if [ -f $DOCKER_CONFIG_DIR/psama/application.truststore ]; then + export PSAMA_TRUSTSTORE_VOLUME="-v $DOCKER_CONFIG_DIR/psama/application.truststore:/usr/local/tomcat/conf/application.truststore" + export PSAMA_TRUSTSTORE_JAVA_OPTS="-Djavax.net.ssl.trustStore=/usr/local/tomcat/conf/application.truststore -Djavax.net.ssl.trustStorePassword=password" +else + echo "pic-sure-auth-micro-app (psama) truststore not found" + exit 2 +fi docker stop hpds && docker rm hpds docker run --name=hpds --restart always --network=picsure \ @@ -71,8 +81,8 @@ docker run --name=psama --restart always \ --network=picsure \ --env-file $DOCKER_CONFIG_DIR/psama/.env \ $EMAIL_TEMPLATE_VOUME \ - $TRUSTSTORE_VOLUME \ - -e JAVA_OPTS="$PSAMA_OPTS $TRUSTSTORE_JAVA_OPTS" \ + $PSAMA_TRUSTSTORE_VOLUME \ + -e JAVA_OPTS="$PSAMA_OPTS $PSAMA_TRUSTSTORE_JAVA_OPTS" \ -d hms-dbmi/psama:LATEST docker stop wildfly && docker rm wildfly From 99dee40275a0b45fee55594fb8f2625fb8bd5d30 Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:57:40 -0400 Subject: [PATCH 17/22] Rename job in Jenkins pipeline configuration (#148) Updated the name of the Jenkins job from 'PIC-SURE Auth Micro-App Build' to 'PIC-SURE Auth Micro-App Build - Jenkinsfile' in the pipeline configuration. This ensures alignment with the correct job naming conventions. --- .../jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml index e77dd38d..cbed4bbf 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml @@ -69,7 +69,7 @@ pipeline { }, picsureauth:{ script{ - def result = build job: 'PIC-SURE Auth Micro-App Build', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSAMA']]] + def result = build job: 'PIC-SURE Auth Micro-App Build - Jenkinsfile', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSAMA']]] } } ) From 7fea8b260f640f40eadcabec81772ebb1552a4cf Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:50:44 -0400 Subject: [PATCH 18/22] [ALS-6921] Open Access Configuration Changes (#149) * Add openAccess configuration properties --- initial-configuration/config/wildfly/standalone.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/initial-configuration/config/wildfly/standalone.xml b/initial-configuration/config/wildfly/standalone.xml index e189ea68..fcba7164 100644 --- a/initial-configuration/config/wildfly/standalone.xml +++ b/initial-configuration/config/wildfly/standalone.xml @@ -443,6 +443,9 @@ + + + From f3cd1b0039e31c469aec005067fc2e47db71bb59 Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Mon, 28 Oct 2024 11:11:36 -0400 Subject: [PATCH 19/22] [ALS-7703] Patches for GIC release - Bump jenkins version - Fix Update User Token job to work with psama >=3.2.x - Long awaited fix to persist DOCKER_CONFIG_DIR, and MYSQL_NETWORK vars through restart --- initial-configuration/jenkins/jenkins-docker/Dockerfile | 2 +- .../jenkins/jenkins-docker/jobs/Update User Token/config.xml | 2 +- update-jenkins.sh | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/initial-configuration/jenkins/jenkins-docker/Dockerfile b/initial-configuration/jenkins/jenkins-docker/Dockerfile index a8ade3e6..12c9d0fd 100644 --- a/initial-configuration/jenkins/jenkins-docker/Dockerfile +++ b/initial-configuration/jenkins/jenkins-docker/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/jenkins:2.440-jdk11 +FROM jenkins/jenkins:2.442-jdk11 COPY plugins.yml /usr/share/jenkins/ref/plugins.yml diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml index 424059ce..9539fc0d 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Update User Token/config.xml @@ -61,7 +61,7 @@ grep client_secret /usr/local/docker-config/wildfly/standalone.xml | cut -d &apo export USER_SUBJECT=`docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -N -e \ "SELECT subject FROM auth.user where email='$USERNAME'; "` -export user_token=`java -jar generateJwt.jar secret.txt sub "${USER_SUBJECT}" ${DAYSUNTILEXPIRATION} day | grep -v "Generating"` +export user_token=`java -jar generateJwt.jar secret.txt sub "LONG_TERM_TOKEN|${USER_SUBJECT}" ${DAYSUNTILEXPIRATION} day | grep -v "Generating"` docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ "update auth.user set long_term_token='$user_token' where email='$USERNAME';" diff --git a/update-jenkins.sh b/update-jenkins.sh index 67f57d7d..28d6b32e 100755 --- a/update-jenkins.sh +++ b/update-jenkins.sh @@ -35,6 +35,8 @@ fi # Pull through previous PICSURE configurations sed -i "s|__PROJECT_SPECIFIC_OVERRIDE_REPO__|`cat /var/jenkins_home_bak/config.xml | grep -A1 project_specific_override_repo | tail -1 | sed 's/<\/*string>//g' | sed 's/ //g' `|g" /var/jenkins_home/config.xml sed -i "s|__RELEASE_CONTROL_REPO__|`cat /var/jenkins_home_bak/config.xml | grep -A1 release_control_repo | tail -1 | sed 's/<\/*string>//g' | sed 's/ //g' `|g" /var/jenkins_home/config.xml +sed -i "s|/usr/local/docker-config/|`cat /var/jenkins_home_bak/config.xml | grep -A1 DOCKER_CONFIG_DIR | tail -1 | sed 's/<\/*string>//g' | sed 's/ //g' `|g" /var/jenkins_home/config.xml +sed -i "s|host|`cat /var/jenkins_home_bak/config.xml | grep -A1 MYSQL_NETWORK | tail -1 | sed 's/<\/*string>//g' | sed 's/ //g' `|g" /var/jenkins_home/config.xml sed -i "s|*/master|`cat /var/jenkins_home_bak/config.xml | grep -A1 release_control_branch | tail -1 | sed 's/<\/*string>//g' | sed 's/ //g' `|g" /var/jenkins_home/config.xml From c2fb431680c52e3e119576a18f693b8b1ebe632e Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Wed, 20 Nov 2024 15:02:49 -0500 Subject: [PATCH 20/22] [ALS-7536] Pull configurations out of versioned files and into DOCKER_CONFIG_DIR - Adjust start script - Fix DOCKER_CONFIG_DIR bug in start script - Add migration script --- README.md | 2 +- initial-configuration/config/hpds/hpds.env | 1 + initial-configuration/config/httpd/httpd.env | 0 initial-configuration/config/psama/.env | 4 +- .../config/wildfly/wildfly.env | 1 + .../install-dependencies-docker.sh | 7 ++- .../jobs/Create Admin User/config.xml | 11 ++--- initial-configuration/migrate-env.sh | 28 +++++++++++ initial-configuration/mysql-docker/setup.sh | 2 + start-picsure.sh | 46 +++++-------------- 10 files changed, 58 insertions(+), 44 deletions(-) create mode 100644 initial-configuration/config/hpds/hpds.env create mode 100644 initial-configuration/config/httpd/httpd.env create mode 100644 initial-configuration/config/wildfly/wildfly.env mode change 100644 => 100755 initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml create mode 100755 initial-configuration/migrate-env.sh diff --git a/README.md b/README.md index 63e98caf..5df75bcf 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ you exact instructions. If you're following the legacy install instructions, you `cd pic-sure-all-in-one/initial-configuration` Choose one of the following use cases: - *Fully dockerized install.* Our current happy path. -`sudo ./install-dependencies-docker.sh /path/to/desired/config/dir/ /path/to/desired/mysql/cnf/dir && source ~/.bashrc` +`sudo ./install-dependencies-docker.sh /path/to/desired/config/dir/ && source ~/.bashrc` - *Legacy install.* I know what I'm doing. `sudo ./install-dependencies.sh` - *Jenkins on https.* This is rare: ```shell diff --git a/initial-configuration/config/hpds/hpds.env b/initial-configuration/config/hpds/hpds.env new file mode 100644 index 00000000..6871e644 --- /dev/null +++ b/initial-configuration/config/hpds/hpds.env @@ -0,0 +1 @@ +CATALINA_OPTS=-XX:+UseParallelGC -XX:SurvivorRatio=250 -Xms1g -Xmx16g -DCACHE_SIZE=1500 -DSMALL_TASK_THREADS=1 -DLARGE_TASK_THREADS=1 -DSMALL_JOB_LIMIT=100 -DID_BATCH_SIZE=2000 -DALL_IDS_CONCEPT=NONE -DID_CUBE_NAME=NONE -Denable_file_sharing=true diff --git a/initial-configuration/config/httpd/httpd.env b/initial-configuration/config/httpd/httpd.env new file mode 100644 index 00000000..e69de29b diff --git a/initial-configuration/config/psama/.env b/initial-configuration/config/psama/.env index 12e613a8..d6f5bb4f 100644 --- a/initial-configuration/config/psama/.env +++ b/initial-configuration/config/psama/.env @@ -25,4 +25,6 @@ AUTH0_HOST=https://__AUTH0_TENANT__.auth0.com/ # Token Expiration Times TOKEN_EXPIRATION_TIME=3600000 -LONG_TERM_TOKEN_EXPIRATION_TIME=2592000000 \ No newline at end of file +LONG_TERM_TOKEN_EXPIRATION_TIME=2592000000 + +JAVA_OPTS="-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true -Djavax.net.ssl.trustStore=/usr/local/tomcat/conf/application.truststore -Djavax.net.ssl.trustStorePassword=password" diff --git a/initial-configuration/config/wildfly/wildfly.env b/initial-configuration/config/wildfly/wildfly.env new file mode 100644 index 00000000..94815e41 --- /dev/null +++ b/initial-configuration/config/wildfly/wildfly.env @@ -0,0 +1 @@ +JAVA_OPTS=-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true -Djavax.net.ssl.trustStore=/opt/jboss/wildfly/standalone/configuration/application.truststore -Djavax.net.ssl.trustStorePassword=password \ No newline at end of file diff --git a/initial-configuration/install-dependencies-docker.sh b/initial-configuration/install-dependencies-docker.sh index f7e4cf6b..82b0d84e 100755 --- a/initial-configuration/install-dependencies-docker.sh +++ b/initial-configuration/install-dependencies-docker.sh @@ -16,12 +16,12 @@ export LC_ALL=C # $1 is the path to the docker-config dir $2 is the path to the rc rc_file function set_docker_config_dir { local docker_config_dir=$1 - local rc_file=$2 + export rc_file=$2 if [ -z "$docker_config_dir" ]; then docker_config_dir="/var/local/docker-config" fi if [ -z "$rc_file" ]; then - rc_file="$HOME/.bashrc" + export rc_file="$HOME/.bashrc" fi #Check if docker_config_dir is a dir and exists if [ ! -d "$docker_config_dir" ]; then @@ -218,6 +218,9 @@ if [ -n "$2" ]; then ./convert-cert.sh $2 $3 $password fi +echo Deleting pass.tmp +rm pass.tmp + echo "Installation script complete. Staring Jenkins." cd .. ./start-jenkins.sh diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml old mode 100644 new mode 100755 index dc684679..4314b49d --- a/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Create Admin User/config.xml @@ -8,8 +8,6 @@ EMAIL - - false @@ -33,14 +31,15 @@ export USER_ID=`uuidgen -r` export USER_ID_HEX=`echo $USER_ID | awk '{ print toupper($0) }'|sed 's/-//g'` docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ -"INSERT INTO auth.user VALUES (unhex('$USER_ID_HEX'), null, '{\"email\":\"$EMAIL\"}', null, (select uuid from connection where label='$CONNECTION_LABEL'),'$EMAIL',0,null,1,null);" auth +"INSERT INTO auth.user (uuid, auth0_metadata, general_metadata, acceptedTOS, connectionId, email, matched, subject, is_active, long_term_token) VALUES (unhex('$USER_ID_HEX'), null, '{\"email\":\"$EMAIL\"}', null, (select uuid from connection where label='$CONNECTION_LABEL'),'$EMAIL',0,null,1,null);" auth docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ -"INSERT INTO auth.user_role VALUES (unhex('$USER_ID_HEX'), unhex('002DC366B0D8420F998F885D0ED797FD'));" auth +"INSERT INTO auth.user_role (user_id, role_id) VALUES (unhex('$USER_ID_HEX'), unhex('002DC366B0D8420F998F885D0ED797FD'));" auth docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e \ -"INSERT INTO auth.user_role VALUES (unhex('$USER_ID_HEX'), unhex('797FD002DC366B0D8420F998F885D0ED'));" auth +"INSERT INTO auth.user_role (user_id, role_id) VALUES (unhex('$USER_ID_HEX'), unhex('797FD002DC366B0D8420F998F885D0ED'));" auth + - + \ No newline at end of file diff --git a/initial-configuration/migrate-env.sh b/initial-configuration/migrate-env.sh new file mode 100755 index 00000000..9bcfe78e --- /dev/null +++ b/initial-configuration/migrate-env.sh @@ -0,0 +1,28 @@ +# This script is used to migrate variable from the old start-picsure.sh +# which housed many configurable environment variables +# Those variables are now stored in their respective .env files in $DOCKER_CONFIG_DIR + +# BEFORE running this script, run source start-picsure.sh + + +echo "Making config dirs for hpds, psama, httpd, and wildfly in $DOCKER_CONFIG_DIR" + +mkdir -p $DOCKER_CONFIG_DIR/hpds +mkdir -p $DOCKER_CONFIG_DIR/psama +mkdir -p $DOCKER_CONFIG_DIR/httpd +mkdir -p $DOCKER_CONFIG_DIR/wildfly + +echo "Populating config files with env vars from old start script" + +echo "" >> $DOCKER_CONFIG_DIR/hpds/hpds.env +echo "CATALINA_OPTS= $HPDS_OPTS" >> $DOCKER_CONFIG_DIR/hpds/hpds.env + +echo "" >> $DOCKER_CONFIG_DIR/psama/.env +echo "JAVA_OPTS=$PSAMA_OPTS" >> $DOCKER_CONFIG_DIR/psama/.env + +echo "" >> $DOCKER_CONFIG_DIR/httpd/httpd.env + +echo "" >> $DOCKER_CONFIG_DIR/wildfly/wildfly.env +echo "JAVA_OPTS=$WILDFLY_JAVA_OPTS $TRUSTSTORE_JAVA_OPTS" >> $DOCKER_CONFIG_DIR/wildfly/wildfly.env + +echo "Done." diff --git a/initial-configuration/mysql-docker/setup.sh b/initial-configuration/mysql-docker/setup.sh index b2d9c336..65fd37f5 100755 --- a/initial-configuration/mysql-docker/setup.sh +++ b/initial-configuration/mysql-docker/setup.sh @@ -24,6 +24,8 @@ if [ -z "$(docker ps --format '{{.Names}}' | grep picsure-db)" ]; then echo "Configuring .my.cnf" # shellcheck disable=SC2129 + mkdir -p "$MYSQL_CONFIG_DIR" + touch "$MYSQL_CONFIG_DIR"/.my.cnf echo "[mysql]" >> "$MYSQL_CONFIG_DIR"/.my.cnf echo "user=root" >> "$MYSQL_CONFIG_DIR"/.my.cnf echo "password=\"$(cat pass.tmp)\"" >> "$MYSQL_CONFIG_DIR"/.my.cnf diff --git a/start-picsure.sh b/start-picsure.sh index 60fb1bca..524a07cd 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -18,35 +18,18 @@ else export EXPORT_SIZE="0"; fi -export PSAMA_OPTS="-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true $PROXY_OPTS" -export WILDFLY_JAVA_OPTS="-Xms2g -Xmx4g -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true $PROXY_OPTS" -export HPDS_OPTS="-XX:+UseParallelGC -XX:SurvivorRatio=250 -Xms1g -Xmx16g -DCACHE_SIZE=1500 -DSMALL_TASK_THREADS=1 -DLARGE_TASK_THREADS=1 -DSMALL_JOB_LIMIT=100 -DID_BATCH_SIZE=$EXPORT_SIZE -DALL_IDS_CONCEPT=NONE -DID_CUBE_NAME=NONE -Denable_file_sharing=true " +# Docker Volumes export PICSURE_SETTINGS_VOLUME="-v $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json:/usr/local/apache2/htdocs/picsureui/settings/settings.json" export PICSURE_BANNER_VOLUME="-v $DOCKER_CONFIG_DIR/httpd/banner_config.json:/usr/local/apache2/htdocs/picsureui/settings/banner_config.json" export PSAMA_SETTINGS_VOLUME="-v $DOCKER_CONFIG_DIR/httpd/psamaui_settings.json:/usr/local/apache2/htdocs/picsureui/psamaui/settings/settings.json" -export EMAIL_TEMPLATE_VOUME="-v $DOCKER_CONFIG_DIR/wildfly/emailTemplates:/opt/jboss/wildfly/standalone/configuration/emailTemplates " - -# these debug options can be added to wildfly or hpds container startup to enable remote debugging or profiling. -# Don't forget to add a port mapping too! -export DEBUG_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:8000" -export PROFILING_OPTS=" -Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=9000 -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9000 " - -if [ -f $DOCKER_CONFIG_DIR/wildfly/application.truststore ]; then - export TRUSTSTORE_VOLUME="-v $DOCKER_CONFIG_DIR/wildfly/application.truststore:/opt/jboss/wildfly/standalone/configuration/application.truststore" - export TRUSTSTORE_JAVA_OPTS="-Djavax.net.ssl.trustStore=/opt/jboss/wildfly/standalone/configuration/application.truststore -Djavax.net.ssl.trustStorePassword=password" -else - echo "wildfly truststore not found" - exit 2 -fi - -if [ -f $DOCKER_CONFIG_DIR/psama/application.truststore ]; then - export PSAMA_TRUSTSTORE_VOLUME="-v $DOCKER_CONFIG_DIR/psama/application.truststore:/usr/local/tomcat/conf/application.truststore" - export PSAMA_TRUSTSTORE_JAVA_OPTS="-Djavax.net.ssl.trustStore=/usr/local/tomcat/conf/application.truststore -Djavax.net.ssl.trustStorePassword=password" -else - echo "pic-sure-auth-micro-app (psama) truststore not found" - exit 2 +export EMAIL_TEMPLATE_VOLUME="-v $DOCKER_CONFIG_DIR/wildfly/emailTemplates:/opt/jboss/wildfly/standalone/configuration/emailTemplates " +export TRUSTSTORE_VOLUME="-v $DOCKER_CONFIG_DIR/wildfly/application.truststore:/opt/jboss/wildfly/standalone/configuration/application.truststore" +export PSAMA_TRUSTSTORE_VOLUME="-v $DOCKER_CONFIG_DIR/psama/application.truststore:/usr/local/tomcat/conf/application.truststore" +if [ -f $DOCKER_CONFIG_DIR/httpd/custom_httpd_volumes ]; then + export CUSTOM_HTTPD_VOLUMES=`cat $DOCKER_CONFIG_DIR/httpd/custom_httpd_volumes` fi +# Start Commands docker stop hpds && docker rm hpds docker run --name=hpds --restart always --network=picsure \ -v $DOCKER_CONFIG_DIR/hpds:/opt/local/hpds \ @@ -54,14 +37,9 @@ docker run --name=hpds --restart always --network=picsure \ -v /var/log/hpds-logs/:/var/log/ \ -v $DOCKER_CONFIG_DIR/hpds_csv/:/usr/local/docker-config/hpds_csv/ \ -v $DOCKER_CONFIG_DIR/aws_uploads/:/gic_query_results/ \ - -e CATALINA_OPTS=" $HPDS_OPTS " \ - -p 5007:5007 \ + --env-file $DOCKER_CONFIG_DIR/hpds/hpds.env \ -d hms-dbmi/pic-sure-hpds:LATEST -if [ -f $DOCKER_CONFIG_DIR/httpd/custom_httpd_volumes ]; then - export CUSTOM_HTTPD_VOLUMES=`cat $DOCKER_CONFIG_DIR/httpd/custom_httpd_volumes` -fi - docker stop httpd && docker rm httpd docker run --name=httpd --restart always --network=picsure \ -v /var/log/httpd-docker-logs/:/usr/local/apache2/logs/ \ @@ -72,6 +50,7 @@ docker run --name=httpd --restart always --network=picsure \ $CUSTOM_HTTPD_VOLUMES \ -p 80:80 \ -p 443:443 \ + --env-file $DOCKER_CONFIG_DIR/httpd/httpd.env \ -d hms-dbmi/pic-sure-ui-overrides:LATEST docker exec httpd sed -i '/^#LoadModule proxy_wstunnel_module/s/^#//' conf/httpd.conf docker restart httpd @@ -80,9 +59,8 @@ docker stop psama && docker rm psama docker run --name=psama --restart always \ --network=picsure \ --env-file $DOCKER_CONFIG_DIR/psama/.env \ - $EMAIL_TEMPLATE_VOUME \ + $EMAIL_TEMPLATE_VOLUME \ $PSAMA_TRUSTSTORE_VOLUME \ - -e JAVA_OPTS="$PSAMA_OPTS $PSAMA_TRUSTSTORE_JAVA_OPTS" \ -d hms-dbmi/psama:LATEST docker stop wildfly && docker rm wildfly @@ -96,10 +74,10 @@ docker run --name=wildfly --restart always --network=picsure -u root \ -v $DOCKER_CONFIG_DIR/wildfly/deployments/:/opt/jboss/wildfly/standalone/deployments/ \ -v $DOCKER_CONFIG_DIR/wildfly/standalone.xml:/opt/jboss/wildfly/standalone/configuration/standalone.xml \ $TRUSTSTORE_VOLUME \ - $EMAIL_TEMPLATE_VOUME \ + $EMAIL_TEMPLATE_VOLUME \ -v $DOCKER_CONFIG_DIR/wildfly/wildfly_mysql_module.xml:/opt/jboss/wildfly/modules/system/layers/base/com/sql/mysql/main/module.xml \ -v $DOCKER_CONFIG_DIR/wildfly/mysql-connector-java-5.1.49.jar:/opt/jboss/wildfly/modules/system/layers/base/com/sql/mysql/main/mysql-connector-java-5.1.49.jar \ - -e JAVA_OPTS="$WILDFLY_JAVA_OPTS $TRUSTSTORE_JAVA_OPTS" \ + --env-file $DOCKER_CONFIG_DIR/wildfly/wildfly.env \ -d hms-dbmi/pic-sure-wildfly:LATEST if [ -d $DOCKER_CONFIG_DIR/dictionary ]; then From 6742f348628570f77de98a9db97c213713ce8fcd Mon Sep 17 00:00:00 2001 From: James Date: Thu, 21 Nov 2024 15:53:06 -0500 Subject: [PATCH 21/22] [ALS-6867] Replace Frontend (#151) * Init new UI * Set repo by default * clean up migration stuff, remove old ui * remove old mounts --- README.md | 4 +- .../config/httpd/httpd-vhosts.conf | 62 ++-- .../jenkins/jenkins-docker/config.xml | 8 +- .../jobs/Add or Edit a Setting/config.xml | 52 --- .../Configure Institution Node/config.xml | 222 ++++++++++-- .../Initial Configuration Pipeline/config.xml | 338 +++++++++++++----- .../PIC-SURE Database Migrations/config.xml | 37 +- .../jobs/PIC-SURE Database Repair/config.xml | 7 +- .../config.xml | 35 +- .../jobs/PIC-SURE Pipeline/config.xml | 33 +- start-picsure.sh | 28 +- 11 files changed, 543 insertions(+), 283 deletions(-) delete mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Add or Edit a Setting/config.xml rename initial-configuration/jenkins/jenkins-docker/jobs/{Project Specific PIC-SURE Overrides => PIC-SURE Frontend Build}/config.xml (68%) diff --git a/README.md b/README.md index 5df75bcf..3ef6aad6 100644 --- a/README.md +++ b/README.md @@ -150,9 +150,7 @@ system variables: - `EMAIL`: This is the Google account that will be the initial admin user. - - `PROJECT_SPECIFIC_OVERRIDE_REPOSITORY`: This is the repo that contains the project specific overrides for your - project. If you just want the default PIC-SURE behavior use this - repo : https://github.com/hms-dbmi/baseline-pic-sure + - `MIGRATION_NAME`: This is the name of the migration that will be run. If you just want the default PIC-SURE behavior use `Baseline` from the repo: https://github.com/hms-dbmi/pic-sure-migrations or fork it and add your migration. If you are a GIC Institution, use `GIC-Institution`. - `RELEASE_CONTROL_REPOSITORY`: This is the repo that contains the build-spec.json file for your project. This file controls what code is built and deployed. If you just want the default PIC-SURE behavior use this diff --git a/initial-configuration/config/httpd/httpd-vhosts.conf b/initial-configuration/config/httpd/httpd-vhosts.conf index ef1241f4..73b8e47e 100644 --- a/initial-configuration/config/httpd/httpd-vhosts.conf +++ b/initial-configuration/config/httpd/httpd-vhosts.conf @@ -17,13 +17,12 @@ AddType application/x-pkcs7-crl .crl SSLCipherSuite ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES128-SHA256:AES256-GCM-SHA384:AES256-SHA256 SSLProxyCipherSuite ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:AES128-GCM-SHA256:AES128-SHA256:AES256-GCM-SHA384:AES256-SHA256 - -SSLHonorCipherOrder on - -SSLProtocol all -TLSv1.2 -SSLProxyProtocol all -TLSv1.2 +SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1 -TLSv1.2 +SSLHonorCipherOrder off +SSLSessionTickets off SSLPassPhraseDialog builtin +SSLUseStapling On SSLSessionCache "shmcb:${HTTPD_PREFIX}/logs/ssl_scache(512000)" SSLSessionCacheTimeout 300 @@ -35,19 +34,30 @@ ServerTokens Prod ServerName localhost RewriteEngine On ProxyPreserveHost On + #Dont allow httpd debug methods + RewriteCond %%{REQUEST_METHOD} ^TRACK + RewriteRule .* - [F] + RewriteCond %%{REQUEST_METHOD} ^TRACE + RewriteRule .* - [F] + RewriteCond %{HTTPS} off [OR] RewriteCond %{HTTP_HOST} ^(?:)?(.+)$ [NC] - RewriteRule ^ https://%{SERVER_NAME}/picsureui/ [L,NE,R=301] + RewriteRule ^ https://%{SERVER_NAME}/$1 [L,NE,R=301] + ServerName %{SERVER_NAME} ProxyTimeout 300 + SSLEngine on SSLProxyEngine on SSLProxyVerify none SSLProxyCheckPeerCN off SSLProxyCheckPeerName off + # enable HTTP/2, if available + Protocols h2 http/1.1 + SSLCertificateFile "${HTTPD_PREFIX}/cert/server.crt" SSLCertificateKeyFile "${HTTPD_PREFIX}/cert/server.key" SSLCertificateChainFile "${HTTPD_PREFIX}/cert/server.chain" @@ -77,15 +87,14 @@ ServerTokens Prod RewriteEngine On ProxyPreserveHost On - - ProxyPass http://jupyterhub:8000/jupyterhub - ProxyPassReverse http://jupyterhub:8000/jupyterhub - + #Dont allow httpd debug methods + RewriteCond %%{REQUEST_METHOD} ^TRACK + RewriteRule .* - [F] + RewriteCond %%{REQUEST_METHOD} ^TRACE + RewriteRule .* - [F] - - ProxyPassMatch ws://jupyterhub:8000/jupyterhub/$1/$2$3 - ProxyPassReverse ws://jupyterhub:8000/jupyterhub/$1/$2$3 - + # Match the request to /health and return a 200 OK status for AWS ELB health checks + RewriteRule ^/picsure/health$ - [R=200,L] RewriteRule ^/picsure/(.*)$ "http://wildfly:8080/pic-sure-api-2/PICSURE/$1" [P] RewriteRule ^/psama/(.*)$ "http://psama:8090/auth/$1" [P] @@ -93,26 +102,23 @@ ServerTokens Prod RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-d - RewriteRule /psamaui/(.*) /psamaui/index_03272020.html - - RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f - RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-d - RewriteRule /picsureui/(.*) /picsureui/index_03272020.html - RewriteRule ^/static/(.*)$ /static/$1 [L] - RedirectMatch ^/$ /picsureui/ - ErrorDocument 404 /index.html - DocumentRoot "${HTTPD_PREFIX}/htdocs" - ErrorLog "${HTTPD_PREFIX}/logs/error_log" - TransferLog "${HTTPD_PREFIX}/logs/access_log" - CustomLog "${HTTPD_PREFIX}/logs/ssl_request_log" \ - "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" + LogFormat "%%{X-Forwarded-For}i %t %%{SSL_PROTOCOL}x %%{SSL_CIPHER}x \"%r\" %b" proxy-ssl + LogFormat "%h %l %u %t \"%r\" %>s %b \"%%{Referer}i\" \"%%{User-Agent}i\"" combined + LogFormat "%%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%%{Referer}i\" \"%%{User-Agent}i\"" proxy + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog "$${HTTPD_PREFIX}/logs/access_log" combined env=!forwarded + CustomLog "$${HTTPD_PREFIX}/logs/access_log" proxy env=forwarded + CustomLog "$${HTTPD_PREFIX}/logs/ssl_request_log" proxy-ssl env=forwarded + CustomLog "$${HTTPD_PREFIX}/logs/ssl_request_log" \ + "%t %h %%{SSL_PROTOCOL}x %%{SSL_CIPHER}x \"%r\" %b" env=!forwarded + ErrorLog "$${HTTPD_PREFIX}/logs/error_log" + TransferLog "$${HTTPD_PREFIX}/logs/access_log" BrowserMatch "MSIE [2-5]" \ nokeepalive ssl-unclean-shutdown \ downgrade-1.0 force-response-1.0 - diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index 509de7a3..e183c33a 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -196,9 +196,7 @@ - 7 - project_specific_override_repo - __PROJECT_SPECIFIC_OVERRIDE_REPO__ + 8 release_control_branch */master release_control_repo @@ -211,6 +209,10 @@ host MYSQL_CONFIG_DIR /usr/local/docker-config/picsure-db/ + MIGRATION_REPO + https://github.com/hms-dbmi/PIC-SURE-Migrations + MIGRATION_NAME + __PROJECT_SPECIFIC_MIGRATION_NAME__ diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Add or Edit a Setting/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Add or Edit a Setting/config.xml deleted file mode 100644 index 81dbd5a1..00000000 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Add or Edit a Setting/config.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - Edit the values in picsureui_settings.json file. You can find the current file in either your Network tab of your browser's developer tools by searching for "settings.json" or in this location: $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json - false - - - - - SETTING_FIELD_NAME - The field name of the setting you would like to edit. Example: helpLink - - false - - - SETTING_VALUE - The value of the setting. This can be a string or json object. NOTE: If it is a string is should be in quotes. For more complex values you may need run it through a json linter. Example: "mailto:example@example.com" OR {"hello": "world", "arrayName": ["array", "test"]} - - false - - - - - - true - false - false - false - - false - - - - #!/bin/bash - # Check for the presence - current_field_name=$(jq -r --arg field_name "$SETTING_FIELD_NAME" 'select(.[$field_name] != null) | .[$field_name]' /usr/local/docker-config/httpd/picsureui_settings.json) - echo $current_field_name - - # Check if the key is missing - if [[ -z $current_field_name ]]; then - # Add the key with the value - jq --arg field_name "$SETTING_FIELD_NAME" --argjson value "$SETTING_VALUE" '. + {($field_name): $value}' /usr/local/docker-config/httpd/picsureui_settings.json > /tmp/temp.json && mv /tmp/temp.json /usr/local/docker-config/httpd/picsureui_settings.json - else - # Replace the old value - jq --arg field_name "$SETTING_FIELD_NAME" --argjson value "$SETTING_VALUE" '.[$field_name] = $value' /usr/local/docker-config/httpd/picsureui_settings.json > /tmp/temp.json && mv /tmp/temp.json /usr/local/docker-config/httpd/picsureui_settings.json - fi - - - - - - diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Institution Node/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Institution Node/config.xml index 3b59dff2..5ea1d25b 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Configure Institution Node/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Configure Institution Node/config.xml @@ -39,10 +39,10 @@ false - PROJECT_SPECIFIC_OVERRIDE_REPOSITORY - This must be the project specific git repository that contains the project specific database migrations and UI overrides in the standard PIC-SURE Override Repository Structure. - - false + PROJECT_MIGRATION_NAME + Name of the migration to run, see https://github.com/hms-dbmi/PIC-SURE-Migrations or fork it and add your migration + GIC-Institution + true RELEASE_CONTROL_REPOSITORY @@ -68,9 +68,114 @@ 0 false - - - + + OPEN + PIC-SURE can be used without forcing the user log in. Unauthorized users will have a limited feature set. See more information on Gitbook: TODO_GITBOOK + true + + + DISCOVER + Enable the Discover page which allows users to search across all data in the project. + true + + + DASHBOARD + Enable the Dashboard page which allows users to search across all data in the project. + true + + + DIST_EXPLORER + Enable the distribution explorer page which creates visualizations of the data. + true + + + ENABLE_GENE_QUERY + Enable the genomic filtering UI which guides users creating genomic queries. Enable if HPDS has been loaded with Genomic data. + true + + + ENABLE_SNP_QUERY + Enable the genomic filtering UI which guides users creating genomic SNP queries. Enable if HPDS has been loaded with Genomic data. + true + + + VITE_ALLOW_EXPORT_ENABLED + Enables the export button on the search results page. + true + + + API + Shows the user information on how to use the API and allows them to access their LONG_TERM_TOKEN. + true + + + ALLOW_EXPORT + This enables the prepare for analysis button which guides the user through your export process. There are a few different settings below to control how the user may receive their data in the end. + true + + + DOWNLOAD_AS_CSV + Allows the user to download their data at the end of the export process as a CSV file. + true + + + SHOW_VARIABLE_EXPORT + Allow adding export variables via the search results UI. + true + + + SHOW_VARIABLE_HIERARCHY + Shows the hierarchy button for each search result and enables the hierarchy screen in the export process. + true + + + DOWNLOAD_AS_PFB + Allows the user to download their data at the end of the export process as a PFB file. + false + + + DATA_REQUESTS + Enables data request functionality. + false + + + VARIANT_EXPLORER + Enable the variant explorer. Only enable if HPDS has been loaded with Genomic data. + true + + + VARIANT_EXPLORER_TYPE + Aggregate is recommended. + + + aggregate + full + + + + + EXPLORER_MAX_COUNT + Max amount of data point for the variant explorer. + 10000 + true + + + VARIANT_EXPLORER_EXCLUDE_COLUMNS + Advanced: An array of columns to exclude in the variant explorer. +The value must be in an array format, for example: ["AN"] + true + + + TOUR + Enable the tour feature which guides the user through the application. + true + + + TOUR_SEARCH_TERM + The tour preforms a search. You can specify the search term here. + age + true + @@ -93,7 +198,6 @@ pipeline { steps { script { sh """ - sed -i "s|__PROJECT_SPECIFIC_OVERRIDE_REPO__|$env.PROJECT_SPECIFIC_OVERRIDE_REPOSITORY|g" /var/jenkins_home/config.xml sed -i "s|__RELEASE_CONTROL_REPO__|$env.RELEASE_CONTROL_REPOSITORY|g" /var/jenkins_home/config.xml """ Jenkins.instance.doReload() @@ -123,6 +227,54 @@ pipeline { [$class: 'StringParameterValue', name: 'OUTBOUND_EMAIL_USER_PASSWORD', value: env.OUTBOUND_EMAIL_USER_PASSWORD]] } + }, + createFrontendEnvFile: { + script { + sh "[ ! -f /usr/local/docker-config/httpd/.env ] || cp -rf /usr/local/docker-config/httpd/.env /usr/local/docker-config/httpd/.env.bak" + sh "touch .env" + sh "chmod +wr .env" + sh "echo 'VITE_PROJECT_HOSTNAME=' >> .env" + sh "echo 'VITE_ORIGIN=' >> .env" + sh "echo 'VITE_LOGO=' >> .env" + sh "echo 'VITE_AUTH0_TENANT=$env.AUTH0_TENANT' >> .env" + sh "echo 'VITE_OPEN=$env.OPEN' >> .env" + sh "echo 'VITE_DISCOVER=$env.DISCOVER' >> .env" + sh "echo 'VITE_DASHBOARD=$env.DASHBOARD' >> .env" + sh "echo 'VITE_DIST_EXPLORER=$env.DIST_EXPLORER' >> .env" + sh "echo 'VITE_API=$env.API' >> .env" + sh "echo 'VITE_ALLOW_EXPORT=$env.ALLOW_EXPORT' >> .env" + sh "echo 'VITE_ALLOW_EXPORT_ENABLED=$env.ALLOW_EXPORT_ENABLED' >> .env" + sh "echo 'VITE_DOWNLOAD_AS_CSV=$env.DOWNLOAD_AS_CSV' >> .env" + sh "echo 'VITE_ENABLE_GENE_QUERY=$env.ENABLE_GENE_QUERY' >> .env" + sh "echo 'VITE_ENABLE_SNP_QUERY=$env.ENABLE_SNP_QUERY' >> .env" + sh "echo 'VITE_SHOW_VARIABLE_EXPORT=$env.SHOW_VARIABLE_EXPORT' >> .env" + sh "echo 'VITE_SHOW_VARIABLE_HIERARCHY=$env.SHOW_VARIABLE_HIERARCHY' >> .env" + sh "echo 'VITE_DOWNLOAD_AS_PFB=$env.DOWNLOAD_AS_PFB' >> .env" + sh "echo 'VITE_DATA_REQUESTS=$env.DATA_REQUESTS' >> .env" + sh "echo 'VITE_VARIANT_EXPLORER=$env.VARIANT_EXPLORER' >> .env" + sh "echo 'VITE_VARIANT_EXPLORER_TYPE=$env.VARIANT_EXPLORER_TYPE' >> .env" + sh "echo 'VITE_EXPLORER_MAX_COUNT=$env.EXPLORER_MAX_COUNT' >> .env" + sh "echo 'VITE_VARIANT_EXPLORER_EXCLUDE_COLUMNS=$env.VARIANT_EXPLORER_EXCLUDE_COLUMNS' >> .env" + sh "echo 'VITE_TOUR=$env.TOUR' >> .env" + sh "echo 'VITE_TOUR_SEARCH_TERM=$env.TOUR_SEARCH_TERM' >> .env" + sh "echo 'VITE_REQUIRE_CONSENTS=false' >> .env" + sh "echo 'VITE_USE_QUERY_TEMPLATE=false' >> .env" + sh "echo '#Login Provider (You may add as many as you would like):' >> .env" + sh "echo 'VITE_AUTH_PROVIDER_MODULE_GOOGLE=true' >> .env" + sh "echo 'VITE_AUTH_PROVIDER_MODULE_GOOGLE_TYPE=AUTH0' >> .env" + sh "echo 'VITE_AUTH_PROVIDER_MODULE_GOOGLE_CLIENTID=$env.AUTH0_CLIENT_ID' >> .env" + sh "echo 'VITE_AUTH_PROVIDER_MODULE_GOOGLE_CONNECTION=google-oauth2' >> .env" + sh "echo 'VITE_AUTH_PROVIDER_MODULE_GOOGLE_DESCRIPTION=Login' >> .env" + sh "echo '#Resource UUIDs:' >> .env" + sh "echo 'VITE_RESOURCE_HPDS=' >> .env" + sh "echo 'VITE_RESOURCE_OPEN_HPDS=' >> .env" + sh "echo 'VITE_RESOURCE_BASE_QUERY=' >> .env" + sh "echo 'VITE_RESOURCE_VIZ=' >> .env" + sh "echo '#Google Analytics settings (leave blank to disable):' >> .env" + sh "echo 'VITE_GOOGLE_ANALYTICS_ID=' >> .env" + sh "echo 'VITE_GOOGLE_TAG_MANAGER_ID=' >> .env" + sh "mv .env '/usr/local/docker-config/httpd/.env'" + } } ) } @@ -150,19 +302,24 @@ pipeline { micro_app_ref = build.git_hash echo 'micro_app_ref ' + micro_app_ref } - if(build.project_job_git_key.equalsIgnoreCase("PSU")) { - psu_ref = build.git_hash - echo 'psu_ref ' + psu_ref + if (build.project_job_git_key.equalsIgnoreCase("PSF")) { + psf_ref = build.git_hash + echo 'psf_ref ' + psf_ref } + if (build.project_job_git_key.equalsIgnoreCase("PSM")) { + psm_ref = build.git_hash + echo 'psm_ref ' + psm_ref + } if(build.project_job_git_key.equalsIgnoreCase("PSA")) { pic_sure_ref = build.git_hash echo 'pic_sure_ref ' + pic_sure_ref } } - if(micro_app_ref.isEmpty() || pic_sure_ref.isEmpty() || psu_ref.isEmpty()) { + if(micro_app_ref.isEmpty() || pic_sure_ref.isEmpty() || psf_ref.isEmpty() || psm_ref.isEmpty()) { currentBuild.result = 'ABORTED' echo 'micro_app_ref = ' + micro_app_ref - echo 'psu_ref = ' + psu_ref + echo 'psf_ref = ' + psf_ref + echo 'psm_ref = ' + psm_ref echo 'pic_sure_ref = ' + pic_sure_ref error('Build Spec Not configured correctly!') } @@ -176,6 +333,8 @@ pipeline { script { checkout([$class: 'GitSCM', branches: [[name: micro_app_ref ]], userRemoteConfigs: [[url: 'https://github.com/hms-dbmi/pic-sure-auth-microapp']]]) + def image = docker.image('dbmi/pic-sure-db-migrations:pic-sure-db-migration_v1.0') + image.pull() //git url: 'https://github.com/hms-dbmi/pic-sure-auth-microapp' docker.image('dbmi/pic-sure-db-migrations:pic-sure-db-migration_v1.0').inside('--network=picsure -v $DOCKER_CONFIG_DIR/flyway/auth:/opt/flyway-migrations/auth "--entrypoint=" ') { sh "rm -rf /opt/flyway-migrations/auth/sql" @@ -222,25 +381,24 @@ pipeline { steps { node("master") { script { - checkout([$class: 'GitSCM', branches: [[name: psu_ref ]], - userRemoteConfigs: [[url: env.project_specific_override_repo]]]) - //git url: env.project_specific_override_repo - sh """ - sed -i "s/__APPLICATION_UUID__/`cat /usr/local/docker-config/APP_ID_HEX`/g" ./custom-migrations/auth/*.sql - sed -i "s/__RESOURCE_UUID__/`cat /usr/local/docker-config/RESOURCE_ID_HEX`/g" ./custom-migrations/picsure/*.sql - """ - docker.image('dbmi/pic-sure-db-migrations:pic-sure-db-migration_v1.0').inside('--network=picsure -v $DOCKER_CONFIG_DIR/flyway/picsure:/opt/flyway-migrations/picsure "--entrypoint=" ') { - sh "rm -rf /opt/flyway-migrations/picsure/sql" - sh "cp -R ./custom-migrations/picsure /opt/flyway-migrations/picsure/sql" - sh "/opt/flyway/flyway -X -baselineOnMigrate=true -configFiles=/opt/flyway-migrations/picsure/flyway-picsure.conf -table=flyway_custom_schema_history migrate" - sleep(time:10,unit:"SECONDS") - } - docker.image('dbmi/pic-sure-db-migrations:pic-sure-db-migration_v1.0').inside('--network=picsure -v $DOCKER_CONFIG_DIR/flyway/auth:/opt/flyway-migrations/auth "--entrypoint=" ') { - sh "rm -rf /opt/flyway-migrations/auth/sql" - sh "cp -R ./custom-migrations/auth /opt/flyway-migrations/auth/sql" - sh "/opt/flyway/flyway -X -baselineOnMigrate=true -configFiles=/opt/flyway-migrations/auth/flyway-auth.conf -table=flyway_custom_schema_history migrate" - sleep(time:10,unit:"SECONDS") - } + checkout([$class: 'GitSCM', branches: [[name: psm_ref ]], + userRemoteConfigs: [[url: MIGRATION_REPO]]]) + sh """ + sed -i "s/__APPLICATION_UUID__/`cat /usr/local/docker-config/APP_ID_HEX`/g" ./${MIGRATION_NAME}/auth/*.sql + sed -i "s/__RESOURCE_UUID__/`cat /usr/local/docker-config/RESOURCE_ID_HEX`/g" ./${MIGRATION_NAME}/picsure/*.sql + """ + docker.image('dbmi/pic-sure-db-migrations:pic-sure-db-migration_v1.0').inside('--network=picsure -v $DOCKER_CONFIG_DIR/flyway/picsure:/opt/flyway-migrations/picsure "--entrypoint=" ') { + sh "rm -rf /opt/flyway-migrations/picsure/sql" + sh "cp -R ./${MIGRATION_NAME}/picsure /opt/flyway-migrations/picsure/sql" + sh "/opt/flyway/flyway -X -baselineOnMigrate=true -configFiles=/opt/flyway-migrations/picsure/flyway-picsure.conf -table=flyway_custom_schema_history migrate" + sleep(time:10,unit:"SECONDS") + } + docker.image('dbmi/pic-sure-db-migrations:pic-sure-db-migration_v1.0').inside('--network=picsure -v $DOCKER_CONFIG_DIR/flyway/auth:/opt/flyway-migrations/auth "--entrypoint=" ') { + sh "rm -rf /opt/flyway-migrations/auth/sql" + sh "cp -R ./${MIGRATION_NAME}/auth /opt/flyway-migrations/auth/sql" + sh "/opt/flyway/flyway -X -baselineOnMigrate=true -configFiles=/opt/flyway-migrations/auth/flyway-auth.conf -table=flyway_custom_schema_history migrate" + sleep(time:10,unit:"SECONDS") + } } } } diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml index d75c72f0..241bd113 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Initial Configuration Pipeline/config.xml @@ -18,49 +18,157 @@ AUTH0_CLIENT_ID __PIC_SURE_CLIENT_ID__ - false + true AUTH0_CLIENT_SECRET __PIC_SURE_CLIENT_SECRET__ - false + true AUTH0_TENANT avillachlab - false + true ADMIN_USER_EMAIL The email address for the initial admin user. This should be a Google account. - false + true - PROJECT_SPECIFIC_OVERRIDE_REPOSITORY - This must be the project specific git repository that contains the project specific database migrations and UI overrides in the standard PIC-SURE Override Repository Structure. - - false + PROJECT_MIGRATION_NAME + Name of the migration to run, see https://github.com/hms-dbmi/PIC-SURE-Migrations or fork it and add your migration + Baseline + true RELEASE_CONTROL_REPOSITORY This must be the project specific release control git repository that contains the git commit hashes that will be built and deployed. - false + true OUTBOUND_EMAIL_USER The email account name that should be used to send outbound email. The default server is Google's SMTP, so this should be a GMail account. This field is optional __user@email.com__ - false + true OUTBOUND_EMAIL_USER_PASSWORD The password for the account used to send outbound. This field is optional __YOUR_EMAIL_PASSWORD__ - false + true + + + OPEN + PIC-SURE can be used without forcing the user log in. Unauthorized users will have a limited feature set. See more information on Gitbook: TODO_GITBOOK + true + + + DISCOVER + Enable the Discover page which allows users to search across all data in the project. + true + + + DASHBOARD + Enable the Dashboard page which allows users to search across all data in the project. + true + + + DIST_EXPLORER + Enable the distribution explorer page which creates visualizations of the data. + true + + + ENABLE_GENE_QUERY + Enable the genomic filtering UI which guides users creating genomic queries. Enable if HPDS has been loaded with Genomic data. + true + + + ENABLE_SNP_QUERY + Enable the genomic filtering UI which guides users creating genomic SNP queries. Enable if HPDS has been loaded with Genomic data. + true + + + VITE_ALLOW_EXPORT_ENABLED + Enables the export button on the search results page. + true + + + API + Shows the user information on how to use the API and allows them to access their LONG_TERM_TOKEN. + true + + + ALLOW_EXPORT + This enables the prepare for analysis button which guides the user through your export process. There are a few different settings below to control how the user may receive their data in the end. + true + + + DOWNLOAD_AS_CSV + Allows the user to download their data at the end of the export process as a CSV file. + true + + + SHOW_VARIABLE_EXPORT + Allow adding export variables via the search results UI. + true + + + SHOW_VARIABLE_HIERARCHY + Shows the hierarchy button for each search result and enables the hierarchy screen in the export process. + true + + + DOWNLOAD_AS_PFB + Allows the user to download their data at the end of the export process as a PFB file. + false + + + DATA_REQUESTS + Enables data request functionality. + false + + + VARIANT_EXPLORER + Enable the variant explorer. Only enable if HPDS has been loaded with Genomic data. + true + + + VARIANT_EXPLORER_TYPE + Aggregate is recommended. + + + aggregate + full + + + + + EXPLORER_MAX_COUNT + Max amount of data point for the variant explorer. + 10000 + true + + + VARIANT_EXPLORER_EXCLUDE_COLUMNS + Advanced: An array of columns to exclude in the variant explorer. +The value must be in an array format, for example: ["AN"] + true + + + TOUR + Enable the tour feature which guides the user through the application. + true + + + TOUR_SEARCH_TERM + The tour preforms a search. You can specify the search term here. + age + true @@ -81,102 +189,152 @@ pipeline { steps { script { sh """ - sed -i "s|__PROJECT_SPECIFIC_OVERRIDE_REPO__|$env.PROJECT_SPECIFIC_OVERRIDE_REPOSITORY|g" /var/jenkins_home/config.xml sed -i "s|__RELEASE_CONTROL_REPO__|$env.RELEASE_CONTROL_REPOSITORY|g" /var/jenkins_home/config.xml + sed -i "s|__PROJECT_SPECIFIC_MIGRATION_NAME__|$env.PROJECT_MIGRATION_NAME|g" /var/jenkins_home/config.xml """ Jenkins.instance.doReload() } } } - stage('Initial Config and Build') { - steps { - parallel ( - dbMigrate: { - script { - def result = build job: 'PIC-SURE Database Migrations' - } - }, - picsureBuild: { - script { - def result = build job: 'PIC-SURE Pipeline' - } - }, - auth0Config: { - script { - def result = build job: 'Configure Auth0 Integration', parameters: [ - [$class: 'StringParameterValue', name: 'AUTH0_CLIENT_ID', value:env.AUTH0_CLIENT_ID], - [$class: 'StringParameterValue', name: 'AUTH0_CLIENT_SECRET', value: env.AUTH0_CLIENT_SECRET]] - } - }, - addRootCerts: { - script { - def result = build job: 'Create Root Certs in TrustStore' - } - }, - emailConfig: { - script { - def result = build job: 'Configure Outbound Email Settings', parameters: [ - [$class: 'StringParameterValue', name: 'OUTBOUND_EMAIL_USER', value:env.OUTBOUND_EMAIL_USER], - [$class: 'StringParameterValue', name: 'EMAIL_FROM', value: env.OUTBOUND_EMAIL_USER], - [$class: 'StringParameterValue', name: 'OUTBOUND_EMAIL_USER_PASSWORD', value: env.OUTBOUND_EMAIL_USER_PASSWORD]] + stage('Initial Config and Build') { + steps { + parallel ( + dbMigrate: { + script { + def result = build job: 'PIC-SURE Database Migrations' + } + }, + picsureBuild: { + script { + def result = build job: 'PIC-SURE Pipeline' + } + }, + auth0Config: { + script { + def result = build job: 'Configure Auth0 Integration', parameters: [ + [$class: 'StringParameterValue', name: 'AUTH0_CLIENT_ID', value:env.AUTH0_CLIENT_ID], + [$class: 'StringParameterValue', name: 'AUTH0_CLIENT_SECRET', value: env.AUTH0_CLIENT_SECRET]] + } + }, + addRootCerts: { + script { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + def result = build job: 'Create Root Certs in TrustStore' + } + } + }, + createFrontendEnvFile: { + script { + sh "[ ! -f /usr/local/docker-config/httpd/.env ] || cp -rf /usr/local/docker-config/httpd/.env /usr/local/docker-config/httpd/.env.bak" + sh "touch .env" + sh "chmod +wr .env" + sh "echo 'VITE_PROJECT_HOSTNAME=' >> .env" + sh "echo 'VITE_ORIGIN=' >> .env" + sh "echo 'VITE_LOGO=' >> .env" + sh "echo 'VITE_AUTH0_TENANT=$env.AUTH0_TENANT' >> .env" + sh "echo 'VITE_OPEN=$env.OPEN' >> .env" + sh "echo 'VITE_DISCOVER=$env.DISCOVER' >> .env" + sh "echo 'VITE_DASHBOARD=$env.DASHBOARD' >> .env" + sh "echo 'VITE_DIST_EXPLORER=$env.DIST_EXPLORER' >> .env" + sh "echo 'VITE_API=$env.API' >> .env" + sh "echo 'VITE_ALLOW_EXPORT=$env.ALLOW_EXPORT' >> .env" + sh "echo 'VITE_ALLOW_EXPORT_ENABLED=$env.ALLOW_EXPORT_ENABLED' >> .env" + sh "echo 'VITE_DOWNLOAD_AS_CSV=$env.DOWNLOAD_AS_CSV' >> .env" + sh "echo 'VITE_ENABLE_GENE_QUERY=$env.ENABLE_GENE_QUERY' >> .env" + sh "echo 'VITE_ENABLE_SNP_QUERY=$env.ENABLE_SNP_QUERY' >> .env" + sh "echo 'VITE_SHOW_VARIABLE_EXPORT=$env.SHOW_VARIABLE_EXPORT' >> .env" + sh "echo 'VITE_SHOW_VARIABLE_HIERARCHY=$env.SHOW_VARIABLE_HIERARCHY' >> .env" + sh "echo 'VITE_DOWNLOAD_AS_PFB=$env.DOWNLOAD_AS_PFB' >> .env" + sh "echo 'VITE_DATA_REQUESTS=$env.DATA_REQUESTS' >> .env" + sh "echo 'VITE_VARIANT_EXPLORER=$env.VARIANT_EXPLORER' >> .env" + sh "echo 'VITE_VARIANT_EXPLORER_TYPE=$env.VARIANT_EXPLORER_TYPE' >> .env" + sh "echo 'VITE_EXPLORER_MAX_COUNT=$env.EXPLORER_MAX_COUNT' >> .env" + sh "echo 'VITE_VARIANT_EXPLORER_EXCLUDE_COLUMNS=$env.VARIANT_EXPLORER_EXCLUDE_COLUMNS' >> .env" + sh "echo 'VITE_TOUR=$env.TOUR' >> .env" + sh "echo 'VITE_TOUR_SEARCH_TERM=$env.TOUR_SEARCH_TERM' >> .env" + sh "echo 'VITE_REQUIRE_CONSENTS=false' >> .env" + sh "echo 'VITE_USE_QUERY_TEMPLATE=false' >> .env" + sh "echo '#Login Provider (You may add as many as you would like):' >> .env" + sh "echo 'VITE_AUTH_PROVIDER_MODULE_GOOGLE=true' >> .env" + sh "echo 'VITE_AUTH_PROVIDER_MODULE_GOOGLE_TYPE=AUTH0' >> .env" + sh "echo 'VITE_AUTH_PROVIDER_MODULE_GOOGLE_CLIENTID=$env.AUTH0_CLIENT_ID' >> .env" + sh "echo 'VITE_AUTH_PROVIDER_MODULE_GOOGLE_CONNECTION=google-oauth2' >> .env" + sh "echo 'VITE_AUTH_PROVIDER_MODULE_GOOGLE_DESCRIPTION=Login' >> .env" + sh "echo '#Resource UUIDs:' >> .env" + sh "echo 'VITE_RESOURCE_HPDS=' >> .env" + sh "echo 'VITE_RESOURCE_OPEN_HPDS=' >> .env" + sh "echo 'VITE_RESOURCE_BASE_QUERY=' >> .env" + sh "echo 'VITE_RESOURCE_VIZ=' >> .env" + sh "echo '#Google Analytics settings (leave blank to disable):' >> .env" + sh "echo 'VITE_GOOGLE_ANALYTICS_ID=' >> .env" + sh "echo 'VITE_GOOGLE_TAG_MANAGER_ID=' >> .env" + sh "mv .env '/usr/local/docker-config/httpd/.env'" + } + }, + emailConfig: { + script { + def result = build job: 'Configure Outbound Email Settings', parameters: [ + [$class: 'StringParameterValue', name: 'OUTBOUND_EMAIL_USER', value:env.OUTBOUND_EMAIL_USER], + [$class: 'StringParameterValue', name: 'EMAIL_FROM', value: env.OUTBOUND_EMAIL_USER], + [$class: 'StringParameterValue', name: 'OUTBOUND_EMAIL_USER_PASSWORD', value: env.OUTBOUND_EMAIL_USER_PASSWORD]] + } } - } - ) + ) + } } - } - stage('Post Migration Configs') { - steps { - parallel ( - adminConfig: { - script { - def result = build job: 'Create Admin User', parameters: [ - [$class: 'StringParameterValue', name: 'EMAIL', value:env.ADMIN_USER_EMAIL], - [$class: 'StringParameterValue', name: 'CONNECTION_LABEL', value: "Google"]] - } - }, - tokenIntroToken: { - script { - def result = build job: 'Configure PIC-SURE Token Introspection Token' + stage('Post Migration Configs') { + steps { + parallel ( + adminConfig: { + script { + def result = build job: 'Create Admin User', parameters: [ + [$class: 'StringParameterValue', name: 'EMAIL', value:env.ADMIN_USER_EMAIL], + [$class: 'StringParameterValue', name: 'CONNECTION_LABEL', value: "Google"]] + } + }, + tokenIntroToken: { + script { + def result = build job: 'Configure PIC-SURE Token Introspection Token' + } + }, + encryptionKey: { + script { + def result = build job: 'Create HPDS Encryption Key' + } } - }, - encryptionKey: { - script { - def result = build job: 'Create HPDS Encryption Key' + ) + } + } + stage('Load Demo Data and Deploy') { + steps { + script { + def result = build job: 'Retrieve Build Spec' + retrieveBuildSpecId = result.number + } + script { + copyArtifacts filter: '*', projectName: 'Retrieve Build Spec', selector: specific(""+retrieveBuildSpecId) + sh 'cat build-spec.json' + sh 'cat pipeline_git_commit.txt' + sh 'pwd' + def buildSpec = new JsonSlurper().parse(new File('/var/jenkins_home/workspace/PIC-SURE Pipeline/build-spec.json')) + pipelineBuildId = new File('/var/jenkins_home/workspace/PIC-SURE Pipeline/pipeline_git_commit.txt').text.trim() + for(def build : buildSpec.application){ + build_hashes[build.project_job_git_key] = build.git_hash } } - ) - } - } - stage('Load Demo Data and Deploy') { - steps { - script { - def result = build job: 'Retrieve Build Spec' - retrieveBuildSpecId = result.number - } - script { - copyArtifacts filter: '*', projectName: 'Retrieve Build Spec', selector: specific(""+retrieveBuildSpecId) - sh 'cat build-spec.json' - sh 'cat pipeline_git_commit.txt' - sh 'pwd' - def buildSpec = new JsonSlurper().parse(new File('/var/jenkins_home/workspace/PIC-SURE Pipeline/build-spec.json')) - pipelineBuildId = new File('/var/jenkins_home/workspace/PIC-SURE Pipeline/pipeline_git_commit.txt').text.trim() - for(def build : buildSpec.application){ - build_hashes[build.project_job_git_key] = build.git_hash + script { + def result = build job: 'Load HPDS Data From CSV' + } + script { + def result = build job: 'Start PIC-SURE', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId]] } - } - script { - def result = build job: 'Load HPDS Data From CSV' - } - script { - def result = build job: 'Start PIC-SURE', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSU']]] } } } - } -} + } true false - + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Database Migrations/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Database Migrations/config.xml index 2d9013e4..bdd535d8 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Database Migrations/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Database Migrations/config.xml @@ -18,6 +18,8 @@ def micro_app_ref = '' def pic_sure_ref = '' def psu_ref = '' +def psm_ref = '' +def psf_ref = '' node { stage('Retrieve Build Spec') { @@ -41,19 +43,24 @@ node { micro_app_ref = build.git_hash echo 'micro_app_ref ' + micro_app_ref } - if(build.project_job_git_key.equalsIgnoreCase("PSU")) { - psu_ref = build.git_hash - echo 'psu_ref ' + psu_ref - } if(build.project_job_git_key.equalsIgnoreCase("PSA")) { pic_sure_ref = build.git_hash echo 'pic_sure_ref ' + pic_sure_ref } + if(build.project_job_git_key.equalsIgnoreCase("PSF")) { + psf_ref = build.git_hash + echo 'psf_ref ' + psf_ref + } + if(build.project_job_git_key.equalsIgnoreCase("PSM")) { + psm_ref = build.git_hash + echo 'psm_ref ' + psm_ref + } } - if(micro_app_ref.isEmpty() || pic_sure_ref.isEmpty() || psu_ref.isEmpty()) { + if(micro_app_ref.isEmpty() || pic_sure_ref.isEmpty() || psf_ref.isEmpty() || psm_ref.isEmpty()) { currentBuild.result = 'ABORTED' echo 'micro_app_ref = ' + micro_app_ref - echo 'psu_ref = ' + psu_ref + echo 'psf_ref = ' + psf_ref + echo 'psm_ref = ' + psm_ref echo 'pic_sure_ref = ' + pic_sure_ref error('Build Spec Not configured correctly!') } @@ -62,7 +69,8 @@ node { stage('Auth Schema Migration') { checkout([$class: 'GitSCM', branches: [[name: micro_app_ref ]], userRemoteConfigs: [[url: 'https://github.com/hms-dbmi/pic-sure-auth-microapp']]]) - //git url: 'https://github.com/hms-dbmi/pic-sure-auth-microapp' + def image = docker.image('dbmi/pic-sure-db-migrations:pic-sure-db-migration_v1.0') + image.pull() docker.image('dbmi/pic-sure-db-migrations:pic-sure-db-migration_v1.0').inside('--network=picsure -v $DOCKER_CONFIG_DIR/flyway/auth:/opt/flyway-migrations/auth "--entrypoint=" ') { sh "rm -rf /opt/flyway-migrations/auth/sql" sh "cp -R ./pic-sure-auth-db/db/sql /opt/flyway-migrations/auth/sql" @@ -82,22 +90,21 @@ node { } } stage('Project Specific Migrations') { - checkout([$class: 'GitSCM', branches: [[name: psu_ref ]], - userRemoteConfigs: [[url: env.project_specific_override_repo]]]) - //git url: env.project_specific_override_repo + checkout([$class: 'GitSCM', branches: [[name: psm_ref ]], + userRemoteConfigs: [[url: MIGRATION_REPO]]]) sh """ - sed -i "s/__APPLICATION_UUID__/`cat /usr/local/docker-config/APP_ID_HEX`/g" ./custom-migrations/auth/*.sql - sed -i "s/__RESOURCE_UUID__/`cat /usr/local/docker-config/RESOURCE_ID_HEX`/g" ./custom-migrations/picsure/*.sql + sed -i "s/__APPLICATION_UUID__/`cat /usr/local/docker-config/APP_ID_HEX`/g" ./${MIGRATION_NAME}/auth/*.sql + sed -i "s/__RESOURCE_UUID__/`cat /usr/local/docker-config/RESOURCE_ID_HEX`/g" ./${MIGRATION_NAME}/picsure/*.sql """ docker.image('dbmi/pic-sure-db-migrations:pic-sure-db-migration_v1.0').inside('--network=picsure -v $DOCKER_CONFIG_DIR/flyway/picsure:/opt/flyway-migrations/picsure "--entrypoint=" ') { sh "rm -rf /opt/flyway-migrations/picsure/sql" - sh "cp -R ./custom-migrations/picsure /opt/flyway-migrations/picsure/sql" + sh "cp -R ./${MIGRATION_NAME}/picsure /opt/flyway-migrations/picsure/sql" sh "/opt/flyway/flyway -X -baselineOnMigrate=true -configFiles=/opt/flyway-migrations/picsure/flyway-picsure.conf -table=flyway_custom_schema_history migrate" sleep(time:10,unit:"SECONDS") } - docker.image('dbmi/pic-sure-db-migrations:pic-sure-db-migration_v1.0').inside('--network=picsure -v $DOCKER_CONFIG_DIR/flyway/auth:/opt/flyway-migrations/auth "--entrypoint=" ') { + docker.image('dbmi/pic-sure-db-migrations:pic-sure-db-migration_v1.0').inside('--network=picsure -v $DOCKER_CONFIG_DIR/flyway/auth:/opt/flyway-migrations/auth "--entrypoint=" ') { sh "rm -rf /opt/flyway-migrations/auth/sql" - sh "cp -R ./custom-migrations/auth /opt/flyway-migrations/auth/sql" + sh "cp -R ./${MIGRATION_NAME}/auth /opt/flyway-migrations/auth/sql" sh "/opt/flyway/flyway -X -baselineOnMigrate=true -configFiles=/opt/flyway-migrations/auth/flyway-auth.conf -table=flyway_custom_schema_history migrate" sleep(time:10,unit:"SECONDS") } diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Database Repair/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Database Repair/config.xml index c11b240f..2384a02a 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Database Repair/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Database Repair/config.xml @@ -50,10 +50,10 @@ node { echo 'pic_sure_ref ' + pic_sure_ref } } - if(micro_app_ref.isEmpty() || pic_sure_ref.isEmpty() || psu_ref.isEmpty()) { + if(micro_app_ref.isEmpty() || pic_sure_ref.isEmpty() || psm_ref.isEmpty()) { currentBuild.result = 'ABORTED' echo 'micro_app_ref = ' + micro_app_ref - echo 'psu_ref = ' + psu_ref + echo 'psm_ref = ' + psm_ref echo 'pic_sure_ref = ' + pic_sure_ref error('Build Spec Not configured correctly!') } @@ -83,8 +83,7 @@ node { } stage('Project Specific Migrations') { checkout([$class: 'GitSCM', branches: [[name: psu_ref ]], - userRemoteConfigs: [[url: env.project_specific_override_repo]]]) - //git url: env.project_specific_override_repo + userRemoteConfigs: [[url: MIGRATION_REPO]]]) sh """ sed -i "s/__APPLICATION_UUID__/`cat /usr/local/docker-config/APP_ID_HEX`/g" ./custom-migrations/auth/*.sql sed -i "s/__RESOURCE_UUID__/`cat /usr/local/docker-config/RESOURCE_ID_HEX`/g" ./custom-migrations/picsure/*.sql diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Project Specific PIC-SURE Overrides/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Frontend Build/config.xml similarity index 68% rename from initial-configuration/jenkins/jenkins-docker/jobs/Project Specific PIC-SURE Overrides/config.xml rename to initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Frontend Build/config.xml index 44aaff5b..6a21c34c 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Project Specific PIC-SURE Overrides/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Frontend Build/config.xml @@ -8,24 +8,22 @@ pipeline_build_id - MANUAL_RUN - false + true git_hash - - */main + main false - + 2 - ${project_specific_override_repo} + https://github.com/hms-dbmi/pic-sure-frontend.git @@ -34,7 +32,7 @@ false - + true @@ -50,28 +48,25 @@ if [ -f "/usr/local/docker-config/setProxy.sh" ]; then . /usr/local/docker-config/setProxy.sh fi -sed -i s/TARGET_BUILD_VERSION/LATEST/g ui/Dockerfile -GIT_BRANCH_SHORT=`echo ${GIT_BRANCH} | cut -d "/" -f 2` +GIT_BRANCH_SHORT=`echo ${GIT_BRANCH} | cut -d "/" -f 2` GIT_COMMIT_SHORT=`echo ${GIT_COMMIT} | cut -c1-7` -if [ -f "./setEnv.sh" ]; then - . ./setEnv.sh -fi +cp /usr/local/docker-config/httpd/.env . -if [ -f "./custom_httpd_volumes" ]; then - cp ./custom_httpd_volumes /usr/local/docker-config/httpd/ +if [ ! -f .env ]; then + echo ".env file not found" + exit 1 fi - -cd ui docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$http_proxy --build-arg no_proxy="$no_proxy" \ ---build-arg HTTP_PROXY=$http_proxy --build-arg HTTPS_PROXY=$http_proxy --build-arg NO_PROXY="$no_proxy" \ --t hms-dbmi/pic-sure-ui-overrides:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} . -docker tag hms-dbmi/pic-sure-ui-overrides:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} hms-dbmi/pic-sure-ui-overrides:LATEST + --build-arg HTTP_PROXY=$http_proxy --build-arg HTTPS_PROXY=$http_proxy --build-arg NO_PROXY="$no_proxy" \ + -f Dockerfile -t hms-dbmi/pic-sure-frontend:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} . +docker tag hms-dbmi/pic-sure-frontend:${GIT_BRANCH_SHORT}_${GIT_COMMIT_SHORT} hms-dbmi/pic-sure-frontend:LATEST + - + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml index cbed4bbf..2945f3b2 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml @@ -43,7 +43,7 @@ pipeline { } } } - stage('PIC-SURE API, PIC-SURE-HPDS UI') { + stage('PIC-SURE API and PSAMA') { steps { parallel ( picsureapi:{ @@ -51,15 +51,15 @@ pipeline { def result = build job: 'PIC-SURE-API Build', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSA']]] } }, - picsurehpdsui:{ - script{ - def result = build job: 'PIC-SURE-HPDS-UI Docker Build', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSHU']]] + picsureauth:{ + script{ + def result = build job: 'PIC-SURE Auth Micro-App Build - Jenkinsfile', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSAMA']]] } } ) } } - stage('PSAMA, Project UI Overrides and PIC-SURE HPDS'){ + stage('PIC-SURE HPDS and Wildfly'){ steps { parallel ( picsurehpds:{ @@ -67,35 +67,24 @@ pipeline { def result = build job: 'PIC-SURE-HPDS Build', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSH']]] } }, - picsureauth:{ - script{ - def result = build job: 'PIC-SURE Auth Micro-App Build - Jenkinsfile', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSAMA']]] - } - } - ) - } - } - stage('Wildfly Build'){ - steps { - parallel ( wildfly:{ script { def result = build job: 'PIC-SURE Wildfly Image Build', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId]] } - }, - projectSpecificUI:{ - script { - def result = build job: 'Project Specific PIC-SURE Overrides', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSU']]] - } } ) } } + stage('Frontend Build'){ + steps { + script { + def result = build job: 'PIC-SURE Frontend Build', parameters: [[$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId],[$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['PSF']]] + } + } stage('Dictionary Build') { steps { script { if (build_hashes['DICTIONARY']) { - echo "Building Dictionary API" def result = build job: 'Build Dictionary API', parameters: [ [$class: 'StringParameterValue', name: 'pipeline_build_id', value: pipelineBuildId], [$class: 'StringParameterValue', name: 'git_hash', value: build_hashes['DICTIONARY']] diff --git a/start-picsure.sh b/start-picsure.sh index 524a07cd..00a7b806 100755 --- a/start-picsure.sh +++ b/start-picsure.sh @@ -12,10 +12,11 @@ if [ -f "$DOCKER_CONFIG_DIR/setProxy.sh" ]; then . $DOCKER_CONFIG_DIR/setProxy.sh fi -if [ -z "$(grep queryExportType $DOCKER_CONFIG_DIR/httpd/picsureui_settings.json | grep DISABLED)" ]; then - export EXPORT_SIZE="2000"; -else - export EXPORT_SIZE="0"; +if [ -z "$(grep "VITE_ALLOW_EXPORT" $DOCKER_CONFIG_DIR/httpd/.env | grep 'false')" ]; then + export EXPORT_SIZE="2000"; + else + export EXPORT_SIZE="0"; + fi fi # Docker Volumes @@ -41,17 +42,16 @@ docker run --name=hpds --restart always --network=picsure \ -d hms-dbmi/pic-sure-hpds:LATEST docker stop httpd && docker rm httpd + docker run --name=httpd --restart always --network=picsure \ - -v /var/log/httpd-docker-logs/:/usr/local/apache2/logs/ \ - $PICSURE_SETTINGS_VOLUME \ - $PICSURE_BANNER_VOLUME \ - $PSAMA_SETTINGS_VOLUME \ - -v $DOCKER_CONFIG_DIR/httpd/cert:/usr/local/apache2/cert/ \ - $CUSTOM_HTTPD_VOLUMES \ - -p 80:80 \ - -p 443:443 \ - --env-file $DOCKER_CONFIG_DIR/httpd/httpd.env \ - -d hms-dbmi/pic-sure-ui-overrides:LATEST + -v /var/log/httpd-docker-logs/:/app/logs/ \ + -v $DOCKER_CONFIG_DIR/httpd/cert:/usr/local/apache2/cert/ \ + -v $DOCKER_CONFIG_DIR/httpd/httpd-vhosts.conf:/usr/local/apache2/conf/extra/httpd-vhosts.conf \ + $CUSTOM_HTTPD_VOLUMES \ + --env-file $DOCKER_CONFIG_DIR/httpd/httpd.env \ + -p 80:80 \ + -p 443:443 \ + -d hms-dbmi/pic-sure-frontend:LATEST docker exec httpd sed -i '/^#LoadModule proxy_wstunnel_module/s/^#//' conf/httpd.conf docker restart httpd From 77f9c0b940a7029905129caf269196a5aad3c3be Mon Sep 17 00:00:00 2001 From: Luke Sikina Date: Mon, 25 Nov 2024 10:13:36 -0500 Subject: [PATCH 22/22] [ALS-7538] Remove Microservice jobs, create bespoke jobs for uploader - Remove build and deploy microsevice - Add Build and Deploy job for uploader - Move Uploader .env logic - Add to build pipeline --- .../jenkins/jenkins-docker/config.xml | 23 ++++ .../Build and Deploy Microservice/config.xml | 106 ------------------ .../jobs/Build and Deploy Uploader/config.xml | 57 ++++++++++ .../jobs/PIC-SURE Pipeline/config.xml | 22 +++- .../jobs/Upload Uploader .env File/config.xml | 31 +++++ 5 files changed, 130 insertions(+), 109 deletions(-) delete mode 100755 initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Uploader/config.xml create mode 100644 initial-configuration/jenkins/jenkins-docker/jobs/Upload Uploader .env File/config.xml diff --git a/initial-configuration/jenkins/jenkins-docker/config.xml b/initial-configuration/jenkins/jenkins-docker/config.xml index e183c33a..60bd2377 100644 --- a/initial-configuration/jenkins/jenkins-docker/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/config.xml @@ -180,6 +180,29 @@ false + + + Uploader + false + false + + + + Build and Deploy Uploader + Upload Uploader .env File + + + + + + + + + + + + false + Deployment 50000 diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml deleted file mode 100755 index e99e67ac..00000000 --- a/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Microservice/config.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - - Build and deploy a microservice. The microservice must live in a directory in - https://github.com/hms-dbmi/pic-sure-services/ - - false - - - - - git_hash - */main - false - - - service_name - The name of the directory for your service in the pic-sure-services repo - false - - - service_description - Describe it - false - - - .env - The .env file needed to run this service. The UUID will be added to the end of this env - - - - - - 2 - - - https://github.com/hms-dbmi/pic-sure-services.git - - - - - ${git_hash} - - - false - - - - true - false - false - false - - false - - - # Get the resource from the db if it exists -export SQL="SELECT LOWER(CONCAT(SUBSTR(HEX(uuid), 1, 8), '-', SUBSTR(HEX(uuid), 9, 4), '-', SUBSTR(HEX(uuid), 13, 4), '-', SUBSTR(HEX(uuid), 17, 4), '-', SUBSTR(HEX(uuid), 21))) from picsure.resource where name = '$service_name'"; -export resource_uuid=$(docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); - -# Add the resource to the database if it doesn't already exist -if [ -z "$resource_uuid" ]; then - echo 'This is the first time building this resource. Adding to db' - echo '' - export SQL="INSERT IGNORE INTO picsure.resource (uuid, name, resourceRSPath, description) \ - VALUES (UUID(), '$service_name', 'http://$service_name/', '$service_description')"; - docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -e "$SQL" picsure -fi - -# Get the resource from the db -export SQL="SELECT LOWER(CONCAT(SUBSTR(HEX(uuid), 1, 8), '-', SUBSTR(HEX(uuid), 9, 4), '-', SUBSTR(HEX(uuid), 13, 4), '-', SUBSTR(HEX(uuid), 17, 4), '-', SUBSTR(HEX(uuid), 21))) from picsure.resource where name = '$service_name'"; -export resource_uuid=$(docker run -i -v $MYSQL_CONFIG_DIR/.my.cnf:/root/.my.cnf --network=${MYSQL_NETWORK:-host} mysql mysql -se "$SQL" picsure); -echo '' -echo "Done adding to db. Using $resource_uuid as uuid"; - -# Add the resource to our env file -if grep -q 'RESOURCE_UUID' .env; then - echo '' - echo 'RESOURCE_UUID exists in .env. Replacing with our value' - sed -i~ '/^RESOURCE_UUID=/s/=.*/="$resource_uuid"/' .env -else - echo '' - echo 'RESOURCE_UUID does NOT exist in .env. Appending our to eof' - echo RESOURCE_UUID=$resource_uuid >> .env -fi - - -# Copy over all the things -echo '' -echo 'Copying source code, partial compose file and .env' -cp -r $service_name /pic-sure-services/ -mv .env /pic-sure-services/$service_name/ -cp docker-compose.yml /pic-sure-services/ -cd /pic-sure-services/ - -# This find command just gets all the docker-compose files in this dir and the child dirs -# All those files get merged together into one compose file -echo '' -echo 'Building and starting container. Good luck!' -docker compose -f docker-compose.yml $(find ./* -maxdepth 2 -name '*docker-compose.yml' | sed -e 's/^/-f /' | xargs) up --build -d - - - - - - \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Uploader/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Uploader/config.xml new file mode 100644 index 00000000..2ea13db4 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Build and Deploy Uploader/config.xml @@ -0,0 +1,57 @@ + + + + + false + + + + + git_hash + */main + false + + + pipeline_build_id + MANUAL_BUILD + false + + + + + + 2 + + + https://github.com/hms-dbmi/pic-sure-services.git + + + + + ${git_hash} + + + false + + + + true + false + false + false + + false + + + mv /usr/local/docker-config/uploader/.env .env +cp -r uploader /usr/local/docker-config/ +mv .env /usr/local/docker-config/uploader/.env +cd /usr/local/docker-config/uploader/ + +docker compose up -d --build uploader + + + + + + \ No newline at end of file diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml index 2945f3b2..2ad4e76e 100644 --- a/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml +++ b/initial-configuration/jenkins/jenkins-docker/jobs/PIC-SURE Pipeline/config.xml @@ -1,8 +1,8 @@ - - + + @@ -12,7 +12,7 @@ false - + true diff --git a/initial-configuration/jenkins/jenkins-docker/jobs/Upload Uploader .env File/config.xml b/initial-configuration/jenkins/jenkins-docker/jobs/Upload Uploader .env File/config.xml new file mode 100644 index 00000000..0a038351 --- /dev/null +++ b/initial-configuration/jenkins/jenkins-docker/jobs/Upload Uploader .env File/config.xml @@ -0,0 +1,31 @@ + + + + + false + + + + + uploader.env + + + + + + true + false + false + false + + false + + + mkdir -p /usr/local/docker-config/uploader/ +cp uploader.env /usr/local/docker-config/uploader/.env + + + + + + \ No newline at end of file