From 4c7ad2b7153d0c841b24bff4ac0df06c7ecdb37f Mon Sep 17 00:00:00 2001 From: Evan Pagryzinski Date: Tue, 8 Oct 2024 13:26:19 -0400 Subject: [PATCH 1/3] updating dockerize script Enable easier building of docker iamges manually --- scripts/dockerize.sh | 115 +++++++++---------------------------------- 1 file changed, 23 insertions(+), 92 deletions(-) diff --git a/scripts/dockerize.sh b/scripts/dockerize.sh index 2b597f23fc..12907e960e 100755 --- a/scripts/dockerize.sh +++ b/scripts/dockerize.sh @@ -7,9 +7,7 @@ # cd # scripts/dockerize.sh # -# Arguments: -# -# First argument, the Jenkins build number, is required. +# Arguments: # # All other arguments are optional, so that if one is not specified, then it is assumed # to be "auto". A version of "auto" will result in the default version, obtained @@ -37,16 +35,6 @@ # # Requires a JOB_URL to be set (avoids environment-specific stuff in the repo) -#============================================================ -# Get the build number. It must be specified. -#============================================================ -if [ $# -eq 0 ] -then - echo "At least one argument, the Jenkins build number, must be specified." - exit 2 -fi -jenkins_build=$1 -echo "The Jenkins build, from which build artificacts will be acquired, is $jenkins_build." #============================================================= # Identify default versions! @@ -75,44 +63,44 @@ wres_writing_version=$writing_version # then gradle will not create a new zip file. So the caller must specify each # version with positional args, or "auto" to retain auto-detected version. +if [[ "$1" != "" && "$1" != "auto" ]] +then + wres_core_version=$1 +fi + if [[ "$2" != "" && "$2" != "auto" ]] then - wres_core_version=$2 + wres_worker_shim_version=$2 fi if [[ "$3" != "" && "$3" != "auto" ]] then - wres_worker_shim_version=$3 + wres_tasker_version=$3 fi if [[ "$4" != "" && "$4" != "auto" ]] then - wres_tasker_version=$4 + broker_version=$4 fi if [[ "$5" != "" && "$5" != "auto" ]] then - broker_version=$5 + redis_version=$5 fi if [[ "$6" != "" && "$6" != "auto" ]] then - redis_version=$6 + eventsbroker_version=$6 fi if [[ "$7" != "" && "$7" != "auto" ]] then - eventsbroker_version=$7 + wres_vis_version=$7 fi if [[ "$8" != "" && "$8" != "auto" ]] then - wres_vis_version=$8 -fi - -if [[ "$9" != "" && "$9" != "auto" ]] -then - wres_writing_version=$9 + wres_writing_version=$8 fi echo "" @@ -143,13 +131,6 @@ tasker_file=wres-tasker-${wres_tasker_version}.zip vis_file=wres-vis-${wres_vis_version}.zip writing_file=wres-writing-${wres_writing_version}.zip -jenkins_workspace=$JOB_URL/$jenkins_build/artifact -core_url=$jenkins_workspace/build/distributions/$wres_core_file -worker_url=$jenkins_workspace/wres-worker/build/distributions/$worker_shim_file -tasker_url=$jenkins_workspace/wres-tasker/build/distributions/$tasker_file -vis_url=$jenkins_workspace/wres-vis/build/distributions/$vis_file -writing_url=$jenkins_workspace/wres-writing/build/distributions/$writing_file - # Ensure the distribution zip files are present for successful docker build if [[ ! -f ./build/distributions/$wres_core_file || \ ! -f ./wres-worker/build/distributions/$worker_shim_file || \ @@ -160,69 +141,15 @@ then echo "" echo "It appears you are not an automated build server (or something went wrong if you are)." echo "" - echo "Please download these files and place them in the stated directory:" - echo "" - - if [[ ! -f ./build/distributions/$wres_core_file ]] - then - echo " $core_url - build/distributions" - fi - - if [[ ! -f ./wres-worker/build/distributions/$worker_shim_file ]] - then - echo " $worker_url - wres-worker/build/distributions" - fi - - if [[ ! -f ./wres-tasker/build/distributions/$tasker_file ]] - then - echo " $tasker_url - wres-tasker/build/distributions" - fi - - if [[ ! -f ./wres-vis/build/distributions/$vis_file ]] - then - echo " $vis_url - wres-vis/build/distributions" - fi - - if [[ ! -f ./wres-writing/build/distributions/$writing_file ]] - then - echo " $writing_url - wres-writing/build/distributions" - fi - echo "" - echo "You can use the following curl commands, with user name and token specified in ~/jenkins_token, to obtain the files:" - echo "" - - if [[ ! -f ./build/distributions/$wres_core_file ]] - then - echo " curl --config ~/jenkins_token -o ./build/distributions/$wres_core_file $core_url" - fi - - if [[ ! -f ./wres-worker/build/distributions/$worker_shim_file ]] - then - echo " curl --config ~/jenkins_token -o ./wres-worker/build/distributions/$worker_shim_file $worker_url" - fi - - if [[ ! -f ./wres-tasker/build/distributions/$tasker_file ]] - then - echo " curl --config ~/jenkins_token -o ./wres-tasker/build/distributions/$tasker_file $tasker_url" - fi - - if [[ ! -f ./wres-vis/build/distributions/$vis_file ]] - then - echo " curl --config ~/jenkins_token -o ./wres-vis/build/distributions/$vis_file $vis_url" - fi - - if [[ ! -f ./wres-writing/build/distributions/$writing_file ]] - then - echo " curl --config ~/jenkins_token -o ./wres-writing/build/distributions/$writing_file $writing_url" - fi - echo "" - echo "You can also use the '-u user:token' option instead of '--config ~/jenkins_token', e.g. '-u :'." + echo "You do not have one of the required files, check the bellow exist" echo "" - echo "The above URLs are only valid if your .zip files are the latest artifact. To pull down old artifacts, identify the Jenkins build number associated with the VLab GIT revision and modify the \"ws\" in the url to be \"/artifact\". For example," + echo "./build/distributions/$wres_core_file" + echo "./wres-worker/build/distributions/$worker_shim_file" + echo "./wres-tasker/build/distributions/$tasker_file" + echo "./wres-writing/build/distributions/$writing_file" + echo "./wres-vis/build/distributions/$vis_file" echo "" - echo "$JOB_URL/3686/artifact/wres-vis/build/distributions/wres-vis-20210225-713c981.zip" echo "" - echo "After they have completely finished downloading and have been completely copied into the local directories, re-run this script." exit 3 fi @@ -397,6 +324,10 @@ then docker tag wres/wres-writing:$writing_version $DOCKER_REGISTRY/wres/wres-writing:$writing_version docker push $DOCKER_REGISTRY/wres/wres-writing:$writing_version fi + + echo "Tagging and pushing wres/nginx as wres/nginx..." + docker tag wres/nginx $DOCKER_REGISTRY/wres/nginx + docker push $DOCKER_REGISTRY/wres/nginx fi else From 4fa71fb34dbe0d1ef46979d6f3547af0affd3121 Mon Sep 17 00:00:00 2001 From: Evan Pagryzinski Date: Thu, 17 Oct 2024 08:45:50 -0400 Subject: [PATCH 2/3] removing input needed This is going to be run as an automation so we don't want to force input --- scripts/dockerize.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/scripts/dockerize.sh b/scripts/dockerize.sh index 12907e960e..1c87e9fbcc 100755 --- a/scripts/dockerize.sh +++ b/scripts/dockerize.sh @@ -348,13 +348,6 @@ echo "About to update the .yml files with the new versions based on a template." echo "If you are only updating some of the images/versions, it is recommended" echo "you skip this step and do that by manually editing the .ymls." echo "" -echo "Do you wish to continue with this last step?" -select yn in "Yes" "No"; do - case $yn in - Yes ) echo "Proceeding..."; break;; - No ) exit;; - esac -done cp compose-entry.template.yml compose-entry.yml sed -i "s/TASKER_IMAGE/${tasker_version}/" compose-entry.yml From 8eb336cf0d77d1130f3bbca67ad2f66333b6c69b Mon Sep 17 00:00:00 2001 From: Evan Pagryzinski Date: Thu, 17 Oct 2024 08:56:33 -0400 Subject: [PATCH 3/3] Caching layers in the build causing issues We ran into issues yesterday where corrupted layers were being cached and adding no-cache helped fix that --- scripts/dockerize.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/dockerize.sh b/scripts/dockerize.sh index 1c87e9fbcc..e54249ee0e 100755 --- a/scripts/dockerize.sh +++ b/scripts/dockerize.sh @@ -177,7 +177,7 @@ echo "Built wres/wres-tasker:$tasker_version -- $tasker_image_id" # Build and tag the broker image echo "Building broker image..." pushd wres-broker -broker_image_id=$( docker build --pull --build-arg version=$broker_version --quiet --tag wres/wres-broker:$broker_version . ) +broker_image_id=$( docker build --pull --no-cache --build-arg version=$broker_version --quiet --tag wres/wres-broker:$broker_version . ) popd echo "Built wres/wres-broker:$broker_version -- $broker_image_id" @@ -185,7 +185,7 @@ echo "Built wres/wres-broker:$broker_version -- $broker_image_id" # Build and tag the redis image echo "Building redis image..." pushd wres-redis -redis_image_id=$( docker build --pull --build-arg version=$redis_version --quiet --tag wres/wres-redis:$redis_version . ) +redis_image_id=$( docker build --pull --no-cache --build-arg version=$redis_version --quiet --tag wres/wres-redis:$redis_version . ) popd echo "Built wres/wres-redis:$redis_version -- $redis_image_id" @@ -193,7 +193,7 @@ echo "Built wres/wres-redis:$redis_version -- $redis_image_id" # Build and tag the eventsbroker image echo "Building events broker image..." pushd wres-eventsbroker -eventsbroker_image_id=$( docker build --build-arg version=$eventsbroker_version --quiet --tag wres/wres-eventsbroker:$eventsbroker_version . ) +eventsbroker_image_id=$( docker build --no-cache --build-arg version=$eventsbroker_version --quiet --tag wres/wres-eventsbroker:$eventsbroker_version . ) popd echo "Built wres/wres-eventsbroker:$eventsbroker_version -- $eventsbroker_image_id"