Fix (set -e) workaround when calling health checker

deployments/README.md

@@ -21,6 +21,10 @@ The `port` is mandatory, but the `env_file_path` and `docker_flags` are optional
 
 The paths provided in the said file **MUST** be **absolute** (trust me you don't want to handle bash's path spaghetti :upside_down_face: :wink:).
 
+#### Deployment Health Checks and Rollbacks
+
+You can set a `project_health_check_url` per project, which will be used to assert the service is running properly after deployment. The deploy script will query the url (using cURL) every 10 seconds until it returns HTTP 200, for a maximum of 5 minutes. It is advised that you pass a url served by the respective service which only returns 200 when the service is in good operating state. This variable is optional. If it is not provided, the health check will not be done. 
+
 ## Notes
 
 `docker system prune` should be run periodically to clean up dangling images and containers. The deployment scripts attempt to minimize the number of these but some are left on purpose due to speeding up multi-stage builds.

deployments/deploy-types.sh

@@ -31,7 +31,7 @@ function deploy_default() {
     old_container_id="$(docker ps -aq --filter ancestor="$image_tag")"
 
     echo -e "Starting docker build\n"
-    docker build -f Dockerfile-prod -t "$image_tag" .
+    local new_image_id=$(docker build -q -f Dockerfile-prod -t "$image_tag" .)
     local build_status="$?"
 
     # Disabled as this meant that no dependencies could be cached. Instead run `docker system prune` periodically to clear up disk space if necessary.
@@ -60,7 +60,7 @@ function deploy_default() {
     else
         echo -e "#-> No docker flags specified.\n"
     fi
-    echo "${docker_flags:-} -d --restart=unless-stopped --env PORT=80 -p $port:80 $image_tag" | xargs docker run
+    local new_container_id=$(echo "${docker_flags:-} -d --restart=unless-stopped --env PORT=80 -p $port:80 $image_tag" | xargs docker run)
     local run_status="$?"
     if [ "$run_status" != 0 ]; then
         >&2 echo -e "\n###-> ERROR! Run failed!"
@@ -78,51 +78,45 @@ function deploy_default() {
         return "$run_status"
     fi
 
-    local health_check_result
-    health_check_result=1
-    if [ $health_check_url ]; then
-        echo -e "Starting health check...\n"
-        health_checker $health_check_url || true
-        health_check_result="$?"
-    fi
-
-    if [ "$health_check_result" != 0 ]; then
-	    >&2 echo -e "\n###-> ERROR! Service did not pass the health check! Rolling back to previous container!"
-
-        local new_container_id
-        new_container_id="$(docker ps -aq --filter ancestor="$image_tag")"
-
-        local new_image_id
-        new_image_id="$(docker images -q "$image_tag")"
-
-        docker stop "$new_container_id" &>/dev/null
-    	printf "New container exit code: "
-    	docker wait "$new_container_id"
-    	echo -e "\n###-> New container stopped.\n"
-
-        >&2 echo "###-> Retagging old image and starting old container back up"
-        if [[ -n "$old_image_id" ]]; then
-            docker tag "$old_image_id" "$image_tag"
-        else
-            echo "###->> No old image found for retagging!!"
-        fi
-        if [[ -n "$old_container_id" ]]; then
-            docker start "$old_container_id"
-        else
-            echo "###->> No old container found for starting back up!!"
+    local health_check_result=0
+    if [[ -n "$health_check_url" ]]; then
+        echo -e "###->Starting health check...\n"
+        # This is done this way due to the use of set -e above.
+        # If the command is successful, the || won't run, so the default value is 0
+        # if the command is not successful, we need the || so that the script does not exit immediately 
+        health_checker $health_check_url || health_check_result="$?"
+
+        if [ "$health_check_result" != 0 ]; then
+            >&2 echo -e "\n###-> ERROR! Service did not pass the health check! Rolling back to previous container!"
+
+            docker stop "$new_container_id" &>/dev/null
+            docker wait "$new_container_id"
+            echo -e "\n###-> New container stopped.\n"
+
+            >&2 echo "###-> Retagging old image and starting old container back up"
+            if [[ -n "$old_image_id" ]]; then
+                docker tag "$old_image_id" "$image_tag"
+            else
+                >&2 echo "###->> No old image found for retagging!!"
+            fi
+            if [[ -n "$old_container_id" ]]; then
+                docker start "$old_container_id"
+            else
+                >&2 echo "###->> No old container found for starting back up!!"
+            fi
+
+            >&2 echo "###->> Removing new (un-healthy) container"
+            docker rm "$new_container_id"
+
+            if [[ "$(docker images -q "$image_tag")" == "$new_image_id" ]]; then
+                >&2 echo "###-> Not removing image, as the container was run using the same one (build did a full cache hit)"
+            else
+                printf "Removed new image with id: "
+                docker rmi "$new_image_id"
+            fi
+
+            return 1
         fi
-
-        echo "###->> Removing new (un-healthy) container"
-        docker rm "$new_container_id"
-
-        if [[ "$(docker images -q "$image_tag")" == "$new_image_id" ]]; then
-	        echo "###-> Not removing image, as the container was run using the same one (build did a full cache hit)"
-	    else
-	        printf "old image id: "
-           docker rmi "$new_image_id"
-	    fi
-
-        return 1
     fi
 
     # Cleanup
@@ -139,7 +133,7 @@ function deploy_default() {
 	        echo "###-> Not removing image, as the container was run using the same one (build did a full cache hit)"
 	    else
 	        printf "old image id: "
-           docker rmi "$old_image_id"
+            docker rmi "$old_image_id"
 	    fi
     else
 	    echo "###-> No old image found, so none removed."

deployments/project-configs.sh

@@ -39,12 +39,12 @@ project_dotenv_location[nijobs-fe---experimental]='/home/ni/niployments/deployme
 project_port[nijobs-be---master]=4010
 project_dotenv_location[nijobs-be---master]='/home/ni/niployments/deployments/env-files/nijobs-be/master/.env.local'
 project_docker_flags[nijobs-be---master]='-v /home/ni/niployments/deployments/volumes-data/nijobs:/usr/src/app/static'
-project_health_check_url[nijobs-be---master]='https://localhost:${project_port[nijobs-be---master]}/'
+project_health_check_url[nijobs-be---master]="https://localhost:${project_port[nijobs-be---master]}/"
 ## nijobs-be staging
 project_port[nijobs-be---develop]=4011
 project_dotenv_location[nijobs-be---develop]='/home/ni/niployments/deployments/env-files/nijobs-be/develop/.env.local'
 project_docker_flags[nijobs-be---develop]='-v /home/ni/niployments/deployments/volumes-data/nijobs-beta:/usr/src/app/static'
-project_health_check_url[nijobs-be---develop]='https://localhost:${project_port[nijobs-be---develop]}/'
+project_health_check_url[nijobs-be---develop]="https://localhost:${project_port[nijobs-be---develop]}/"
 
 # debug example:
 # project_dotenv_location[nijobs-be---develop]='/home/miguel/Coding/NIAEFEUP/niployments/deployments/env-files/nijobs-be/develop/.env.local'