Merge pull request #54 from KevFan/INTLY-3934-Timestamps

INTLY-3934 - add timestamp to scripts

Makefile

@@ -5,7 +5,7 @@ IMAGE=backup-container
 TAG=latest
 
 image/build:
-	@(cd image; docker build -t ${REG}/${ORG}/${IMAGE}:${TAG} .)
+	@docker build -t ${REG}/${ORG}/${IMAGE}:${TAG} ./image
 
 image/push:
 	@docker push ${REG}/${ORG}/${IMAGE}:${TAG}

image/tools/entrypoint.sh

@@ -29,7 +29,7 @@ if [[ "$debug" ]]; then
 fi
 
 if [[ -z "$component" ]]; then
-    (>&2 echo 'Please specify a component using "-c"')
+    (>&2 timestamp_echo 'Please specify a component using "-c"')
     exit 1
 fi
 
@@ -46,26 +46,28 @@ ARCHIVES_DEST=$DEST/archives
 mkdir -p $DEST $ARCHIVES_DEST
 export HOME=$DEST
 
+timestamp_echo "Starting backup"
+
 component_dump_data $DEST
 if [ "$?" -ne "0" ]; then
-    echo "==> Component data dump failed"
+    timestamp_echo "Component data dump failed"
     exit 1
 fi
-echo '==> Component data dump completed'
+timestamp_echo "Component data dump completed"
 
 if [[ "$encryption_engine" ]]; then
     check_encryption_enabled
     if [ "$?" -eq "0" ]; then
         encrypt_prepare ${DEST}
         encrypted_files="$(encrypt_archive $ARCHIVES_DEST)"
         if [ "$?" -ne "0" ]; then
-            echo "==> Encryption failed"
+            timestamp_echo "Encryption failed"
             exit 1
         fi
 
-        echo '==> Data encryption completed'
+        timestamp_echo "Data encryption completed"
     else
-        echo "==> encryption secret not found. Skipping"
+        timestamp_echo "Encryption secret not found. Skipping"
         encrypted_files="$ARCHIVES_DEST/*"
     fi
 else
@@ -74,13 +76,15 @@ fi
 
 upload_archive "${encrypted_files}" $DATESTAMP backups/$PRODUCT_NAME/$component
 if [ "$?" -ne "0" ]; then
-    echo "==> Archive upload failed"
+    timestamp_echo "Archive upload failed"
     exit 1
 fi
 
-echo "[$DATESTAMP] Backup completed"
+timestamp_echo "[$DATESTAMP] Backup completed"
 
 if [[ "$debug" ]]; then
-    echo '==> Debug flag detected - will sleep for all eternity'
+    timestamp_echo "Debug flag detected - will sleep for all eternity"
     sleep infinity
 fi
+
+timestamp_echo "Finished Backup"

image/tools/lib/backend/s3.sh

@@ -20,9 +20,11 @@ function get_s3_access_key {
 }
 
 function upload_archive {
+    timestamp_echo "Starting upload"
+
     check_backup_enabled
     if [ "$?" -eq "1" ]; then
-        echo "==> backend secret not found. Skipping"
+        timestamp_echo "backend secret not found. Skipping"
         return 0
     fi
 
@@ -42,12 +44,12 @@ function upload_archive {
             s3cmd put --access_key ${AWS_ACCESS_KEY_ID} --secret_key ${AWS_SECRET_ACCESS_KEY} --progress ${fname} "s3://$AWS_S3_BUCKET_NAME/$bucket_folder/$datestamp/$(basename ${fname})"
             rc=$?
             if [ "${rc}" -ne "0" ]; then
-                echo "==> Upload $fname: FAILED"
+                timestamp_echo "Upload $fname: FAILED"
                 exit 1
             fi
-            echo "==> Upload ${fname} completed"
+            timestamp_echo "Upload ${fname} completed"
         else
-            echo "==> No backups in ${fname} to upload"
+            timestamp_echo "No backups in ${fname} to upload"
         fi
     done
 }

image/tools/lib/component/codeready_pv.sh

@@ -8,12 +8,14 @@ function dump_pod_data {
 }
 
 function component_dump_data {
+    timestamp_echo "Dumping codeready component data"
+
     if [[ -z "${PRODUCT_NAMESPACE:-}" ]]; then
         PRODUCT_NAMESPACE="${PRODUCT_NAMESPACE_PREFIX}codeready"
     fi
     local pods="$(oc get pods -n ${PRODUCT_NAMESPACE} --no-headers=true -l "che.workspace_id,che.original_name notin (che-jwtproxy)" | awk '{print $1}')"
     if [ "${#pods}" -eq "0" ]; then
-        echo "=>> No workspaces found to backup"
+        timestamp_echo "No workspaces found to backup"
         exit 0
     fi
 

image/tools/lib/component/enmasse_pv.sh

@@ -23,14 +23,14 @@ function component_dump_data {
     local pods=$(get_broker_pods)
 
     if [ "${#pods}" -eq "0" ]; then
-        echo "=>> No broker pods found to backup"
+        timestamp_echo "No broker pods found to backup"
         exit 0
     fi
 
     mkdir -p ${dump_dest}
 
     for pod in ${pods}; do
-        echo "==> processing enmasse broker pod ${pod}"
+        timestamp_echo "Processing enmasse broker pod ${pod}"
         dump_pod_data ${pod} ${dump_dest}
     done
 
@@ -40,6 +40,6 @@ function component_dump_data {
         tar -zcvf "$archive_path/enmasse-pv-data-${ts}.tar.gz" -C $dump_dest .
         rm -rf $dump_dest
     else
-        echo "==> no enmasse broker data to backup"
+        timestamp_echo "No enmasse broker data to backup"
     fi
 }

image/tools/lib/component/mysql.sh

@@ -24,7 +24,7 @@ function component_dump_data {
 
     check_mysql_backup_enabled
     if [[ $? -eq 1 ]]; then
-        echo "==> mysql secret not found in default namespace. Skipping"
+        timestamp_echo "Mysql secret not found in default namespace. Skipping"
         exit 0
     fi
 
@@ -39,7 +39,7 @@ function component_dump_data {
         mysqldump --single-transaction -h${MYSQL_HOST} -u${MYSQL_USER} -p${MYSQL_PASSWORD} -R ${database} | gzip > ${dest}/archives/${database}-${ts}.dump.gz
         local rc=$?
         if [ "${rc}" -ne "0" ]; then
-            echo "==> Dump $database: FAILED"
+            timestamp_echo "Dump $database: FAILED"
             exit 1
         fi
     done

image/tools/lib/component/postgres.sh

@@ -50,7 +50,7 @@ function component_dump_data {
     ${POSTGRES_PREFIX}/pg_dumpall --clean --if-exists --oids -U ${POSTGRES_USERNAME} -h ${POSTGRES_HOST} | gzip > ${dest}/archives/${namespace}.${ts}.pg_dumpall.gz
     rc=$?
     if [[ "${rc}" -ne "0" ]]; then
-      echo "backup of postgresql: FAILED"
+      timestamp_echo "Backup of postgresql: FAILED"
       exit 1
     fi
   else
@@ -61,11 +61,11 @@ function component_dump_data {
         continue
       fi
 
-      echo "dumping ${db}"
+      timestamp_echo "dumping ${db}"
       ${POSTGRES_PREFIX}/pg_dump --clean --if-exists --oids -U ${POSTGRES_USERNAME} -h ${POSTGRES_HOST} ${db} | gzip > ${dest}/archives/${namespace}.${db}-${ts}.pg_dump.gz
       rc=$?
       if [[ "${rc}" -ne "0" ]]; then
-          echo "==> Dump $db: FAILED"
+          timestamp_echo "Dump $db: FAILED"
           exit 1
       fi
     done

image/tools/lib/component/resources.sh

@@ -14,7 +14,7 @@ function check_resource {
     # least two lines: one for the header and one for each
     # resource found
     if [ "$result"  -eq "1" ]; then
-        echo "==> No $type in $ns to back up"
+        timestamp_echo "No $type in $ns to back up"
         return 1
     else
         return 0
@@ -34,7 +34,7 @@ function backup_resource {
     set +eo pipefail
     check_resource ${type} ${ns}
     if [ "$?" -eq "0" ]; then
-        echo "==> backing up $type in $ns"
+        timestamp_echo "backing up $type in $ns"
         if [ "$loop" == "y" ]; then
             for obj in $(oc get ${type} -n ${ns} | tr -s ' ' | cut -d ' ' -f 1 |  tail -n +2); do
                 echo "$(oc get ${type}/${obj} -n ${ns} -o yaml --export)" > ${dest}/archives/${ns}-${type}.${obj}.yaml
@@ -51,7 +51,7 @@ function backup_resource {
 function backup_namespace {
     local ns=$1
     local dest=$2
-    echo "==> backing up namespace $ns"
+    timestamp_echo "Backing up namespace $ns"
     oc get namespace ${ns} -o yaml --export | gzip > ${dest}/archives/${ns}-namespace.yaml.gz
 }
 
@@ -60,23 +60,23 @@ function backup_namespace {
 function backup_service_accounts {
     local ns=$1
     local dest=$2
-    echo "==> backing up service accounts in $ns"
+    timestamp_echo "Backing up service accounts in $ns"
     oc get serviceaccounts -n ${ns} --field-selector='metadata.name!=builder,metadata.name!=deployer,metadata.name!=default' -o yaml --export | gzip > ${dest}/archives/${ns}-sa.yaml.gz
 }
 
 # Backs up all rolebindings in a namespace but excludes the system ones
 function backup_role_bindings {
     local ns=$1
     local dest=$2
-    echo "==> backing up role bindings in $ns"
+    timestamp_echo "Backing up role bindings in $ns"
     oc get rolebindings -n ${ns} --field-selector='metadata.name!=system:deployers,metadata.name!=system:image-builders,metadata.name!=system:image-pullers' -o yaml --export | gzip > ${dest}/archives/${ns}-rb.yaml.gz
 }
 
 # Backs up a cluster level resource
 function backup_cluster_resource {
     local type=$1
     local dest=$2
-    echo "==> backing up cluster resource $type"
+    timestamp_echo "Backing up cluster resource $type"
     oc get ${type} -o yaml --export | gzip > ${dest}/archives/${type}.yaml.gz
 }
 
@@ -96,7 +96,7 @@ function component_dump_data {
     local namespaces=$(get_middleware_namespaces)
 
     for ns in ${namespaces} "default"; do
-        echo "==> processing namespace $ns"
+        timestamp_echo "Processing namespace $ns"
         backup_resource secrets ${ns} ${dest}
         backup_resource configmaps ${ns} ${dest}
         backup_resource services ${ns} ${dest}
@@ -150,7 +150,7 @@ function component_dump_data {
         backup_namespace ${ns} ${dest}
     done
 
-    echo "==> processing cluster resources"
+    timestamp_echo "Processing cluster resources"
 
     # These resources are not located in a particular namespace
     backup_cluster_resource oauthclients ${dest}

image/tools/lib/encryption/gpg.sh

@@ -39,7 +39,7 @@ function encrypt_archive {
         gpg --no-tty --batch --yes --encrypt --recipient "$recipient" --trust-model ${trust} ${fname}
         rc=$?
         if [[ ${rc} -ne 0 ]]; then
-            echo "==> Encrypt $fname: FAILED"
+            timestamp_echo "Encrypt $fname: FAILED"
             exit 1
         fi
     done

image/tools/lib/utils.sh

@@ -10,9 +10,13 @@ function cp_pod_data {
     # Check if any files were rewritten to during oc cp, and copy it again if it was.
     while [[ $copy_output == *"file changed as we read it"* ]] && [ $num_attempted_copy -lt $max_tries ]
     do
-       echo "A file has been overwritten during copying, executing 'oc cp' again"
+       timestamp_echo "A file has been overwritten during copying, executing 'oc cp' again"
        sleep 5
        copy_output=$(oc cp $pod_data_src $cp_dest)
        ((num_attempted_copy++))
     done
+}
+
+function timestamp_echo {
+    echo `(date -u '+%Y-%m-%d %H:%M:%S')` '==>' $1
 }