From 2447514ba83b6579b99fcb40f6c39c920b5b8bfa Mon Sep 17 00:00:00 2001 From: roleyfoley Date: Thu, 28 Sep 2023 16:36:10 +1000 Subject: [PATCH] refactor: remove legacy bash scripts - Removes all automation bash scripts as they are no longer supported and have been replaced with other processes - Moves the saveCMDBRepos script into the execution side of the bash scripts as it is still in use - Removes any execution scripts that are no longer support and have been replaced with other commands --- README.md | 2 - automation/common.sh | 66 -- automation/constructTree.sh | 395 ---------- automation/contextTree.sh | 1 - automation/jenkins/aws/acceptRelease.sh | 20 - automation/jenkins/aws/acceptReleaseSetup.sh | 12 - automation/jenkins/aws/backup.sh | 27 - .../jenkins/aws/buildContentModelBender.sh | 47 -- automation/jenkins/aws/buildContentStatic.sh | 27 - automation/jenkins/aws/buildDataSetS3.sh | 63 -- automation/jenkins/aws/buildInfraDocs.sh | 77 -- automation/jenkins/aws/buildJS.sh | 170 ----- automation/jenkins/aws/buildJekyll.sh | 78 -- automation/jenkins/aws/buildLaravel.sh | 23 - automation/jenkins/aws/buildMeteor.sh | 39 - automation/jenkins/aws/buildNode.sh | 92 --- automation/jenkins/aws/buildOpenapi.sh | 181 ----- automation/jenkins/aws/buildPython.sh | 237 ------ automation/jenkins/aws/buildRDSSnapshot.sh | 65 -- automation/jenkins/aws/buildSPAStatic.sh | 25 - automation/jenkins/aws/buildScripts.sh | 31 - automation/jenkins/aws/buildSetup.sh | 176 ----- automation/jenkins/aws/buildSwagger.sh | 11 - .../jenkins/aws/buildTenantBlueprints.sh | 64 -- automation/jenkins/aws/confirmBuilds.sh | 21 - automation/jenkins/aws/constructTree.sh | 11 - automation/jenkins/aws/deploy.sh | 23 - automation/jenkins/aws/deployRelease.sh | 23 - automation/jenkins/aws/deploySetup.sh | 12 - automation/jenkins/aws/manageAccount.sh | 24 - .../jenkins/aws/manageBuildReferences.sh | 682 ----------------- .../jenkins/aws/manageDataSetRDSSnapshot.sh | 8 - automation/jenkins/aws/manageDataSetS3.sh | 9 - automation/jenkins/aws/manageDocker.sh | 504 ------------- automation/jenkins/aws/manageEnvironment.sh | 25 - automation/jenkins/aws/manageImages.sh | 305 -------- automation/jenkins/aws/manageRdssnapshot.sh | 357 --------- automation/jenkins/aws/manageRepo.sh | 318 -------- automation/jenkins/aws/manageS3Registry.sh | 439 ----------- automation/jenkins/aws/manageUnits.sh | 194 ----- automation/jenkins/aws/plan.sh | 28 - automation/jenkins/aws/prepareRelease.sh | 29 - automation/jenkins/aws/prepareReleaseSetup.sh | 15 - automation/jenkins/aws/promoteRelease.sh | 43 -- automation/jenkins/aws/runLambda.sh | 29 - automation/jenkins/aws/runTasks.sh | 35 - automation/jenkins/aws/saveCMDBRepos.sh | 93 --- automation/jenkins/aws/setCredentials.sh | 1 - .../jenkins/aws/updateBuildReference.sh | 6 - .../jenkins/aws/updateBuildReferences.sh | 34 - .../aws/validateAcceptReleaseParameters.sh | 17 - .../aws/validateDeployReleaseParameters.sh | 21 - .../validateUpdateBuildReferenceParameters.sh | 18 - ...validateUpdateBuildReferencesParameters.sh | 18 - automation/setContext.sh | 684 ------------------ automation/utility.sh | 1 - cli/createBlueprint.sh | 83 --- cli/createBuildblueprint.sh | 83 --- cli/createSSLCertificateRequest.sh | 62 -- cli/integrator/manageCredentials.sh | 108 --- cli/manageCredentialCrypto.sh | 164 ----- cli/manageCrypto.sh | 429 ----------- cli/manageFileCrypto.sh | 100 --- cli/manageSSLCertificate.sh | 193 ----- cli/rebootRDSDatabase.sh | 113 --- cli/runLambda.sh | 169 ----- cli/runTask.sh | 295 -------- cli/saveCMDBRepos.sh | 272 +++++++ cli/snapshotRDSDatabase.sh | 157 ---- cli/updateObjectACL.sh | 98 --- 70 files changed, 272 insertions(+), 8010 deletions(-) delete mode 100755 automation/common.sh delete mode 100755 automation/constructTree.sh delete mode 120000 automation/contextTree.sh delete mode 100755 automation/jenkins/aws/acceptRelease.sh delete mode 100755 automation/jenkins/aws/acceptReleaseSetup.sh delete mode 100755 automation/jenkins/aws/backup.sh delete mode 100755 automation/jenkins/aws/buildContentModelBender.sh delete mode 100755 automation/jenkins/aws/buildContentStatic.sh delete mode 100755 automation/jenkins/aws/buildDataSetS3.sh delete mode 100755 automation/jenkins/aws/buildInfraDocs.sh delete mode 100755 automation/jenkins/aws/buildJS.sh delete mode 100755 automation/jenkins/aws/buildJekyll.sh delete mode 100755 automation/jenkins/aws/buildLaravel.sh delete mode 100755 automation/jenkins/aws/buildMeteor.sh delete mode 100755 automation/jenkins/aws/buildNode.sh delete mode 100755 automation/jenkins/aws/buildOpenapi.sh delete mode 100755 automation/jenkins/aws/buildPython.sh delete mode 100755 automation/jenkins/aws/buildRDSSnapshot.sh delete mode 100755 automation/jenkins/aws/buildSPAStatic.sh delete mode 100755 automation/jenkins/aws/buildScripts.sh delete mode 100755 automation/jenkins/aws/buildSetup.sh delete mode 100755 automation/jenkins/aws/buildSwagger.sh delete mode 100755 automation/jenkins/aws/buildTenantBlueprints.sh delete mode 100755 automation/jenkins/aws/confirmBuilds.sh delete mode 100755 automation/jenkins/aws/constructTree.sh delete mode 100755 automation/jenkins/aws/deploy.sh delete mode 100755 automation/jenkins/aws/deployRelease.sh delete mode 100755 automation/jenkins/aws/deploySetup.sh delete mode 100755 automation/jenkins/aws/manageAccount.sh delete mode 100755 automation/jenkins/aws/manageBuildReferences.sh delete mode 100755 automation/jenkins/aws/manageDataSetRDSSnapshot.sh delete mode 100755 automation/jenkins/aws/manageDataSetS3.sh delete mode 100755 automation/jenkins/aws/manageDocker.sh delete mode 100755 automation/jenkins/aws/manageEnvironment.sh delete mode 100755 automation/jenkins/aws/manageImages.sh delete mode 100755 automation/jenkins/aws/manageRdssnapshot.sh delete mode 100755 automation/jenkins/aws/manageRepo.sh delete mode 100755 automation/jenkins/aws/manageS3Registry.sh delete mode 100755 automation/jenkins/aws/manageUnits.sh delete mode 100644 automation/jenkins/aws/plan.sh delete mode 100755 automation/jenkins/aws/prepareRelease.sh delete mode 100755 automation/jenkins/aws/prepareReleaseSetup.sh delete mode 100755 automation/jenkins/aws/promoteRelease.sh delete mode 100644 automation/jenkins/aws/runLambda.sh delete mode 100755 automation/jenkins/aws/runTasks.sh delete mode 100755 automation/jenkins/aws/saveCMDBRepos.sh delete mode 120000 automation/jenkins/aws/setCredentials.sh delete mode 100755 automation/jenkins/aws/updateBuildReference.sh delete mode 100755 automation/jenkins/aws/updateBuildReferences.sh delete mode 100755 automation/jenkins/aws/validateAcceptReleaseParameters.sh delete mode 100755 automation/jenkins/aws/validateDeployReleaseParameters.sh delete mode 100755 automation/jenkins/aws/validateUpdateBuildReferenceParameters.sh delete mode 100755 automation/jenkins/aws/validateUpdateBuildReferencesParameters.sh delete mode 100755 automation/setContext.sh delete mode 120000 automation/utility.sh delete mode 100755 cli/createBlueprint.sh delete mode 100755 cli/createBuildblueprint.sh delete mode 100755 cli/createSSLCertificateRequest.sh delete mode 100755 cli/integrator/manageCredentials.sh delete mode 100755 cli/manageCredentialCrypto.sh delete mode 100755 cli/manageCrypto.sh delete mode 100755 cli/manageFileCrypto.sh delete mode 100755 cli/manageSSLCertificate.sh delete mode 100755 cli/rebootRDSDatabase.sh delete mode 100755 cli/runLambda.sh delete mode 100755 cli/runTask.sh create mode 100755 cli/saveCMDBRepos.sh delete mode 100755 cli/snapshotRDSDatabase.sh delete mode 100755 cli/updateObjectACL.sh diff --git a/README.md b/README.md index 2275d8dc..bed44954 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,6 @@ These options must be set in order for Hamlet Deploy Executor to function correc | Variable | Value | |-------------------------|----------------------------------------| -| AUTOMATION_BASE_DIR | `/automation` | -| AUTOMATION_DIR | `automation/jenkins/aws` | | GENERATION_BASE_DIR | `` | | GENERATION_DIR | `/cli` | diff --git a/automation/common.sh b/automation/common.sh deleted file mode 100755 index a5ec81fc..00000000 --- a/automation/common.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash - -# Automation framework common definitions -# -# This script is designed to be sourced into other scripts - -# TODO(mfl): Remove symlinks in /automation once all explicit usage -# of these in jenkins jobs has been modified to use -# /execution instead. - -. "${GENERATION_BASE_DIR}/execution/utility.sh" -. "${GENERATION_BASE_DIR}/execution/contextTree.sh" - -# Set hamlet local store -export HAMLET_HOME_DIR="${HAMLET_HOME_DIR:-"${HOME}/.hamlet"}" -export HAMLET_EVENT_DIR="${HAMLET_HOME_DIR}/events" -export HAMLET_EVENT_LOG="${HAMLET_EVENT_DIR}/event_log.json" - -# -- Repositories -- - -function save_repo() { - local directory="$1"; shift - local name="$1"; shift - local message="$1"; shift - local reference="$1"; shift - local tag="$1"; shift - - local optional_arguments=() - [[ -n "${reference}" ]] && optional_arguments+=("-b" "${reference}") - [[ -n "${tag}" ]] && optional_arguments+=("-t" "${tag}") - - ${AUTOMATION_DIR}/manageRepo.sh -p \ - -d "${directory}" \ - -l "${name}" \ - -m "${message}" \ - "${optional_arguments[@]}" -} - -function save_product_config() { - local arguments=("$@") - - save_repo "${PRODUCT_DIR}" "config" "${arguments[@]}" -} - -function save_product_infrastructure() { - local arguments=("$@") - - save_repo "${PRODUCT_INFRASTRUCTURE_DIR}" "infrastructure" "${arguments[@]}" -} - -function save_product_state() { - local arguments=("$@") - - save_repo "${PRODUCT_STATE_DIR}" "state" "${arguments[@]}" -} - -function save_product_code() { - local arguments=("$@") - - save_repo "${AUTOMATION_BUILD_DIR}" "code" "${arguments[@]}" -} - -# -- Logging -- -function getLogLevel() { - checkLogLevel "${AUTOMATION_LOG_LEVEL}" -} diff --git a/automation/constructTree.sh b/automation/constructTree.sh deleted file mode 100755 index 2861a451..00000000 --- a/automation/constructTree.sh +++ /dev/null @@ -1,395 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -REFERENCE_MASTER="master" - -# Defaults -PRODUCT_CONFIG_REFERENCE_DEFAULT="${REFERENCE_MASTER}" -PRODUCT_INFRASTRUCTURE_REFERENCE_DEFAULT="${REFERENCE_MASTER}" -ACCOUNT_CONFIG_REFERENCE_DEFAULT="${REFERENCE_MASTER}" -ACCOUNT_INFRASTRUCTURE_REFERENCE_DEFAULT="${REFERENCE_MASTER}" - -function usage() { - cat <> "${BASE_DIR_TEMP}/.gitignore" - - # The config repo may contain - # - config +/- infrastructure - # - product(s) +/- account(s) - if [[ -n $(findDir "${BASE_DIR_TEMP}" "infrastructure") ]]; then - # Mix of infrastructure and config - ACCOUNT_CANDIDATE_DIR="$(findDir "${BASE_DIR_TEMP}" "${ACCOUNT}")" - # Ensure we definitely have an account - if [[ ( -n "${ACCOUNT_CANDIDATE_DIR}" ) && - ( - ( -d "${ACCOUNT_CANDIDATE_DIR}/account.json" ) || - ( -d "${ACCOUNT_CANDIDATE_DIR}/config/account.json" ) - ) ]]; then - # Everything in one repo - PRODUCT_CONFIG_DIR="${BASE_DIR}/cmdb" - else - PRODUCT_CANDIDATE_DIR="$(findDir "${BASE_DIR_TEMP}" "${PRODUCT}")" - # Ensure we definitely have a product - if [[ ( -n "${PRODUCT_CANDIDATE_DIR}" ) && - ( - ( -d "${PRODUCT_CANDIDATE_DIR}/product.json" ) || - ( -d "${PRODUCT_CANDIDATE_DIR}/config/product.json" ) - ) ]]; then - # Multi-product repo - PRODUCT_CONFIG_DIR="${BASE_DIR}/products" - MULTI_PRODUCT_REPO=true - else - # Single product repo - PRODUCT_CONFIG_DIR="${BASE_DIR}/${PRODUCT}" - fi - fi - else - # Just config - ACCOUNT_CANDIDATE_DIR="$(findDir "${BASE_DIR_TEMP}" "${ACCOUNT}")" - # Ensure we definitely have an account - if [[ ( -n "${ACCOUNT_CANDIDATE_DIR}" ) && - ( - ( -d "${ACCOUNT_CANDIDATE_DIR}/account.json" ) || - ( -d "${ACCOUNT_CANDIDATE_DIR}/config/account.json" ) - ) ]]; then - # products and accounts - PRODUCT_CONFIG_DIR="${BASE_DIR}/config" - else - PRODUCT_CANDIDATE_DIR="$(findDir "${BASE_DIR_TEMP}" "${PRODUCT}")" - # Ensure we definitely have a product - if [[ ( -n "${PRODUCT_CANDIDATE_DIR}" ) && - ( - ( -d "${PRODUCT_CANDIDATE_DIR}/product.json" ) || - ( -d "${PRODUCT_CANDIDATE_DIR}/config/product.json" ) - ) ]]; then - # Multi-product repo - PRODUCT_CONFIG_DIR="${BASE_DIR}/config/products" - MULTI_PRODUCT_REPO=true - else - # Single product repo - PRODUCT_CONFIG_DIR="${BASE_DIR}/config/${PRODUCT}" - fi - fi - fi - - mkdir -p $(filePath "${PRODUCT_CONFIG_DIR}") - mv "${BASE_DIR_TEMP}" "${PRODUCT_CONFIG_DIR}" - save_context_property PRODUCT_CONFIG_COMMIT "$(git -C "${PRODUCT_CONFIG_DIR}" rev-parse HEAD)" - - PRODUCT_INFRASTRUCTURE_DIR=$(findGen3ProductInfrastructureDir "${BASE_DIR}" "${PRODUCT}") - if [[ -z "${PRODUCT_INFRASTRUCTURE_DIR}" ]]; then - # Pull in the infrastructure repo - ${AUTOMATION_DIR}/manageRepo.sh -c -l "product infrastructure" \ - -n "${PRODUCT_INFRASTRUCTURE_REPO}" -v "${PRODUCT_GIT_PROVIDER}" \ - -d "${BASE_DIR_TEMP}" -b "${PRODUCT_INFRASTRUCTURE_REFERENCE}" - RESULT=$? && [[ ${RESULT} -ne 0 ]] && exit - - # Ensure temporary files are ignored - [[ (! -f "${BASE_DIR_TEMP}/.gitignore") || ($(grep -q "temp_\*" "${BASE_DIR_TEMP}/.gitignore") -ne 0) ]] && \ - echo "temp_*" >> "${BASE_DIR_TEMP}/.gitignore" - - ACCOUNT_CANDIDATE_DIR="$(findDir "${BASE_DIR_TEMP}" "${ACCOUNT}")" - # Ensure we definitely have an account - if [[ ( -n "${ACCOUNT_CANDIDATE_DIR}" ) && - ( - ( -d "${ACCOUNT_CANDIDATE_DIR}/account.json" ) || - ( -d "${ACCOUNT_CANDIDATE_DIR}/config/account.json" ) - ) ]]; then - # products and accounts - PRODUCT_INFRASTRUCTURE_DIR="${BASE_DIR}/infrastructure" - else - # Is product repo contains multiple products, assume the infrastructure repo does too - if [[ "${MULTI_PRODUCT_REPO}" == "true" ]]; then - # Multi-product repo - PRODUCT_INFRASTRUCTURE_DIR="${BASE_DIR}/infrastructure/products" - else - # Single product repo - PRODUCT_INFRASTRUCTURE_DIR="${BASE_DIR}/infrastructure/${PRODUCT}" - fi - fi - mkdir -p $(filePath "${PRODUCT_INFRASTRUCTURE_DIR}") - mv "${BASE_DIR_TEMP}" "${PRODUCT_INFRASTRUCTURE_DIR}" - fi - - save_context_property PRODUCT_INFRASTRUCTURE_COMMIT "$(git -C "${PRODUCT_INFRASTRUCTURE_DIR}" rev-parse HEAD)" - fi - - if [[ !("${EXCLUDE_ACCOUNT_DIRECTORIES}" == "true") ]]; then - - # Check for required context - [[ -z "${ACCOUNT}" ]] && fatal "ACCOUNT not defined" && exit - - # Multiple accounts in the account config repo - MULTI_ACCOUNT_REPO=false - - # Pull in the account config repo - ACCOUNT_CONFIG_DIR=$(findGen3AccountDir "${BASE_DIR}" "${ACCOUNT}") - if [[ -z "${ACCOUNT_CONFIG_DIR}" ]]; then - ${AUTOMATION_DIR}/manageRepo.sh -c -l "account config" \ - -n "${ACCOUNT_CONFIG_REPO}" -v "${ACCOUNT_GIT_PROVIDER}" \ - -d "${BASE_DIR_TEMP}" -b "${ACCOUNT_CONFIG_REFERENCE}" - RESULT=$? && [[ ${RESULT} -ne 0 ]] && exit - - # Ensure temporary files are ignored - [[ (! -f "${BASE_DIR_TEMP}/.gitignore") || ($(grep -q "temp_\*" "${BASE_DIR_TEMP}/.gitignore") -ne 0) ]] && \ - echo "temp_*" >> "${BASE_DIR_TEMP}/.gitignore" - - if [[ -n $(findDir "${BASE_DIR_TEMP}" "infrastructure") ]]; then - # Mix of infrastructure and config - ACCOUNT_CANDIDATE_DIR="$(findDir "${BASE_DIR_TEMP}" "${ACCOUNT}")" - # Ensure we definitely have an account - if [[ ( -n "${ACCOUNT_CANDIDATE_DIR}" ) && - ( - ( -d "${ACCOUNT_CANDIDATE_DIR}/account.json" ) || - ( -d "${ACCOUNT_CANDIDATE_DIR}/config/account.json" ) - ) ]]; then - # Multi-account repo - ACCOUNT_CONFIG_DIR="${BASE_DIR}/accounts" - MULTI_ACCOUNT_REPO=true - else - # Single account repo - ACCOUNT_CONFIG_DIR="${BASE_DIR}/${ACCOUNT}" - fi - else - ACCOUNT_CANDIDATE_DIR="$(findDir "${BASE_DIR_TEMP}" "${ACCOUNT}")" - # Ensure we definitely have an account - if [[ ( -n "${ACCOUNT_CANDIDATE_DIR}" ) && - ( - ( -d "${ACCOUNT_CANDIDATE_DIR}/account.json" ) || - ( -d "${ACCOUNT_CANDIDATE_DIR}/config/account.json" ) - ) ]]; then - # Multi-account repo - ACCOUNT_CONFIG_DIR="${BASE_DIR}/config/accounts" - MULTI_ACCOUNT_REPO=true - else - # Single account repo - ACCOUNT_CONFIG_DIR="${BASE_DIR}/config/${ACCOUNT}" - fi - fi - mkdir -p $(filePath "${ACCOUNT_CONFIG_DIR}") - mv "${BASE_DIR_TEMP}" "${ACCOUNT_CONFIG_DIR}" - save_context_property ACCOUNT_CONFIG_COMMIT "$(git -C "${ACCOUNT_CONFIG_DIR}" rev-parse HEAD)" - fi - - ACCOUNT_STATE_DIR=$(findGen3AccountStateDir "${BASE_DIR}" "${ACCOUNT}") - if [[ -z "${ACCOUNT_STATE_DIR}" ]]; then - # Pull in the account infrastructure repo - ${AUTOMATION_DIR}/manageRepo.sh -c -l "account infrastructure" \ - -n "${ACCOUNT_INFRASTRUCTURE_REPO}" -v "${ACCOUNT_GIT_PROVIDER}" \ - -d "${BASE_DIR_TEMP}" -b "${ACCOUNT_INFRASTRUCTURE_REFERENCE}" - RESULT=$? && [[ ${RESULT} -ne 0 ]] && exit - - # Ensure temporary files are ignored - [[ (! -f "${BASE_DIR_TEMP}/.gitignore") || ($(grep -q "temp_\*" "${BASE_DIR_TEMP}/.gitignore") -ne 0) ]] && \ - echo "temp_*" >> "${BASE_DIR_TEMP}/.gitignore" - - # Is account repo contains multiple accounts, assume the infrastructure repo does too - if [[ "${MULTI_ACCOUNT_REPO}" == "true" ]]; then - # Multi-account repo - ACCOUNT_STATE_DIR="${BASE_DIR}/infrastructure/accounts" - else - # Single account repo - ACCOUNT_STATE_DIR="${BASE_DIR}/infrastructure/${ACCOUNT}" - fi - mkdir -p $(filePath "${ACCOUNT_STATE_DIR}") - mv "${BASE_DIR_TEMP}" "${ACCOUNT_STATE_DIR}" - fi - fi - - # Examine the structure and define key directories - findGen3Dirs "${BASE_DIR}" - RESULT=$? && [[ ${RESULT} -ne 0 ]] && exit -fi - -if [[ "${USE_EXISTING_TREE}" == "true" ]]; then - - if [[ -z "${ROOT_DIR}" ]]; then - export ROOT_DIR="$(findGen3RootDir "${ROOT_DIR:-$(pwd)}")" - fi - - current_dir="$(pwd)" - pushd "${current_dir}" >/dev/null - - solutions_ancestor_dir="$(findAncestorDir "solutions" "${current_dir}")" - solutionsv2_ancestor_dir="$(findAncestorDir "solutionsv2" "${current_dir}")" - if [[ (-z "${solutions_ancestor_dir}") && (-z "${solutionsv2_ancestor_dir}") ]]; then - # We are not in the solutions part of the tree - # Assume we are in the >=v2.0.0 cmdb config or operations trees - infrastructure_dir="${current_dir//settings/solutions}" - infrastructure_dir="${infrastructure_dir//operations/infrastructure}" - infrastructure_dir="${infrastructure_dir//config/infrastructure}" - debug "Not in solutions tree - checking ${infrastructure_dir} ..." - if [[ -d "${infrastructure_dir}" ]]; then - cd "${infrastructure_dir}" - fi - fi - - if [[ -f "segment.json" ]]; then - export SEGMENT="$(fileName "$(pwd)")" - if [[ -f "../environment.json" ]]; then - cd .. - else - export ENVIRONMENT="${SEGMENT}" - export SEGMENT="default" - cd ../../../config - fi - fi - save_context_property "SEGMENT" "${SEGMENT}" - - if [[ -f "environment.json" ]]; then - export ENVIRONMENT="$(fileName "$(pwd)")" - cd ../../../config - fi - save_context_property "ENVIRONMENT" "${ENVIRONMENT}" - - # handle the different structure for account and product sections of the CMDB - if [[ "$(pwd)" == "${infrastructure_dir}" && -d "../config" ]]; then - cd ../config - fi - - if [[ -f "account.json" ]]; then - [[ -z "${ACCOUNT}" ]] && export ACCOUNT="$(cd ..; fileName "$(pwd)")" - fi - - save_context_property "ACCOUNT" "${ACCOUNT}" - - if [[ -f "product.json" ]]; then - export PRODUCT="$(fileName "$(pwd)")" - [[ "${PRODUCT}" == "config" ]] && - export PRODUCT="$(cd ..; fileName "$(pwd)")" - fi - - save_context_property "PRODUCT" "${PRODUCT}" - - # Back to where we started - popd >/dev/null - - if [[ -z "${ROOT_DIR}" || ! (-d "${ROOT_DIR}") ]]; then - fatal "ROOT_DIR: ${ROOT_DIR} - could not be found for existing tree" - exit - fi - BASE_DIR="${ROOT_DIR}" - - # Examine the structure and define key directories - findGen3Dirs "${ROOT_DIR}" "${TENANT}" "${ACCOUNT}" "${PRODUCT}" "${ENVIRONMENT}" "${SEGMENT}" - RESULT=$? && [[ ${RESULT} -ne 0 ]] && exit -fi - -# A couple of the older upgrades need GENERATION_DATA_DIR set to -# locate the AWS account number to account id mappings -export GENERATION_DATA_DIR="${BASE_DIR}" - -# Remember directories for future steps -save_gen3_dirs_in_context diff --git a/automation/contextTree.sh b/automation/contextTree.sh deleted file mode 120000 index 4a38125c..00000000 --- a/automation/contextTree.sh +++ /dev/null @@ -1 +0,0 @@ -../execution/contextTree.sh \ No newline at end of file diff --git a/automation/jenkins/aws/acceptRelease.sh b/automation/jenkins/aws/acceptRelease.sh deleted file mode 100755 index da7e2a71..00000000 --- a/automation/jenkins/aws/acceptRelease.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -# Ensure mandatory arguments have been provided -[[ (-z "${RELEASE_MODE}") || - (-z "${RELEASE_TAG}") ]] && fatalMandatory - -# Include the build information in the detail message -${AUTOMATION_DIR}/manageBuildReferences.sh -l -RESULT=$? && [[ "${RESULT}" -ne 0 ]] && exit - -# Tag the builds -${AUTOMATION_DIR}/manageBuildReferences.sh -a "${RELEASE_TAG}" -RESULT=$? diff --git a/automation/jenkins/aws/acceptReleaseSetup.sh b/automation/jenkins/aws/acceptReleaseSetup.sh deleted file mode 100755 index 5dbf6621..00000000 --- a/automation/jenkins/aws/acceptReleaseSetup.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -# Get all the deployment unit commit information -${AUTOMATION_DIR}/manageBuildReferences.sh -f -RESULT=$? diff --git a/automation/jenkins/aws/backup.sh b/automation/jenkins/aws/backup.sh deleted file mode 100755 index 095107e0..00000000 --- a/automation/jenkins/aws/backup.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${GENERATION_BASE_DIR}/execution/common.sh" - -# DEPRECATED -deprecated_script - -# Formulate parameters - any provided to this script are also passed trhough -SNAPSHOT_OPTS= -if [[ -n "${SNAPSHOT_COUNT}" ]]; then - SNAPSHOT_OPTS="${SNAPSHOT_OPTS} -r ${SNAPSHOT_COUNT}" -fi -if [[ -n "${SNAPSHOT_AGE}" ]]; then - SNAPSHOT_OPTS="${SNAPSHOT_OPTS} -a ${SNAPSHOT_AGE}" -fi -if [[ -n "${COMPONENT}" ]]; then - SNAPSHOT_OPTS="${SNAPSHOT_OPTS} -i ${COMPONENT}" -fi - -# Snapshot the database -cd "${SEGMENT_SOLUTIONS_DIR}" - -${GENERATION_DIR}/snapshotRDSDatabase.sh -s b${AUTOMATION_JOB_IDENTIFIER} ${SNAPSHOT_OPTS} "$@" -RESULT=$? -[[ ${RESULT} -ne 0 ]] && fatal "Snapshot of ${ENVIRONMENT}/${SEGMENT} failed" diff --git a/automation/jenkins/aws/buildContentModelBender.sh b/automation/jenkins/aws/buildContentModelBender.sh deleted file mode 100755 index 3bc63306..00000000 --- a/automation/jenkins/aws/buildContentModelBender.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") ; exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -dockerstagedir="$(getTempDir "cota_docker_XXXXXX" "${DOCKER_STAGE_DIR}")" -chmod a+rwx "${dockerstagedir}" - -function main() { - # Make sure we are in the build source directory - cd ${AUTOMATION_BUILD_SRC_DIR} - - # Create Build folders for Jenkins Permissions - mkdir -p ${AUTOMATION_BUILD_SRC_DIR}/stage - mkdir -p "${dockerstagedir}/indir" - mkdir -p "${dockerstagedir}/stage" - - cp -r "${AUTOMATION_BUILD_SRC_DIR}"/* "${dockerstagedir}/indir/" - - # run Model Bender build using Docker Build image - info "Running ModelBender enterprise tasks..." - docker run --rm \ - --volume="${dockerstagedir}/indir:/work/indir" \ - --volume="${dockerstagedir}/stage:/work/outdir" \ - codeontap/modelbender:latest \ - enterprise --indir=indir --outdir=outdir - - info "Rendering ModelBender content..." - docker run --rm \ - --volume="${dockerstagedir}/stage:/work/indir" \ - codeontap/modelbender:latest \ - render --indir=indir - - cd "${dockerstagedir}/stage" - - mkdir -p "${AUTOMATION_BUILD_SRC_DIR}/dist" - zip -r "${AUTOMATION_BUILD_SRC_DIR}/dist/contentnode.zip" * - - # All good - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/buildContentStatic.sh b/automation/jenkins/aws/buildContentStatic.sh deleted file mode 100755 index da409754..00000000 --- a/automation/jenkins/aws/buildContentStatic.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") ; exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -function main() { - # Make sure we are in the build source directory - cd ${AUTOMATION_BUILD_SRC_DIR} - - # packge for content node - if [[ -d "${AUTOMATION_BUILD_SRC_DIR}" ]]; then - mkdir -p "${AUTOMATION_BUILD_SRC_DIR}/dist" - - cd "${AUTOMATION_BUILD_SRC_DIR}" - zip -r "${AUTOMATION_BUILD_SRC_DIR}/dist/contentnode.zip" * - - fi - - # All good - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/buildDataSetS3.sh b/automation/jenkins/aws/buildDataSetS3.sh deleted file mode 100755 index d46a004c..00000000 --- a/automation/jenkins/aws/buildDataSetS3.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") ; exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -tmpdir="$(getTempDir "cota_inf_XXX")" -data_manifest_filename="cot_data_file_manifest.json" - -function main() { - - # Get the generation context so we can run template generation - . "${GENERATION_BASE_DIR}/execution/setContext.sh" - . ${AUTOMATION_DIR}/setCredentials.sh "${ACCOUNT}" - - info "Building Deployment ${DEPLOYMENT_UNIT_LIST}" - for DEPLOYMENT_UNIT in ${DEPLOYMENT_UNIT_LIST[0]}; do - - DEPLOYMENT_GROUP="${DEPLOYMENT_GROUP:-"application"}" - - # Generate a build blueprint so that we can find out the source S3 bucket - info "Generating blueprint to find details..." - ${GENERATION_DIR}/createTemplate.sh -e "buildblueprint" -p "aws" -l "${DEPLOYMENT_GROUP}" -u "${DEPLOYMENT_UNIT}" -o "${tmpdir}" > /dev/null - BUILD_BLUEPRINT="${tmpdir}/buildblueprint-${DEPLOYMENT_GROUP}-${DEPLOYMENT_UNIT}-config.json" - - if [[ ! -f "${BUILD_BLUEPRINT}" || -z "$(cat ${BUILD_BLUEPRINT} )" ]]; then - fatal "Could not generate blueprint for task details" - return 255 - fi - - mkdir -p "${tmpdir}/${DEPLOYMENT_UNIT}" - data_manifest_file="${tmpdir}/${DEPLOYMENT_UNIT}/${data_manifest_filename}" - - dataset_master_location="$( jq -r '.Occurrence.State.Attributes.DATASET_MASTER_LOCATION' < "${BUILD_BLUEPRINT}" )" - dataset_prefix="$( jq -r '.Occurrence.State.Attributes.DATASET_PREFIX' < "${BUILD_BLUEPRINT}" )" - master_data_bucket_name="$( jq -r '.Occurrence.State.Attributes.NAME' < "${BUILD_BLUEPRINT}" )" - dataset_region="$( jq -r '.Occurrence.State.Attributes.REGION' < "${BUILD_BLUEPRINT}" )" - - info "Generating master data reference from bucket: ${master_data_bucket_name} - prefix: ${dataset_prefix}" - aws --region "${dataset_region}" s3api list-objects-v2 --bucket "${master_data_bucket_name}" --prefix "${dataset_prefix}" --query 'Contents[*].{Key:Key,ETag:ETag,LastModified:LastModified}' > "${data_manifest_file}" || return $? - - if [[ -f "${data_manifest_file}" ]]; then - - build_reference="$( shasum -a 1 "${data_manifest_file}" | cut -d " " -f 1 )" - save_context_property CODE_COMMIT_LIST "${build_reference}" - save_context_property S3_DATA_STAGE "${dataset_master_location}" - - save_chain_property GIT_COMMIT "${build_reference}" - - cp "${data_manifest_file}" "${AUTOMATION_BUILD_SRC_DIR}/${data_manifest_filename}" - - info "Commit: ${build_reference}" - - else - fatal "Could not generate data manifest file or no files could be found" - return 128 - fi - - done - - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/buildInfraDocs.sh b/automation/jenkins/aws/buildInfraDocs.sh deleted file mode 100755 index 988817c3..00000000 --- a/automation/jenkins/aws/buildInfraDocs.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") ; exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -tmpdir="$(getTempDir "cota_inf_XXX")" -dockerstagedir="$(getTempDir "cota_docker_XXXXXX" "${DOCKER_STAGE_DIR}")" -chmod a+rwx "${dockerstagedir}" - -function main() { - # Make sure we are in the build source directory - cd ${AUTOMATION_BUILD_SRC_DIR} - - # Unless specified use the latest InfraDocs version - if [[ -z "${INFRADOCS_VERSION}" ]]; then - INFRADOCS_VERSION=latest - fi - - # Default Document for generation testing - if [[ -z "${JEKYLL_DEFAULT_PAGE}" ]]; then - JEKYLL_DEFAULT_PAGE=index.html - fi - - # Default Timezone - if [[ -z "${JEKYLL_TIMEZONE}" ]]; then - JEKYLL_TIMEZONE="Australia/Sydney" - fi - - # Default build Env - if [[ -z "${JEKYLL_ENV}" ]]; then - JEKYLL_ENV="production" - fi - - # Create Build folders for Jenkins Permissions - touch ${AUTOMATION_BUILD_SRC_DIR}/Gemfile.lock - chmod a+w ${AUTOMATION_BUILD_SRC_DIR}/Gemfile.lock - - mkdir -p ${tmpdir}/_site - - # run Jekyll build using Docker Build image - info "Running Jeykyll build" - - mkdir -p "${dockerstagedir}/indir" - mkdir -p "${dockerstagedir}/outdir" - cp -r "${AUTOMATION_BUILD_SRC_DIR}"/* "${dockerstagedir}/indir/" - - docker run --rm \ - --env JEKYLL_ENV="${JEKYLL_ENV}" \ - --env TZ="${JEKYLL_TIMEZONE}" \ - --volume="${dockerstagedir}/indir:/indir" \ - --volume="${dockerstagedir}/outdir:/outdir" \ - codeontap/infradocs:"${INFRADOCS_VERSION}" - - # Package for spa if required - if [[ -f "${dockerstagedir}/outdir/${JEKYLL_DEFAULT_PAGE}" ]]; then - - cd "${dockerstagedir}/outdir" - - mkdir -p "${AUTOMATION_BUILD_SRC_DIR}/dist" - zip -r "${AUTOMATION_BUILD_SRC_DIR}/dist/spa.zip" * - - else - - fatal "No default page avaialable" - return 1 - - fi - - # All good - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/buildJS.sh b/automation/jenkins/aws/buildJS.sh deleted file mode 100755 index 03c062f5..00000000 --- a/automation/jenkins/aws/buildJS.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env bash -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") && (-d "${NVM_DIR}") ]] && nvm deactivate; rm -rf "${NVM_DIR}" ; exit $?' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -function runTask() { - local REQUIRED_TASK=$1; shift - - # The build file existence checks below rely on nullglob - # to return nothing if no match - shopt -s nullglob - BUILD_FILES=(?runtfile.js ?ulpfile.js package.json) - - for BUILD_FILE in "${BUILD_FILES[@]}"; do - - BUILD_TASKS=() - case ${BUILD_FILE} in - ?runtfile.js) - BUILD_TASKS=( $(grunt -h --no-color | sed -n '/^Available tasks/,/^$/ {s/^ *\([^ ]\+\) [^ ]\+.*$/\1/p}') ) - BUILD_UTILITY="grunt" - ;; - - ?ulpfile.js) - BUILD_TASKS=( $(gulp --tasks-simple) ) - BUILD_UTILITY="gulp" - ;; - - package.json) - BUILD_TASKS=( $(jq -r '.scripts | select(.!=null) | keys[]' < package.json) ) - BUILD_UTILITY="${NODE_PACKAGE_MANAGER} run" - ;; - esac - - for BUILD_TASK in "${BUILD_TASKS[@]}"; do - if [[ "${BUILD_TASK}" == "${REQUIRED_TASK}" ]]; then - ${BUILD_UTILITY} ${REQUIRED_TASK} || - { exit_status=$?; fatal "${BUILD_UTILITY} \"${TASK}\" task failed"; return ${exit_status}; } - - # Task complete so stop looking for build file supporting it - return 0 - fi - done - done - - # Task not found - return 255 -} - -function main() { - # Make sure we are in the build source directory - cd ${AUTOMATION_BUILD_SRC_DIR} - - #Check for package.json - [[ ! -f package.json ]] && - { fatal "no package.json file found. Is this a node repo?"; return 1; } - - # setup nvm environment if required - if [[ -n "${AUTOMATION_NODEJS_VERSION}" ]]; then - - NVM_DIR="$(getTempDir "cota_nvm_XXX")" - curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | NVM_DIR="${NVM_DIR}" bash - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" - - nvm install "${AUTOMATION_NODEJS_VERSION}" || - { exit_status=$?; fatal "NVM install for node ${AUTOMATION_NODEJS_VERSION} install failed" ; return ${exit_status}; } - nvm use "${AUTOMATION_NODEJS_VERSION}" - - fi - - # Select the package manage to use - if [[ -z "${NODE_PACKAGE_MANAGER}" ]]; then - if $(which yarn > /dev/null 2>&1) ; then - NODE_PACKAGE_MANAGER="yarn" - else - NODE_PACKAGE_MANAGER="npm" - fi - fi - - ${NODE_PACKAGE_MANAGER} install || - { exit_status=$?; fatal "npm install failed"; return ${exit_status}; } - - # Run bower as part of the build if required - if [[ -f bower.json ]]; then - bower install --allow-root || - { exit_status=$?; fatal "bower install failed"; return ${exit_status}; } - fi - - # Determine required tasks - # Build is always first - if [[ -n "${BUILD_TASKS}" ]]; then - REQUIRED_TASKS=( ${BUILD_TASKS} ) - else - REQUIRED_TASKS=( "build" ) - fi - - # Perform format specific tasks if defined - IMAGE_FORMATS_ARRAY=(${IMAGE_FORMATS_LIST}) - IFS="${IMAGE_FORMAT_SEPARATORS}" read -ra FORMATS <<< "${IMAGE_FORMATS_ARRAY[0]}" - - # The build file existence checks below rely on nullglob - # to return nothing if no match - shopt -s nullglob - BUILD_FILES=(?runtfile.js ?ulpfile.js package.json) - - # Perform the required build tasks in the order specified - for REQUIRED_TASK in "${REQUIRED_TASKS[@]}"; do - runTask "${REQUIRED_TASK}"; exit_status=$? - case $exit_status in - 255) - if [[ "${IGNORE_MISSING_TASKS}" == "true" ]]; then - # Nothing more to do for this task - continue - fi - - # Missing required task - fatal "Required task ${REQUIRED_TASK} not found in build files" - return 1 - ;; - 0) - # Task found and executed successfully - continue - ;; - *) - # Some other error - return ${exit_status} - ;; - esac - done - - # Perform optional format tasks - for REQUIRED_TASK in "${FORMATS[@]}"; do - runTask "${REQUIRED_TASK}"; exit_status=$? - case $exit_status in - 255|0) - # Task not found or executed successfully - continue - ;; - *) - # Some other error - return ${exit_status} - ;; - esac - done - - # Clean up dev dependencies - case ${NODE_PACKAGE_MANAGER} in - yarn) - yarn install --production || - { exit_status=$?; fatal "yarn prune failed"; return ${exit_status}; } - ;; - *) - npm prune --production || - { exit_status=$?; fatal "npm prune failed"; return ${exit_status}; } - ;; - esac - - # deactivate nvm if it was used and cleanup - if [[ -n "${AUTOMATION_NODEJS_VERSION}" ]]; then - nvm deactivate - rm -rf "${NVM_DIR}" - fi - - # All good - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/buildJekyll.sh b/automation/jenkins/aws/buildJekyll.sh deleted file mode 100755 index f6ce8569..00000000 --- a/automation/jenkins/aws/buildJekyll.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") ; exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -dockerstagedir="$(getTempDir "cota_docker_XXXXXX" "${DOCKER_STAGE_DIR}")" -chmod a+rwx "${dockerstagedir}" - -function main() { - # Make sure we are in the build source directory - cd ${AUTOMATION_BUILD_SRC_DIR} - - # Is this really a jekyll based project - [[ ! -f _config.yml ]] && - { fatal "No _config.yml - is this really a jekyll based repo?"; return 1; } - - # Unless specified use the latest Jekyll version - if [[ -z "${JEKYLL_VERSION}" ]]; then - JEKYLL_VERSION=latest - fi - - # Default Document for generation testing - if [[ -z "${JEKYLL_DEFAULT_PAGE}" ]]; then - JEKYLL_DEFAULT_PAGE=index.html - fi - - # Default Timezone - if [[ -z "${JEKYLL_TIMEZONE}" ]]; then - JEKYLL_TIMEZONE="Australia/Sydney" - fi - - # Default build Env - if [[ -z "${JEKYLL_ENV}" ]]; then - JEKYLL_ENV="production" - fi - - #TODO(rossmurr4y): There is an issue in Jekyll where it may not create - # the cache dir correctly. This is listed as resolved in the next Jekyll - # version. At that point, this should be returned to creating the empty - # indir directory. - mkdir -p "${dockerstagedir}/indir/.jekyll-cache" - cp -r "${AUTOMATION_BUILD_SRC_DIR}"/* "${dockerstagedir}/indir/" - - # Create Build folders for Jenkins Permissions - touch "${dockerstagedir}/indir/Gemfile.lock" - chmod a+w "${dockerstagedir}/indir/Gemfile.lock" - - mkdir -p "${dockerstagedir}/indir/_site" - chmod a+rwx "${dockerstagedir}/indir/_site" - - # run Jekyll build using Docker Build image - info "Running Jeykyll build" - docker run --rm \ - --env JEKYLL_ENV="${JEKYLL_ENV}" \ - --env TZ="${JEKYLL_TIMEZONE}" \ - --volume="${dockerstagedir}/indir:/srv/jekyll" \ - jekyll/builder:"${JEKYLL_VERSION}" \ - jekyll build --verbose - - # Package for spa if required - if [[ -f "${dockerstagedir}/indir/_site/${JEKYLL_DEFAULT_PAGE}" ]]; then - - # Allow access to all files that have been generated so they can be cleaned up. - cd "${dockerstagedir}/indir/_site" - - mkdir -p "${AUTOMATION_BUILD_SRC_DIR}/dist" - zip -r "${AUTOMATION_BUILD_SRC_DIR}/dist/spa.zip" * - fi - - # All good - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/buildLaravel.sh b/automation/jenkins/aws/buildLaravel.sh deleted file mode 100755 index e90ccde8..00000000 --- a/automation/jenkins/aws/buildLaravel.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -cd laravel/ - -/usr/local/bin/composer install --prefer-source --no-interaction -RESULT=$? -[[ $RESULT -ne 0 ]] && fatal "Composer install fails with the exit code $RESULT" - -/usr/local/bin/composer update -RESULT=$? -[[ $RESULT -ne 0 ]] && fatal "Composer update fails with the exit code $RESULT" - -cd ../ - -${AUTOMATION_DIR}/manageImages.sh -RESULT=$? diff --git a/automation/jenkins/aws/buildMeteor.sh b/automation/jenkins/aws/buildMeteor.sh deleted file mode 100755 index b70a4ed4..00000000 --- a/automation/jenkins/aws/buildMeteor.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -# Change to the app directory -cd app - -# Select the package manage to use -NODE_PACKAGE_MANAGER="${NODE_PACKAGE_MANAGER:-yarn}" - -# Install required node modules -${NODE_PACKAGE_MANAGER} install -RESULT=$? -[[ $RESULT -ne 0 ]] && fatal "npm install failed" - -# Build meteor but don't tar it -meteor build ../dist --directory -RESULT=$? -[[ $RESULT -ne 0 ]] && fatal "Meteor build failed" - -cd .. - -# Install the required node modules -(cd dist/bundle/programs/server && ${NODE_PACKAGE_MANAGER} install --production) -RESULT=$? -[[ $RESULT -ne 0 ]] && "Installation of app node modules failed" - -# Sanity check on final size of build -MAX_METEOR_BUILD_SIZE=${MAX_METEOR_BUILD_SIZE:-100} -[[ $(du -s -m ./dist | cut -f 1) -gt ${MAX_METEOR_BUILD_SIZE} ]] && RESULT=1 && - fatal "Build size exceeds ${MAX_METEOR_BUILD_SIZE}M" - -${AUTOMATION_DIR}/manageImages.sh -RESULT=$? diff --git a/automation/jenkins/aws/buildNode.sh b/automation/jenkins/aws/buildNode.sh deleted file mode 100755 index 0ce9daf1..00000000 --- a/automation/jenkins/aws/buildNode.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -# Make sure we are in the build source directory -cd ${AUTOMATION_BUILD_SRC_DIR} - -# Select the package manage to use -if [[ -z "${NODE_PACKAGE_MANAGER}" ]]; then - if $(which yarn > /dev/null 2>&1) ; then - NODE_PACKAGE_MANAGER="yarn" - else - NODE_PACKAGE_MANAGER="npm" - fi -fi - -${NODE_PACKAGE_MANAGER} install -RESULT=$? -[[ $RESULT -ne 0 ]] && fatal "npm install failed" - -# Run bower as part of the build if required -if [[ -f bower.json ]]; then - bower install --allow-root - RESULT=$? - [[ $RESULT -ne 0 ]] && fatal "Bower install failed" -fi - -# Determine required tasks -# Build is always first -REQUIRED_TASKS=( "build" ) - -# Perform format specific tasks if defined -IMAGE_FORMATS_ARRAY=(${IMAGE_FORMATS_LIST}) -IFS="${IMAGE_FORMAT_SEPARATORS}" read -ra FORMATS <<< "${IMAGE_FORMATS_ARRAY[0]}" -REQUIRED_TASKS=( "${REQUIRED_TASKS[@]}" "${FORMATS[@]}" ) - -# The build file existence checks below rely on nullglob -# to return nothing if no match -shopt -s nullglob -BUILD_FILES=( ?runtfile.js ?ulpfile.js package.json) - -# Perform build tasks in the order specified -for REQUIRED_TASK in "${REQUIRED_TASKS[@]}"; do - for BUILD_FILE in "${BUILD_FILES[@]}"; do - BUILD_TASKS=() - case ${BUILD_FILE} in - ?runtfile.js) - BUILD_TASKS=( $(grunt -h --no-color | sed -n '/^Available tasks/,/^$/ {s/^ *\([^ ]\+\) [^ ]\+.*$/\1/p}') ) - BUILD_UTILITY="grunt" - ;; - - ?ulpfile.js) - BUILD_TASKS=( $(gulp --tasks-simple) ) - BUILD_UTILITY="gulp" - ;; - - package.json) - BUILD_TASKS=( $(jq -r '.scripts | select(.!=null) | keys[]' < package.json) ) - BUILD_UTILITY="${NODE_PACKAGE_MANAGER} run" - ;; - esac - - if [[ "${BUILD_TASKS[*]/${REQUIRED_TASK}/XXfoundXX}" != "${BUILD_TASKS[*]}" ]]; then - ${BUILD_UTILITY} ${REQUIRED_TASK} - RESULT=$? - [[ $RESULT -ne 0 ]] && fatal "${BUILD_UTILITY} \"${TASK}\" task failed" - - # Task complete so stop looking for build file supporting it - break - fi - done -done - -# Clean up dev dependencies -case ${NODE_PACKAGE_MANAGER} in - yarn) - yarn install --production - ;; - *) - npm prune --production - ;; -esac -RESULT=$? -[[ $RESULT -ne 0 ]] && fatal "Prune failed" - -${AUTOMATION_DIR}/manageImages.sh -f "${IMAGE_FORMATS_ARRAY[0]}" -RESULT=$? diff --git a/automation/jenkins/aws/buildOpenapi.sh b/automation/jenkins/aws/buildOpenapi.sh deleted file mode 100755 index 2af98d91..00000000 --- a/automation/jenkins/aws/buildOpenapi.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env bash - -# Augment an openapi file with AWS API gateway integration semantics - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") && (-d "${tmpdir}") ]] && rm -rf "${tmpdir}";exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -# Determine the registry - it impacts some file names -IMAGE_FORMATS_ARRAY=(${IMAGE_FORMATS_LIST}) -REGISTRY_TYPE="${IMAGE_FORMATS_ARRAY[0]}" - -# We need to use a docker staging dir to provide docker-in-docker support -tmpdir="$(getTempDir "cota_swag_XXXXXX" "${DOCKER_STAGE_DIR}")" -chmod a+rwx "${tmpdir}" - -# Determine build dir in case of multiple specs in subdirs -BUILD_DIR="$(fileName "${AUTOMATION_BUILD_DIR}" )" - -# If a bundle context is provided, it defines the subtree in which the -# spec file can be located. The search for files thus needs to be limited -# to the bundle context if it is provided. -# -# The logic used may result in the repeated searching of the same directory if -# a bundle context is specified, but its a once off process and it keeps the -# search logic the same regardless of bundle context. -AUTOMATION_BUILD_DIR_PARENT="$(cd ${AUTOMATION_BUILD_DIR}/..; pwd)" -AUTOMATION_BUILD_DIR_GPARENT="$(cd ${AUTOMATION_BUILD_DIR}/../..; pwd)" -AUTOMATION_BUILD_DIR_GGPARENT="$(cd ${AUTOMATION_BUILD_DIR}/../../..; pwd)" -if [[ -n "${BUNDLE_CONTEXT_DIR}" ]]; then - OPENAPI_BUNDLE_CONTEXT_DIR="${AUTOMATION_DATA_DIR}/${BUNDLE_CONTEXT_DIR}" - - if [[ "${AUTOMATION_BUILD_DIR_PARENT#${OPENAPI_BUNDLE_CONTEXT_DIR}}" == "${AUTOMATION_BUILD_DIR_PARENT}" ]]; then - AUTOMATION_BUILD_DIR_PARENT="${OPENAPI_BUNDLE_CONTEXT_DIR}" - fi - if [[ "${AUTOMATION_BUILD_DIR_GPARENT#${OPENAPI_BUNDLE_CONTEXT_DIR}}" == "${AUTOMATION_BUILD_DIR_GPARENT}" ]]; then - AUTOMATION_BUILD_DIR_GPARENT="${OPENAPI_BUNDLE_CONTEXT_DIR}" - fi - if [[ "${AUTOMATION_BUILD_DIR_GGPARENT#${OPENAPI_BUNDLE_CONTEXT_DIR}}" == "${AUTOMATION_BUILD_DIR_GGPARENT}" ]]; then - AUTOMATION_BUILD_DIR_GGPARENT="${OPENAPI_BUNDLE_CONTEXT_DIR}" - fi -fi - -# Possible input files -OPENAPI_SPEC_FILE=$(findFile \ - "${AUTOMATION_BUILD_DIR}/build/openapi.json" \ - "${AUTOMATION_BUILD_DIR}/build/openapi.yml" \ - "${AUTOMATION_BUILD_DIR}/build/openapi.yaml" \ - "${AUTOMATION_BUILD_DIR}/build/swagger.json" \ - "${AUTOMATION_BUILD_DIR}/build/swagger.yml" \ - "${AUTOMATION_BUILD_DIR}/build/swagger.yaml" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/${BUILD_DIR}/openapi.json" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/${BUILD_DIR}/openapi.yml" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/${BUILD_DIR}/openapi.yaml" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/${BUILD_DIR}/swagger.json" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/${BUILD_DIR}/swagger.yml" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/${BUILD_DIR}/swagger.yaml" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/${BUILD_DIR}/openapi.json" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/${BUILD_DIR}/openapi.yml" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/${BUILD_DIR}/openapi.yaml" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/${BUILD_DIR}/swagger.json" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/${BUILD_DIR}/swagger.yml" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/${BUILD_DIR}/swagger.yaml" \ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/${BUILD_DIR}/openapi.json" \ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/${BUILD_DIR}/openapi.yml" \ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/${BUILD_DIR}/openapi.yaml" \ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/${BUILD_DIR}/swagger.json" \ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/${BUILD_DIR}/swagger.yml" \ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/${BUILD_DIR}/swagger.yaml" \ - "${AUTOMATION_BUILD_DIR}/openapi.json" \ - "${AUTOMATION_BUILD_DIR}/openapi.yml" \ - "${AUTOMATION_BUILD_DIR}/openapi.yaml" \ - "${AUTOMATION_BUILD_DIR}/swagger.json" \ - "${AUTOMATION_BUILD_DIR}/swagger.yml" \ - "${AUTOMATION_BUILD_DIR}/swagger.yaml" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/openapi.json" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/openapi.yml" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/openapi.yaml" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/swagger.json" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/swagger.yml" \ - "${AUTOMATION_BUILD_DIR_PARENT}/**/*spec/swagger.yaml" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/openapi.json" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/openapi.yml" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/openapi.yaml" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/swagger.json" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/swagger.yml" \ - "${AUTOMATION_BUILD_DIR_GPARENT}/**/*spec/swagger.yaml" \ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/openapi.json" \ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/openapi.yml"\ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/openapi.yaml"\ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/swagger.json" \ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/swagger.yml" \ - "${AUTOMATION_BUILD_DIR_GGPARENT}/**/*spec/swagger.yaml") - -# Was a spec file found? -[[ ! -f "${OPENAPI_SPEC_FILE}" ]] && fatal "Can't find source openAPI file within the bundle context" && exit 1 - -# Bundle context if not explicitly defined starts where the spec file is -if [[ -z "${BUNDLE_CONTEXT_DIR}" ]]; then - OPENAPI_BUNDLE_CONTEXT_DIR="$(filePath "${OPENAPI_SPEC_FILE}")" -fi - -# Determine attributes of spec file -OPENAPI_SPEC_FILE_BASE="$(fileBase "${OPENAPI_SPEC_FILE}")" -OPENAPI_SPEC_FILE_EXTENSION="$(fileExtension "${OPENAPI_SPEC_FILE}")" -OPENAPI_SPEC_FILE_RELATIVE_PATH="${OPENAPI_SPEC_FILE#${OPENAPI_BUNDLE_CONTEXT_DIR}}" - -# Collect the files that could be bundled -OPENAPI_BUNDLE_DIR="${tmpdir}/bundle" -mkdir "${OPENAPI_BUNDLE_DIR}" - -pushd "${OPENAPI_BUNDLE_CONTEXT_DIR}" > /dev/null 2>&1 -find . -name "*.json" -exec cp -p --parents {} "${OPENAPI_BUNDLE_DIR}" ";" -find . -name "*.yml" -exec cp -p --parents {} "${OPENAPI_BUNDLE_DIR}" ";" -find . -name "*.yaml" -exec cp -p --parents {} "${OPENAPI_BUNDLE_DIR}" ";" -popd > /dev/null - -# Bundle the spec file -TEMP_OPENAPI_SPEC_FILE="${tmpdir}/openapi.json" -docker run --rm \ - -v "${OPENAPI_BUNDLE_DIR}:/app/indir" \ - -v "${tmpdir}:/app/outdir" \ - codeontap/utilities swagger-cli bundle -r \ - --outfile "/app/outdir/openapi.${OPENAPI_SPEC_FILE_EXTENSION}" \ - "/app/indir/${OPENAPI_SPEC_FILE_RELATIVE_PATH}" || - { exit_status=$?; fatal "Unable to bundle ${OPENAPI_SPEC_FILE}"; exit ${exit_status}; } - -# Convert yaml to json -case "${OPENAPI_SPEC_FILE_EXTENSION}" in - yml|yaml) - # Need to use a yaml to json converter that preserves comments in YAML multi-line blocks, as - # AWS uses these are directives in API Gateway templates - COMBINE_COMMAND="import sys, yaml, json; json.dump(yaml.load(open('/app/indir/openapi.${OPENAPI_SPEC_FILE_EXTENSION}','r')), open('/app/outdir/$(fileName ${TEMP_OPENAPI_SPEC_FILE})','w'), indent=4)" - docker run --rm \ - -v "${tmpdir}:/app/indir" -v "${tmpdir}:/app/outdir" \ - codeontap/python-utilities \ - -c "${COMBINE_COMMAND}" - ;; - -esac - -[[ ! -f "${TEMP_OPENAPI_SPEC_FILE}" ]] && fatal "Can't find source OpenAPI file" && exit 1 - -# Validate it -# We use swagger-cli because it supports openapi3 and bundling -VALIDATORS=( "swagger-cli validate /app/indir/$(fileName ${TEMP_OPENAPI_SPEC_FILE})" ) -for VALIDATOR in "${VALIDATORS[@]}"; do - docker run --rm -v "${tmpdir}:/app/indir" codeontap/utilities ${VALIDATOR} || - { exit_status=$?; fatal "OpenAPI file is not valid"; exit ${exit_status}; } -done - -# Check for any legacy api extensions -APIGW_CONFIG=$(findFile \ - "${AUTOMATION_BUILD_DIR}/apigw.json" \ - "${AUTOMATION_BUILD_DEVOPS_DIR}/apigw.json" \ - "${AUTOMATION_BUILD_DEVOPS_DIR}/codeontap/apigw.json") - -if [[ -f "${APIGW_CONFIG}" ]]; then - # Build time openapi extensions are no longer supported - # They should be implemented as config provided during - # the generation of templates for the deployment process. - fatal "Openapi extension at build time is no longer supported. Please convert to deployment based extensions." - exit 1 -else - # Rename result to that expected by deployment extension process - OPENAPI_EXTENDED_BASE_FILE="${tmpdir}/${REGISTRY_TYPE}-extended-base.json" - cp "${TEMP_OPENAPI_SPEC_FILE}" "${OPENAPI_EXTENDED_BASE_FILE}" - - # Define the desired result location - DIST_DIR="${AUTOMATION_BUILD_DIR}/dist" - mkdir -p ${DIST_DIR} - OPENAPI_RESULT_FILE="${DIST_DIR}/${REGISTRY_TYPE}.zip" - - zip -j "${OPENAPI_RESULT_FILE}" "${OPENAPI_EXTENDED_BASE_FILE}" -fi - -# All good -RESULT=0 diff --git a/automation/jenkins/aws/buildPython.sh b/automation/jenkins/aws/buildPython.sh deleted file mode 100755 index 35cc7876..00000000 --- a/automation/jenkins/aws/buildPython.sh +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") && (-d "${venv_dir}") ]] && rm -rf "${venv_dir}"; exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -function main() { - # Make sure we are in the build source directory - cd ${AUTOMATION_BUILD_SRC_DIR} - - # Update git origin url for product code repo to specify automation user credentials for successful push - [[ -n "${PRODUCT_CODE_REPO}" ]] && git remote set-url origin https://${GITHUB_CREDENTIALS}@${GITHUB_GIT_DNS}/${GITHUB_GIT_ORG}/${PRODUCT_CODE_REPO}.git - - # Determine required tasks - [[ -n "${BUILD_TASKS}" ]] && REQUIRED_TASKS=( ${BUILD_TASKS} ) || REQUIRED_TASKS=( "build" "unit" ) - - # virtual environment is needed not only for build, but for unit and swagger tasks - if inArray "REQUIRED_TASKS" "build|unit|swagger"; then - # Is this really a python based project - [[ ! -f requirements.txt ]] && [[ ! -d requirements ]] && [[ ! -n "${PYTHON_REQUIREMENTS_FILES}" ]] && - { fatal "No requirements.txt or requirements - is this really a python base repo?"; return 1; } - - # Set up the virtual build environment - venv_dir="$(getTempDir "cota_venv_XXX")" - PYTHON_VERSION="${AUTOMATION_PYTHON_VERSION:+ -p } ${AUTOMATION_PYTHON_VERSION}" - - # Note that python version below should NOT be in quotes to ensure arguments parsed correctly - virtualenv ${PYTHON_VERSION} "${venv_dir}" || - { exit_status=$?; fatal "Creation of virtual build environment failed"; return ${exit_status}; } - - . ${venv_dir}/bin/activate - - # Pin pip if required - [[ -n "${AUTOMATION_PIP_VERSION}" ]] && pip install "pip==${AUTOMATION_PIP_VERSION}" - - # Process requirements files - # If there is a root requirements.txt file install it and if there are any other matching requirements*.txt pattern - # Otherwise use *.txt files from the requirements directory - shopt -s nullglob - if [[ -n "${PYTHON_REQUIREMENTS_FILES}" ]]; then - REQUIREMENTS_FILES=( ${PYTHON_REQUIREMENTS_FILES} ) - else - [[ -f requirements.txt ]] && REQUIREMENTS_FILES=( requirements*.txt ) || REQUIREMENTS_FILES=( requirements/*.txt ) - fi - - for REQUIREMENTS_FILE in "${REQUIREMENTS_FILES[@]}"; do - pip install -r ${REQUIREMENTS_FILE} --upgrade || - { exit_status=$?; fatal "Installation of requirements failed"; return ${exit_status}; } - done - - # Patch the virtual env if packages have not been installed into site-packages dir - # This is a defect in zappa 0.42, in that it doesn't allow for platforms that install - # packages into dist-packages. Remove this patch once zappa is fixed - if [[ -n ${VIRTUAL_ENV} ]]; then - for lib in "lib" "lib64"; do - if [[ -d "${VIRTUAL_ENV}/${lib}" ]]; then - SITE_PACKAGES_DIR=$(find ${VIRTUAL_ENV}/${lib} -name site-packages) - if [[ -n ${SITE_PACKAGES_DIR} ]]; then - if [[ $(find ${SITE_PACKAGES_DIR} -type d | wc -l) < 2 ]]; then - cp -rp ${SITE_PACKAGES_DIR}/../dist-packages/* ${SITE_PACKAGES_DIR} - fi - fi - fi - done - fi - - if [[ -f package.json ]]; then - # Select the package manage to use - if [[ -z "${NODE_PACKAGE_MANAGER}" ]]; then - NODE_PACKAGE_MANAGER="npm" - fi - # Set install options - case ${NODE_PACKAGE_MANAGER} in - npm) - NODE_PACKAGE_MANAGER_INSTALL_OPTIONS="--unsafe-perm" - ;; - *) - NODE_PACKAGE_MANAGER_INSTALL_OPTIONS="" - ;; - esac - ${NODE_PACKAGE_MANAGER} install ${NODE_PACKAGE_MANAGER_INSTALL_OPTIONS} || - { exit_status=$?; fatal "${NODE_PACKAGE_MANAGER} install failed"; return ${exit_status}; } - fi - - # Run bower as part of the build if required - if [[ -f bower.json ]]; then - bower install --allow-root || - { exit_status=$?; fatal "Bower install failed"; return ${exit_status}; } - fi - fi - - if inArray "REQUIRED_TASKS" "unit"; then - # Run unit tests - there should always be a task even if it does nothing. Checking for pytest.ini file first. - MANAGE_OPTIONS="" - if [[ -n ${TEST_REPORTS_DIR} ]]; then - if [[ -n ${TEST_JUNIT_DIR} ]]; then - # Set path for test results in xml format if TEST_REPORTS_DIR and TEST_JUNIT_DIR are set - MANAGE_OPTIONS+="${TEST_REPORTS_DIR}/${TEST_JUNIT_DIR}/unit-test-results.xml" - fi - fi - if [[ -f pytest.ini ]]; then - info "Running unit tests with pytest..." - if [[ -n ${MANAGE_OPTIONS} ]]; then - # Set --junitxml option if TEST_REPORTS_DIR and TEST_JUNIT_DIR are set - MANAGE_OPTIONS=" --junitxml=${MANAGE_OPTIONS}" - fi - if [[ -n ${COVERAGE_REPORT} ]]; then - # Note: coverage and pytest-cov are required to run `pytest` with `--cov` option - # COVERAGE_REPORT specifies output format - xml, html or annotate - MANAGE_OPTIONS+=" --cov --cov-report ${COVERAGE_REPORT}" - if [[ -n ${COVERAGE_REPORT_OUTPUT} ]]; then - # COVERAGE_REPORT_OUTPUT specifies output path - # see https://pypi.org/project/pytest-cov/ for details - MANAGE_OPTIONS+=":${COVERAGE_REPORT_OUTPUT}" - fi - fi - if [[ -n ${UNIT_OPTIONS} ]]; then - MANAGE_OPTIONS+=" ${UNIT_OPTIONS}" - fi - pytest ${MANAGE_OPTIONS} || - { exit_status=$?; fatal "Tests failed"; return ${exit_status}; } - else - if [[ -f manage.py ]]; then - info "Running unit tests with manage.py test..." - if [[ -n ${MANAGE_OPTIONS} ]]; then - # Set --junit-xml argument if TEST_REPORTS_DIR and TEST_JUNIT_DIR are set - MANAGE_OPTIONS=" --junit-xml ${MANAGE_OPTIONS}" - fi - if [[ -n ${UNIT_OPTIONS} ]]; then - MANAGE_OPTIONS+=" ${UNIT_OPTIONS}" - fi - ENV_FILE=${PYTHON_UNIT_TEST_ENV_FILE} python manage.py test ${MANAGE_OPTIONS} || - { exit_status=$?; fatal "Tests failed"; return ${exit_status}; } - else - warning "Neither pytest.ini nor manage.py found - no tests run" - fi - fi - fi - - if inArray "REQUIRED_TASKS" "integration"; then - # Run integration tests - if [[ -f "${AUTOMATION_BUILD_DEVOPS_DIR}/docker-test/Dockerfile-test" ]]; then - info "Running integration tests ..." - cd ${AUTOMATION_BUILD_DEVOPS_DIR}/docker-test/ - ./scripts/runDockerComposeTests.sh || - { exit_status=$?; fatal "Integration tests failed"; return ${exit_status}; } - cd ${AUTOMATION_BUILD_SRC_DIR} - fi - fi - - if inArray "REQUIRED_TASKS" "testviafile"; then - # Run tests with a script file - TEST_SCRIPT_FILE="${TEST_SCRIPT_FILE:-run_tests_ci.sh}" - if [[ -f "${AUTOMATION_BUILD_SRC_DIR}/${TEST_SCRIPT_FILE}" ]]; then - info "Running tests with ${TEST_SCRIPT_FILE} ..." - ./${TEST_SCRIPT_FILE} || - { exit_status=$?; fatal "Tests failed"; return ${exit_status}; } - fi - fi - - if inArray "REQUIRED_TASKS" "swagger"; then - # Generate swagger documents - if [[ -f manage.py ]]; then - info "Generate swagger documents ..." - - MANAGE_OPTIONS="" - if [[ -n ${SWAGGER_OPTIONS} ]]; then - MANAGE_OPTIONS+=" ${SWAGGER_OPTIONS}" - fi - if [[ -n ${COMPONENT_INSTANCES} ]]; then - # Iterate component instance list if it is specified. Case for the projects with different API channels. - # COMPONENT_INSTANCE must be environment variable - for COMPONENT_INSTANCE in ${COMPONENT_INSTANCES}; do - # spec directory is on the same level with the build directory - SWAGGER_TARGET_FILE="${AUTOMATION_BUILD_DIR}"/../spec/${COMPONENT_INSTANCE}/swagger.yaml - ENV_FILE=${PYTHON_SWAGGER_ENV_FILE} COMPONENT_INSTANCE=${COMPONENT_INSTANCE} python manage.py swagger ${SWAGGER_TARGET_FILE} ${MANAGE_OPTIONS} || - { exit_status=$?; fatal "Generate swagger documents failed"; return ${exit_status}; } - done - else - SWAGGER_TARGET_FILE="${AUTOMATION_BUILD_DIR}"/../spec/swagger.yaml - ENV_FILE=${PYTHON_SWAGGER_ENV_FILE} python manage.py swagger ${SWAGGER_TARGET_FILE} ${MANAGE_OPTIONS} || - { exit_status=$?; fatal "Generate swagger documents failed"; return ${exit_status}; } - fi - # set code reference to master if a - [[ ! -n "${SWAGGER_CODE_REFERENCE}" ]] && SWAGGER_CODE_REFERENCE="master" - save_product_code "swagger documents generated based on ${GIT_COMMIT}" "${SWAGGER_CODE_REFERENCE}" - else - warning "No manage.py" - fi - fi - - if inArray "REQUIRED_TASKS" "build"; then - # Clean up pyc files before packaging into zappa - find ${AUTOMATION_BUILD_SRC_DIR} -name '*.pyc' -delete - - # Package for lambda if required - for ZAPPA_DIR in "${AUTOMATION_BUILD_DEVOPS_DIR}/lambda" "./"; do - if [[ -f "${ZAPPA_DIR}/zappa_settings.json" ]]; then - info "Packaging for lambda ..." - BUILD=$(zappa package default -s ${ZAPPA_DIR}/zappa_settings.json | tail -1 | cut -d' ' -f3) - if [[ -f ${BUILD} ]]; then - mkdir -p "${AUTOMATION_BUILD_SRC_DIR}/dist" - mv ${BUILD} "${AUTOMATION_BUILD_SRC_DIR}/dist/lambda.zip" - else - { exit_status=$?; fatal "Packaging for lambda failed"; return ${exit_status}; } - fi - fi - done - fi - - if inArray "REQUIRED_TASKS" "build|unit|swagger"; then - # Clean up - if [[ -f package.json ]]; then - case ${NODE_PACKAGE_MANAGER} in - yarn) - yarn install --production || - { exit_status=$?; fatal "yarn install --production failed"; return ${exit_status}; } - ;; - *) - npm prune --production || - { exit_status=$?; fatal "npm prune failed"; return ${exit_status}; } - ;; - esac - fi - - # Clean up the virtual env - [[ -d "${venv_dir}" ]] && rm -rf "${venv_dir}" - fi - - # All good - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/buildRDSSnapshot.sh b/automation/jenkins/aws/buildRDSSnapshot.sh deleted file mode 100755 index d6f3f809..00000000 --- a/automation/jenkins/aws/buildRDSSnapshot.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") ; exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -function main() { - - tmpdir="$(getTempDir "cota_inf_XXX")" - - # Get the generation context so we can run template generation - . "${GENERATION_BASE_DIR}/execution/setContext.sh" - . ${AUTOMATION_DIR}/setCredentials.sh "${ACCOUNT}" - - info "Building Deployment ${DEPLOYMENT_UNIT_LIST}" - for DEPLOYMENT_UNIT in ${DEPLOYMENT_UNIT_LIST[0]}; do - - DEPLOYMENT_GROUP="${DEPLOYMENT_GROUP:-"application"}" - # Generate a build blueprint so that we can find out the source S3 bucket - info "Generating blueprint to find details..." - ${GENERATION_DIR}/createTemplate.sh -e "buildblueprint" -p "aws" -l "${DEPLOYMENT_GROUP}" -u "${DEPLOYMENT_UNIT}" -o "${tmpdir}" > /dev/null - BUILD_BLUEPRINT="${tmpdir}/buildblueprint-${DEPLOYMENT_GROUP}-${DEPLOYMENT_UNIT}-config.json" - - if [[ ! -f "${BUILD_BLUEPRINT}" || -z "$(cat ${BUILD_BLUEPRINT} )" ]]; then - fatal "Could not generate blueprint for task details" - return 255 - fi - - mkdir -p "${tmpdir}/${DEPLOYMENT_UNIT}" - data_manifest_file="${tmpdir}/${DEPLOYMENT_UNIT}/${data_manifest_filename}" - - rdssnapshot_database_id="$( jq -r '.Occurrence.State.Attributes.INSTANCEID' < "${BUILD_BLUEPRINT}" )" - rdssnapshot_region="$( jq -r '.Occurrence.State.Attributes.REGION' < "${BUILD_BLUEPRINT}" )" - rdssnapshot_type="$( jq -r '.Occurrence.State.Attributes.TYPE' < "${BUILD_BLUEPRINT}" )" - - info "Creating Snapshot of RDS Instance: ${rdssnapshot_database_id} ..." - - snapshot_id="buildSnapshot-${DEPLOYMENT_UNIT}-$(date +%Y%m%d%H%M)" - - create_snapshot "${rdssnapshot_region}" "${rdssnapshot_type}" "${rdssnapshot_database_id}" "${snapshot_id}" - RESULT=$? - - if [[ "${RESULT}" -eq 0 ]]; then - - snapshot_create_time="$( aws --region "${rdssnapshot_region}" rds describe-db-snapshots --db-snapshot-identifier "${snapshot_id}" --query "DBSnapshots[0].SnapshotCreateTime" --output text )" - build_reference="$( echo "${snapshot_create_time}" | shasum -a 1 | cut -d " " -f 1 )" - - save_context_property CODE_COMMIT_LIST "${build_reference}" - save_context_property SNAPSHOT_SOURCE "${snapshot_id}" - save_chain_property GIT_COMMIT "${build_reference}" - - info "Commit: ${build_reference}" - - else - fatal "Could not create snapshot of rds instance ${rdssnapshot_database_id}" - return 128 - fi - done - - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/buildSPAStatic.sh b/automation/jenkins/aws/buildSPAStatic.sh deleted file mode 100755 index 70822ffb..00000000 --- a/automation/jenkins/aws/buildSPAStatic.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# - Basic build for an SPA that is static content - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") ; exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -function main() { - # Make sure we are in the build source directory - cd ${AUTOMATION_BUILD_SRC_DIR} - - # packge for content node - mkdir -p "${AUTOMATION_BUILD_SRC_DIR}/dist" - - cd "${AUTOMATION_BUILD_SRC_DIR}" - zip -r "${AUTOMATION_BUILD_SRC_DIR}/dist/spa.zip" * - - # All good - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/buildScripts.sh b/automation/jenkins/aws/buildScripts.sh deleted file mode 100755 index 3a6d0a1b..00000000 --- a/automation/jenkins/aws/buildScripts.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") ; exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -tmpdir="$(getTempDir "cota_inf_XXX")" - -function main() { - # Make sure we are in the build source directory - cd ${AUTOMATION_BUILD_SRC_DIR} - - # Mkae sure we have a script to start from - [[ ! -f init.sh ]] && - { fatal "No init.sh found - this is the entry point for this build type"; return 1; } - - zip -r "${tmpdir}/scripts.zip" * - - if [[ -f ${tmpdir}/scripts.zip ]]; then - mkdir "${AUTOMATION_BUILD_SRC_DIR}/dist" - cp ${tmpdir}/scripts.zip "${AUTOMATION_BUILD_SRC_DIR}/dist/scripts.zip" - fi - - # All good - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/buildSetup.sh b/automation/jenkins/aws/buildSetup.sh deleted file mode 100755 index 9176326f..00000000 --- a/automation/jenkins/aws/buildSetup.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -# Ensure we are in the directory where the repo was checked out -cd ${AUTOMATION_BUILD_DIR} - -# Check for a build qualifier -BUILD_TASK_QUALIFIER= -if [[ -n "${BUILD_TASKS}" ]]; then - REQUIRED_TASKS=( ${BUILD_TASKS} ) - for REQUIRED_TASK in "${REQUIRED_TASKS[@]}"; do - REQUIRED_TASK_BASE="${REQUIRED_TASK%%:*}" - REQUIRED_TASK_QUALIFIER="${REQUIRED_TASK##*:}" - if [[ ("${REQUIRED_TASK_BASE}" == "build") && - ("${REQUIRED_TASK}" != "${REQUIRED_TASK_BASE}") ]]; then - BUILD_TASK_QUALIFIER="${REQUIRED_TASK_QUALIFIER,,}_" - fi - done -fi - -DU_FILES=( - "${AUTOMATION_BUILD_DEVOPS_DIR}/${BUILD_TASK_QUALIFIER}deployment_units.json" \ - "${AUTOMATION_BUILD_DEVOPS_DIR}/codeontap/${BUILD_TASK_QUALIFIER}deployment_units.json" \ - "${BUILD_TASK_QUALIFIER}deployment_units.json" \ - slices.json slices.ref slice.ref \ -) -# Check for repo provided deployment unit list -# slice(s).ref and slices.json are legacy - always use deployment_units.json -if [[ -z "${DEPLOYMENT_UNIT_LIST}" ]]; then - for DU_FILE in "${DU_FILES[@]}"; do - if [[ -f "${DU_FILE}" ]]; then - case "${DU_FILE##*.}" in - json) - for ATTRIBUTE in units slices formats; do - ATTRIBUTE_VALUE=$(jq -r ".${ATTRIBUTE} | select(.!=null) | .[]" < "${DU_FILE}" | tr -s "\r\n" " ") - if [[ -z "${ATTRIBUTE_VALUE}" ]]; then - ATTRIBUTE_VALUE=$(jq -r ".${ATTRIBUTE^} | select(.!=null) | .[]" < "${DU_FILE}" | tr -s "\r\n" " ") - fi - declare "${ATTRIBUTE^^}"="${ATTRIBUTE_VALUE}" - done - for ATTRIBUTE in scope; do - ATTRIBUTE_VALUE=$(jq -r ".${ATTRIBUTE} | select(.!=null)" < "${DU_FILE}" | tr -d "\r\n") - if [[ -z "${ATTRIBUTE_VALUE}" ]]; then - ATTRIBUTE_VALUE=$(jq -r ".${ATTRIBUTE^} | select(.!=null)" < "${DU_FILE}" | tr -d "\r\n") - fi - declare "${ATTRIBUTE^^}"="${ATTRIBUTE_VALUE}" - done - export DEPLOYMENT_UNIT_LIST="${UNITS}" - break - ;; - - ref) - export DEPLOYMENT_UNIT_LIST=$(cat "${DU_FILE}") - break - ;; - esac - fi - done - - save_context_property DEPLOYMENT_UNIT_LIST -fi - -# Already set image format overrides that in the repo -IMAGE_FORMATS="${IMAGE_FORMATS:-${IMAGE_FORMAT}}" -IMAGE_FORMATS="${IMAGE_FORMATS:-${FORMATS:-docker}}" -IFS="${IMAGE_FORMAT_SEPARATORS}, " read -ra IMAGE_FORMATS_ARRAY <<< "${IMAGE_FORMATS}" -export IMAGE_FORMATS_LIST=$(IFS="${IMAGE_FORMAT_SEPARATORS}"; echo "${IMAGE_FORMATS_ARRAY[*]}") -save_context_property IMAGE_FORMATS_LIST - -DEPLOYMENT_UNIT_ARRAY=(${DEPLOYMENT_UNIT_LIST}) -DEPLOYMENT_UNIT="${DEPLOYMENT_UNIT_ARRAY[0]}" -CODE_COMMIT_ARRAY=(${CODE_COMMIT_LIST}) -CODE_COMMIT="${CODE_COMMIT_ARRAY[0]}" - -# Already set registry scope overrides that in the repo -REGISTRY_SCOPE="${REGISTRY_SCOPE:-${SCOPE}}" -export REGISTRY_SCOPE_LIST="${REGISTRY_SCOPE}" -save_context_property REGISTRY_SCOPE_LIST - -# Record key parameters for downstream jobs -save_chain_property DEPLOYMENT_UNITS "${DEPLOYMENT_UNIT_LIST}" -save_chain_property GIT_COMMIT "${CODE_COMMIT}" -save_chain_property IMAGE_FORMATS -save_chain_property REGISTRY_SCOPE - -# Include the build information in the detail message -${AUTOMATION_DIR}/manageBuildReferences.sh -l -RESULT=$? -[[ "${RESULT}" -ne 0 ]] && exit - -# Ensure no builds exist regardless of format -PRESENT=0 - -for IMAGE_FORMAT in "${IMAGE_FORMATS_ARRAY[@]}"; do - case ${IMAGE_FORMAT,,} in - dataset) - ${AUTOMATION_DIR}/manageDataSetS3.sh -v -u "${DEPLOYMENT_UNIT}" -g "undefined" -c "${REGISTRY_SCOPE}" - RESULT=$? - [[ "${RESULT}" -eq 0 ]] && PRESENT=1 - ;; - - rdssnapshot) - ${AUTOMATION_DIR}/manageDataSetRDSSnapshot.sh -v -u "${DEPLOYMENT_UNIT}" -g "undefined" -c "${REGISTRY_SCOPE}" - RESULT=$? - [[ "${RESULT}" -eq 0 ]] && PRESENT=1 - ;; - - docker) - ${AUTOMATION_DIR}/manageDocker.sh -v -s "${DEPLOYMENT_UNIT}" -g "${CODE_COMMIT}" -c "${REGISTRY_SCOPE}" - RESULT=$? - [[ "${RESULT}" -eq 0 ]] && PRESENT=1 - ;; - - lambda|scripts|openapi|swagger|spa|contentnode) - ${AUTOMATION_DIR}/manageS3Registry.sh -v \ - -u "${DEPLOYMENT_UNIT}" -g "${CODE_COMMIT}" -c "${REGISTRY_SCOPE}" \ - -y "${IMAGE_FORMAT,,}" -f "${IMAGE_FORMAT,,}.zip" - RESULT=$? - [[ "${RESULT}" -eq 0 ]] && PRESENT=1 - ;; - - lambda_jar) - ${AUTOMATION_DIR}/manageS3Registry.sh -v \ - -u "${DEPLOYMENT_UNIT}" -g "${CODE_COMMIT}" -c "${REGISTRY_SCOPE}" \ - -y "${IMAGE_FORMAT,,}" -f "${IMAGE_FORMAT,,}.jar" - RESULT=$? - [[ "${RESULT}" -eq 0 ]] && PRESENT=1 - ;; - - *) - fatal "Unsupported image format \"${IMAGE_FORMAT}\"" - ;; - esac -done - -RESULT=${PRESENT} -[[ "${RESULT}" -ne 0 ]] && exit - -# Perform prebuild actions -if [[ -f prebuild.json ]]; then - # Include repos - for ((INDEX=0; ; INDEX++)); do - ENTRY=$(jq -c ".IncludeRepos[${INDEX}] | select(.!=null)" < prebuild.json) - if [[ -n "${ENTRY}" ]]; then - # Extract key attributes - REPO_PROVIDER=$(jq -r '.provider' <<< $ENTRY) - REPO_NAME=$(jq -r '.name' <<< $ENTRY) - - if [[ (-n "${REPO_PROVIDER}") && (-n "${REPO_NAME}") ]]; then - if [[ ! -e "./${REPO_NAME}" ]]; then - ${AUTOMATION_DIR}/manageRepo.sh -c -l "${REPO_NAME}" \ - -n "${REPO_NAME}" -v "${REPO_PROVIDER^^}" \ - -d "./${REPO_NAME}" - RESULT=$? - [[ ${RESULT} -ne 0 ]] && exit - else - warning "\"${REPO_NAME}\" repo already exists - using existing local rather than fetching again" - fi - else - warning "Incorrectly formatted include repo information: ${ENTRY}" - fi - else - # No more entries to process - break - fi - done -fi - -# All good -RESULT=0 diff --git a/automation/jenkins/aws/buildSwagger.sh b/automation/jenkins/aws/buildSwagger.sh deleted file mode 100755 index 6937f27f..00000000 --- a/automation/jenkins/aws/buildSwagger.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM - -# DEPRECATED -deprecated_script - -# All the logic is in the openapi build -${AUTOMATION_DIR}/buildOpenapi.sh "$@" -RESULT=$? diff --git a/automation/jenkins/aws/buildTenantBlueprints.sh b/automation/jenkins/aws/buildTenantBlueprints.sh deleted file mode 100755 index ec019b3e..00000000 --- a/automation/jenkins/aws/buildTenantBlueprints.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ (-z "${AUTOMATION_DEBUG}") ; exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -# Usage - This script should be called from the directory where a generated blueprint is stored. -# This would ideally be tirggered using a git hook from an automation server - -function main() { - if [[ -z "${AUTOMATION_REGISTRY_REPO}" ]]; then - fatal "No automation registry available" - return 255 - fi - - # Make sure we are in the build source directory - cd ${AUTOMATION_BUILD_SRC_DIR} - - local BLUEPRINT_CONSOLIDATION_DIR="${AUTOMATION_BUILD_SRC_DIR}/registry" - - ${AUTOMATION_DIR}/manageRepo.sh -c -l "blueprint consolidation" \ - -n "${AUTOMATION_REGISTRY_REPO}" -v "${ACCOUNT_GIT_PROVIDER}" \ - -d "${BLUEPRINT_CONSOLIDATION_DIR}" - - if [[ -n "${INFRADOCS_PREFIX}" ]]; then - local BLUEPRINT_CONSOLIDATION_DIR="${AUTOMATION_BUILD_SRC_DIR}/registry/" - fi - - local BLUEPRINT_DESTINATION_DIR="${BLUEPRINT_CONSOLIDATION_DIR}/blueprints/content" - local BLUEPRINT_DESTINATION_FILE="${BLUEPRINT_DESTINATION_DIR}/${TENANT}-${PRODUCT}-${ENVIRONMENT}-${SEGMENT}-blueprint.json" - - info "blueprint repo ${BLUEPRINT_DESTINATION_DIR}" - - if [[ -f "${AUTOMATION_BUILD_SRC_DIR}/blueprint.json" ]]; then - - if [[ ! -d "${BLUEPRINT_DESTINATION_DIR}" ]]; then - mkdir -p "${BLUEPRINT_DESTINATION_DIR}" - fi - - echo "Adding Blueprint to Tenant Infrastructure..." - cp "${AUTOMATION_BUILD_SRC_DIR}/blueprint.json" "${BLUEPRINT_DESTINATION_FILE}" - - else - - if [[ -f "${BLUEPRINT_DESTINATION_FILE}" ]]; then - echo "Removing Blueprint from Tenant Infrastructure..." - rm "${BLUEPRINT_DESTINATION_FILE}" - fi - fi - - DETAIL_MESSAGE="${DETAIL_MESSAGE}, blueprint consolidation" - ${AUTOMATION_DIR}/manageRepo.sh -p \ - -d "${BLUEPRINT_CONSOLIDATION_DIR}" \ - -l "blueprint consolidation" \ - -m "${DETAIL_MESSAGE}" - - # All good - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/confirmBuilds.sh b/automation/jenkins/aws/confirmBuilds.sh deleted file mode 100755 index b7bcdcac..00000000 --- a/automation/jenkins/aws/confirmBuilds.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -# Ensure mandatory arguments have been provided -[[ (-z "${RELEASE_MODE}") || - (-z "${ACCEPTANCE_TAG}") ]] && fatalMandatory - -# Verify the reference updates -${AUTOMATION_DIR}/manageBuildReferences.sh -v ${ACCEPTANCE_TAG} -RESULT=$? -[[ "${RESULT}" -ne 0 ]] && exit - -# Include the build information in the detail message -${AUTOMATION_DIR}/manageBuildReferences.sh -l -RESULT=$? diff --git a/automation/jenkins/aws/constructTree.sh b/automation/jenkins/aws/constructTree.sh deleted file mode 100755 index 068ff260..00000000 --- a/automation/jenkins/aws/constructTree.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -${AUTOMATION_BASE_DIR}/constructTree.sh "$@" -RESULT=$? diff --git a/automation/jenkins/aws/deploy.sh b/automation/jenkins/aws/deploy.sh deleted file mode 100755 index 17a94a89..00000000 --- a/automation/jenkins/aws/deploy.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -function main() { - # Add conventional commit details - DETAIL_MESSAGE="${DETAIL_MESSAGE}, cctype=deploy, ccdesc=${AUTOMATION_JOB_IDENTIFIER}" - save_context_property DETAIL_MESSAGE - - # Create the templates - ${AUTOMATION_DIR}/manageUnits.sh -l "application" -a "${DEPLOYMENT_UNIT_LIST}" -r "${PRODUCT_CONFIG_COMMIT}" || return $? - - # Commit the generated application templates/stacks - # It is assumed no changes have been made to the config part of the cmdb - save_product_infrastructure "${DETAIL_MESSAGE}" "${PRODUCT_INFRASTRUCTURE_REFERENCE}" || return $? -} - -main "$@" diff --git a/automation/jenkins/aws/deployRelease.sh b/automation/jenkins/aws/deployRelease.sh deleted file mode 100755 index fd85117f..00000000 --- a/automation/jenkins/aws/deployRelease.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -function main() { - # Add conventional commit and deploy/release tag to details - DETAIL_MESSAGE="deployment=${DEPLOYMENT_TAG}, release=${RELEASE_TAG}, ${DETAIL_MESSAGE}, cctype=deploy, ccdesc=${AUTOMATION_JOB_IDENTIFIER}" - save_context_property DETAIL_MESSAGE - - # Update the stacks - ${AUTOMATION_DIR}/manageUnits.sh -l "application" -a "${DEPLOYMENT_UNIT_LIST}" || return $? - - # Commit the generated application templates/stacks - # It is assumed no changes have been made to the config part of the cmdb - save_product_infrastructure "${DETAIL_MESSAGE}" "${PRODUCT_INFRASTRUCTURE_REFERENCE}" || return $? -} - -main "$@" diff --git a/automation/jenkins/aws/deploySetup.sh b/automation/jenkins/aws/deploySetup.sh deleted file mode 100755 index c66d572c..00000000 --- a/automation/jenkins/aws/deploySetup.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -# Include the build information in the detail message -${AUTOMATION_DIR}/manageBuildReferences.sh -l -RESULT=$? diff --git a/automation/jenkins/aws/manageAccount.sh b/automation/jenkins/aws/manageAccount.sh deleted file mode 100755 index 71fbfb97..00000000 --- a/automation/jenkins/aws/manageAccount.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# DEPRECATED -deprecated_script - -function main() { - # Add conventional commit details - DETAIL_MESSAGE="${DETAIL_MESSAGE}, cctype=manacc, ccdesc=${AUTOMATION_JOB_IDENTIFIER}" - - ${AUTOMATION_DIR}/manageUnits.sh -r "${ACCOUNT_CONFIG_COMMIT}" || return $? - - # With the removal of tagging, this shouldn't be needed as no changes should be made to the config part of the cmdb - # All ok so tag the config repo - # save_repo "${ACCOUNT_DIR}" "account config" "${DETAIL_MESSAGE}" "${PRODUCT_CONFIG_REFERENCE}" || return $? - - # Commit the generated application templates/stacks - save_repo "${ACCOUNT_STATE_DIR}" "account infrastructure" "${DETAIL_MESSAGE}" "${ACCOUNT_INFRASTRUCTURE_REFERENCE}" || return $? -} - -main "$@" diff --git a/automation/jenkins/aws/manageBuildReferences.sh b/automation/jenkins/aws/manageBuildReferences.sh deleted file mode 100755 index 9cba6519..00000000 --- a/automation/jenkins/aws/manageBuildReferences.sh +++ /dev/null @@ -1,682 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} - -. "${AUTOMATION_BASE_DIR}/common.sh" - -# Defaults -REFERENCE_OPERATION_ACCEPT="accept" -REFERENCE_OPERATION_LIST="list" -REFERENCE_OPERATION_LISTFULL="listfull" -REFERENCE_OPERATION_UPDATE="update" -REFERENCE_OPERATION_VERIFY="verify" -REFERENCE_OPERATION_DEFAULT="${REFERENCE_OPERATION_LIST}" - -function usage() { - cat < "${BUILD_FILE}" - - log_write_event "update_build_ref_${CURRENT_DEPLOYMENT_UNIT}_${CODE_COMMIT}" "update_build_reference" "$(dirname "${SEGMENT_BUILDS_DIR}/${BUILD_FILE}")" \ - "deployment_unit=${CURRENT_DEPLOYMENT_UNIT}" "code_commit=${CODE_COMMIT}" "code_tag=${CODE_TAG}" "image_format=${IMAGE_FORMATS}" "registry_scope=${REGISTRY_SCOPE}" - ;; - - ${REFERENCE_OPERATION_VERIFY}) - # Ensure code repo defined if tag provided only if commit not provided - if [[ "${CODE_COMMIT}" == "?" ]]; then - if [[ "${CODE_TAG}" != "?" ]]; then - if [[ ("${CODE_REPO}" == "?") || - ("${CODE_PROVIDER}" == "?") ]]; then - fatal "Ignoring tag for the \"${CURRENT_DEPLOYMENT_UNIT}\" deployment unit - no code repo and/or provider defined" - return 1 - fi - # Determine the details of the provider hosting the code repo - defineGitProviderAttributes "${CODE_PROVIDER}" "CODE" - # Get the commit corresponding to the tag - TAG_COMMIT=$(git ls-remote -t https://${!CODE_CREDENTIALS_VAR}@${CODE_DNS}/${CODE_ORG}/${CODE_REPO} \ - "${CODE_TAG}" | cut -f 1) - CODE_COMMIT=$(git ls-remote -t https://${!CODE_CREDENTIALS_VAR}@${CODE_DNS}/${CODE_ORG}/${CODE_REPO} \ - "${CODE_TAG}^{}" | cut -f 1) - - [[ -z "${CODE_COMMIT}" ]] && - fatal "Tag ${CODE_TAG} not found in the ${CODE_REPO} repo. Was an annotated tag used?" && return 1 - - # Fetch other info about the tag - # We are using a github api here to avoid having to pull in the whole repo - - # git currently doesn't have a command to query the message of a remote tag - CODE_TAG_MESSAGE=$(curl -s https://${!CODE_CREDENTIALS_VAR}@${CODE_API_DNS}/repos/${CODE_ORG}/${CODE_REPO}/git/tags/${TAG_COMMIT} | jq .message | tr -d '"') - [[ (-z "${CODE_TAG_MESSAGE}") || - ("${CODE_TAG_MESSAGE}" == "Not Found") ]] && - fatal "Message for tag ${CODE_TAG} not found in the ${CODE_REPO} repo" && return 1 - # else - # TODO: Confirm commit is in remote repo - for now we'll assume its there if an image exists - else - # Nothing to do for this deployment unit - # Note that it is permissible to not have a tag for a deployment unit - # that is associated with a code repo. This situation arises - # if application settings are changed and a new release is - # thus required. - continue - fi - fi - - # If no formats explicitly defined, use those in the build reference if defined - if [[ ("${IMAGE_FORMATS}" == "?") && - (-f ${BUILD_FILE}) ]]; then - getBuildReferenceParts "$(cat ${BUILD_FILE})" - IMAGE_FORMATS="${BUILD_REFERENCE_FORMATS}" - fi - # Format may be shared - [[ "${IMAGE_FORMATS}" == "?" ]] && IMAGE_FORMATS="${SHARED_IMAGE_FORMATS}" - IFS="${IMAGE_FORMAT_SEPARATORS}" read -ra CODE_IMAGE_FORMATS_ARRAY <<< "${IMAGE_FORMATS}" - - # If no scope explicitly defined, use the scope in the build reference if defined - if [[ ("${REGISTRY_SCOPE}" == "?") && - (-f ${BUILD_FILE}) ]]; then - getBuildReferenceParts "$(cat ${BUILD_FILE})" - REGISTRY_SCOPE="${BUILD_REFERENCE_SCOPE}" - fi - # Scope may be shared - [[ "${REGISTRY_SCOPE}" == "?" ]] && REGISTRY_SCOPE="${SHARED_REGISTRY_SCOPE}" - - # If we don't know the image type, then there is a problem - # Most likely it is the first time this unit has been mentioned and no format was - # included as part of the prepare operation. - [[ "${IMAGE_FORMATS}" == "?" ]] && - { - fatal "Image format(s) not known for \"${CURRENT_DEPLOYMENT_UNIT}\" deployment unit" - fatal "Valid image formats are ${REGISTRY_TYPES_LIST}" - fatal "Automation scripts: Provide the format after the code reference separated by \"!\" if unit is being mentioned for the first time." - fatal "hamlet cli: provide the --image-format option" - return 1 - } - - for IMAGE_FORMAT in "${CODE_IMAGE_FORMATS_ARRAY[@]}"; do - IMAGE_PROVIDER_VAR="PRODUCT_${IMAGE_FORMAT^^}_PROVIDER" - IMAGE_PROVIDER="${!IMAGE_PROVIDER_VAR}" - FROM_IMAGE_PROVIDER_VAR="FROM_PRODUCT_${IMAGE_FORMAT^^}_PROVIDER" - FROM_IMAGE_PROVIDER="${!FROM_IMAGE_PROVIDER_VAR}" - - local_image_available="true" - - case ${IMAGE_FORMAT,,} in - dataset) - ${AUTOMATION_DIR}/manageDataSetS3.sh -v \ - -a "${IMAGE_PROVIDER}" \ - -u "${REGISTRY_DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -c "${REGISTRY_SCOPE}" || local_image_available="false" - ;; - rdssnapshot) - ${AUTOMATION_DIR}/manageDataSetRDSSnapshot.sh -v \ - -a "${IMAGE_PROVIDER}" \ - -u "${REGISTRY_DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -c "${REGISTRY_SCOPE}" || local_image_available="false" - ;; - docker) - ${AUTOMATION_DIR}/manageDocker.sh -v \ - -a "${IMAGE_PROVIDER}" \ - -s "${REGISTRY_DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -c "${REGISTRY_SCOPE}" || local_image_available="false" - ;; - lambda|scripts|openapi|swagger|spa|contentnode) - ${AUTOMATION_DIR}/manageS3Registry.sh -v \ - -y "${IMAGE_FORMAT,,}" \ - -f "${IMAGE_FORMAT,,}.zip" \ - -a "${IMAGE_PROVIDER}" \ - -u "${REGISTRY_DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -c "${REGISTRY_SCOPE}" || local_image_available="false" - ;; - - lambda_jar) - ${AUTOMATION_DIR}/manageS3Registry.sh -v \ - -y "${IMAGE_FORMAT,,}" \ - -f "${IMAGE_FORMAT,,}.jar" \ - -a "${IMAGE_PROVIDER}" \ - -u "${REGISTRY_DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -c "${REGISTRY_SCOPE}" || local_image_available="false" - ;; - *) - fatal "Unknown image format \"${IMAGE_FORMAT}\"" - return 1 - ;; - esac - if [[ "${local_image_available}" == "false" ]]; then - if [[ -n "${FROM_IMAGE_PROVIDER}" ]]; then - # Attempt to pull image in from remote provider - - remote_image_avaialble="true" - case ${IMAGE_FORMAT,,} in - dataset) - ${AUTOMATION_DIR}/manageDataSetS3.sh -p \ - -a "${IMAGE_PROVIDER}" \ - -u "${REGISTRY_DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -r "${VERIFICATION_TAG}" \ - -z "${FROM_IMAGE_PROVIDER}" \ - -b "REGISTRY_CONTENT" \ - -c "${REGISTRY_SCOPE}" || remote_image_avaialble="false" - ;; - rdssnapshot) - ${AUTOMATION_DIR}/manageDataSetRDSSnapshot.sh -p \ - -a "${IMAGE_PROVIDER}" \ - -u "${REGISTRY_DEPLOYMENT_UNIT}" \ - -r "${VERIFICATION_TAG}" \ - -z "${FROM_IMAGE_PROVIDER}" \ - -g "${CODE_COMMIT}" \ - -c "${REGISTRY_SCOPE}" || remote_image_avaialble="false" - ;; - docker) - ${AUTOMATION_DIR}/manageDocker.sh -p \ - -a "${IMAGE_PROVIDER}" \ - -s "${REGISTRY_DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -r "${VERIFICATION_TAG}" \ - -z "${FROM_IMAGE_PROVIDER}" \ - -c "${REGISTRY_SCOPE}" || remote_image_avaialble="false" - ;; - lambda|scripts|openapi|swagger|spa|contentnode) - ${AUTOMATION_DIR}/manageS3Registry.sh -p \ - -y "${IMAGE_FORMAT,,}" \ - -f "${IMAGE_FORMAT,,}.zip" \ - -a "${IMAGE_PROVIDER}" \ - -u "${REGISTRY_DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -r "${VERIFICATION_TAG}" \ - -z "${FROM_IMAGE_PROVIDER}" \ - -c "${REGISTRY_SCOPE}" || remote_image_avaialble="false" - ;; - lambda_jar) - ${AUTOMATION_DIR}/manageS3Registry.sh -p \ - -y "${IMAGE_FORMAT,,}" \ - -f "${IMAGE_FORMAT,,}.jar" \ - -a "${IMAGE_PROVIDER}" \ - -u "${REGISTRY_DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -r "${VERIFICATION_TAG}" \ - -z "${FROM_IMAGE_PROVIDER}" \ - -c "${REGISTRY_SCOPE}" || remote_image_avaialble="false" - ;; - *) - fatal "Unknown image format \"${IMAGE_FORMAT}\"" - return 1 - ;; - esac - [[ "${remote_image_avaialble}" == "false" ]] && - fatal "Unable to pull ${IMAGE_FORMAT,,} image for deployment unit ${CURRENT_DEPLOYMENT_UNIT} and commit ${CODE_COMMIT} from provider ${FROM_IMAGE_PROVIDER}. Was the build successful?" && return 1 - else - fatal "${IMAGE_FORMAT^} image for deployment unit ${CURRENT_DEPLOYMENT_UNIT} and commit ${CODE_COMMIT} not found. Was the build successful?" && return 1 - fi - fi - - done - - # Save details of this deployment unit - CODE_COMMIT_ARRAY[${INDEX}]="${CODE_COMMIT}" - ;; - - esac - done - - # Capture any changes to context - case ${REFERENCE_OPERATION} in - ${REFERENCE_OPERATION_LIST}) - save_context_property DETAIL_MESSAGE - ;; - - ${REFERENCE_OPERATION_LISTFULL}) - save_context_property DEPLOYMENT_UNIT_LIST "${DEPLOYMENT_UNIT_ARRAY[*]}" - save_context_property CODE_COMMIT_LIST "${CODE_COMMIT_ARRAY[*]}" - save_context_property CODE_TAG_LIST "${CODE_TAG_ARRAY[*]}" - save_context_property IMAGE_FORMATS_LIST "${IMAGE_FORMATS_ARRAY[*]}" - save_context_property REGISTRY_SCOPE_LIST "${REGISTRY_SCOPE_ARRAY[*]}" - save_context_property DETAIL_MESSAGE "${DETAIL_MESSAGE}" - ;; - - ${REFERENCE_OPERATION_VERIFY}) - save_context_property CODE_COMMIT_LIST "${CODE_COMMIT_ARRAY[*]}" - ;; - - esac -} - -main "$@" diff --git a/automation/jenkins/aws/manageDataSetRDSSnapshot.sh b/automation/jenkins/aws/manageDataSetRDSSnapshot.sh deleted file mode 100755 index 4b0ad2dd..00000000 --- a/automation/jenkins/aws/manageDataSetRDSSnapshot.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM - -# Note that filename can still overridden via provided parameters -${AUTOMATION_DIR}/manageRdssnapshot.sh -y "rdssnapshot" "$@" -RESULT=$? diff --git a/automation/jenkins/aws/manageDataSetS3.sh b/automation/jenkins/aws/manageDataSetS3.sh deleted file mode 100755 index 870ac485..00000000 --- a/automation/jenkins/aws/manageDataSetS3.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM - -# Note that filename can still overridden via provided parameters -${AUTOMATION_DIR}/manageS3Registry.sh -y "dataset" -f "cot_data_file_manifest.json" "$@" -RESULT=$? - diff --git a/automation/jenkins/aws/manageDocker.sh b/automation/jenkins/aws/manageDocker.sh deleted file mode 100755 index 8afe1ef8..00000000 --- a/automation/jenkins/aws/manageDocker.sh +++ /dev/null @@ -1,504 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# Defaults -DOCKER_TAG_DEFAULT="latest" -DOCKER_IMAGE_SOURCE_REMOTE="remote" -DOCKER_IMAGE_SOURCE_DEFAULT="${DOCKER_IMAGE_SOURCE_REMOTE}" -DOCKER_OPERATION_BUILD="build" -DOCKER_OPERATION_VERIFY="verify" -DOCKER_OPERATION_TAG="tag" -DOCKER_OPERATION_PULL="pull" -DOCKER_OPERATION_DEFAULT="${DOCKER_OPERATION_VERIFY}" -DOCKER_CONTEXT_DIR_DEFAULT="${AUTOMATION_BUILD_DIR}" - -function usage() { - cat < /dev/null - USER_IMAGE="${PATHS[index]}" - IMAGE_FILE="${image_dir}/${IMAGE_FILENAME}" - if [[ -f "${USER_IMAGE}" ]]; then - cp "${USER_IMAGE}" "${IMAGE_FILE}" - fi - else - pushd "${AUTOMATION_BUILD_DIR}" > /dev/null - IMAGE_FILE="${AUTOMATION_BUILD_SRC_DIR}/${IMAGE_FILENAME}" - fi - - if [[ -f "${IMAGE_FILE}" ]]; then - ${AUTOMATION_DIR}/manageDataSetS3.sh -s \ - -u "${DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -f "${IMAGE_FILE}" \ - -b "${S3_DATA_STAGE}" \ - -c "${REGISTRY_SCOPE}" || return $? - popd > /dev/null - else - fatal "dataset manifest ${USER_IMAGE:-${IMAGE_FILE}} missing" - return 1 - fi - ;; - - rdssnapshot) - ${AUTOMATION_DIR}/manageDataSetRDSSnapshot.sh -s \ - -u "${DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -c "${REGISTRY_SCOPE}" || return $? - ;; - - docker) - - docker_args=() - if [[ -z "${DOCKERFILE}" ]]; then - - # Find a dockerfile based on the build src - pushd "${AUTOMATION_BUILD_DIR}" > /dev/null - DOCKERFILE="${AUTOMATION_BUILD_SRC_DIR}/Dockerfile" - if [[ -f "${AUTOMATION_BUILD_DEVOPS_DIR}/docker/Dockerfile" ]]; then - DOCKERFILE="${AUTOMATION_BUILD_DEVOPS_DIR}/docker/Dockerfile" - fi - if [[ -n "${DOCKER_FILE}" && -f "${AUTOMATION_DATA_DIR}/${DOCKER_FILE}" ]]; then - DOCKERFILE="${AUTOMATION_DATA_DIR}/${DOCKER_FILE}" - fi - docker_args+=("-y" "${DOCKER_FILE}") - else - - if [[ -z "${DOCKER_CONTEXT}" ]]; then - fatal "DOCKER_CONTEXT ( -e ) must be set when providing a DOCKERFILE ( -d )" - return 1 - fi - - # Override the standard Dockerfile with your own - docker_args+=("-y" "${DOCKERFILE}") - docker_args+=("-x" "${DOCKER_CONTEXT}") - - pushd "$(pwd)" > /dev/null - fi - - # Skip the build process and just use the provided image - if [[ -n "${DOCKER_IMAGE}" ]]; then - docker_args+=("-w" "${DOCKER_IMAGE}") - fi - - if [[ -f "${DOCKERFILE}" || -n "${DOCKER_IMAGE}" ]]; then - ${AUTOMATION_DIR}/manageDocker.sh -b \ - -s "${DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -c "${REGISTRY_SCOPE}" \ - "${docker_args[@]}" || return $? - popd > /dev/null - else - fatal "Dockerfile ${DOCKERFILE} missing" - return 1 - fi - ;; - - lambda|scripts|openapi|swagger|spa|contentnode) - IMAGE_FILENAME="${FORMAT,,}.zip" - if [[ -n "${PATHS[index]}" ]]; then - pushd "$(pwd)" > /dev/null - USER_IMAGE="${PATHS[index]}" - IMAGE_FILE="${image_dir}/${IMAGE_FILENAME}" - - if [[ -f "${USER_IMAGE}" ]]; then - if [[ "$(fileExtension "${USER_IMAGE}")" != "zip" ]]; then - fatal "${FORMAT..} image must be a zip file or a directory - ${USER_IMAGE} does not have zip extension" - return 1 - fi - cp "${USER_IMAGE}" "${IMAGE_FILE}" - fi - - if [[ -d "${USER_IMAGE}" ]]; then - pushd "${USER_IMAGE}" > /dev/null - if [[ -f "${IMAGE_FILENAME}" ]]; then - cp "${IMAGE_FILENAME}" "${IMAGE_FILE}" - else - zip -r "${IMAGE_FILE}" * - fi - popd > /dev/null - fi - else - pushd "${AUTOMATION_BUILD_DIR}" > /dev/null - IMAGE_FILE="${AUTOMATION_BUILD_SRC_DIR}/dist/${IMAGE_FILENAME}" - fi - - if [[ -f "${IMAGE_FILE}" ]]; then - ${AUTOMATION_DIR}/manageS3Registry.sh -s \ - -y "${FORMAT,,}" -f "${IMAGE_FILENAME}" \ - -u "${DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -f "${IMAGE_FILE}" \ - -c "${REGISTRY_SCOPE}" || return $? - popd > /dev/null - else - fatal "${FORMAT,,} image ${USER_IMAGE:-${IMAGE_FILE}} missing" - return 1 - fi - ;; - - lambda_jar) - IMAGE_FILENAME="${FORMAT,,}.jar" - if [[ -n "${PATHS[index]}" ]]; then - pushd "$(pwd)" > /dev/null - USER_IMAGE="${PATHS[index]}" - IMAGE_FILE="${image_dir}/${IMAGE_FILENAME}" - - if [[ -f "${USER_IMAGE}" ]]; then - if [[ "$(fileExtension "${USER_IMAGE}")" != "jar" ]]; then - fatal "${FORMAT..} image must be a jar file - ${USER_IMAGE} does not have jar extension" - return 1 - fi - cp "${USER_IMAGE}" "${IMAGE_FILE}" - fi - else - pushd "${AUTOMATION_BUILD_DIR}" > /dev/null - IMAGE_FILE="${AUTOMATION_BUILD_SRC_DIR}/dist/${IMAGE_FILENAME}" - fi - - if [[ -f "${IMAGE_FILE}" ]]; then - ${AUTOMATION_DIR}/manageS3Registry.sh -s \ - -y "${FORMAT,,}" -f "${IMAGE_FILENAME}" \ - -u "${DEPLOYMENT_UNIT}" \ - -g "${CODE_COMMIT}" \ - -f "${IMAGE_FILE}" \ - -c "${REGISTRY_SCOPE}" || return $? - popd > /dev/null - else - fatal "${FORMAT,,} image ${USER_IMAGE:-${IMAGE_FILE}} missing" - return 1 - fi - ;; - - *) - fatal "Unsupported image format \"${FORMAT}\"" - return 1 - ;; - esac - done - - return 0 -} - -main "$@"; RESULT=$? -exit $RESULT diff --git a/automation/jenkins/aws/manageRdssnapshot.sh b/automation/jenkins/aws/manageRdssnapshot.sh deleted file mode 100755 index b2cf0c64..00000000 --- a/automation/jenkins/aws/manageRdssnapshot.sh +++ /dev/null @@ -1,357 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ -z ${AUTOMATION_DEBUG} ]] && rm -rf ./temp_*; exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# Defaults -SNAPSHOT_TAG_DEFAULT="latest" -SNAPSHOT_TYPE_DEFAULT="rdssnapshot" -SNAPSHOT_OPERATION_SAVE="save" -SNAPSHOT_OPERATION_VERIFY="verify" -SNAPSHOT_OPERATION_TAG="tag" -SNAPSHOT_OPERATION_PULL="pull" -SNAPSHOT_OPERATION_DEFAULT="${SNAPSHOT_OPERATION_VERIFY}" -SNAPSHOT_EXPAND_DEFAULT="false" -SNAPSHOT_REMOVE_SOURCE_DEFAULT="false" - -function usage() { - cat </dev/null 2>&1 -RESULT=$? -[[ "$RESULT" -ne 0 ]] && - fatal "Can't access ${SNAPSHOT_TYPE} registry at ${SNAPSHOT_PROVIDER}" && RESULT=1 && exit - -# Perform the required action -case ${SNAPSHOT_OPERATION} in - ${SNAPSHOT_OPERATION_SAVE}) - - info "Copying Snapshot from: ${SNAPSHOT_SOURCE} to: ${SNAPSHOT_IMAGE}" - - # A build will create a new snapshot we just need to bring it into the registry - aws --region "${SNAPSHOT_PROVIDER_REGION}" rds copy-db-snapshot --source-db-snapshot-identifier "${SNAPSHOT_SOURCE}" --target-db-snapshot-identifier "${SNAPSHOT_IMAGE}" --no-copy-tags --tags Key=RegistrySnapshot,Value="true" || exit $? - - info "Waiting for snapshot to become available..." - sleep 2 - aws --region "${SNAPSHOT_PROVIDER_REGION}" rds wait db-snapshot-completed --db-snapshot-identifier "${SNAPSHOT_IMAGE}" - - # remove the source snapshot once we have it in the registry - This makes sure a new build will be ok - info "Deleting Snapshot ${SNAPSHOT_SOURCE}" - aws --region "${SNAPSHOT_PROVIDER_REGION}" rds delete-db-snapshot --db-snapshot-identifier "${SNAPSHOT_SOURCE}" || exit $? - info "Waiting for snapshot to delete" - sleep 2 - aws --region "${SNAPSHOT_PROVIDER_REGION}" rds wait db-snapshot-deleted --db-snapshot-identifier "${SNAPSHOT_SOURCE}" - - ;; - - ${SNAPSHOT_OPERATION_VERIFY}) - # Check whether the image is already in the local registry - SNAPSHOT_ARN="$(aws --region "${SNAPSHOT_PROVIDER_REGION}" rds describe-db-snapshots --db-snapshot-identifier "${SNAPSHOT_IMAGE}" --query "DBSnapshots[0].DBSnapshotArn" --output text)" - if [[ -n "${SNAPSHOT_ARN}" ]]; then - SNAPSHOT_TAG="$(aws --region "${SNAPSHOT_PROVIDER_REGION}" rds list-tags-for-resource --resource-name "${SNAPSHOT_ARN}" --query "TagList[?Key==\`RegistryTag\`].Value|[0]" --output text)" - if [[ "${SNAPSHOT_TAG}" == "${SNAPSHOT_TAG}" ]]; then - info "${SNAPSHOT_TYPE^} image ${SNAPSHOT} present in the local registry" - RESULT=0 - exit - fi - fi - - info "${SNAPSHOT_TYPE^} image ${SNAPSHOT_IMAGE} with tag ${SNAPSHOT_TAG} not present in the local registry" - RESULT=1 - exit - ;; - - ${SNAPSHOT_OPERATION_TAG}) - # Check for the local image - SNAPSHOT_ARN="$(aws --region "${SNAPSHOT_PROVIDER_REGION}" rds describe-db-snapshots --db-snapshot-identifier "${SNAPSHOT_IMAGE}" --query "DBSnapshots[0].DBSnapshotArn" --output text)" - if [[ -z "${SNAPSHOT_ARN}" || "${SNAPSHOT_ARN}" == 'null' ]]; then - fatal "Can't find ${SNAPSHOT_IMAGE} in ${SNAPSHOT_PROVIDER}" - exit - else - aws --region "${SNAPSHOT_PROVIDER_REGION}" rds add-tags-to-resource --resource-name "${SNAPSHOT_ARN}" --tags Key=RegistryTag,Value="${REMOTE_SNAPSHOT_TAG}" - RESULT=$? - if [[ "${RESULT}" -ne 0 ]]; then - fatal "Couldn't tag image ${SNAPSHOT_IMAGE} with tag ${REMOTE_SNAPSHOT_TAG}" - exit $RESULT - fi - fi - ;; - - ${SNAPSHOT_OPERATION_PULL}) - # Get access to the remote registry - . ${AUTOMATION_DIR}/setCredentials.sh "${REMOTE_SNAPSHOT_PROVIDER}" - - # Confirm image is present - SNAPSHOT_SNAPSHOT_ARN="$(aws --region "${SNAPSHOT_PROVIDER_REGION}" rds describe-db-snapshots --db-snapshot-identifier "${SNAPSHOT_IMAGE}" --query "DBSnapshots[0].DBSnapshotArn" --output text )" - RESULT=$? - if [[ "$RESULT" -ne 0 ]]; then - fatal "Can't find ${SNAPSHOT_IMAGE} in ${REMOTE_SNAPSHOT_PROVIDER}" - exit 255 - else - - # Now see if its available in the local registry - . ${AUTOMATION_DIR}/setCredentials.sh "${SNAPSHOT_PROVIDER}" - - aws --region "${SNAPSHOT_PROVIDER_REGION}" rds describe-db-snapshots --db-snapshot-identifier "${SNAPSHOT_IMAGE}" >/dev/null 2>&1 - RESULT=$? - if [[ "$RESULT" -eq 0 ]]; then - info "Image ${SNAPSHOT_IMAGE} already available" - exit 0 - else - # share the snapshot from the remote registry to the local registry - . ${AUTOMATION_DIR}/setCredentials.sh "${REMOTE_SNAPSHOT_PROVIDER}" - aws --region "${REMOTE_SNAPSHOT_PROVIDER_REGION}" rds modify-db-snapshot-attribute --db-snapshot-identifier "${REMOTE_SNAPSHOT_IMAGE}" --attribute-name restore --values-to-add "${!SNAPSHOT_PROVIDER_AWS_ACCOUNT_ID_VAR}" >/dev/null 2>&1 - - RESULT=$? - if [[ "${RESULT}" -ne 0 ]]; then - fatal "Could not share image ${REMOTE_SNAPSHOT_IMAGE} with account ${AWS_CRED_AWS_ACCOUNT_ID_VAR}" - exit $RESULT - fi - - # now copy the snapshot to the local registry so we have our own copy - . ${AUTOMATION_DIR}/setCredentials.sh "${SNAPSHOT_PROVIDER}" - - # A build will create a new snapshot we just need to bring it into the registry - LOCAL_SNAPSHOT_IMAGE_ARN="$(aws --region "${SNAPSHOT_PROVIDER_REGION}" rds copy-db-snapshot --source-db-snapshot-identifier "${SNAPSHOT_SNAPSHOT_ARN}" --target-db-snapshot-identifier "${SNAPSHOT_IMAGE}" --query 'DBSnapshot.DBSnapshotArn' --tags Key=RegistrySnapshot,Value="true" --no-copy-tags --output text || exit $?)" - - if [[ -n "${LOCAL_SNAPSHOT_IMAGE_ARN}" ]]; then - info "Waiting for snapshot to become available..." - sleep 2 - aws --region "${REMOTE_SNAPSHOT_PROVIDER_REGION}" rds wait db-snapshot-completed --db-snapshot-identifier "${LOCAL_SNAPSHOT_IMAGE_ARN}" || exit $? - info "Registry image ${SNAPSHOT_IMAGE} should now be available" - else - fatal "Registry image ${SNAPSHOT_IMAGE} could not be copied" - exit 255 - fi - fi - fi - ;; - - *) - fatal "Unknown operation \"${SNAPSHOT_OPERATION}\"" && RESULT=1 && exit - ;; -esac - -# All good -RESULT=0 diff --git a/automation/jenkins/aws/manageRepo.sh b/automation/jenkins/aws/manageRepo.sh deleted file mode 100755 index 52561f9f..00000000 --- a/automation/jenkins/aws/manageRepo.sh +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit -1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -REPO_OPERATION_CLONE="clone" -REPO_OPERATION_INIT="init" -REPO_OPERATION_PUSH="push" - -# Defaults -REPO_OPERATION_DEFAULT="${REPO_OPERATION_PUSH}" -REPO_REMOTE_DEFAULT="origin" -REPO_BRANCH_DEFAULT="master" -DEFER_REPO_PUSH_DEFAULT="false" - -function usage() { - cat </dev/null 2>&1 - if [[ $? -ne 0 ]]; then - # Convert directory into a repo - git init . - fi - - check_for_invalid_environment_variables "REPO_REMOTE" || return $? - - git remote show "${REPO_REMOTE}" >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - check_for_invalid_environment_variables "REPO_URL" || return $? - - if ! git remote add "${REPO_REMOTE}" "${REPO_URL}"; then - fatal "Can't add remote ${REPO_REMOTE} to ${REPO_LOG_NAME} repo" - return 1 - fi - fi - - git log -n 1 >/dev/null 2>&1 - if [[ $? -ne 0 ]]; then - # Create basic files - echo -e "# ${REPO_LOG_NAME}" > README.md - touch .gitignore LICENSE.md - - # Commit to repo in preparation for first push - REPO_MESSAGE="${REPO_MESSAGE:-Initial commit}" - push - fi -} - -function clone() { - debug "Cloning the ${REPO_LOG_NAME} repo and checking out the ${REPO_BRANCH} branch ..." - check_for_invalid_environment_variables "REPO_URL" "REPO_BRANCH" || return $? - - if ! git clone -b "${REPO_BRANCH}" "${REPO_URL}" .; then - fatal "Can't clone ${REPO_LOG_NAME} repo" - return 1 - fi -} - -function push() { - - if ! git -C "${REPO_DIR}" rev-parse --is-inside-work-tree &>/dev/null; then - warning "Directory ${REPO_DIR} is not part of a git repo - skipping push" - return 0 - fi - - # Break the message in name/value pairs - conventional_commit_base_body="$(format_conventional_commit_body "${REPO_MESSAGE}")" - - # Separate the values based on the conventional commit format - conventional_commit_type="$( format_conventional_commit_body_summary "${conventional_commit_base_body}" "cctype" )" - conventional_commit_scope="$( format_conventional_commit_body_summary "${conventional_commit_base_body}" "account product environment segment" )" - conventional_commit_description="$( format_conventional_commit_body_summary "${conventional_commit_base_body}" "ccdesc" )" - conventional_commit_body="$( format_conventional_commit_body_subset "${conventional_commit_base_body}" "cctype ccdesc account product environment segment" )" - - formatted_commit_message="$(format_conventional_commit \ - "${conventional_commit_type:-hamlet}" \ - "${conventional_commit_scope}" \ - "${conventional_commit_description:-automation}" \ - "${conventional_commit_body}" )" - - # Extract relevant events and remove the event log - repo_event_log="$( getTempFile XXXXXXX )" - pull_events_from_state "directory" "$(git -C "${REPO_DIR}" rev-parse --show-toplevel)" "${repo_event_log}" "starts_with" - - if [[ -s "${repo_event_log}" ]]; then - - commit_logs=("$( jq -rc '.events[] | del(._id, .directory)' "${repo_event_log}")") - - if [[ -n "${commit_logs}" ]]; then - - formatted_commit_message+=$'\n\n' - - while read msg; do - formatted_commit_message+="$(echo "${msg}" | jq -r 'to_entries|map("\(.key): \(.value|tostring)")|.[]')" - formatted_commit_message+=$'\n--------\n\n' - - done <<< "${commit_logs}" - fi - fi - - check_for_invalid_environment_variables "GIT_USER" "GIT_EMAIL" "REPO_MESSAGE" "REPO_REMOTE" || return $? - - # Make sure we can access the remote and that the branch exists - git ls-remote -q "${REPO_REMOTE}" "${REPO_BRANCH}" 1> /dev/null || return $? - - # Ensure git knows who we are - git config user.name "${GIT_USER}" - git config user.email "${GIT_EMAIL}" - - # Add anything that has been added/modified/deleted - git add -A - - if [[ -n "$(git status --porcelain)" ]]; then - # Commit changes - debug "Committing to the ${REPO_LOG_NAME} repo..." - - if ! git commit -m "${formatted_commit_message}"; then - fatal "Can't commit to the ${REPO_LOG_NAME} repo" - return 1 - fi - - REPO_PUSH_REQUIRED="true" - else - info "no changes to ${REPO_DIR}" - fi - - # Tag the commit if required - if [[ -n "${REPO_TAG}" ]]; then - EXISTING_TAG=$(git ls-remote --tags 2>/dev/null | grep "refs/tags/${REPO_TAG}$") - if [[ -n "${EXISTING_TAG}" ]]; then - warning "Tag ${REPO_TAG} not added to the ${REPO_LOG_NAME} repo - it is already present" - else - debug "Adding tag \"${REPO_TAG}\" to the ${REPO_LOG_NAME} repo..." - if ! git tag -a "${REPO_TAG}" -m "${REPO_MESSAGE}"; then - fatal "Can't tag the ${REPO_LOG_NAME} repo" - return 1 - fi - - REPO_PUSH_REQUIRED="true" - fi - fi - - # Update upstream repo - GENERATION_REPO_PUSH_RETRIES="${GENERATION_REPO_PUSH_RETRIES:-6}" - REPO_PUSHED=false - if [[ ("${DEFER_REPO_PUSH}" != "true") && ("${REPO_PUSH_REQUIRED}" == "true") ]]; then - for TRY in $( seq 1 ${GENERATION_REPO_PUSH_RETRIES} ); do - # Check if remote branch exists - EXISTING_BRANCH=$(git ls-remote --heads 2>/dev/null | grep "refs/heads/${REPO_BRANCH}$") - if [[ -n "${EXISTING_BRANCH}" ]]; then - debug "Rebasing ${REPO_LOG_NAME} in case of changes..." - if ! git pull --rebase ${REPO_REMOTE} ${REPO_BRANCH}; then - fatal "Can't rebase the ${REPO_LOG_NAME} repo from upstream ${REPO_REMOTE}" - return 1 - fi - fi - - debug "Pushing the ${REPO_LOG_NAME} repo upstream..." - if git symbolic-ref -q HEAD; then - if git push --tags ${REPO_REMOTE} ${REPO_BRANCH}; then - REPO_PUSHED=true - break - else - info "Waiting to retry push to ${REPO_LOG_NAME} repo ..." - sleep 5 - fi - else - # If push failed HEAD might be detached. Create a temp branch and merge it to the target to fix it. - git branch temp-${REPO_BRANCH} && \ - git checkout ${REPO_BRANCH} && \ - git merge temp-${REPO_BRANCH} && \ - git branch -D temp-${REPO_BRANCH} && \ - git push --tags ${REPO_REMOTE} ${REPO_BRANCH} && REPO_PUSHED=true - fi - done - if [[ "${REPO_PUSHED}" == "false" ]]; then - fatal "Can't push the ${REPO_LOG_NAME} repo changes to upstream repo ${REPO_REMOTE}" - return 1 - fi - fi -} - -# Define git provider attributes -# $1 = provider -# $2 = variable prefix -function defineGitProviderAttributes() { - DGPA_PROVIDER="${1^^}" - DGPA_PREFIX="${2^^}" - - # Attribute variable names - for DGPA_ATTRIBUTE in "DNS" "API_DNS" "ORG" "CREDENTIALS_VAR"; do - DGPA_PROVIDER_VAR="${DGPA_PROVIDER}_GIT_${DGPA_ATTRIBUTE}" - declare -g ${DGPA_PREFIX}_${DGPA_ATTRIBUTE}="${!DGPA_PROVIDER_VAR}" - done -} - -function set_context() { - # Parse options - while getopts ":b:cd:e:hil:m:n:pqr:s:t:u:v:" opt; do - case $opt in - b) REPO_BRANCH="${OPTARG}" ;; - c) REPO_OPERATION="${REPO_OPERATION_CLONE}" ;; - d) REPO_DIR="${OPTARG}" ;; - e) GIT_EMAIL="${OPTARG}" ;; - h) usage; return 1 ;; - i) REPO_OPERATION="${REPO_OPERATION_INIT}" ;; - l) REPO_LOG_NAME="${OPTARG}" ;; - m) REPO_MESSAGE="${OPTARG}" ;; - n) REPO_NAME="${OPTARG}" ;; - p) REPO_OPERATION="${REPO_OPERATION_PUSH}" ;; - q) DEFER_REPO_PUSH="true" ;; - r) REPO_REMOTE="${OPTARG}" ;; - s) GIT_USER="${OPTARG}" ;; - t) REPO_TAG="${OPTARG}" ;; - u) REPO_URL="${OPTARG}" ;; - v) REPO_PROVIDER="${OPTARG}" ;; - \?) fatalOption; return 1 ;; - :) fatalOptionArgument; return 1 ;; - esac - done - - # Apply defaults - DEFER_REPO_PUSH="${DEFER_REPO_PUSH:-${DEFER_REPO_PUSH_DEFAULT}}" - REPO_OPERATION="${REPO_OPERATION:-${REPO_OPERATION_DEFAULT}}" - REPO_REMOTE="${REPO_REMOTE:-${REPO_REMOTE_DEFAULT}}" - REPO_BRANCH="${REPO_BRANCH:-${REPO_BRANCH_DEFAULT}}" - if [[ -z "${REPO_URL}" ]]; then - if [[ (-n "${REPO_PROVIDER}") && - (-n "${REPO_NAME}") ]]; then - defineGitProviderAttributes "${REPO_PROVIDER}" "REPO_PROVIDER" - if [[ -n "${!REPO_PROVIDER_CREDENTIALS_VAR}" ]]; then - REPO_URL="https://${!REPO_PROVIDER_CREDENTIALS_VAR}@${REPO_PROVIDER_DNS}/${REPO_PROVIDER_ORG}/${REPO_NAME}" - else - REPO_URL="https://${REPO_PROVIDER_DNS}/${REPO_PROVIDER_ORG}/${REPO_NAME}" - fi - fi - fi - - # Ensure mandatory arguments have been provided - check_for_invalid_environment_variables "REPO_DIR" "REPO_LOG_NAME" || return $? - - # Ensure we are inside the repo directory - if [[ ! -d "${REPO_DIR}" ]]; then - if ! mkdir -p "${REPO_DIR}"; then - fatal "Can't create repo directory ${REPO_DIR}" - return 1 - fi - fi - - return 0 -} - -function main() { - - set_context "$@" || return 1 - - cd "${REPO_DIR}" - - # Perform the required action - case ${REPO_OPERATION} in - ${REPO_OPERATION_INIT}) init || return $? ;; - ${REPO_OPERATION_CLONE}) clone || return $? ;; - ${REPO_OPERATION_PUSH}) push || return $? ;; - esac - - # All good - return 0 -} - -main "$@" diff --git a/automation/jenkins/aws/manageS3Registry.sh b/automation/jenkins/aws/manageS3Registry.sh deleted file mode 100755 index a8b75103..00000000 --- a/automation/jenkins/aws/manageS3Registry.sh +++ /dev/null @@ -1,439 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap '[[ -z ${AUTOMATION_DEBUG} ]] && rm -rf ./temp_*; exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - -# Defaults -REGISTRY_TAG_DEFAULT="latest" -REGISTRY_TYPE_DEFAULT="lambda" -REGISTRY_FILENAME_DEFAULT="lambda.zip" -REGISTRY_OPERATION_SAVE="save" -REGISTRY_OPERATION_VERIFY="verify" -REGISTRY_OPERATION_TAG="tag" -REGISTRY_OPERATION_PULL="pull" -REGISTRY_OPERATION_DEFAULT="${REGISTRY_OPERATION_VERIFY}" -REGISTRY_EXPAND_DEFAULT="false" -REGISTRY_REMOVE_SOURCE_DEFAULT="false" - -function usage() { - cat </dev/null 2>&1 - RESULT=$? - [[ "$RESULT" -ne 0 ]] && - fatal "Can't access ${FILE_TO_COPY}" && RESULT=1 && exit - - aws --region "${REGISTRY_PROVIDER_REGION}" s3 cp --no-progress --recursive "${FILE_TO_COPY}" "${FULL_REGISTRY_IMAGE_PATH}/" - - else - - rm -rf "${FILES_TEMP_DIR}" - mkdir -p "${FILES_TEMP_DIR}" - cp "${FILE_TO_COPY}" "${FILES_TEMP_DIR}/${SAVE_AS}" - RESULT=$? - [[ $RESULT -ne 0 ]] && fatal "Unable to copy ${FILE_TO_COPY}" && RESULT=1 && exit - - if [[ ("${REGISTRY_EXPAND}" == "true") && - ("${FILE_TO_COPY##*.}" == "zip") ]]; then - unzip "${FILE_TO_COPY}" -d "${FILES_TEMP_DIR}" - RESULT=$? - [[ $RESULT -ne 0 ]] && - fatal "Unable to unzip ${FILE_TO_COPY}" && RESULT=1 && exit - fi - - aws --region "${REGISTRY_PROVIDER_REGION}" s3 cp --no-progress --recursive "${FILES_TEMP_DIR}/" "${FULL_REGISTRY_IMAGE_PATH}/" - RESULT=$? - [[ $RESULT -ne 0 ]] && - fatal "Unable to save ${BASE_REGISTRY_FILENAME} in the local registry" && RESULT=1 && exit - - fi - - aws --region "${REGISTRY_PROVIDER_REGION}" s3 cp --no-progress "${TAG_FILE}" "${FULL_TAGGED_REGISTRY_IMAGE}" - RESULT=$? - [[ $RESULT -ne 0 ]] && - fatal "Unable to tag ${BASE_REGISTRY_FILENAME} as latest" && RESULT=1 && exit -} - -# Remove the source S3 content. This is used to keep the S3 Stage clean for new uploads -function removeSource() { - local FILE_TO_REMOVE="${1}" - - info "removing ${FILE_TO_REMOVE}" - - if [[ "${FILE_TO_REMOVE}" =~ ^s3:// ]]; then - - aws --region "${REGISTRY_PROVIDER_REGION}" s3 ls "${FILE_TO_REMOVE}" >/dev/null 2>&1 - RESULT=$? - [[ "$RESULT" -ne 0 ]] && - fatal "Can't access ${FILE_TO_REMOVE}" && return 128 - - aws --region "${REGISTRY_PROVIDER_REGION}" s3 rm --recursive "${FILE_TO_REMOVE}" - - else - info "Local data not removed as it is temporary anyway" - fi -} - -# Apply local registry defaults -REGISTRY_TYPE="${REGISTRY_TYPE,,:-${REGISTRY_TYPE_DEFAULT}}" -REGISTRY_PROVIDER_VAR="PRODUCT_${REGISTRY_TYPE^^}_PROVIDER" -REGISTRY_PROVIDER="${REGISTRY_PROVIDER:-${!REGISTRY_PROVIDER_VAR}}" -REGISTRY_FILENAME="${REGISTRY_FILENAME:-${REGISTRY_FILENAME_DEFAULT}}" -BASE_REGISTRY_FILENAME="${REGISTRY_FILENAME##*/}" -REGISTRY_TAG="${REGISTRY_TAG:-${REGISTRY_TAG_DEFAULT}}" -REGISTRY_OPERATION="${REGISTRY_OPERATION:-${REGISTRY_OPERATION_DEFAULT}}" -REGISTRY_PRODUCT="${REGISTRY_PRODUCT:-${PRODUCT}}" -REGISTRY_REMOVE_SOURCE="${REGISTRY_REMOVE_SOURCE:-${REGISTRY_REMOVE_SOURCE_DEFAULT}}" - -# Handle registry scope values -REGISTRY_SUBTYPE="" -case "${REGISTRY_SCOPE}" in - account) - if [[ -n "${ACCOUNT}" ]]; then - REGISTRY_PRODUCT="account" - fi - ;; - segment) - if [[ -n "${SEGMENT}" ]]; then - REGISTRY_SUBTYPE="/${SEGMENT}" - else - fatal "Segment scoped registry required but SEGMENT not defined" && RESULT=1 && exit - fi - ;; - *) - [[ "${REGISTRY_SCOPE:-unset}" != "unset" && "${REGISTRY_SCOPE}" != "?" ]] && REGISTRY_SUBTYPE="/${REGISTRY_SCOPE}" - ;; -esac - -# Default local repository is based on standard image naming conventions -if [[ (-n "${REGISTRY_PRODUCT}") && - (-n "${REGISTRY_CODE_COMMIT}") ]]; then - if [[ (-n "${REGISTRY_DEPLOYMENT_UNIT}" ) ]]; then - REGISTRY_REPO="${REGISTRY_REPO:-${REGISTRY_PRODUCT}${REGISTRY_SUBTYPE}/${REGISTRY_DEPLOYMENT_UNIT}/${REGISTRY_CODE_COMMIT}}" - else - REGISTRY_REPO="${REGISTRY_REPO:-${REGISTRY_PRODUCT}${REGISTRY_SUBTYPE}/${REGISTRY_CODE_COMMIT}}" - fi -fi - -# Empty file for tagging operations -TAG_FILE="./temp_${REGISTRY_TAG}" -touch "${TAG_FILE}" - - -# Determine registry provider details -defineRegistryProviderAttributes "${REGISTRY_PROVIDER}" "${REGISTRY_TYPE}" "REGISTRY_PROVIDER" - -# Ensure the local repository has been determined -[[ -z "${REGISTRY_REPO}" ]] && - fatal "Job requires the local repository name, or the product/deployment unit/commit" && RESULT=1 && exit - -# Apply remote registry defaults -REMOTE_REGISTRY_PROVIDER_VAR="PRODUCT_REMOTE_${REGISTRY_TYPE^^}_PROVIDER" -REMOTE_REGISTRY_PROVIDER="${REMOTE_REGISTRY_PROVIDER:-${!REMOTE_REGISTRY_PROVIDER_VAR}}" -REMOTE_REGISTRY_REPO="${REMOTE_REGISTRY_REPO:-$REGISTRY_REPO}" -REMOTE_REGISTRY_TAG="${REMOTE_REGISTRY_TAG:-$REGISTRY_TAG}" - -# Determine remote registry provider details -defineRegistryProviderAttributes "${REMOTE_REGISTRY_PROVIDER}" "${REGISTRY_TYPE}" "REMOTE_REGISTRY_PROVIDER" - -# pull = tag if local provider = remote provider -if [[ ("${REGISTRY_PROVIDER}" == "${REMOTE_REGISTRY_PROVIDER}") && - ("${REGISTRY_OPERATION}" == "${REGISTRY_OPERATION_PULL}") ]]; then - REGISTRY_OPERATION="${REGISTRY_OPERATION_TAG}" -fi - -# Formulate the local registry details -REGISTRY_IMAGE="${REGISTRY_TYPE}/${REGISTRY_REPO}/${BASE_REGISTRY_FILENAME}" -TAGGED_REGISTRY_IMAGE="${REGISTRY_TYPE}/${REGISTRY_REPO}/tags/${REGISTRY_TAG}" -FULL_REGISTRY_IMAGE="s3://${REGISTRY_PROVIDER_DNS}/${REGISTRY_IMAGE}" -FULL_REGISTRY_IMAGE_PATH="s3://${REGISTRY_PROVIDER_DNS}/${REGISTRY_TYPE}/${REGISTRY_REPO}" -FULL_TAGGED_REGISTRY_IMAGE="s3://${REGISTRY_PROVIDER_DNS}/${TAGGED_REGISTRY_IMAGE}" - -# Set up credentials for registry access -. ${AUTOMATION_DIR}/setCredentials.sh "${REGISTRY_PROVIDER}" - -# Confirm access to the local registry -aws --region "${REGISTRY_PROVIDER_REGION}" s3 ls "s3://${REGISTRY_PROVIDER_DNS}/${REGISTRY_TYPE}" >/dev/null 2>&1 -RESULT=$? -[[ "$RESULT" -ne 0 ]] && - fatal "Can't access ${REGISTRY_TYPE} registry at ${REGISTRY_PROVIDER_DNS}" && RESULT=1 && exit - -# Perform the required action -case ${REGISTRY_OPERATION} in - ${REGISTRY_OPERATION_SAVE}) - copyToRegistry "${REGISTRY_FILENAME}" "${BASE_REGISTRY_FILENAME}" - if [[ -n "${REGISTRY_ADDITIONAL_DIRECTORY}" ]]; then - copyToRegistry "${REGISTRY_ADDITIONAL_DIRECTORY}" - fi - - # Clean out the source staging directory - if [[ "${REGISTRY_REMOVE_SOURCE}" == "true" ]]; then - removeSource "${REGISTRY_FILENAME}" - if [[ -n "${REGISTRY_ADDITIONAL_DIRECTORY}" ]]; then - removeSource "${REGISTRY_ADDITIONAL_DIRECTORY}" - fi - fi - - log_write_event "s3_reg_save_${REGISTRY_FILENAME//\//_}" "registry_save_s3" "$(dirname ${REGISTRY_FILENAME})" \ - "aws_account=$(aws sts get-caller-identity --query 'Account' --output text)" \ - "image=${FULL_REGISTRY_IMAGE_PATH}" "registry_commit=${REGISTRY_CODE_COMMIT}" \ - "registry_tag=${REGISTRY_TAG}" "registry_type=${REGISTRY_TYPE}" - ;; - - ${REGISTRY_OPERATION_VERIFY}) - # Check whether the image is already in the local registry - aws --region "${REGISTRY_PROVIDER_REGION}" s3 ls "${FULL_TAGGED_REGISTRY_IMAGE}" >/dev/null 2>&1 - RESULT=$? - if [[ "${RESULT}" -eq 0 ]]; then - info "${REGISTRY_TYPE^} image ${REGISTRY_IMAGE} present in the local registry" - exit - else - info "${REGISTRY_TYPE^} image ${REGISTRY_IMAGE} with tag ${REGISTRY_TAG} not present in the local registry" - exit - fi - ;; - - ${REGISTRY_OPERATION_TAG}) - # Formulate the remote registry details - REMOTE_TAGGED_REGISTRY_IMAGE="${REGISTRY_TYPE}/${REMOTE_REGISTRY_REPO}/tags/${REMOTE_REGISTRY_TAG}" - FULL_REMOTE_TAGGED_REGISTRY_IMAGE="s3://${REGISTRY_PROVIDER_DNS}/${REMOTE_TAGGED_REGISTRY_IMAGE}" - - # Check for the local image - aws --region "${REGISTRY_PROVIDER_REGION}" s3 ls "${FULL_REGISTRY_IMAGE}" >/dev/null 2>&1 - RESULT=$? - if [[ "$RESULT" -ne 0 ]]; then - fatal "Can't find ${REGISTRY_IMAGE} in ${REGISTRY_PROVIDER_DNS}" && RESULT=1 && exit - else - # Copy to S3 - aws --region "${REGISTRY_PROVIDER_REGION}" s3 cp --no-progress "${TAG_FILE}" "${FULL_REMOTE_TAGGED_REGISTRY_IMAGE}" - RESULT=$? - [[ "${RESULT}" -ne 0 ]] && - fatal "Couldn't tag image ${FULL_REGISTRY_IMAGE} with tag ${REMOTE_REGISTRY_TAG}" && RESULT=1 && exit - fi - ;; - - ${REGISTRY_OPERATION_PULL}) - # Formulate the remote registry details - REMOTE_REGISTRY_IMAGE="${REGISTRY_TYPE}/${REMOTE_REGISTRY_REPO}/${BASE_REGISTRY_FILENAME}" - REMOTE_REGISTRY_PATH="${REGISTRY_TYPE}/${REMOTE_REGISTRY_REPO}" - REMOTE_TAGGED_REGISTRY_IMAGE="${REGISTRY_TYPE}/${REMOTE_REGISTRY_REPO}/tags/${REMOTE_REGISTRY_TAG}" - FULL_REMOTE_REGISTRY_IMAGE="s3://${REMOTE_REGISTRY_PROVIDER_DNS}/${REMOTE_REGISTRY_IMAGE}" - FULL_REMOTE_TAGGED_REGISTRY_IMAGE="s3://${REMOTE_REGISTRY_PROVIDER_DNS}/${REMOTE_TAGGED_REGISTRY_IMAGE}" - FULL_REMOTE_REGISTRY_PATH="s3://${REMOTE_REGISTRY_PROVIDER_DNS}/${REMOTE_REGISTRY_PATH}" - IMAGE_FILE="./temp_${BASE_REGISTRY_FILENAME}" - - # Get access to the remote registry - . ${AUTOMATION_DIR}/setCredentials.sh "${REMOTE_REGISTRY_PROVIDER}" - - # Confirm image is present - aws --region "${REMOTE_REGISTRY_PROVIDER_REGION}" s3 ls "${FULL_REMOTE_TAGGED_REGISTRY_IMAGE}" >/dev/null 2>&1 - RESULT=$? - if [[ "$RESULT" -ne 0 ]]; then - fatal "Can't find ${REMOTE_REGISTRY_IMAGE} in ${REMOTE_REGISTRY_PROVIDER_DNS}" && RESULT=1 && exit - else - # Copy image - aws --region "${REGISTRY_PROVIDER_REGION}" s3 cp --no-progress "${FULL_REMOTE_REGISTRY_IMAGE}" "${IMAGE_FILE}" - RESULT=$? - [[ "$RESULT" -ne 0 ]] && - fatal "Can't copy remote image ${FULL_REMOTE_REGISTRY_IMAGE}" && RESULT=1 && exit - fi - - # Now copy to local rgistry - . ${AUTOMATION_DIR}/setCredentials.sh "${REGISTRY_PROVIDER}" - - copyToRegistry "${IMAGE_FILE}" "${BASE_REGISTRY_FILENAME}" - if [[ -n "${REGISTRY_ADDITIONAL_DIRECTORY}" ]]; then - if [[ "${REGISTRY_ADDITIONAL_DIRECTORY}" == "REGISTRY_CONTENT" ]]; then - copyToRegistry "${FULL_REMOTE_REGISTRY_PATH}" - else - copyToRegistry "${REGISTRY_ADDITIONAL_DIRECTORY}" - fi - fi - - log_write_event "s3_reg_pull_${IMAGE_FILE//\//_}" "registry_save_s3" "$(dirname ${IMAGE_FILE})" \ - "aws_account=$(aws sts get-caller-identity --query 'Account' --output text)" \ - "image=${FULL_REGISTRY_IMAGE_PATH}" "remote_registry_image=${FULL_REMOTE_REGISTRY_IMAGE}" \ - "registry_type=${REGISTRY_TYPE}" - ;; - - *) - fatal "Unknown operation \"${REGISTRY_OPERATION}\"" && RESULT=1 && exit - ;; -esac - -# All good -RESULT=0 diff --git a/automation/jenkins/aws/manageUnits.sh b/automation/jenkins/aws/manageUnits.sh deleted file mode 100755 index b05a65b2..00000000 --- a/automation/jenkins/aws/manageUnits.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} -trap 'exit 1' SIGHUP SIGINT SIGTERM -. "${AUTOMATION_BASE_DIR}/common.sh" - - -# DEPRECATED -deprecated_script - -#Defaults - -function usage() { - cat < /tmp/ssl.conf <> /tmp/ssl.conf < /dev/null) - RESULT=$? - if [[ "${RESULT}" -eq 0 ]]; then - echo -e "${!VAR_ATTRIBUTE}=${VALUE}" - else -# if [[ "${!VAR_ATTRIBUTE}" == "AccessKey" ]]; then - # AccessKey value matches base64 regex so show raw value - echo -e "${!VAR_ATTRIBUTE}=${RAW_VALUE}" -# fi - fi - else - echo -e "${!VAR_ATTRIBUTE}=${RAW_VALUE}" - fi - fi -done - -# All good -RESULT=0 diff --git a/cli/manageCrypto.sh b/cli/manageCrypto.sh deleted file mode 100755 index e93b9901..00000000 --- a/cli/manageCrypto.sh +++ /dev/null @@ -1,429 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${GENERATION_DEBUG}" ]] && set ${GENERATION_DEBUG} -trap '. ${GENERATION_BASE_DIR}/execution/cleanupContext.sh; exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${GENERATION_BASE_DIR}/execution/common.sh" - - -# DEPRECATED -deprecated_script - -BASE64_REGEX="^[A-Za-z0-9+/=\n]\+$" - -# Defaults -CRYPTO_OPERATION_DEFAULT="decrypt" -CRYPTO_FILENAME_DEFAULT="credentials.json" -PREFIX_DEFAULT="base64" -KEY_SCOPE_DEFAULT="segment" - -tmp_dir="$(getTempDir "cote_crypto_XXX")" - -function usage() { - cat < /dev/null - BUILD_BLUEPRINT="${tmp_dir}/buildblueprint-${DEPLOYMENT_GROUP}-${DEPLOYMENT_UNIT}-config.json" - - if [[ ! -f "${BUILD_BLUEPRINT}" || -z "$(cat ${BUILD_BLUEPRINT} )" ]]; then - fatal "Could not generate blueprint for task details" - return 255 - fi - - case "${KEY_SCOPE}" in - "segment") - - arrayFromList "KEY_IDS" "$(jq -r '.Occurrence.Occurrences[] | select( .Core.Type == "baselinekey" and .Configuration.Solution.Engine == "cmk" ) | .State.Attributes.ARN' < ${BUILD_BLUEPRINT})" - - if [[ "$(arraySize "KEY_IDS" )" > 1 ]]; then - fatal "Multiple keys found - please run again using the -k parameter" - fatal "Keys Found: $(listFromArray "KEYID" )" - return 255 - else - KEYID="${KEY_IDS[0]}" - fi - ;; - - "account"|"root"|"integrator") - - KEYID="$(jq -r '.Occurrence.Occurrences[] | select( .Core.Type == "baselinekey" and .Configuration.Solution.Engine == "cmk-account" ) | .State.Attributes.ARN' < ${BUILD_BLUEPRINT})" - - ;; - esac - - if [[ -z "${KEYID}" ]]; then - fatal "No key material available" - return 255 - fi - - debug "Key to be used is ${KEYID}" - ;; - esac - fi - - # Location base file search - case "${KEY_SCOPE}" in - "segment") - if [[ -n "${CRYPTO_FILE}" ]]; then - FILES+=("${SEGMENT_OPERATIONS_DIR}/${CRYPTO_FILE}") - fi - FILES+=("${SEGMENT_OPERATIONS_DIR}/${CRYPTO_FILENAME_DEFAULT}") - ;; - - "account") - if [[ -n "${CRYPTO_FILE}" ]]; then - FILES+=("${ACCOUNT_OPERATIONS_DIR}/${CRYPTO_FILENAME_DEFAULT}") - fi - FILES+=("${ACCOUNT_OPERATIONS_DIR}/${CRYPTO_FILENAME_DEFAULT}") - esac - - # Try and locate file - for F in "${FILES[@]}"; do - if [[ -f "${F}" ]]; then - TARGET_FILE="${F}" - debug "Target file is ${TARGET_FILE}" - break - fi - done - - # Ensure mandatory arguments have been provided - if [[ (-n "${JSON_PATH}") ]]; then - - if [[ -z "${TARGET_FILE}" ]]; then - fatal "Can't locate target file" - return 255 - fi - - PATH_PARTS=(${JSON_PATH//./ }) - - # Use jq [] syntax to handle dash in parts - ESCAPED_JSON_PATH="." - for PATH_PART in "${PATH_PARTS[@]}"; do - ESCAPED_JSON_PATH="${ESCAPED_JSON_PATH}[\"${PATH_PART}\"]" - done - - debug "jq path in file is ${ESCAPED_JSON_PATH}" - - # Default cipherdata to that in the element - JSON_TEXT=$(jq -r "${ESCAPED_JSON_PATH} | select (.!=null)" < "${TARGET_FILE}") - CRYPTO_TEXT="${CRYPTO_TEXT:-$JSON_TEXT}" - - [[ (("${CRYPTO_OPERATION}" == "encrypt") && (-z "${CRYPTO_TEXT}")) ]] && - fatal "Nothing to encrypt" && return 255 - else - if [[ -z "${CRYPTO_TEXT}" ]]; then - [[ -z "${CRYPTO_FILE}" ]] && insufficientArgumentsError && return 255 - [[ -z "${TARGET_FILE}" ]] && fatal "Can't locate file based on provided path" && return 255 - - # Default cipherdata to the file contents - FILE_TEXT=$( cat "${TARGET_FILE}") - CRYPTO_TEXT="${CRYPTO_TEXT:-$FILE_TEXT}" - fi - fi - - # Force options if required - case ${CRYPTO_OPERATION} in - encrypt) - CRYPTO_VISIBLE="false" - ;; - decrypt) - CRYPTO_DECODE="true" - ;; - reencrypt) - CRYPTO_VISIBLE="false" - CRYPTO_DECODE="true" - ;; - listcmk) - CRYPTO_VISIBLE="false" - CRYPTO_DECODE="true" - CRYPTO_UPDATE="false" - ;; - esac - - - # Strip any explicit prefix indication of encoding/encryption engine - if [[ $(grep "^${PREFIX}:" <<< "${CRYPTO_TEXT}") ]]; then - [[ "${PREFIX,,}" == "base64" ]] && CRYPTO_DECODE="true" - CRYPTO_TEXT="${CRYPTO_TEXT#${PREFIX}:}" - fi - - ciphertext_src="${tmp_dir}/ciphertext.src" - ciphertext_bin="${tmp_dir}/ciphertext.bin" - - # Prepare ciphertext for processing - echo -n "${CRYPTO_TEXT}" > "${ciphertext_src}" - - # base64 decode if necessary - if [[ (-n "${CRYPTO_DECODE}") ]]; then - # Sanity check on input - grep -q "${BASE64_REGEX}" "${ciphertext_src}" - RESULT=$? - if [[ "${RESULT}" -eq 0 ]]; then - base64 -d "${ciphertext_src}" > "${ciphertext_bin}" - else - fatal "Input doesn't appear to be base64 encoded" - return 255 - fi - else - mv "${ciphertext_src}" "${ciphertext_bin}" - fi - - # Perform the operation - case ${CRYPTO_OPERATION} in - encrypt) - cli_v1="$(aws --version | grep 'aws-cli/1.')" - if [[ -n "$cli_v1" ]] ; then - cli_encrypt="kms encrypt" - else - cli_encrypt="kms encrypt --cli-binary-format raw-in-base64-out" - fi - - CRYPTO_TEXT=$(cd "${tmp_dir}"; aws --region ${REGION} --output text ${cli_encrypt} \ - --key-id "${KEYID}" --query CiphertextBlob \ - --plaintext "fileb://ciphertext.bin") - ;; - - decrypt) - CRYPTO_TEXT=$(cd "${tmp_dir}"; aws --region ${REGION} --output text kms decrypt \ - --query Plaintext \ - --ciphertext-blob "fileb://ciphertext.bin") - ;; - reencrypt) - CRYPTO_TEXT=$(cd "${tmp_dir}"; aws --region ${REGION} --output text kms re-encrypt \ - --query CiphertextBlob \ - --destination-key-id "${KEYID}" \ - --ciphertext-blob "fileb://ciphertext.bin") - ;; - listcmk) - CMK_ARN=$(cd "${tmp_dir}"; aws --region ${REGION} --output text kms re-encrypt \ - --query SourceKeyId \ - --destination-key-id "${KEYID}" \ - --ciphertext-blob "fileb://ciphertext.bin") - CMK_ALIAS=$(cd "${tmp_dir}"; aws --region ${REGION} --output text kms list-aliases \ - --key-id "${CMK_ARN}" \ - --query "Aliases[0].AliasName") - # List only - force settings accordingly - CRYPTO_TEXT="ALIAS=${CMK_ALIAS#alias/} ARN=${CMK_ARN}" - ;; - noop) - # Don't touch CRYPTO_TEXT so either existing value will be displayed, or - # unchanged value will be saved. - RESULT=0 - ;; - esac - RESULT=$? - - if [[ "${RESULT}" -eq 0 ]]; then - - # Decode if required - if [[ "${CRYPTO_VISIBLE}" == "true" ]]; then - CRYPTO_TEXT="$(base64 -d <<< "${CRYPTO_TEXT}" )" - fi - - # Update if required - if [[ "${CRYPTO_UPDATE}" == "true" ]]; then - if [[ -n "${JSON_PATH}" ]]; then - case ${CRYPTO_OPERATION} in - encrypt|reencrypt) - CRYPTO_TEXT="${PREFIX:+${PREFIX}:}${CRYPTO_TEXT}" - ;; - esac - jq --indent 4 "${ESCAPED_JSON_PATH}=\"${CRYPTO_TEXT}\"" < "${TARGET_FILE}" > "${tmp_dir}/${CRYPTO_FILENAME_DEFAULT}" - RESULT=$? - if [[ "${RESULT}" -eq 0 ]]; then - mv "${tmp_dir}/${CRYPTO_FILENAME_DEFAULT}" "${TARGET_FILE}" - fi - else - echo "${CRYPTO_TEXT}" > "${tmp_dir}/${CRYPTO_FILENAME_DEFAULT}" - RESULT=$? - if [[ "${RESULT}" -eq 0 ]]; then - if [[ "${CRYPTO_OPERATION}" == "decrypt" ]]; then - mv "${tmp_dir}/${CRYPTO_FILENAME_DEFAULT}" "${TARGET_FILE}.decrypted" - else - mv "${tmp_dir}/${CRYPTO_FILENAME_DEFAULT}" "${TARGET_FILE}" - fi - fi - fi - fi - fi - - if [[ ("${RESULT}" -eq 0) && ( "${CRYPTO_QUIET}" != "true") ]]; then - # Display result - echo "${CRYPTO_TEXT}" - fi - - return ${RESULT} -} - -main "$@" diff --git a/cli/manageFileCrypto.sh b/cli/manageFileCrypto.sh deleted file mode 100755 index 54ad7bb2..00000000 --- a/cli/manageFileCrypto.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${GENERATION_DEBUG}" ]] && set ${GENERATION_DEBUG} -trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${GENERATION_BASE_DIR}/execution/common.sh" - -# DEPRECATED -deprecated_script - -# Defaults - -function usage() { - cat < temp_ssl_list.out 2>&1 - RESULT=$? - if [[ ("${QUIET}" != "true") || ( "${RESULT}" -ne 0 ) ]]; then cat temp_ssl_list.out; fi - # For list - ? - ;; - ${CERTIFICATE_OPERATION_DELETE}) - aws --region ${REGION} iam delete-server-certificate --server-certificate-name ${CERTIFICATE_ID}-ssl > temp_ssl_delete.out 2>&1 - RESULT=$? - if [[ ("${QUIET}" != "true") || ( "${RESULT}" -ne 0 ) ]]; then cat temp_ssl_delete.out; fi - aws --region ${REGION} iam delete-server-certificate --server-certificate-name ${CERTIFICATE_ID}-cloudfront > temp_cloudfront_delete.out 2>&1 - RESULT=$? - if [[ ("${QUIET}" != "true") || ( "${RESULT}" -ne 0 ) ]]; then cat temp_cloudfront_delete.out; fi - # For delete - ? - ;; - ${CERTIFICATE_OPERATION_UPLOAD}) - # Copy files locally to keep aws call simple - LOCAL_CERTIFICATE_PUBLIC="temp_$(basename ${CERTIFICATE_PUBLIC})" - LOCAL_CERTIFICATE_PRIVATE="temp_$(basename ${CERTIFICATE_PRIVATE})" - LOCAL_CERTIFICATE_CHAIN="temp_$(basename ${CERTIFICATE_CHAIN})" - cp "${CERTIFICATE_PUBLIC}" "${LOCAL_CERTIFICATE_PUBLIC}" - cp "${CERTIFICATE_PRIVATE}" "${LOCAL_CERTIFICATE_PRIVATE}" - cp "${CERTIFICATE_CHAIN}" "${LOCAL_CERTIFICATE_CHAIN}" - - - aws --region ${REGION} iam get-server-certificate --server-certificate-name ${CERTIFICATE_ID}-ssl > temp_ssl_check.out 2>&1 - RESULT=$? - if [[ "${QUIET}" != "true" ]]; then cat temp_ssl_check.out; fi - if [[ "${RESULT}" -ne 0 ]]; then - if [[ "${MINGW64}" == "true" ]]; then - MSYS_NO_PATHCONV=1 aws --region ${REGION} iam upload-server-certificate \ - --server-certificate-name ${CERTIFICATE_ID}-ssl \ - --path "/ssl/${CERTIFICATE_ID}/" \ - --certificate-body file://${LOCAL_CERTIFICATE_PUBLIC} \ - --private-key file://${LOCAL_CERTIFICATE_PRIVATE} \ - --certificate-chain file://${LOCAL_CERTIFICATE_CHAIN} - else - aws --region ${REGION} iam upload-server-certificate \ - --server-certificate-name ${CERTIFICATE_ID}-ssl \ - --path "/ssl/${CERTIFICATE_ID}/" \ - --certificate-body file://${LOCAL_CERTIFICATE_PUBLIC} \ - --private-key file://${LOCAL_CERTIFICATE_PRIVATE} \ - --certificate-chain file://${LOCAL_CERTIFICATE_CHAIN} - fi - RESULT=$? - if [[ "${RESULT}" -ne 0 ]]; then exit; fi - fi - - aws --region ${REGION} iam get-server-certificate --server-certificate-name ${CERTIFICATE_ID}-cloudfront > temp_cloudfront_check.out 2>&1 - RESULT=$? - if [[ "${QUIET}" != "true" ]]; then cat temp_cloudfront_check.out; fi - if [[ "${RESULT}" -ne 0 ]]; then - if [[ "${MINGW64}" == "true" ]]; then - MSYS_NO_PATHCONV=1 aws --region ${REGION} iam upload-server-certificate \ - --server-certificate-name ${CERTIFICATE_ID}-cloudfront \ - --path "/cloudfront/${CERTIFICATE_ID}/" \ - --certificate-body file://${LOCAL_CERTIFICATE_PUBLIC} \ - --private-key file://${LOCAL_CERTIFICATE_PRIVATE} \ - --certificate-chain file://${LOCAL_CERTIFICATE_CHAIN} - else - aws --region ${REGION} iam upload-server-certificate - --server-certificate-name ${CERTIFICATE_ID}-cloudfront \ - --path "/cloudfront/${CERTIFICATE_ID}/" \ - --certificate-body file://${LOCAL_CERTIFICATE_PUBLIC} \ - --private-key file://${LOCAL_CERTIFICATE_PRIVATE} \ - --certificate-chain file://${LOCAL_CERTIFICATE_CHAIN} - fi - RESULT=$? - fi - ;; -esac diff --git a/cli/rebootRDSDatabase.sh b/cli/rebootRDSDatabase.sh deleted file mode 100755 index d7795b29..00000000 --- a/cli/rebootRDSDatabase.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${GENERATION_DEBUG}" ]] && set ${GENERATION_DEBUG} -trap '. ${GENERATION_BASE_DIR}/execution/cleanupContext.sh; exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${GENERATION_BASE_DIR}/execution/common.sh" - -# DEPRECATED -deprecated_script - -# Defaults -DELAY_DEFAULT=30 -TIER_DEFAULT="database" - -function usage() { - cat </dev/null | grep "DBInstanceStatus" > "${status_file}" - cat "${status_file}" - grep "available" "${status_file}" >/dev/null 2>&1 - RESULT=$? - if [ "$RESULT" -eq 0 ]; then break; fi - grep "rebooting" "${status_file}" >/dev/null 2>&1 - RESULT=$? - if [ "$RESULT" -ne 0 ]; then break; fi - sleep $DELAY - done -fi diff --git a/cli/runLambda.sh b/cli/runLambda.sh deleted file mode 100755 index 2b7fba37..00000000 --- a/cli/runLambda.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${GENERATION_DEBUG}" ]] && set ${GENERATION_DEBUG} -trap '. ${GENERATION_BASE_DIR}/execution/cleanupContext.sh; exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${GENERATION_BASE_DIR}/execution/common.sh" - -# DEPRECATED -deprecated_script - -#Defaults -INCLUDE_LOG_TAIL_DEFAULT="true" -DEPLOYMENT_GROUP_DEFAULT="application" - -tmpdir="$(getTempDir "hamlete_inf_XXX")" - - -function usage() { - cat < /dev/null - BUILD_BLUEPRINT="${tmpdir}/buildblueprint-${DEPLOYMENT_GROUP}-${DEPLOYMENT_UNIT}-config.json" - - if [[ ! -f "${BUILD_BLUEPRINT}" || -z "$(cat ${BUILD_BLUEPRINT} )" ]]; then - fatal "Could not generate blueprint for task details" - return 255 - fi - - DEPLOYMENT_UNIT_TYPE="$(jq -r '.Type' < "${BUILD_BLUEPRINT}" )" - LAMBDA_OUTPUT_FILE="${tmpdir}/${DEPLOYMENT_UNIT}_output.txt" - - if [[ "${DEPLOYMENT_UNIT_TYPE}" -ne "lambda" ]]; then - fatal "Component type is not a lambda function" - return 255 - fi - - if [[ -n "${INPUT_PAYLOAD}" ]]; then - INPUT_PAYLOAD="$( echo "${INPUT_PAYLOAD}" | jq -r '.' )" - if [[ -z "${INPUT_PAYLOAD}" ]]; then - fatal "Invalid input payload - must be in JSON format" - return 255 - fi - fi - - LAMBDA_ARN="$( jq --arg functionId ${FUNCTION_ID} -r '.Occurrence.Occurrences[] | select( .Core.SubComponent.Id==$functionId ) | .State.Attributes.ARN' < "${BUILD_BLUEPRINT}")" - - if [[ -n "${LAMBDA_ARN}" ]]; then - - LAMBDA_EXECUTE=$(aws --region ${REGION} lambda invoke --function-name "${LAMBDA_ARN}" \ - --invocation-type "RequestResponse" --log-type "Tail" \ - --payload "${INPUT_PAYLOAD}" \ - "${LAMBDA_OUTPUT_FILE}" || return $? ) - - FUNCTION_ERROR="$( echo "${LAMBDA_EXECUTE}" | jq -r '.FunctionError' )" - LOG_RESULTS="$( echo "${LAMBDA_EXECUTE}" | jq -r '. | .LogResult' | base64 --decode )" - RETURN_PAYLOAD="$( echo "${LAMBDA_EXECUTE}" | jq -r '.Payload' )" - - if [[ "$(cat "${LAMBDA_OUTPUT_FILE}")" -ne 'null' ]]; then - info "Output File:" - cat "${LAMBDA_OUTPUT_FILE}" - info "---------------------" - fi - - if [[ "${RETURN_PAYLOAD}" -ne 'null' ]]; then - info "Lambda Return Payload:" - info "${RETURN_PAYLOAD}" - info "-------------------------" - fi - - if [[ "${INCLUDE_LOG_TAIL}" == "true" ]]; then - info "Lambda Execution Logs:" - info "${LOG_RESULTS}" - info "-----------------------" - fi - - if [[ "${FUNCTION_ERROR}" -ne "null" ]]; then - fatal "An error occurred in the lambda function - Details provided in the return payload" - return 128 - fi - - else - error "Lambda ARN not found for ${DEPLOYMENT_UNIT} has it been deployed?" - return 128 - fi - - # All good - info "Lambda execute complete" - return 0 -} - -main "$@" diff --git a/cli/runTask.sh b/cli/runTask.sh deleted file mode 100755 index 69e7c8be..00000000 --- a/cli/runTask.sh +++ /dev/null @@ -1,295 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${GENERATION_DEBUG}" ]] && set ${GENERATION_DEBUG} -trap '. ${GENERATION_BASE_DIR}/execution/cleanupContext.sh; exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${GENERATION_BASE_DIR}/execution/common.sh" - -# DEPRECATED -deprecated_script - -# Defaults -DELAY_DEFAULT=30 -RETRY_COUNT_DEFAULT=120 -ENV_NAMES=() -ENV_VALUES=() - -tmpdir="$(getTempDir "hamlet_runTask_XXX")" - -function usage() { - cat < /dev/null -SEGMENT_BLUEPRINT="${tmpdir}/blueprint-config.json" - -if [[ ! -f "${SEGMENT_BLUEPRINT}" || -z "$(cat ${SEGMENT_BLUEPRINT} )" ]]; then - fatal "Could not generate blueprint for task details" - exit 255 -fi - -# Search through the blueprint to find the cluster and the task -CLUSTER_BLUEPRINT="$(getJSONValue "${SEGMENT_BLUEPRINT}" \ - " .Tenants[0] | objects \ - | .Products[0] | objects \ - | .Environments[0] | objects \ - | .Segments[0] | objects \ - | .Tiers[] | objects | select(.Name==\"${TIER}\") \ - | .Components[] | objects | select(.Name==\"${COMPONENT}\") \ - | .Occurrences[] | objects | \ - select( \ - .Core.Type==\"ecs\" \ - and .Core.Instance.Name==\"${COMPONENT_INSTANCE}\" \ - and .Core.Version.Name==\"${COMPONENT_VERSION}\" \ - )")" - -if [[ -z "${CLUSTER_BLUEPRINT}" ]]; then - error "Could not find ECS Component - Tier: ${TIER} - Component: ${COMPONENT} - Component_Instance: ${COMPONENT_INSTANCE} - Component_Version: ${COMPONENT_VERSION}" - exit 255 -fi - -COMPONENT_BLUEPRINT="$(echo "${CLUSTER_BLUEPRINT}" | jq \ - ".Occurrences[] | objects | \ - select( \ - .Core.Type==\"task\" \ - and .Core.SubComponent.RawName==\"${TASK}\" \ - and .Core.Instance.Name==\"${INSTANCE}\" \ - and .Core.Version.Name==\"${VERSION}\" \ - )")" - -if [[ -z "${COMPONENT_BLUEPRINT}" ]]; then - error "Could not find ECS Task - Task: ${TASK} - Instance: ${INSTANCE} - Version: ${VERSION}" - exit 255 -fi - -CLUSTER_ARN="$( echo "${COMPONENT_BLUEPRINT}" | jq -r '.State.Attributes.ECSHOST' )" - -ENGINE="$( echo "${COMPONENT_BLUEPRINT}" | jq -r '.Configuration.Solution.Engine' )" -PLATFORM_VERSION="$( echo "${COMPONENT_BLUEPRINT}" | jq -r '.Configuration.Solution["aws:FargatePlatform"] | select (.!=null)' )" -if [[ -z "${PLATFORM_VERSION}" ]]; then - PLATFORM_VERSION="LATEST" -fi - -NETWORK_MODE="$( echo "${COMPONENT_BLUEPRINT}" | jq -r '.Configuration.Solution.NetworkMode' )" - -DEFAULT_CONTAINER="$( echo "${COMPONENT_BLUEPRINT}" | jq -r '.Configuration.Solution.Containers | keys | .[0]' )" -TASK_DEFINITION_ID="-$( echo "${COMPONENT_BLUEPRINT}" | jq -r '.State.ResourceGroups.default.Resources.task.Id' )-" - -# Handle container name -if [[ -n "${CONTAINER_ID}" ]]; then - CONTAINER="${CONTAINER_ID}" -else - CONTAINER="${DEFAULT_CONTAINER%-*}" -fi - -if [[ "${SEGMENT}" == "default" ]]; then - SEGMENT="" -fi - -TASK_DEFINITION_ARN="$(aws --region "${REGION}" ecs list-task-definitions --query "taskDefinitionArns[?contains(@, '${TASK_DEFINITION_ID}') == \`true\`]|[?contains(@, '${PRODUCT}-${ENVIRONMENT}-${SEGMENT}') == \`true\`] | [0]" --output text )" - -info "Found the following task details \n * ClusterARN=${CLUSTER_ARN} \n * TaskDefinitionArn=${TASK_DEFINITION_ARN} \n * Container=${CONTAINER}" - -# Check the cluster -if [[ -n "${CLUSTER_ARN}" ]]; then - CLUSTER_STATUS="$(aws --region "${REGION}" ecs describe-clusters --clusters "${CLUSTER_ARN}" --output text --query 'clusters[0].status')" - debug "Cluster Status ${CLUSTER_STATUS}" - if [[ "${CLUSTER_STATUS}" != "ACTIVE" ]]; then - fatal "ECS Cluster ${CLUSTER_ARN} could not be found or was not active" - exit - fi -else - fatal "ECS Cluster not found - Component=${COMPONENT}" - exit -fi - -# Find the task definition -if [[ -z "${TASK_DEFINITION_ARN}" ]]; then - fatal "Unable to locate task definition" - exit -fi - -# Configuration Overrides -CLI_CONFIGURATION="{}" - -# Task hosting engine -case $ENGINE in - fargate) - CLI_CONFIGURATION="$( echo "${CLI_CONFIGURATION}" | jq --arg platformVersion "${PLATFORM_VERSION}" '. * { launchType: "FARGATE", platformVersion : $platformVersion }' )" - ;; -esac - -# Task Networking -case $NETWORK_MODE in - awsvpc) - SECURITY_GROUP="$( echo "${COMPONENT_BLUEPRINT}" | jq -r '.State.Attributes.SECURITY_GROUP' )" - SUBNET="$( echo "${COMPONENT_BLUEPRINT}" | jq -r '.State.Attributes.SUBNET' )" - NETWORK_CONFIGURATION="$( echo "{}" | jq --arg sec_group "${SECURITY_GROUP}" --arg subnet "${SUBNET}" '. | { networkConfiguration : { awsvpcConfiguration : { subnets : [ $subnet ], securityGroups: [ $sec_group ]}}}' )" - - CLI_CONFIGURATION="$( echo "${CLI_CONFIGURATION}" | jq --argjson network "${NETWORK_CONFIGURATION}" '. * $network' )" - ;; -esac - -# Environment Var Configuration -if [[ -n "${ENV_NAMES}" && -n "${ENV_VALUES}" ]]; then - - ENV_CONFIG="[]" - - for i in "${!ENV_NAMES[@]}"; do - ENV_NAME="${ENV_NAMES[$i]}" - ENV_VALUE="${ENV_VALUES[$i]}" - ENV_CONFIG="$(echo "${ENV_CONFIG}" | jq --arg env_name "${ENV_NAME}" --arg env_value "${ENV_VALUE}" '. += [ { name : $env_name, value: $env_value } ]' )" - done - - CLI_CONFIGURATION="$( echo "${CLI_CONFIGURATION}" | jq --arg container "${CONTAINER}" --argjson envvars "${ENV_CONFIG}" '. * { overrides : { containerOverrides : [ { name : $container, environment : $envvars }]}}' )" -fi - -CLI_CONFIGURATION="$(echo "${CLI_CONFIGURATION}" | jq -c '.' )" - -TASK_START="$(aws --region "${REGION}" ecs run-task --cluster "${CLUSTER_ARN}" --task-definition "${TASK_DEFINITION_ARN}" --count 1 ${TASK_ARGS} --cli-input-json "${CLI_CONFIGURATION}" --output json )" -TASK_ARN="$( echo "${TASK_START}" | jq -r '.tasks[0].taskArn' )" - -info "Starting Task..." - -if [[ -z "${TASK_ARN}" || "${TASK_ARN}" == "null" ]]; then - fatal "Task could not be started" - echo "${TASK_START}" - exit 255 -fi - -info "Watching task..." -CURRENT_RETRIES=0 -while true; do - LAST_STATUS="$(aws --region ${REGION} ecs describe-tasks --cluster "${CLUSTER_ARN}" --tasks "${TASK_ARN}" --query "tasks[?taskArn=='${TASK_ARN}'].lastStatus" --output text || break )" - - echo "...${LAST_STATUS}" - - if [[ "${LAST_STATUS}" == "STOPPED" ]]; then - break - fi - - CURRENT_RETRIES=$((CURRENT_RETRIES+1)) - - if [[ "${CURRENT_RETRIES}" == "${RETRY_COUNT}" ]]; then - fatal "Task has not completed in $(( RETRY_COUNT * DELAY )) seconds and has reached the DELAY and RETRY_COUNT limit" - fatal "Stopping monitoring of the task - the task will keep running" - exit 255 - fi - - sleep $DELAY -done - -# Show the exit codes if they are not 0 -TASK_FINAL_STATUS="$( aws --region "${REGION}" ecs describe-tasks --cluster "${CLUSTER_ARN}" --tasks "${TASK_ARN}" --query "tasks[?taskArn=='${TASK_ARN}'].{taskArn: taskArn, overrides: overrides, containers: containers }" || exit $? )" - -info "Task Results" -echo "${TASK_FINAL_STATUS}" - -# Use the exit status of the override container to determine the result -RESULT=$( echo "${TASK_FINAL_STATUS}" | jq -r ".[].containers[] | select(.name=\"${CONTAINER}\") | .exitCode" ) -RESULT=${RESULT:-0} diff --git a/cli/saveCMDBRepos.sh b/cli/saveCMDBRepos.sh new file mode 100755 index 00000000..eac5bceb --- /dev/null +++ b/cli/saveCMDBRepos.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash + +[[ -n "${AUTOMATION_DEBUG}" ]] && set ${AUTOMATION_DEBUG} +trap 'exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM + +. "${GENERATION_BASE_DIR}/execution/common.sh" + +ACCOUNT_REPOS_DEFAULT="false" +PRODUCT_REPOS_DEFAULT="false" + +function options() { + + # Parse options + while getopts ":ahm:pr:t:" option; do + case "${option}" in + a) ACCOUNT_REPOS="true" ;; + h) + usage + return 1 + ;; + m) COMMIT_MESSAGE="${OPTARG}" ;; + p) PRODUCT_REPOS="true" ;; + r) REFERENCE="${OPTARG}" ;; + t) TAG="${OPTARG}" ;; + \?) + fatalOption + return 1 + ;; + esac + done + + ACCOUNT_REPOS="${ACCOUNT_REPOS:-${ACCOUNT_REPOS_DEFAULT}}" + PRODUCT_REPOS="${PRODUCT_REPOS:-${PRODUCT_REPOS_DEFAULT}}" + + exit_on_invalid_environment_variables "COMMIT_MESSAGE" + + return 0 +} + +function usage() { + cat </dev/null; then + warning "Directory ${repo_dir} is not part of a git repo - skipping push" + return 0 + fi + + # Break the message in name/value pairs + conventional_commit_base_body="$(format_conventional_commit_body "${repo_message}")" + + # Separate the values based on the conventional commit format + conventional_commit_type="$(format_conventional_commit_body_summary "${conventional_commit_base_body}" "cctype")" + conventional_commit_scope="$(format_conventional_commit_body_summary "${conventional_commit_base_body}" "account product environment segment")" + conventional_commit_description="$(format_conventional_commit_body_summary "${conventional_commit_base_body}" "ccdesc")" + conventional_commit_body="$(format_conventional_commit_body_subset "${conventional_commit_base_body}" "cctype ccdesc account product environment segment")" + + formatted_commit_message="$(format_conventional_commit \ + "${conventional_commit_type:-hamlet}" \ + "${conventional_commit_scope}" \ + "${conventional_commit_description:-automation}" \ + "${conventional_commit_body}")" + + # Extract relevant events and remove the event log + repo_event_log="$(getTempFile XXXXXXX)" + + pull_events_from_state "directory" "$(git -C "${repo_dir}" rev-parse --show-toplevel)" "${repo_event_log}" "starts_with" + + if [[ -s "${repo_event_log}" ]]; then + + commit_logs=("$(jq -rc '.events[] | del(._id, .directory)' "${repo_event_log}")") + + if [[ -n "${commit_logs}" ]]; then + + formatted_commit_message+=$'\n\n' + + while read msg; do + formatted_commit_message+="$(echo "${msg}" | jq -r 'to_entries|map("\(.key): \(.value|tostring)")|.[]')" + formatted_commit_message+=$'\n--------\n\n' + + done <<<"${commit_logs}" + fi + fi + + # Make sure we can access the remote and that the branch exists + git -C "${repo_dir}" ls-remote -q "${repo_remote}" "${repo_branch}" 1>/dev/null || return $? + + # Add anything that has been added/modified/deleted + git -C "${repo_dir}" add -A + + if [[ -n "$(git -C "${repo_dir}" status --porcelain)" ]]; then + # Commit changes + debug "Committing changes" + + if ! git -C "${repo_dir}" commit -m "${formatted_commit_message}"; then + fatal "Can't commit to the repo" + return 1 + fi + + REPO_PUSH_REQUIRED="true" + else + info "no changes to ${repo_dir}" + fi + + # Tag the commit if required + if [[ -n "${repo_tag}" ]]; then + EXISTING_TAG=$(git -C "${repo_dir}" ls-remote --tags 2>/dev/null | grep "refs/tags/${repo_tag}$") + if [[ -n "${EXISTING_TAG}" ]]; then + warning "Tag ${repo_tag} is already present - skipping tag " + else + debug "Adding tag \"${repo_tag}\"" + if ! git -C "${repo_dir}" tag -a "${repo_tag}" -m "${repo_message}"; then + fatal "Can't create tag" + return 1 + fi + + REPO_PUSH_REQUIRED="true" + fi + fi + + # Update upstream repo + GENERATION_REPO_PUSH_RETRIES="${GENERATION_REPO_PUSH_RETRIES:-6}" + REPO_PUSHED=false + if [[ ("${defer_repo_push}" != "true") && ("${REPO_PUSH_REQUIRED}" == "true") ]]; then + for TRY in $(seq 1 ${GENERATION_REPO_PUSH_RETRIES}); do + # Check if remote branch exists + EXISTING_BRANCH=$(git -C "${repo_dir}" ls-remote --heads 2>/dev/null | grep "refs/heads/${repo_branch}$") + if [[ -n "${EXISTING_BRANCH}" ]]; then + debug "Rebasing in case of changes" + if ! git -C "${repo_dir}" pull --rebase ${repo_remote} ${repo_branch}; then + fatal "Can't rebase from upstream ${repo_remote}" + return 1 + fi + fi + + debug "Pushing the repo upstream" + if git -C "${repo_dir}" symbolic-ref -q HEAD; then + if git push --tags ${repo_remote} ${repo_branch}; then + REPO_PUSHED=true + break + else + info "Waiting to retry push" + sleep 5 + fi + else + # If push failed HEAD might be detached. Create a temp branch and merge it to the target to fix it. + git -C "${repo_dir}" branch temp-${repo_branch} && + git -C "${repo_dir}" checkout ${repo_branch} && + git -C "${repo_dir}" merge temp-${repo_branch} && + git -C "${repo_dir}" branch -D temp-${repo_branch} && + git -C "${repo_dir}" push --tags ${repo_remote} ${repo_branch} && REPO_PUSHED=true + fi + done + if [[ "${REPO_PUSHED}" == "false" ]]; then + fatal "Can't push the to upstream repo ${repo_remote}" + return 1 + fi + fi + + # All good + return 0 +} + +function save_repo() { + local directory="$1" + shift + local message="$1" + shift + local reference="$1" + shift + local tag="$1" + shift + + commit_tag_push "${reference:-"master"}" "${directory}" "${message}" "${DEFER_REPO_PUSH:-"false"}" "${REPO_REMOTE:-"origin"}" "${tag}" +} + +function save_product_config() { + local arguments=("$@") + + save_repo "${PRODUCT_DIR}" "${arguments[@]}" +} + +function save_product_infrastructure() { + local arguments=("$@") + + save_repo "${PRODUCT_INFRASTRUCTURE_DIR}" "${arguments[@]}" +} + +function save_product_state() { + local arguments=("$@") + + save_repo "${PRODUCT_STATE_DIR}" "${arguments[@]}" +} + +function main() { + + options "$@" || return $? + + . "${GENERATION_BASE_DIR}/execution/setContext.sh" + + # save account details + if [[ "${ACCOUNT_REPOS}" == "true" && -n "${ACCOUNT}" ]]; then + + info "Committing changes to account repositories" + save_repo "${ACCOUNT_DIR}" "${COMMIT_MESSAGE}" "${REFERENCE}" "${TAG}" || return $? + save_repo "${ACCOUNT_STATE_DIR}" "${COMMIT_MESSAGE}" "${REFERENCE}" "${TAG}" || return $? + + fi + + # save product details + if [[ "${PRODUCT_REPOS}" == "true" && -n "${PRODUCT}" ]]; then + + info "Committing changes to product repositories" + + save_product_config "${COMMIT_MESSAGE}" "${REFERENCE}" "${TAG}" || return $? + save_product_infrastructure "${COMMIT_MESSAGE}" "${REFERENCE}" "${TAG}" || return $? + + if [[ -n "${PRODUCT_STATE_DIR}" ]]; then + save_product_state "${COMMIT_MESSAGE}" "${REFERENCE}" "${TAG}" || return $? + fi + fi + + RESULT=$? + return "${RESULT}" +} + +main "$@" diff --git a/cli/snapshotRDSDatabase.sh b/cli/snapshotRDSDatabase.sh deleted file mode 100755 index 9e44581e..00000000 --- a/cli/snapshotRDSDatabase.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${GENERATION_DEBUG}" ]] && set ${GENERATION_DEBUG} -trap '. ${GENERATION_BASE_DIR}/execution/cleanupContext.sh; exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${GENERATION_BASE_DIR}/execution/common.sh" - -# DEPRECATED -deprecated_script - -# Defaults -TIER_DEFAULT="database" - -function usage() { - cat < $SNAPSHOTDATE ]]; then - LIST="${LIST} ${SNAPSHOT}" - fi - done - fi - if [[ "${LIST}" != "" ]]; then - for SNAPSHOT in $(echo $LIST); do - info "Removing old snapshot - ${SNAPSHOT}" - aws --region "${REGION}" rds delete-db-snapshot --db-snapshot-identifier "${SNAPSHOT}" - done - fi -fi - -if [[ "${WAIT}" == "true" ]]; then - sleep 5 - while [ "${exit_status}" != "0" ]; do - SNAPSHOT_STATE="$(aws --region "${REGION}" rds describe-db-snapshots --db-snapshot-identifier "${DB_SNAPSHOT_IDENTIFIER}" --query 'DBSnapshots[0].Status' || return $? )" - SNAPSHOT_PROGRESS="$(aws --region "${REGION}" rds describe-db-snapshots --db-snapshot-identifier "${DB_SNAPSHOT_IDENTIFIER}" --query 'DBSnapshots[0].PercentProgress' || return $? )" - info "Snapshot id ${DB_SNAPSHOT_IDENTIFIER} creation: state is ${SNAPSHOT_STATE}, ${SNAPSHOT_PROGRESS}%..." - - aws --region "${REGION}" rds wait db-snapshot-available --db-snapshot-identifier "${DB_SNAPSHOT_IDENTIFIER}" - exit_status="$?" - done - - info "Snapshot Created - ${DB_SNAPSHOT_IDENTIFIER} - $(aws --region "${REGION}" rds describe-db-snapshots --db-snapshot-identifier "${DB_SNAPSHOT_IDENTIFIER}" --query 'DBSnapshots[].SnapshotCreateTime' --output text || return $?)" - -fi diff --git a/cli/updateObjectACL.sh b/cli/updateObjectACL.sh deleted file mode 100755 index e65c25c1..00000000 --- a/cli/updateObjectACL.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash - -[[ -n "${GENERATION_DEBUG}" ]] && set ${GENERATION_DEBUG} -trap '. ${GENERATION_BASE_DIR}/execution/cleanupContext.sh; exit ${RESULT:-1}' EXIT SIGHUP SIGINT SIGTERM -. "${GENERATION_BASE_DIR}/execution/common.sh" - -# DEPRECATED -deprecated_script - -# Defaults -ACL_DEFAULT="private" -PREFIX_DEFAULT="/" -DISPLAY_ACLS_DEFAULT="false" - -function usage() { - cat <