diff --git a/ci/.#settings.yml b/ci/.#settings.yml new file mode 120000 index 0000000..b4e60b6 --- /dev/null +++ b/ci/.#settings.yml @@ -0,0 +1 @@ +mrferris@snws-MacBook-Pro-5.local.5095 \ No newline at end of file diff --git a/ci/pipeline.yml b/ci/pipeline.yml new file mode 100644 index 0000000..4158f04 --- /dev/null +++ b/ci/pipeline.yml @@ -0,0 +1,433 @@ +--- +# +# ci/pipeline.yml +# +# Pipeline structure file for a BOSH Release pipeline +# +# DO NOT MAKE CHANGES TO THIS FILE. Instead, modify +# ci/settings.yml and override what needs overridden. +# This uses spruce, so you have some options there. +# +# author: James Hunt +# created: 2016-03-30 + +meta: + name: (( param "Please name your pipeline" )) + release: (( grab meta.name )) + target: (( param "Please identify the name of the target Concourse CI" )) + url: (( param "Please specify the full url of the target Concourse CI" )) + pipeline: (( concat meta.name "-boshrelease" )) + manifest: + path: (( concat "manifests/" meta.name ".yml" )) + vars: "--- {}" + vars-pr: (( grab meta.manifest.vars )) + operator_file_paths: "" # comma separated list relative to repo root + + git: + email: (( param "Please provide the git email for automated commits" )) + name: (( param "Please provide the git name for automated commits" )) + + image: + name: starkandwayne/concourse + tag: latest + + aws: + bucket: (( concat meta.pipeline "-pipeline" )) + region_name: us-east-1 + access_key: (( param "Please set your AWS Access Key ID for your pipeline S3 Bucket" )) + secret_key: (( param "Please set your AWS Secret Key ID for your pipeline S3 Bucket" )) + + github: + uri: (( concat "git@github.com:" meta.github.owner "/" meta.github.repo )) + owner: (( param "Please specify the name of the user / organization that owns the Github repository" )) + repo: (( param "Please specify the name of the Github repository" )) + branch: master + private_key: (( param "Please generate an SSH Deployment Key for this repo and specify it here" )) + access_token: (( param "Please generate a Personal Access Token to be used for creating github releases (do you have a ci-bot?)" )) + + bosh-lite: + target: (( param "Please specify the BOSH target URI for the bosh-lite to run test deployments against" )) + cacert: (( param "Please specify the BOSH Director Root CA cert" )) + username: admin + password: (( param "Please specify the BOSH Director admin password" )) + deployment: (( concat meta.name "-testflight" )) + deployment-pr: (( concat meta.name "-testflight-pr" )) + + slack: + webhook: (( param "Please specify your Slack Incoming Webhook Integration URL" )) + success_moji: ":airplane_departure:" + fail_moji: ":airplane_arriving:" + upset_moji: ":sad_panda:" + channel: (( param "Please specify the channel (#name) or user (@user) to send messages to" )) + username: concourse + icon: https://cl.ly/2F421Y300u07/concourse-logo-blue-transparent.png + fail_url: '(( concat "<" meta.url "/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME| Concourse Failure! " meta.slack.upset_moji ">" ))' + +groups: + - name: (( grab meta.pipeline )) + jobs: + - testflight + - testflight-pr + - rc + - shipit + - name: versioning + jobs: + - major + - minor + - patch + +jobs: + - name: testflight + public: true + serial: true + plan: + - name: main + do: + - name: get + aggregate: + - { get: git, trigger: true } + - name: testflights + aggregate: + - name: testflight + task: testflight + config: + platform: linux + image_resource: + type: docker-image + source: + repository: (( grab meta.image.name )) + tag: (( grab meta.image.tag )) + inputs: + - { name: git } + run: + path: ./git/ci/scripts/testflight + args: [] + params: + REPO_ROOT: git + BOSH_ENVIRONMENT: (( grab meta.bosh-lite.target )) + BOSH_CA_CERT: (( grab meta.bosh-lite.cacert )) + BOSH_CLIENT: (( grab meta.bosh-lite.username )) + BOSH_CLIENT_SECRET: (( grab meta.bosh-lite.password )) + BOSH_DEPLOYMENT: (( grab meta.bosh-lite.deployment )) + TEST_ERRANDS: (( grab meta.test-errands || meta.test-errand || ~ )) + AWS_ACCESS_KEY: (( grab meta.aws.access_key )) + AWS_SECRET_KEY: (( grab meta.aws.secret_key )) + MANIFEST_PATH: (( grab meta.manifest.path )) + MANIFEST_VARS: (( grab meta.manifest.vars )) + MANIFEST_OP_PATHS: (( grab meta.manifest.operator_file_paths )) + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": testflight job failed" ))' + + - name: testflight-pr + public: true + serial: true + plan: + - name: main + do: + - name: get + aggregate: + - { get: git-pull-requests, trigger: true, version: every } + - name: pending-status + put: git-pull-requests + params: + path: git-pull-requests + status: pending + - name: testflights + aggregate: + - name: testflight + task: testflight + config: + platform: linux + image_resource: + type: docker-image + source: + repository: (( grab meta.image.name )) + tag: (( grab meta.image.tag )) + inputs: + - { name: git-pull-requests } + run: + path: ./git-pull-requests/ci/scripts/testflight + args: [] + params: + REPO_ROOT: git-pull-requests + BOSH_ENVIRONMENT: (( grab meta.bosh-lite.target )) + BOSH_CA_CERT: (( grab meta.bosh-lite.cacert )) + BOSH_CLIENT: (( grab meta.bosh-lite.username )) + BOSH_CLIENT_SECRET: (( grab meta.bosh-lite.password )) + BOSH_DEPLOYMENT: (( grab meta.bosh-lite.deployment-pr )) + TEST_ERRANDS: (( grab meta.test-errands || meta.test-errand || ~ )) + AWS_ACCESS_KEY: (( grab meta.aws.access_key )) + AWS_SECRET_KEY: (( grab meta.aws.secret_key )) + MANIFEST_PATH: (( grab meta.manifest.path )) + MANIFEST_VARS: (( grab meta.manifest.vars-pr )) + MANIFEST_OP_PATHS: (( grab meta.manifest.operator_file_paths )) + on_success: + put: git-pull-requests + params: + path: git-pull-requests + status: success + on_failure: + put: git-pull-requests + params: + path: git-pull-requests + status: failure + - name: pr-success-message + task: pr-success-message + config: + platform: linux + image_resource: + type: docker-image + source: + repository: (( grab meta.image.name )) + tag: (( grab meta.image.tag )) + inputs: + - { name: git-pull-requests } + outputs: + - { name: message } + run: + path: sh + args: + - -ce + - | + cd git-pull-requests + pr_url=$(git config --get pullrequest.url) + cd - + echo "<${pr_url}|Pull request passed testflight> Merge when ready: ${pr_url}" > message/body + on_success: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text_file: message/body + + - name: rc + public: true + serial: true + plan: + - do: + - aggregate: + - { get: git, trigger: true, passed: [testflight] } + - { get: version, trigger: true, params: {pre: rc} } + - task: release-notes + config: + platform: linux + image_resource: + type: docker-image + source: + repository: (( grab meta.image.name )) + tag: (( grab meta.image.tag )) + inputs: + - { name: git } + run: + path: sh + args: + - -ce + - | + cd git + if [ -f ci/release_notes.md ]; then + echo "###### RELEASE NOTES ###############" + echo + cat ci/release_notes.md + echo + echo "########################################" + echo + else + echo "NO RELEASE NOTES HAVE BEEN WRITTEN" + echo "You *might* want to do that before" + echo "hitting (+) on that shipit job..." + echo + fi + - put: version + params: {file: version/number} + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": rc job failed" ))' + + - name: minor + public: true + plan: + - do: + - { get: version, trigger: false, params: {bump: minor} } + - { put: version, params: {file: version/number} } + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": minor job failed" ))' + + - name: patch + public: true + plan: + - do: + - { get: version, trigger: false, params: {bump: patch} } + - { put: version, params: {file: version/number} } + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": patch job failed" ))' + + - name: major + public: true + plan: + - do: + - { get: version, trigger: false, params: {bump: major} } + - { put: version, params: {file: version/number} } + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": major job failed" ))' + + - name: shipit + public: true + serial: true + plan: + - do: + - name: inputs + aggregate: + - { get: version, passed: [rc], params: {bump: final} } + - { get: git, passed: [rc] } + - name: release + task: release + config: + platform: linux + image_resource: + type: docker-image + source: + repository: (( grab meta.image.name )) + tag: (( grab meta.image.tag )) + inputs: + - name: version + - name: git + outputs: + - name: gh + - name: pushme + - name: notifications + run: + path: ./git/ci/scripts/shipit + args: [] + params: + REPO_ROOT: git + VERSION_FROM: version/number + RELEASE_ROOT: gh + REPO_OUT: pushme + NOTIFICATION_OUT: notifications + BRANCH: (( grab meta.github.branch )) + GITHUB_OWNER: (( grab meta.github.owner )) + GIT_EMAIL: (( grab meta.git.email )) + GIT_NAME: (( grab meta.git.name )) + AWS_ACCESS_KEY: (( grab meta.aws.access_key )) + AWS_SECRET_KEY: (( grab meta.aws.secret_key )) + + - name: upload-git + put: git + params: + rebase: true + repository: pushme/git + - name: tarball + put: s3-tarball + params: + file: (( concat "gh/artifacts/" meta.name "-*.tgz" )) + - name: github-release + put: github + params: + name: gh/name + tag: gh/tag + body: gh/notes.md + globs: [gh/artifacts/*] + - name: version-bump + put: version + params: + bump: patch + - name: notify + aggregate: + - put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text_file: notifications/message + on_failure: + put: notify + params: + channel: (( grab meta.slack.channel )) + username: (( grab meta.slack.username )) + icon_url: (( grab meta.slack.icon )) + text: '(( concat meta.slack.fail_url " " meta.pipeline ": shipit job failed" ))' + +resource_types: + - name: slack-notification + type: docker-image + source: + repository: cfcommunity/slack-notification-resource + + - name: pull-request + type: docker-image + source: + repository: jtarchie/pr + +resources: + - name: git + type: git + source: + uri: (( grab meta.github.uri )) + branch: (( grab meta.github.branch )) + private_key: (( grab meta.github.private_key )) + + - name: git-pull-requests + type: pull-request + check_every: 15m # Required due to API throttling. + source: + access_token: (( grab meta.github.access_token )) + private_key: (( grab meta.github.private_key )) + repo: (( concat meta.github.owner "/" meta.github.repo )) + base: (( grab meta.github.branch )) + + - name: version + type: semver + source : + driver: s3 + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + key: version + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + initial_version: (( grab meta.initial_version || "0.0.1" )) + + - name: notify + type: slack-notification + source: + url: (( grab meta.slack.webhook )) + + - name: github + type: github-release + source: + user: (( grab meta.github.owner )) + repository: (( grab meta.github.repo )) + access_token: (( grab meta.github.access_token )) + + - name: s3-tarball + type: s3 + source: + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + regexp: (( concat meta.name "-(.*).tgz" )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) diff --git a/ci/repipe b/ci/repipe new file mode 100755 index 0000000..ef29251 --- /dev/null +++ b/ci/repipe @@ -0,0 +1,114 @@ +#!/bin/bash +# +# ci/repipe +# +# Script for merging together pipeline configuration files +# (via Spruce!) and configuring Concourse. +# +# author: James Hunt +# Dennis Bell +# created: 2016-03-04 + +need_command() { + local cmd=${1:?need_command() - no command name given} + + if [[ ! -x "$(command -v $cmd)" ]]; then + echo >&2 "${cmd} is not installed." + if [[ "${cmd}" == "spruce" ]]; then + echo >&2 "Please download it from https://github.com/geofffranks/spruce/releases" + fi + exit 2 + fi +} + +NO_FLY= +SAVE_MANIFEST= +VALIDATE_PIPELINE= +NON_INTERACTIVE= + +cleanup() { + rm -f save-manifest.yml + if [[ -n ${SAVE_MANIFEST} && -e .deploy.yml ]]; then + mv .deploy.yml save-manifest.yml + fi + rm -f .deploy.yml +} + +usage() { + echo Command line arguments: + echo "no-fly Do not execute any fly commands" + echo "save-manifest Save manifest to file save-manifest" + echo "validate Validate pipeline instead of set pipeline" + echo "validate-strict Validate pipeline with strict mode" + echo "non-interactive Run set-pipeline in non-interactive mode" +} + +for arg do + case "${arg}" in + no-fly|no_fly) NO_FLY="yes" ;; + save-manifest|save_manifest) SAVE_MANIFEST="yes" ;; + validate) VALIDATE_PIPELINE="normal" ;; + validate-strict|validate_strict) VALIDATE_PIPELINE="strict" ;; + non-interactive|non_interactive) NON_INTERACTIVE="--non-interactive" ;; + help|-h|--help) usage; exit 0 ;; + *) echo Invalid argument + usage + exit 1 + esac +done + +cd $(dirname $BASH_SOURCE[0]) +echo "Working in $(pwd)" +need_command spruce + +# Allow for target-specific settings +settings_file="$(ls -1 settings.yml ${CONCOURSE_TARGET:+"settings-${CONCOURSE_TARGET}.yml"} 2>/dev/null | tail -n1)" +if [[ -z "$settings_file" ]] +then + echo >&2 "Missing local settings in ci/settings.yml${CONCOURSE_TARGET:+" or ci/settings-${CONCOURSE_TARGET}.yml"}!" + exit 1 +fi + +echo >&2 "Using settings found in ${settings_file}" + +set -e +trap "cleanup" QUIT TERM EXIT INT +spruce merge pipeline.yml ${settings_file} > .deploy.yml +PIPELINE=$(spruce json .deploy.yml | jq -r '.meta.pipeline // ""') +if [[ -z ${PIPELINE} ]]; then + echo >&2 "Missing pipeline name in ci/settings.yml!" + exit 1 +fi + +TARGET_FROM_SETTINGS=$(spruce json .deploy.yml | jq -r '.meta.target // ""') +if [[ -z ${CONCOURSE_TARGET} ]]; then + TARGET=${TARGET_FROM_SETTINGS} +elif [[ "$CONCOURSE_TARGET" != "$TARGET_FROM_SETTINGS" ]] +then + echo >&2 "Target in {$settings_file} differs from target in \$CONCOURSE_TARGET" + echo >&2 " \$CONCOURSE_TARGET: $CONCOURSE_TARGET" + echo >&2 " Target in file: $TARGET_FROM_SETTINGS" + exit 1 +else + TARGET=${CONCOURSE_TARGET} +fi + +if [[ -z ${TARGET} ]]; then + echo >&2 "Missing Concourse Target in ci/settings.yml!" + exit 1 +fi + +fly_cmd="${FLY_CMD:-fly}" + +[[ -n ${NO_FLY} ]] && { echo no fly execution requested ; exit 0; } + +case "${VALIDATE_PIPELINE}" in + normal) fly_opts="validate-pipeline" ;; + strict) fly_opts="validate-pipeline --strict" ;; + *) fly_opts="set-pipeline ${NON_INTERACTIVE} --pipeline ${PIPELINE}" ;; +esac + +set +x +$fly_cmd --target ${TARGET} ${fly_opts} --config .deploy.yml +[[ -n ${VALIDATE_PIPELINE} ]] && exit 0 +$fly_cmd --target ${TARGET} unpause-pipeline --pipeline ${PIPELINE} diff --git a/ci/scripts/bump-release b/ci/scripts/bump-release new file mode 100755 index 0000000..93a1307 --- /dev/null +++ b/ci/scripts/bump-release @@ -0,0 +1,81 @@ +#!/bin/bash + +# The goal of this script is to retain the pretty nature of manifests, whilst updating +# the "releases:" section. +# +# If we use `spruce merge` to do the job, the entire manifest will be reordered - +# keys will be sorted alphabetically. Comments will be lost. +# +# Instead, we assume that "releases:" is the last section of the manifest. +set -e + +: ${REPO_ROOT:?required} +: ${REPO_OUT:?required} +: ${RELEASE:?required} +: ${NAME:?required} + +if [[ ! -f ${RELEASE}/version ]]; then + echo "Director ${RELEASE} must have file /version" + exit 1 +fi + +git clone ${REPO_ROOT} ${REPO_OUT} + +version=$(cat ${RELEASE}/version) + +# So, this is a nested bash/spruce/jq combo. +# What is happening here is that the "releases:" section of each deployment manifest +# is being updated with the new version/sha1 for the release. +# +# We use "spruce json manifest.yml | jq '.releases'" to extract the existing releases array +# and the '.releases | map(if .name == $name)' will modify a specific element of the array +# +# This gives us a modified "releases: [{...}, {...}]" segment of the final deployment manifest. +# We now need to merge this back into the original manifest. +# +# But, I don't want to just use `spruce merge` for this as it will reorder the manifest and +# make it ugly. If I didn't care about the manifest's aesthetics then this whole script +# would be simpler. I want the original layout of the manifest to be retained; and so +# we will just chomp out the original "releases:" section at the end of the file and +# paste in the updated releases section. +function bump_version { + manifest_path=$1 + releases_updated=$(spruce merge < $manifest_path <>ci/release_notes.md +* Bumped ${NAME} to v${version} +EOF + +git merge --no-edit ${BRANCH} +git add -A +git status +git commit -m "bump ${NAME} v${version}" diff --git a/ci/scripts/shipit b/ci/scripts/shipit new file mode 100755 index 0000000..51e7d93 --- /dev/null +++ b/ci/scripts/shipit @@ -0,0 +1,119 @@ +#!/bin/bash +# +# ci/scripts/shipit +# +# Script for generating Github release / tag assets +# and managing release notes for a BOSH Release pipeline +# +# author: James Hunt +# created: 2016-03-30 + +set -eu + +header() { + echo + echo "###############################################" + echo + echo $* + echo +} + +: ${REPO_ROOT:?required} +: ${RELEASE_ROOT:?required} +: ${REPO_OUT:?required} +: ${BRANCH:?required} +: ${GITHUB_OWNER:?required} +: ${VERSION_FROM:?required} +: ${AWS_ACCESS_KEY:?required} +: ${AWS_SECRET_KEY:?required} +: ${GIT_EMAIL:?required} +: ${GIT_NAME:?required} + +if [[ ! -f ${VERSION_FROM} ]]; then + echo >&2 "Version file (${VERSION_FROM}) not found. Did you misconfigure Concourse?" + exit 2 +fi +VERSION=$(cat ${VERSION_FROM}) +if [[ -z ${VERSION} ]]; then + echo >&2 "Version file (${VERSION_FROM}) was empty. Did you misconfigure Concourse?" + exit 2 +fi + +if [[ ! -f ${REPO_ROOT}/ci/release_notes.md ]]; then + echo >&2 "ci/release_notes.md not found. Did you forget to write them?" + exit 1 +fi + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +############################################################### + +cd ${REPO_ROOT} +RELEASE_NAME=$(bosh2 int config/final.yml --path /final_name) + +cat > config/private.yml < ${RELEASE_ROOT}/tag +echo "${RELEASE_NAME} v${VERSION}" > ${RELEASE_ROOT}/name +mv ${REPO_ROOT}/releases/*/*-${VERSION}.tgz ${RELEASE_ROOT}/artifacts +mv ${REPO_ROOT}/ci/release_notes.md ${RELEASE_ROOT}/notes.md +cat >> ${RELEASE_ROOT}/notes.md < ${RELEASE_ROOT}/notification < New ${RELEASE_NAME} v${VERSION} released! +EOF + + +header "Update git repo with final release..." +if [[ -z $(git config --global user.email) ]]; then + git config --global user.email "${GIT_EMAIL}" +fi +if [[ -z $(git config --global user.name) ]]; then + git config --global user.name "${GIT_NAME}" +fi + +(cd ${REPO_ROOT} + for MANIFEST_PATH in $(ls manifests/*.yml); do + $DIR/update-manifest $GITHUB_OWNER $RELEASE_NAME $VERSION $SHA1 $MANIFEST_PATH + done + git merge --no-edit ${BRANCH} + git add -A + git status + git commit -m "release v${VERSION}") + +# so that future steps in the pipeline can push our changes +cp -a ${REPO_ROOT} ${REPO_OUT} + +cat > ${NOTIFICATION_OUT:-notifications}/message <. +EOS diff --git a/ci/scripts/testflight b/ci/scripts/testflight new file mode 100755 index 0000000..cab641d --- /dev/null +++ b/ci/scripts/testflight @@ -0,0 +1,170 @@ +#!/bin/bash +# +# ci/scripts/testflight +# +# Script for testing a BOSH release using bosh2 +# +# author: James Hunt + +set -eu + +: ${BOSH_ENVIRONMENT:?required} +: ${BOSH_CA_CERT:?required} +: ${BOSH_CLIENT:?required} +: ${BOSH_CLIENT_SECRET:?required} +: ${BOSH_DEPLOYMENT:?required} +: ${MANIFEST_PATH:?required} +: ${AWS_ACCESS_KEY:?required} +: ${AWS_SECRET_KEY:?required} +DEBUG=${DEBUG:-} +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +header() { + echo + echo "###############################################" + echo + echo $* + echo +} + +delete_deployment() { + header "Cleaning up deployment..." + bosh2 -n delete-deployment --force +} +trap "echo; echo; echo; sleep 10; delete_deployment" EXIT SIGINT SIGTERM + + +cd ${REPO_ROOT:?required} +header "Pulling in any git submodules..." +git submodule update --init --recursive --force + +header "Confirming testflight inputs" + +echo "Confirming deployment manifest ${MANIFEST_PATH} exists" +if [[ ! -f ${MANIFEST_PATH} ]]; then + echo "Deployment manifest ${MANIFEST_PATH} does not exist" + exit 1 +fi + +echo "Confirming that \$MANIFEST_VARS is valid" +mkdir -p tmp + +# $MANIFEST_VARS can either be "key: value" or "key=value" +# and will be converted to "key: value" YAML +echo "${MANIFEST_VARS:-"{} ---"}" | sed -e "s/ *= */: /g" > tmp/vars.yml +if [[ ! -z $DEBUG ]]; then + echo "Variables passed to deployment manifest:" + cat tmp/vars.yml +fi +# convert YAML to JSON to check YAML validity +spruce json tmp/vars.yml > /dev/null + +echo "Confirming each operator file exists: ${MANIFEST_OP_PATHS:-ok, none specified.}" +op_patch_file_errors= +for op_patch_file in ${MANIFEST_OP_PATHS//,/ } ; do + if [[ ! -f $op_patch_file ]]; then + op_patch_file_errors=1 + echo "Operator file missing: ${op_patch_file}" + fi +done +if [[ ! -z ${op_patch_file_errors} ]]; then + exit 1 +fi + +$DIR/wait-until-proxy-bosh-cf-running +if [[ "${PROXY_IP:-X}" != "X" ]] ; then export BOSH_ALL_PROXY=socks5://localhost:9999 ; fi + +delete_deployment + +header "Creating candidate BOSH release..." +bosh2 -n reset-release # in case dev_releases/ is in repo accidentally + +cat > config/private.yml < tmp/deployment.yml <> tmp/deployment.yml <> tmp/deployment.yml < tmp/manifest.yml + +bosh2 -n deploy tmp/manifest.yml + +TEST_ERRAND=${TEST_ERRAND:-} # backwards compatibility +TEST_ERRANDS=${TEST_ERRANDS:-$TEST_ERRAND} +if [[ -n ${TEST_ERRANDS} ]]; then + for errand in ${TEST_ERRANDS}; do + header "Running '${errand}' errand" + bosh2 -n run-errand ${errand} + done +else + echo "No test errands specified, skipping." +fi + +echo +echo "SUCCESS" +exit 0 diff --git a/ci/scripts/update-blob b/ci/scripts/update-blob new file mode 100755 index 0000000..26fca4d --- /dev/null +++ b/ci/scripts/update-blob @@ -0,0 +1,61 @@ +#!/bin/bash + +set -e + +: ${BLOB_DIR:?required} +: ${BLOB_NAME:?required} +: ${BLOB_BINARY:?required} +: ${BLOB_CLEANUP:?required} +: ${BLOB_DESTINATION:?required} + +VERSION=$(cat ${BLOB_DIR}/version) + +pushd ${REPO_ROOT:?required} + +cat <config/private.yml +--- +blobstore: + provider: s3 + options: + access_key_id: ${AWS_ACCESS_KEY:?required} + secret_access_key: ${AWS_SECRET_KEY:?required} +EOF + +blobs_to_remove=$(spruce json config/blobs.yml | jq -r "keys[] | select(test(\"${BLOB_CLEANUP}\"))") +if [[ ! -z $blobs_to_remove ]]; then + echo "$blobs_to_remove" | xargs -L1 bosh2 remove-blob +fi + +# expand ${VERSION} env var into file path +eval "blob_destination=${BLOB_DESTINATION}" +bosh2 add-blob ../${BLOB_DIR}/${BLOB_BINARY} "${blob_destination}" +bosh2 -n upload-blobs +rm config/private.yml +popd + +if [[ -n "$(cd ${REPO_ROOT}; git status --porcelain)" ]]; then + pushd ${REPO_ROOT} + cat <>ci/release_notes.md + +# ${BLOB_NAME} +Bumped ${BLOB_URL} to v${VERSION} +EOF + popd + + # GIT! + if [[ -z $(git config --global user.email) ]]; then + git config --global user.email "ci@starkandwayne.com" + fi + if [[ -z $(git config --global user.name) ]]; then + git config --global user.name "CI Bot" + fi + + (cd ${REPO_ROOT} + git merge --no-edit ${BRANCH} + git add -A + git status + git commit -m "Bumped ${BLOB_NAME} to v${VERSION}") +fi + +# so that future steps in the pipeline can push our changes +cp -a ${REPO_ROOT} ${REPO_OUT} diff --git a/ci/scripts/update-manifest b/ci/scripts/update-manifest new file mode 100755 index 0000000..40a58af --- /dev/null +++ b/ci/scripts/update-manifest @@ -0,0 +1,20 @@ +#!/bin/bash + +GITHUB_OWNER=$1 +RELEASE_NAME=$2 +VERSION=$3 +SHA1=$4 +MANIFEST_PATH=$5 +: ${MANIFEST_PATH:?USAGE: ./ci/scripts/update-manifest GITHUB_OWNER RELEASE_NAME VERSION SHA1 MANIFEST_PATH} + +set -e -u + +manifest_len=$(wc -l $MANIFEST_PATH | awk '{print $1}') +manifest_head=$(head -n `expr $manifest_len - 4` $MANIFEST_PATH) +cat > $MANIFEST_PATH < proxy/ssh/private_key + chmod 600 proxy/ssh/private_key + + header "Checking jumpbox available..." + set +e + until ssh ${PROXY_USERNAME}@${PROXY_IP} -i proxy/ssh/private_key \ + -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=60 \ + "whoami" + do + echo "Waiting until jumpbox/proxy available..." + done + set -e + + header "Starting socks5 proxy..." + ssh ${PROXY_USERNAME}@${PROXY_IP} -i proxy/ssh/private_key -N -D 9999 & + sleep 10 + + echo BOSH_ALL_PROXY=socks5://localhost:9999 + export BOSH_ALL_PROXY=socks5://localhost:9999 +fi + +header "Checking bosh available..." +until bosh2 env; do + echo "Waiting until bosh available..." + sleep 60 +done + +if [[ -f git/tmp/vars.yml ]]; then + cf_api_url=$(bosh2 int git/tmp/vars.yml --path /cf-api-url) + if [[ "${cf_api_url:-X}" != "X" ]]; then + header "Checking Cloud Foundry available..." + until cf api $cf_api_url --skip-ssl-validation; do + echo "Waiting until Cloud Foundry available..." + sleep 60 + done + fi +fi + +echo +echo diff --git a/ci/settings.yml b/ci/settings.yml new file mode 100644 index 0000000..b6f86fc --- /dev/null +++ b/ci/settings.yml @@ -0,0 +1,40 @@ +--- +meta: + name: weave-scope + target: sw + test-errand: [] + url: https://ci2.starkandwayne.com + + manifest: + path: ci/manifest.yml + + git: + email: ci@starkandwayne.com + name: Stark & Wayne CI Bot + + bosh-lite: + target: https://10.58.111.44:25555 + username: (( vault "secret/bosh-lites/lite44/users/admin:username" )) + password: (( vault "secret/bosh-lites/lite44/users/admin:password" )) + cacert: (( vault "secret/bosh-lites/lite44/certs:rootCA.pem" )) + + aws: + access_key: (( vault "secret/aws/cfcommunity:access" )) + secret_key: (( vault "secret/aws/cfcommunity:secret" )) + + github: + owner: cloudfoundry-community + repo: weavescope-boshrelease + branch: master + private_key: (( vault "secret/pipelines/jumpbox-boshrelease/github:private" )) + access_token: (( vault "secret/pipelines/jumpbox-boshrelease/github:token" )) + +# slack: +# blob_failure: '(( concat "$BUILD_PIPELINE_NAME: :airplane_arriving: <" meta.url "/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME| Failed to update the blob for $BUILD_JOB_NAME>" ))' + # webhook: (( vault "secret/pipelines/jumpbox-boshrelease/slack:webhook" )) +# channel: "#cf-community-pipeline" +# blob_success: '(( concat "$BUILD_PIPELINE_NAME: New version of $BUILD_JOB_NAME was detected, and updated in master. <" meta.url "/pipelines/$BUILD_PIPELINE_NAME| Cut a new release?>" ))' + + + +