upgrade CI; fix github access token

shieldproject · Sep 5, 2017 · cf28c6f · cf28c6f
1 parent 7fdcc06
commit cf28c6f
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 15 deletions.
diff --git a/ci/pipeline.yml b/ci/pipeline.yml
@@ -20,6 +20,7 @@ meta:
   manifest:
     path:   (( concat "manifests/" meta.name ".yml" ))
     vars:   "--- {}"
+    vars-pr: (( grab meta.manifest.vars ))
     operator_file_paths: "" # comma separated list relative to repo root
 
   image:
@@ -28,6 +29,7 @@ meta:
 
   aws:
     bucket:     (( concat meta.pipeline "-pipeline" ))
+    region_name: us-east-1
     access_key: (( param "Please set your AWS Access Key ID for your pipeline S3 Bucket" ))
     secret_key: (( param "Please set your AWS Secret Key ID for your pipeline S3 Bucket" ))
 
@@ -95,7 +97,9 @@ jobs:
             BOSH_CLIENT:          (( grab meta.bosh-lite.username ))
             BOSH_CLIENT_SECRET:   (( grab meta.bosh-lite.password ))
             BOSH_DEPLOYMENT:      (( grab meta.bosh-lite.deployment ))
-            TEST_ERRANDS:         (( grab meta.test-errands || ~ ))
+            TEST_ERRANDS:         (( grab meta.test-errands || meta.test-errand || ~ ))
+            AWS_ACCESS_KEY:       (( grab meta.aws.access_key ))
+            AWS_SECRET_KEY:       (( grab meta.aws.secret_key ))
             MANIFEST_PATH:        (( grab meta.manifest.path ))
             MANIFEST_VARS:        (( grab meta.manifest.vars ))
             MANIFEST_OP_PATHS:    (( grab meta.manifest.operator_file_paths ))
@@ -137,9 +141,11 @@ jobs:
             BOSH_CLIENT:          (( grab meta.bosh-lite.username ))
             BOSH_CLIENT_SECRET:   (( grab meta.bosh-lite.password ))
             BOSH_DEPLOYMENT:      (( grab meta.bosh-lite.deployment-pr ))
-            TEST_ERRANDS:         (( grab meta.test-errands || ~ ))
+            TEST_ERRANDS:         (( grab meta.test-errands || meta.test-errand || ~ ))
+            AWS_ACCESS_KEY:       (( grab meta.aws.access_key ))
+            AWS_SECRET_KEY:       (( grab meta.aws.secret_key ))
             MANIFEST_PATH:        (( grab meta.manifest.path ))
-            MANIFEST_VARS:        (( grab meta.manifest.vars ))
+            MANIFEST_VARS:        (( grab meta.manifest.vars-pr ))
             MANIFEST_OP_PATHS:    (( grab meta.manifest.operator_file_paths ))
         on_success:
           put: git-pull-requests
@@ -249,6 +255,7 @@ jobs:
           outputs:
             - name: gh
             - name: pushme
+            - name: notifications
           run:
             path: ./git/ci/scripts/shipit
             args: []
@@ -257,9 +264,9 @@ jobs:
             VERSION_FROM: version/number
             RELEASE_ROOT: gh
             REPO_OUT:     pushme
-            BRANCH:       (( grab meta.github.branch ))
-            GITHUB_OWNER: (( grab meta.github.owner ))
-            MANIFEST_PATH: (( concat "manifests/" meta.name ".yml" ))
+            NOTIFICATION_OUT: notifications
+            BRANCH:        (( grab meta.github.branch ))
+            GITHUB_OWNER:  (( grab meta.github.owner ))
 
             AWS_ACCESS_KEY:       (( grab meta.aws.access_key ))
             AWS_SECRET_KEY:       (( grab meta.aws.secret_key ))
@@ -284,6 +291,14 @@ jobs:
         put: version
         params:
           bump: patch
+      - name: notify
+        aggregate:
+        - put: notify
+          params:
+            channel:  (( grab meta.slack.channel ))
+            username: (( grab meta.slack.username ))
+            icon_url: (( grab meta.slack.icon ))
+            text_file: notifications/message
       on_failure:
         put: notify
         params:
@@ -324,6 +339,7 @@ resources:
     source :
       driver:            s3
       bucket:            (( grab meta.aws.bucket ))
+      region_name:       (( grab meta.aws.region_name ))
       key:               version
       access_key_id:     (( grab meta.aws.access_key ))
       secret_access_key: (( grab meta.aws.secret_key ))
@@ -345,6 +361,7 @@ resources:
     type: s3
     source:
       bucket:            (( grab meta.aws.bucket ))
+      region_name:       (( grab meta.aws.region_name ))
       regexp:            (( concat meta.name "-(.*).tgz" ))
       access_key_id:     (( grab meta.aws.access_key ))
       secret_access_key: (( grab meta.aws.secret_key ))
diff --git a/ci/release_notes.md b/ci/release_notes.md
@@ -1,3 +1,4 @@
+* `shield-daemon` job `shield-db` link is now optional (thanks @karampok)
 * `shield-daemon` job has two new properties that can be overridden if you need to tweak these settings (thanks https://github.com/starkandwayne/shield-boshrelease/pull/90 from @karampok)
 
     ```

diff --git a/ci/scripts/shipit b/ci/scripts/shipit
@@ -24,7 +24,9 @@ header() {
 : ${BRANCH:?required}
 : ${GITHUB_OWNER:?required}
 : ${VERSION_FROM:?required}
-: ${MANIFEST_PATH:?required}
+: ${AWS_ACCESS_KEY:?required}
+: ${AWS_SECRET_KEY:?required}
+
 if [[ ! -f ${VERSION_FROM} ]]; then
   echo >&2 "Version file (${VERSION_FROM}) not found.  Did you misconfigure Concourse?"
   exit 2
@@ -47,14 +49,14 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd ${REPO_ROOT}
 RELEASE_NAME=$(bosh2 int config/final.yml --path /final_name)
 
-cat > config/private.yml << EOF
+cat > config/private.yml <<YAML
 ---
 blobstore:
   provider: s3
   options:
-    access_key_id: ${AWS_ACCESS_KEY:?required}
-    secret_access_key: ${AWS_SECRET_KEY:?required}
-EOF
+    access_key_id: ${AWS_ACCESS_KEY}
+    secret_access_key: ${AWS_SECRET_KEY}
+YAML
 
 header "Pulling in any git submodules..."
 git submodule update --init --recursive --force
@@ -99,11 +101,17 @@ if [[ -z $(git config --global user.name) ]]; then
 fi
 
 (cd ${REPO_ROOT}
- $DIR/update-manifest $GITHUB_OWNER $RELEASE_NAME $VERSION $SHA1 $MANIFEST_PATH
+ for MANIFEST_PATH in $(ls manifests/*.yml); do
+   $DIR/update-manifest $GITHUB_OWNER $RELEASE_NAME $VERSION $SHA1 $MANIFEST_PATH
+ done
  git merge --no-edit ${BRANCH}
  git add -A
  git status
  git commit -m "release v${VERSION}")
 
 # so that future steps in the pipeline can push our changes
 cp -a ${REPO_ROOT} ${REPO_OUT}
+
+cat > ${NOTIFICATION_OUT:-notifications}/message <<EOS
+New ${RELEASE_NAME} v${VERSION} released. <https://github.com/${GITHUB_OWNER}/${RELEASE_NAME}-boshrelease/releases/tag/v${VERSION}|Release notes>.
+EOS
diff --git a/ci/scripts/testflight b/ci/scripts/testflight
@@ -24,6 +24,8 @@ set -e
 : ${BOSH_CLIENT_SECRET:?required}
 : ${BOSH_DEPLOYMENT:?required}
 : ${MANIFEST_PATH:?required}
+: ${AWS_ACCESS_KEY:?required}
+: ${AWS_SECRET_KEY:?required}
 
 cd ${REPO_ROOT:?required}
 header "Pulling in any git submodules..."
@@ -67,6 +69,16 @@ header "Cleaning up from any previous deployments..."
 bosh2 -n delete-deployment
 
 header "Creating candidate BOSH release..."
+bosh2 -n reset-release # in case dev_releases/ is in repo accidentally
+
+cat > config/private.yml <<YAML
+---
+blobstore:
+  provider: s3
+  options:
+    access_key_id: ${AWS_ACCESS_KEY}
+    secret_access_key: ${AWS_SECRET_KEY}
+YAML
 bosh2 create-release
 bosh2 upload-release --rebase
 
@@ -78,6 +90,9 @@ vm_type=$(bosh2 int <(bosh2 cloud-config) --path /vm_types/0/name)
 disk_type=$(bosh2 int <(bosh2 cloud-config) --path /disk_types/0/name)
 network=$(bosh2 int <(bosh2 cloud-config) --path /networks/0/name)
 
+release_final_version=$(spruce json dev_releases/*/index.yml | jq -r ".builds[].version" | sed -e "s%+.*%%")
+release_dev_version="${release_final_version}.latest"
+
 cat > tmp/deployment.yml <<YAML
 ---
 - type: replace
@@ -88,7 +103,7 @@ cat > tmp/deployment.yml <<YAML
   path: /releases/name=${release_name}
   value:
     name: ${release_name}
-    version: latest
+    version: ${release_dev_version}
 YAML
 
 for ig_name in $(spruce json ${MANIFEST_PATH} | jq -r ".instance_groups[].name"); do

diff --git a/ci/settings.yml b/ci/settings.yml
@@ -30,8 +30,8 @@ meta:
     repo:   shield-boshrelease
     branch: master
     branch-6x: 6.x
-    private_key:  (( vault "secret/pipelines/shield-boshrelease/github:private" ))
-    access_token: (( vault "secret/pipelines/shield-boshrelease/github:token" ))
+    private_key:  (( vault "secret/pipelines/shared/github:private_key" ))
+    access_token: (( vault "secret/pipelines/shared/github:access_token" ))
 
   slack:
     webhook: (( vault "secret/pipelines/shield-boshrelease/slack:webhook" ))