From 645d0ea074fd466af394a2fb8155fce6821f1b41 Mon Sep 17 00:00:00 2001 From: CI Bot Date: Mon, 4 Dec 2017 23:27:42 +0000 Subject: [PATCH] release v8.0.0 --- .final_builds/jobs/core/index.yml | 6 + .final_builds/jobs/import/index.yml | 6 + .final_builds/jobs/shield-agent/index.yml | 4 + .final_builds/jobs/store/index.yml | 6 + .final_builds/license/index.yml | 9 +- .final_builds/packages/nginx/index.yml | 9 +- .final_builds/packages/shield/index.yml | 4 + .final_builds/packages/vault/index.yml | 6 + ci/release_notes.md | 368 ---------------------- manifests/shield.yml | 6 +- releases/shield/index.yml | 2 + releases/shield/shield-8.0.0.yml | 41 +++ 12 files changed, 90 insertions(+), 377 deletions(-) create mode 100644 .final_builds/jobs/core/index.yml create mode 100644 .final_builds/jobs/import/index.yml create mode 100644 .final_builds/jobs/store/index.yml create mode 100644 .final_builds/packages/vault/index.yml delete mode 100644 ci/release_notes.md create mode 100644 releases/shield/shield-8.0.0.yml diff --git a/.final_builds/jobs/core/index.yml b/.final_builds/jobs/core/index.yml new file mode 100644 index 00000000..ba1ddac8 --- /dev/null +++ b/.final_builds/jobs/core/index.yml @@ -0,0 +1,6 @@ +builds: + c38e700d82fa31e399193fa6ece41076ba670767: + version: c38e700d82fa31e399193fa6ece41076ba670767 + blobstore_id: 6e6e3868-acf0-472f-6558-46f3921d1b20 + sha1: 37a0087196a59c2553a31b5ad0e2dff9d285e01a +format-version: "2" diff --git a/.final_builds/jobs/import/index.yml b/.final_builds/jobs/import/index.yml new file mode 100644 index 00000000..4f315014 --- /dev/null +++ b/.final_builds/jobs/import/index.yml @@ -0,0 +1,6 @@ +builds: + ca3cb29830e91422e3cddc42584d70545850cdbd: + version: ca3cb29830e91422e3cddc42584d70545850cdbd + blobstore_id: 96607d1f-1993-4bc0-7a21-6e59be8101c0 + sha1: 207c322aa376b98744e45334903bd9a1e2d83cbb +format-version: "2" diff --git a/.final_builds/jobs/shield-agent/index.yml b/.final_builds/jobs/shield-agent/index.yml index 533ac2d5..9b951366 100644 --- a/.final_builds/jobs/shield-agent/index.yml +++ b/.final_builds/jobs/shield-agent/index.yml @@ -35,6 +35,10 @@ builds: version: 5a02735092b02a1a0d737c4343425e3291c1e306 blobstore_id: 2661c242-df3c-470f-9342-c7edef376837 sha1: 49dcf7bc061270849ace6937e105e5fe905e19c8 + 5bb5be59d93008633523c6f3e7c32070fc9e2451: + version: 5bb5be59d93008633523c6f3e7c32070fc9e2451 + blobstore_id: b942c360-2442-406a-77bf-562dbd19e222 + sha1: 4e5058914a4da694bd4e2a18ad136102370bce13 5bdaad52cd2c780cff315a11522b3b9ec1f74eb8: version: 5bdaad52cd2c780cff315a11522b3b9ec1f74eb8 blobstore_id: e1a09684-148b-461d-8eed-35508c5c92b0 diff --git a/.final_builds/jobs/store/index.yml b/.final_builds/jobs/store/index.yml new file mode 100644 index 00000000..cd74945b --- /dev/null +++ b/.final_builds/jobs/store/index.yml @@ -0,0 +1,6 @@ +builds: + 0d9d35e4319c0f11ac42592a5e284d6eb6176463: + version: 0d9d35e4319c0f11ac42592a5e284d6eb6176463 + blobstore_id: c65a1e61-6397-4fc3-735d-3a902003a390 + sha1: 1e0cdf9c099bdfa32e8b9ecf724764aa74fa075f +format-version: "2" diff --git a/.final_builds/license/index.yml b/.final_builds/license/index.yml index 8dc33c88..50c801d6 100644 --- a/.final_builds/license/index.yml +++ b/.final_builds/license/index.yml @@ -1,7 +1,10 @@ ---- builds: 79267db2e7516e504dfe773afdfd03a5ecfaf43a: version: 79267db2e7516e504dfe773afdfd03a5ecfaf43a - sha1: 79cecf4b9642b40459bfb5d7de7f29048ba5bdfb blobstore_id: a1b3d539-9fa0-4a85-812b-3a28bd559ca8 -format-version: '2' + sha1: 79cecf4b9642b40459bfb5d7de7f29048ba5bdfb + fc24a259da4ed145bfa70db6bc689b7dfb91673c: + version: fc24a259da4ed145bfa70db6bc689b7dfb91673c + blobstore_id: 858e5b45-9442-4a66-69db-a77d85336b98 + sha1: c7c3ea002ab3a6a32019cb39099ca59e2ec73175 +format-version: "2" diff --git a/.final_builds/packages/nginx/index.yml b/.final_builds/packages/nginx/index.yml index ce529a77..e2549e10 100644 --- a/.final_builds/packages/nginx/index.yml +++ b/.final_builds/packages/nginx/index.yml @@ -1,7 +1,10 @@ ---- builds: + 3ba0635a08bd2a2c8c65ae885607feedf24ad0ff: + version: 3ba0635a08bd2a2c8c65ae885607feedf24ad0ff + blobstore_id: f64e2333-93e6-40c4-56a8-71b21a93cc22 + sha1: 88d5ed2b8c4fadb62797797c4a7d06202519a2bf 3e69a75504769646a78bc7464722f10c73489129: version: 3e69a75504769646a78bc7464722f10c73489129 - sha1: a4b06094253406d7a91f15ecf4164298251cf7f4 blobstore_id: 8a428eeb-b94b-482c-a37e-7378510ff53e -format-version: '2' + sha1: a4b06094253406d7a91f15ecf4164298251cf7f4 +format-version: "2" diff --git a/.final_builds/packages/shield/index.yml b/.final_builds/packages/shield/index.yml index 69a14316..e485d8c5 100644 --- a/.final_builds/packages/shield/index.yml +++ b/.final_builds/packages/shield/index.yml @@ -59,6 +59,10 @@ builds: version: 73265946be62a4616322e12880ab37642b86b3eb blobstore_id: a5f21809-b4b4-4b73-b9ef-36bbf7f41099 sha1: d4e7689b5871172ba637bcbf964a4fada604d093 + 737bcec47f6494d262530fedc3fad83d9bef8bfa: + version: 737bcec47f6494d262530fedc3fad83d9bef8bfa + blobstore_id: 766ec50f-9f0e-4310-4462-03f1650ad72c + sha1: 8ebbdfb152759c470044ec80dc5131bb2b56c188 79776a1fe2888eae0a5829b2e1047986ee6981a4: version: 79776a1fe2888eae0a5829b2e1047986ee6981a4 blobstore_id: 9858ec7f-3b9c-462f-a712-63aa4599cbf2 diff --git a/.final_builds/packages/vault/index.yml b/.final_builds/packages/vault/index.yml new file mode 100644 index 00000000..cdd18f93 --- /dev/null +++ b/.final_builds/packages/vault/index.yml @@ -0,0 +1,6 @@ +builds: + 3e8858808909a77022801c4658251bd8104c58ef: + version: 3e8858808909a77022801c4658251bd8104c58ef + blobstore_id: 4f229bce-7a85-4fe9-6a37-9a5e19ead094 + sha1: cab7741012ad26c4eda7c6de54cb753cd263d9d6 +format-version: "2" diff --git a/ci/release_notes.md b/ci/release_notes.md deleted file mode 100644 index 474380b6..00000000 --- a/ci/release_notes.md +++ /dev/null @@ -1,368 +0,0 @@ -This release packages SHIELD v8. -SHIELD v8 is a dramatic overhaul of the SHIELD Data Protection -System, and this BOSH release version is a severe departure from -the previous methods of deploying SHIELD. - - -Upgrading from v6.x / v7.x --------------------------- - -This section details deployment manifest changes that operators -will need to apply in order to migate from v6 / v7 of SHIELD to -this (v8) release. - -### Changes to the `shield-daemon` job (now `core`) - -The `shield-daemon` job is now just `core`. - -The `name` property is gone. In its place are the following -properties for identifying your SHIELD: - - - `core.env` - The name of the environment, like "sandbox" or - "production", or "a testing shield instance", or "fred". - - `core.color` - A CSS color name, or hexadecimal RGB color to - use for the environment name in the new web UI. `yellow` - and `green` look nice. - - `core.motd` - A (possible multi-line) message that will be - displayed to users logging into SHIELD. Useful for whatever - messages of the day are generally useful for (compliance, - advertising maintenance windows, etc.) - -`workers` has been renamed to `core.workers`, but otherwise retains -its semantic meaning. - -`max_timeout` has been renamed to `core.task-timeout`, but -otherwise retains its semantic meaning. - -`ssl.key`, `ssl.crt`, and `ssl.timeout` have been renamed to -`tls.key`, `tls.certificate`, and `tls.reuse-after`, respectively. -The default value of `tls.reuse-after` was dropped from 12 (hours) -to 2 (hours). - -The `ssh_private_key` has been renamed to `agent.key`, because -it's not used for SSH in the same sense as most SSH (RSA) keys. -Its value should stay the same for a smooth upgrade. - -The `database.*` properties have been removed; SHIELD v8 has its -own internal data store that does not need to be configured by the -operator. See the **Database Migration** subsection, later, for -details on migrating your data into this new data store. - -The `auth.oauth.*` properties have been removed; SHIELD v8 -supports multiple (possibly OAuth2-based) authentication -providers. These are configured under the new top-level -`authentication` key. - -The `auth.username` and `auth.password` properties have been -removed; SHIELD v8 no longer supports simple HTTP Basic -Authentication. Instead, it features a robust user authentication -system backed by an internal local user database. Two new -properties, `failsafe.username` and `failsafe.password` kind of -take over for these deprecated properties. They specify a -username and (cleartext) password that SHIELD will insert into the -local user database if it boots up and finds that no users exist -yet. This "failsafe" is intended to provide a secure way of -bootstrapping a SHIELD environment, without being stuck with a -user whose password is in a BOSH manifest somewhere. -Administrators are free to delete the failsafe user once they have -set up other accounts. - -The `auth.api_keys` property has been removed; SHIELD v8 does not -support API Keys in the same fashion as its predecessors. -Instead, user accounts are free to issue _Auth Tokens_ that behave -a stand-ins for their issuer (not unlike Github Personal Access -Tokens). - -`nginx.worker_processes` has been shortened to `nginx.workers`. - -`nginx.worker_connections` has been shortened to -`nginx.connections`. - -`nginx.keepalive_timeout` has been shortened to `nginx.keepalive`. - -The `log_level` property has been renamed to `log-level`. - -### Changes to the `shield-agent` job - -This job is still called `shield-agent`, since it needs to be -unique across a wide variety of other deployments. - -`name` is a new property for specifying the name this agent will -use when registering with the SHIELD Core. - -`autoprovision` has been removed. Its usage was always -problematic, and with the introduction of proper BOSH links, we -only need to specify where and how to talk to the SHIELD Core in -the event that our Core lives on another BOSH director (which is -rare). - -`shield-url` is a new property that kind of takes the place of -`autoprovision`, by allowing operators to identify where their -SHIELD Core lives, as a full URL (i.e. -"https://shield.example.com") - -`require-shield-core` is a new property that lets operators ignore -an error condition whereby an agent is unable to communicate with -the SHIELD Core. In theory, that is a show-stopping problem, but -in practice, we found that it held up too many deployments for -legitimate reasons, ranging from simple network connectivity -issues and firewalling to more mundane problems like "we haven't -deployed SHIELD itself yet." - -The `daemon_public_key` property has been removed. In its place -is the new `agent.key` property. The most important difference -between these two properties is that `daemon_public_key` was the -authorized_keys-formatted public key, and `agent.key` is the -**private key** that the SHIELD Core also specifies as -`agent.key`. Internally, the BOSH release will extract the -correctly formatted public fingerprint from the private key. - -Note that if the `shield` link is in use, you don't need to -explicitly set `agent.key` -- the agent startup scripts will just -retrieve the public key from the SHIELD Core automagically. This -allows SHIELD site operators to rotate that key with minimal fuss. - -The `recovery.*` properties have been removed, since SHIELD v8's -new encryption feature makes it difficult to restore backups -outside of the watchful eye of a running SHIELD Core. - -For SHIELD Agents that need to operate behind HTTP proxies, three -new `env.*` properties were added. `env.http_proxy` and -`env.https_proxy` allow you to specify the full URL for an -upstream proxy that will handle (respectively) cleartext HTTP -requests and TLS-encrypted HTTPS requests. The `env.no_proxy` -property is a list of FQDNs, domain fragments, and IP addresses -that will be flattened and joined by commas to fashion an -exclusion list to put in the `$no_proxy` environment variable. - -The new `env.path`, `env.libs`, and `env.auto` properties control -how the SHIELD agent process will set up its environment, for the -benefit of executed plugins. - -`env.path` is a list of auxiliary paths to bin/ and sbin/ -directories that you want to manually inject into the `$PATH` of -the running shield-agent / plugins. - -`env.libs` is a list of auxiliary paths to lib/ directories that -you want / need in your `$LD_LIBRARY_PATH` for dynamic shared -object runtime loading. - -`env.auto` is a boolean; if set, the shield-agent job will go -looking for installed BOSH packages named `shield-addon-*`, add -any bin/ and sbin/ directories to `$PATH`, and add any lib/ -directories to `$LD_LIBRARY_PATH`. This allows you to augment an -agent with additional command-line tools it might need, like -specific versions of `psql`, or `xtrabackup`. `env.auto` is on by -default. - -The auto-provisioning properties `stores`, `targets`, -`retention-policies`, and `jobs` have all been removed, in favor -of the new `buckler import`-based `import` errand. - -The `log_level` property has been renamed to `log-level`. - -### Removed Jobs - -The `agent-mysql` and `xtrabackup` jobs have been removed. If you -need to augment a SHIELD agent with MySQL / MariaDB tools, you can -try the nee [SHIELD MySQL Addon][mysql-addon], which contains all -of these packages. - -The `agent-pgtools` job has been removed. If you need to augment -a SHIELD agent with PostgreSQL tools, you can try the new [SHIELD -PostgreSQL Addon][postgres-addon]. - -The `mongo-tools3.2` and `mongo-tools3.4` jobs have been removed. -They too have moved into a separate BOSH release, the [SHIELD -MongoDB Addon][mongodb-addon]. - -The `postgres` and `mariadb` jobs have been removed. SHIELD v8 -now leverages a standalone, dedicated database that is baked into -the new `core` job. See the subsection **Database Migration**, -below, for details on migrating your SHIELD data. - -### The New Import Errand - -Previous versions of the SHIELD BOSH release used a post-start -script and `shield-agent` properties to facilitate a form of -configuration auto-provisioning. - -In v8, this has all been replaced by the new `import` errand, -which drives the much more powerful and flexible `buckler import` -command-line tool. - -The `import` errand takes a single property, `import`, which is a -full recipe of things to import into SHIELD, as understood by the -`buckler` tool's `import` sub-command. - -Here's an example that sets up a bunch of stuff: - -```yaml -- name: import - lifecycle: errand - instances: 1 - azs: [z1] - vm_type: default - stemcell: default - networks: {name: default} - jobs: - - name: import - release: shield - properties: - import: - core: https://shield.example.com - token: ... # an auth token, from `buckler create-auth-token` - - global: - storage: - - name: S3 Cloud Storage - summary: | - Public S3 cloud storage for all SHIELD tenants to use - agent: 127.0.0.1:5444 - plugin: s3 - config: - access_key_id: AKI12 - secret_access_key: secret - - policies: - - name: Long-Term Storage - days: 90 - - users: - - name: James Hunt - username: jhunt - password: sekrit - sysrole: admin - tenants: - - name: Stark & Wayne - role: admin - - tenants: - - name: CF Community - members: - - user: jhunt@local - role: admin - storage: - - name: Scality - agent: 10.32.45.10:5444 - plugin: scality - config: - s3_host: 10.32.45.1 - s3_port: 8200 - bucket: my-bucket - - policies: - - name: Short-Term - days: 7 - - name: Long-Term - days: 90 - - systems: - - name: BOSH - agent: 10.4.0.1:5444 - plugin: postgres - config: {} - jobs: - - name: Daily - when: daily 4am - policy: Short-Term - storage: Scality - paused: true - - - name: Monthly - when: every month on the 1st at 3am - policy: Long-Term - storage: Scality -``` - - -### New Encryption Vault - -SHIELD v8 encrypts all backup archives, and it uses a unique, -randomly generated initialization vector and encryption key for -each new archive. These secrets are required for restoration, and -they have to be stored somewhere safe, so we store then in an -encrypted vault. - -For the most part, the care and feeding of this vault is entirely -handled for you. However, the deployment needs to configure an -X.509 Certificate Authority, and issue an X.509 Certificate for -the IP SAN 127.0.0.1. - - -### Database Migration - -SHIELD v8 introduces several new features, including a new -built-in data store. Chances are if you are upgrading from a -previous version of SHIELD (either v6 or v7), you are going to -want to migrate all that data. To do so safely and effectively, -you just need to specify the `migrate-from.type` and -`migrate-from.dsn` properties in your SHIELD deployment manifest. - -For example, if you had a v6 SHIELD BOSH deployment manifest with -`shield-daemon` properties that looked like this: - -```yaml -# old school -properties: - database: - type: postgres - host: 10.5.6.7 - port: 5524 - username: dba - password: sekrit - database: shield1 -``` - -Then your `migrate-from.type` should be "postgres", and -`migrate-from.dsn` should roll up all that connectivity -information in a PostgreSQL data source name, like this: - -```yaml -# new school -properties: - migrate-from: - type: postgres - dsn: postgres://dba:sekrit@10.5.6.7:5524/shield1?sslmode=disable -``` - -Likewise, if you used to use MySQL for SHIELD, and had this in -your manifest: - -```yaml -# old school -properties: - type: mysql - host: 172.15.3.4 - port: 3316 - username: scyld - password: sekrit - database: shielddb -``` - -You would want to specify this to engage data migration: - -```yaml -# new school -properties: - type: mysql - dsn: scyld:sekrit@tcp(172.15.3.4:3316)/shielddb -``` - -Refer to the [lib/pq documentation][pq-dsn] and the -[go-sql-driver/mysql documentation][my-dsn] for more details. - -Note that database migration is a once-only affair. If the -internal database file exists, the release will skip migration -altogether. This takes some of the urgency out of needing to -"clean up" your deployment manifest to remove the `migrate-from.*` -properties. - - - -[mysql-addon]: https://github.com/shieldproject/shield-mysql-addon-boshrelease -[postgres-addon]: https://github.com/shieldproject/shield-postgres-addon-boshrelease -[mongodb-addon]: https://github.com/shieldproject/shield-mongodb-addon-boshrelease - -[pq-dsn]: https://godoc.org/github.com/lib/pq -[my-dsn]: https://github.com/go-sql-driver/mysql#dsn-data-source-name diff --git a/manifests/shield.yml b/manifests/shield.yml index 77e05eb9..fdc02a12 100644 --- a/manifests/shield.yml +++ b/manifests/shield.yml @@ -94,6 +94,6 @@ stemcells: releases: - name: shield - version: 7.0.3 - url: https://github.com/starkandwayne/shield-boshrelease/releases/download/v7.0.3/shield-7.0.3.tgz - sha1: a1f705e0ad0130ac78d851332d97905a3f8cbce2 + version: 8.0.0 + url: https://github.com/starkandwayne/shield-boshrelease/releases/download/v8.0.0/shield-8.0.0.tgz + sha1: 75f663c61bc3903d02ce2820977ea54282ba53c0 diff --git a/releases/shield/index.yml b/releases/shield/index.yml index 3cc7c56c..445b973b 100644 --- a/releases/shield/index.yml +++ b/releases/shield/index.yml @@ -35,6 +35,8 @@ builds: version: 6.6.4 83b596c6-c271-41d9-b6bf-b8c53ae80071: version: "2" + 853317a4-2688-40b3-5f5e-ddc289d3d88f: + version: 8.0.0 91c4f18c-8275-40bd-a459-f0c41c2d5859: version: 6.3.4 928b8be6-6542-431f-a962-7199abe66f0f: diff --git a/releases/shield/shield-8.0.0.yml b/releases/shield/shield-8.0.0.yml new file mode 100644 index 00000000..684ba08d --- /dev/null +++ b/releases/shield/shield-8.0.0.yml @@ -0,0 +1,41 @@ +name: shield +version: 8.0.0 +commit_hash: 3a0b35f +uncommitted_changes: false +jobs: +- name: core + version: c38e700d82fa31e399193fa6ece41076ba670767 + fingerprint: c38e700d82fa31e399193fa6ece41076ba670767 + sha1: 37a0087196a59c2553a31b5ad0e2dff9d285e01a +- name: import + version: ca3cb29830e91422e3cddc42584d70545850cdbd + fingerprint: ca3cb29830e91422e3cddc42584d70545850cdbd + sha1: 207c322aa376b98744e45334903bd9a1e2d83cbb +- name: shield-agent + version: 5bb5be59d93008633523c6f3e7c32070fc9e2451 + fingerprint: 5bb5be59d93008633523c6f3e7c32070fc9e2451 + sha1: 4e5058914a4da694bd4e2a18ad136102370bce13 +- name: store + version: 0d9d35e4319c0f11ac42592a5e284d6eb6176463 + fingerprint: 0d9d35e4319c0f11ac42592a5e284d6eb6176463 + sha1: 1e0cdf9c099bdfa32e8b9ecf724764aa74fa075f +packages: +- name: nginx + version: 3ba0635a08bd2a2c8c65ae885607feedf24ad0ff + fingerprint: 3ba0635a08bd2a2c8c65ae885607feedf24ad0ff + sha1: 88d5ed2b8c4fadb62797797c4a7d06202519a2bf + dependencies: [] +- name: shield + version: 737bcec47f6494d262530fedc3fad83d9bef8bfa + fingerprint: 737bcec47f6494d262530fedc3fad83d9bef8bfa + sha1: 8ebbdfb152759c470044ec80dc5131bb2b56c188 + dependencies: [] +- name: vault + version: 3e8858808909a77022801c4658251bd8104c58ef + fingerprint: 3e8858808909a77022801c4658251bd8104c58ef + sha1: cab7741012ad26c4eda7c6de54cb753cd263d9d6 + dependencies: [] +license: + version: fc24a259da4ed145bfa70db6bc689b7dfb91673c + fingerprint: fc24a259da4ed145bfa70db6bc689b7dfb91673c + sha1: c7c3ea002ab3a6a32019cb39099ca59e2ec73175