Merge pull request #326 from tapis-project/staging

Staging

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 Notable changes between versions.
 
+## 1.5.2
+
+### Services Updates
+
+- [Systems: 1.5.0 to 1.5.10 (tapis/systems)](https://github.com/tapis-project/tapis-systems/blob/dev/CHANGELOG.md)
+- [Apps: 1.5.0 to 1.5.10 (tapis/apps)](https://github.com/tapis-project/tapis-apps/blob/dev/CHANGELOG.md)
+- [Notifications: 1.5.0 to 1.5.10 (tapis/notifications, notifications-dispatcher)](https://github.com/tapis-project/tapis-notifications/blob/dev/CHANGELOG.md)
+- [Files: 1.5.0 to 1.5.10 (tapis/tapis-files, tapis/tapis-files-workers)](https://github.com/tapis-project/tapis-files/blob/dev/CHANGELOG.md)
+- [Jobs: 1.5.0 to 1.5.10 (tapis/jobsworker, jobsmigrate, jobsapi)](https://github.com/tapis-project/tapis-jobs/blob/dev/tapis-jobsapi/CHANGELOG.md)
+- [Security: 1.5.0 to 1.5.10 (tapis/securitymigrate, securityadmin, securityapi, securityexport)](https://github.com/tapis-project/tapis-security/blob/dev/tapis-securityapi/CHANGELOG.md)
+- [Meta: 1.5.0 to 1.5.10 (tapis/metaapi)](https://github.com/tapis-project/tapis-meta/blob/dev/CHANGELOG.md)
+- Rebuild java services with latest shared code to fix JWT validation issue.
+
 
 ## 1.5.1
 

playbooks/roles/actors/defaults/main/images.yml

@@ -7,3 +7,4 @@ actors_mongo_image: mongo:4.2.6
 actors_alpine_image: alpine:3.17
 actors_mongobackup_image: tapis/mongobackup:0.1
 actors_rabbitmq_image: rabbitmq:3.6.12-management
+actors_util_image: tapis/ubutil2204:1.5.0

playbooks/roles/actors/templates/docker/burnup

@@ -4,7 +4,7 @@ echo "burnup actors:"
 
 mkdir -p {{ tapisdatadir }}/actors
 MYUID=`id -u`
-docker run -it --rm -v {{ tapisdatadir }}/actors:/actors tapis/ubutil2204:1.4.0 chown $MYUID /actors
+docker run -it --rm -v {{ tapisdatadir }}/actors:/actors {{ actors_util_image }} chown $MYUID /actors
 
 python3 ../admin/util/parse_skexport -c actors -d {{ tapisdatadir }}
 

playbooks/roles/admin/templates/docker/util/parse_skexport

@@ -5,7 +5,8 @@ import json
 
 
 # var mapping for all comps
-# e.g. {"{skexport_name}": "{internal_component_name}"}
+# e.g. {"{internal_component_name}": "{skexport_name}"}
+
 actors = {
     "RABBITMQ_DEFAULT_PASS":"DBCREDENTIAL_RABBITMQ_ABACO_RABBIT_MQ_HOST_RABBITMQ_ABACO_PASSWORD",
     "MONGO_INITDB_ROOT_PASSWORD": "DBCREDENTIAL_MONGO_ABACO_MONGO_DB_HOST_MONGODB_ABACO_PASSWORD",
@@ -19,6 +20,7 @@ apps = {
     "POSTGRES_USER_PASSWORD": "DBCREDENTIAL_POSTGRES_APPS_POSTGRES_TAPISAPPDB_TAPIS_APP_PASSWORD",
     "service_password": "SERVICEPWD_APPS_PASSWORD",
     "MONITOR_PASSWORD": "DBCREDENTIAL_POSTGRES_APPS_POSTGRES_TAPISAPPDB_TAPIS_APP_PASSWORD"
+    }
 
 authenticator = {
     "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_POSTGRES_AUTHENTICATOR_AUTHENTICATOR_PASSWORD",
@@ -57,7 +59,13 @@ meta = {
     "alt_mongo_password": "DBCREDENTIAL_MONGO_ALT_RESTHEART_MONGO_NA_RESTHEART_PASSWORD"
     }
 
-monitoring = {"":""}
+monitoring = {
+    "GF_SECURITY_ADMIN_PASSWORD":"USER_ADMIN_GRAFANA_ADMIN_SECRETS_GRAFANA_PASSWORD",
+    "GF_SERVER_DOMAIN": "{{  global_service_url |replace("https://", "") }}",
+    "TAPIS_DB_MONITOR_PASSWORD":"USER_ADMIN_GRAFANA_ADMIN_POSTGRES_RO_PASSWORD_PASSWORD",
+    "AWS_SECRET_ACCESS_KEY": "USER_ADMIN_GRAFANA_ADMIN_THANOS_S3_SECRET_KEY_PASSWORD",
+    "META_PASSWORD": "DBCREDENTIAL_MONGO_RESTHEART_MONGO_NA_RESTHEART_PASSWORD"
+}
 
 notifications = {
     "PGADMIN_DEFAULT_PASSWORD": "DBCREDENTIAL_PGADMIN_NOTIFICATIONS_POSTGRES_TAPISNTFDB_WOW_PASSWORD",
@@ -91,18 +99,23 @@ security = {
 skadmin = {"":""}
 
 streams = {
-    "DBCREDENTIAL_INFLUX_INFLUX_HOST_CHORDS_TS_PRODUCTION_ADMIN_PASSWORD": "influxdb_password",  
-    "DBCREDENTIAL_INFLUX_INFLUX_HOST_CHORDS_TS_PRODUCTION_ADMIN_PASSWORD": "INFLUXDB_PASSWORD",
-    "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD": "mysql_password",
-    "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD": "mysql-password",
-    "USER_ADMIN_ADMIN_SECRETS_CHORDS_API_KEY": "chords_api_key",
-    "USER_ADMIN_ADMIN_SECRETS_CHORDS_API_KEY": "CHORDS_API_KEY",
-    "DBCREDENTIAL_INFLUX_INFLUX_HOST_CHORDS_TS_PRODUCTION_ADMIN_PASSWORD": "influx-password",
-    "SERVICEPWD_STREAMS_PASSWORD": "service-password",
-    "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD": "CHORDS_ADMIN_PW",
-    "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD": "INFLUXDB_ADMIN_PW",
-    "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD": "MYSQL_PASSWORD",
-    "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD": "CHRODS_MYSQL_PASSWORD"
+    "influxdb_password": "DBCREDENTIAL_INFLUX_INFLUX_HOST_CHORDS_TS_PRODUCTION_ADMIN_PASSWORD",
+    "INFLUXDB_PASSWORD": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "INFLUXDB_ADMIN_PW": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "mysql_password": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "mysql-password": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "MYSQL_PASSWORD": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "CHORDS_MYSQL_PASSWORD": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "CHORDS_ADMIN_PW": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "chords_api_key": "USER_ADMIN_ADMIN_SECRETS_CHORDS_API_KEY",
+    "CHORDS_API_KEY": "USER_ADMIN_ADMIN_SECRETS_CHORDS_API_KEY",
+    "influx-password": "DBCREDENTIAL_INFLUX_INFLUX_HOST_CHORDS_TS_PRODUCTION_ADMIN_PASSWORD",
+    "service-password": "SERVICEPWD_STREAMS_PASSWORD",
+    "service_password": "SERVICEPWD_STREAMS_PASSWORD",
+    "CHORDS_ADMIN_PW": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "INFLUXDB_ADMIN_PW": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "MYSQL_PASSWORD": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "CHORDS_MYSQL_PASSWORD": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD"
 }
 
 systems = {
@@ -161,13 +174,19 @@ except FileExistsError:
 
 # set local vars
 infile = tapisdatadir_absolute + '/skadmin/env'
-local = locals()[component]
+try:
+    local = locals()[component]
+except KeyError:
+    if args.verbose:
+        print(f'invalid component name. Exiting')
+        exit(1)
+
 try:
     outfile = open((tapisdatadir_absolute + '/' + args.comp + '/env'), 'w')
     if args.verbose:
-            print(f'successfuly opened {outfile}')
+            print(f'successfuly opened outfile: {outfile}')
 except Exception as e:
-    print(f'''error opening outfile {outfile}::
+    print(f'''error opening outfile: {outfile}::
     {e}''')
 output_dict = {}
 
@@ -204,7 +223,7 @@ if component == 'security':
 
 with open(infile, 'r') as lines:
     if args.verbose:
-        print(f'successfuly opened {infile}')
+        print(f'successfuly opened infile: {infile}')
     for line in lines:
         if len(line) <= 1:
             continue # ignore empty line
@@ -214,14 +233,17 @@ with open(infile, 'r') as lines:
             for key in local:
                 if local[key] == skexport_key:
                     if args.verbose:
-                        print(f'writing {key}="{skexport_value}"')
+                        print(f'mapped {key}="{skexport_value}"')
                     # outfile.write(f"{key}='{skexport_value}'\n")
                     output_dict[key] = skexport_value
                     if key == 'MONGO_INITDB_ROOT_PASSWORD':
                         output_dict["MONGO_URI"] = f"mongodb://restheart:{skexport_value}@restheart-mongo:27017/?authSource=admin"
         except KeyError:
             pass
 
+if args.verbose:
+    print(f'Completed mapping. Writing {output_dict}')
+
 for key in output_dict:
     outfile.write(f'{key}="{output_dict[key]}"\n')
 

playbooks/roles/admin/templates/docker/verification/authenticator-test

@@ -1,7 +1,8 @@
 #!/bin/bash
 #
 ## authenticator:
-curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{admin_devtenant_url}}/v3/oauth2/tokens
+source functions
+gettoken
 
 # should return 200 and JSON with token
 

playbooks/roles/admin/templates/docker/verification/files-test

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-TOK=`curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{ admin_service_url }}/v3/oauth2/tokens | jq -r '.result.access_token.access_token'`
+source functions
+gettoken
 
 curl {{ admin_devtenant_url }}/v3/files/healthcheck | jq 
 

playbooks/roles/admin/templates/docker/verification/functions

@@ -9,6 +9,10 @@ SERVICEURL="{{admin_service_url}}"
 
 ### util
 
+gettoken(){
+  tok=$(curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{ admin_service_url }}/v3/oauth2/tokens | jq -r .result.access_token.access_token)
+}
+
 ### checks
 
 checkcert(){
@@ -29,7 +33,6 @@ pingsite(){
   ping -c 1 -W 5 $1
 }
 
-
 preflight_check(){
   getconfig
   echo "site: $SERVICEURL"

playbooks/roles/admin/templates/docker/verification/jobs-test

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-TOK=`curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{ admin_service_url }}/v3/oauth2/tokens | jq -r '.result.access_token.access_token'`
+source functions
+gettoken
 
-curl {{ admin_devtenant_url }}/v3/jobs/healthcheck | jq 
+curl {{ admin_devtenant_url }}/v3/apps/healthcheck | jq 
 

playbooks/roles/admin/templates/docker/verification/meta-test

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+
+source functions
+gettoken
+
+curl -H X-Tapis-Token:$tok {{admin_devtenant_url}}/v3/meta/hello 
+
+
+curl {{admin_devtenant_url}}/v3/meta/ready

playbooks/roles/admin/templates/docker/verification/notifications-test

@@ -1,5 +1,5 @@
 #!/bin/bash
-
-TOK=`curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{ admin_service_url }}/v3/oauth2/tokens | jq -r '.result.access_token.access_token'`
+source functions
+gettoken
 
 curl  {{ admin_devtenant_url }}/v3/notifications/healthcheck | jq

playbooks/roles/admin/templates/docker/verification/sk-test

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-
+source functions
+gettoken
 
 echo "hello"
 curl -s {{ admin_devtenant_url }}/v3/security/hello | jq .status

playbooks/roles/admin/templates/docker/verification/streams-test

@@ -1,9 +1,9 @@
 #!/bin/bash
 
-curl {{admin_devtenant_url}}/v3/streams/hello 
-
+source functions
+gettoken
 
-curl {{admin_devtenant_url}}/v3/streams/hello
+curl {{admin_devtenant_url}}/v3/streams/hello 
 
 
 

playbooks/roles/admin/templates/docker/verification/systems-test

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-TOK=`curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{ admin_service_url }}/v3/oauth2/tokens | jq -r '.result.access_token.access_token'`
+source functions
+gettoken
 
 curl {{ admin_devtenant_url }}/v3/systems/healthcheck | jq 
 

playbooks/roles/admin/templates/docker/verification/tenants-test

@@ -1,6 +1,8 @@
 #!/bin/bash
 
 # tenants
+source functions
+gettoken
 # should return 200 and JSON with tenants list.
 
 curl {{admin_devtenant_url}}/v3/tenants | jq

playbooks/roles/admin/templates/docker/verification/tokens-test

@@ -4,6 +4,8 @@
 s=`grep SERVICEPWD_TENANTS_PASSWORD {{ tapisdatadir }}/skadmin/env | awk -F= '{print $2}' | tr -d " \t\n\r"`
 
 # tokens
+source functions
+gettoken
 # should return 200 and JSON with token
 
 curl -u "tenants:$s" -H "Content-type: application/json" -d '{"token_tenant_id": "admin", "account_type": "service", "token_username": "tenants", "target_site_id": "{{ admin_service_site_id }}"}'  {{ admin_devtenant_url }}/v3/tokens

playbooks/roles/apps/defaults/main/images.yml

@@ -1,3 +1,4 @@
-apps_api_image: tapis/apps:1.5.0
+apps_api_image: tapis/apps:1.5.10
 apps_postgres_image: postgres:12.4
 apps_pgadmin_image: dpage/pgadmin4:6.20
+apps_util_image: tapis/ubutil2204:1.5.0

playbooks/roles/apps/templates/docker/burnup

@@ -9,7 +9,7 @@ MYUID=`id -u`
 
 export UID_GID="$(id -u):$(id -g)"
 
-docker run -it --rm -v {{ tapisdatadir }}/apps:/apps tapis/ubutil2204:1.4.0 chown $MYUID /apps
+docker run -it --rm -v {{ tapisdatadir }}/apps:/apps {{ apps_util_image }} chown $MYUID /apps
 
 python3 ../admin/util/parse_skexport -c apps -d {{ tapisdatadir }}
 

playbooks/roles/files/defaults/main/images.yml

@@ -1,9 +1,10 @@
-files_api_image: tapis/tapis-files:1.5.0
-files_workers_image: tapis/tapis-files-workers:1.5.0
+files_api_image: tapis/tapis-files:1.5.10
+files_workers_image: tapis/tapis-files-workers:1.5.10
 files_postgres_image: postgres:11
 files_migrations_image: postgres:11
 files_minio_image: minio/minio
 files_irods_provider_postgres_image: mjstealey/irods-provider-postgres:4.2.4
 files_pgadmin_image: dpage/pgadmin4:6.20
 files_rabbitmq_image: rabbitmq:3.8.11-management
 files_rabbitmq_management_image: rabbitmq:3-management-alpine
+files_util_image: tapis/ubutil2204:1.5.0

playbooks/roles/files/templates/docker/burnup

@@ -12,7 +12,7 @@ MYUID=`id -u`
 
 export UID_GID="$(id -u):$(id -g)"
 
-docker run -it --rm -v {{ tapisdatadir }}/files:/files tapis/ubutil2204:1.4.0 chown $MYUID /files
+docker run -it --rm -v {{ tapisdatadir }}/files:/files {{ files_util_image }} chown $MYUID /files
 
 python3 ../admin/util/parse_skexport -c files -d {{ tapisdatadir }}
 

playbooks/roles/get_defaults/defaults/main/vars.yml

@@ -31,5 +31,5 @@ components_to_deploy_default:
   - tokens
   - vault
 #  - workflows  
-  - test-resources
+#  - test-resources
 

playbooks/roles/jobs/defaults/main/images.yml

@@ -1,6 +1,7 @@
-jobs_api_image: tapis/jobsapi:1.5.0
-jobs_migrations_image: tapis/jobsmigrate:1.5.0
-jobs_worker_image: tapis/jobsworker:1.5.0
+jobs_api_image: tapis/jobsapi:1.5.10
+jobs_migrations_image: tapis/jobsmigrate:1.5.10
+jobs_worker_image: tapis/jobsworker:1.5.10
 jobs_postgres_image: postgres:12.4
 jobs_pgadmin_image: dpage/pgadmin4:6.20
 jobs_rabbitmq_management_image: rabbitmq:3.8.11-management
+jobs_util_image: tapis/ubutil2204:1.5.0

playbooks/roles/jobs/templates/docker/burnup

@@ -8,7 +8,7 @@ mkdir -p {{ tapisdatadir }}/jobs/postgres
 export MYUID="$(id -u)"
 export UID_GID="$(id -u):$(id -g)"
 
-docker run -it --rm -v {{ tapisdatadir }}/jobs:/jobs tapis/ubutil2204:1.4.0 chown $MYUID /jobs
+docker run -it --rm -v {{ tapisdatadir }}/jobs:/jobs {{ jobs_util_image }} chown $MYUID /jobs
 
 python3 ../admin/util/parse_skexport -c jobs -d {{ tapisdatadir }}
 

playbooks/roles/meta/defaults/main/images.yml

@@ -1,7 +1,8 @@
-meta_api_image: tapis/metaapi:1.5.0
+meta_api_image: tapis/metaapi:1.5.10
 meta_rh_server_image: tapis/tapis-meta-rh-server:1.5.0
 meta_mongo_exporter_image: tapis/mqe:0.1
 meta_mongo_singlenode_image: tapis/mongo-singlenode:4.4.6-bionic
 meta_mongodb_backup_image: tapis/mongodb-backup:1
 meta_mongobackup_image: tapis/mongobackup:1.5.0
 meta_alpine_image: alpine:3.17
+meta_util_image: tapis/ubutil2204:1.5.0

playbooks/roles/meta/templates/docker/burnup

@@ -7,14 +7,7 @@ mkdir -p {{ tapisdatadir }}/meta/mongodb/data
 export MYUID="$(id -u)"
 export UID_GID="$(id -u):$(id -g)"
 
-docker run -it --rm -v {{ tapisdatadir }}/meta:/meta tapis/ubutil2204:1.3.0 chown $MYUID /meta
-
-python3 {{ tapisdir }}/admin/util/parse_skexport -c meta -d {{ tapisdatadir }}
-
-myuid=`id -u`
-export UID=`id -u`
-export GID=`id -g`
-docker run -it --rm -v {{ tapisdatadir }}/meta:/meta tapis/ubutil2204:1.3.0 chown $myuid /meta
+docker run -it --rm -v {{ tapisdatadir }}/meta:/meta {{ meta_util_image }} chown $MYUID /meta
 
 python3 {{ tapisdir }}/admin/util/parse_skexport -c meta -d {{ tapisdatadir }}
 

playbooks/roles/monitoring/defaults/main/images.yml

@@ -1,4 +1,4 @@
-monitoring_stern_fluentd_image: tapis/stern-fluentd:1.5.0
+monitoring_stern_fluentd_image: tapis/stern-fluentd:1.5.1
 monitoring_tapis_exporter_image: tapis/exporter:1.5.0
 monitoring_alpine_image: alpine:3.6
 monitoring_postgres_image: bitnami/postgresql:14
@@ -7,3 +7,4 @@ monitoring_kibana_image: docker.elastic.co/kibana/kibana:7.17.9
 monitoring_grafana_image: grafana/grafana:8.5.5
 monitoring_prometheus_image: prom/prometheus:v2.38.0
 monitoring_thanos_image: quay.io/thanos/thanos:v0.8.0
+monitoring_util_image: tapis/ubutil2204:1.5.0

playbooks/roles/monitoring/templates/docker/burnup

@@ -1,7 +1,16 @@
 #!/bin/bash
 
-echo "burnup monitoring:"
+echo burnup monitoring
 
-mkdir -p {{ tapisdatadir }}/monitoring
-docker compose up -d
+mkdir -p {{ tapisdatadir }}/monitoring/prometheus/data
+mkdir -p {{ tapisdatadir }}/monitoring/grafana/data
+mkdir -p {{ tapisdatadir }}/monitoring/elasticsearch/data
 
+export MYUID="$(id -u)"
+export UID_GID="$(id -u):$(id -g)"
+
+docker run -it --rm -v {{ tapisdatadir }}/monitoring:/monitoring {{ monitoring_util_image }} chown $MYUID /monitoring
+
+python3 {{ tapisdir }}/admin/util/parse_skexport -c monitoring -d {{ tapisdatadir }}
+
+docker compose up -d