Merge pull request #377 from tapis-project/staging

Staging

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog 
 
+## 1.6.3 
+
+### Service Updates
+
+- [Authenticator: 1.6.1 to 1.6.2 (tapis/authenticator, tapis/authenticator-migrations)](https://github.com/tapis-project/authenticator/blob/staging/CHANGELOG.md)
+- [Systems: 1.6.2 to 1.6.3 (tapis/systems)](https://github.com/tapis-project/tapis-systems/blob/1.6.3/CHANGELOG.md)
+- [Files: 1.6.2 to 1.6.3 (tapis/tapis-files, tapis/tapis-files-workers)](https://github.com/tapis-project/tapis-files/blob/dev/CHANGELOG.md)
+- [Jobs: 1.6.2 to 1.6.3 (tapis/jobsworker, jobsmigrate, jobsapi)](https://github.com/tapis-project/tapis-jobs/blob/dev/tapis-jobsapi/CHANGELOG.md)
+- [Globus-Proxy: 1.6.0 to 1.6.2 (tapis/globus-proxy)](https://github.com/tapis-project/globus-proxy/blob/dev/CHANGELOG.md)
+- Removed Monitoring-Kibana & Monitoring-Elasticsearch components. These may be left in burndown scripts for now but will be removed in a future release.
+
 ## 1.6.2
 
 ### Service Updates

playbooks/roles/actors/templates/docker/burnup

@@ -3,9 +3,16 @@
 echo "burnup actors:"
 
 mkdir -p {{ tapisdatadir }}/actors
+mkdir -p {{ tapisdatadir }}/actors/mongo
+mkdir -p {{ tapisdatadir }}/actors/mongo/keyfile
+
 MYUID=`id -u`
 docker run -it --rm -v {{ tapisdatadir }}/actors:/actors {{ actors_util_image }} chown $MYUID /actors
 
 python3 ../admin/util/parse_skexport -c actors -d {{ tapisdatadir }}
 
+cat {{ tapisdatadir }}/actors/env | grep MONGO_INITDB_ROOT_PASSWORD | cut -d'"' -f 2 > {{ tapisdatadir }}/actors/mongo/keyfiles/metakeyfile
+
 docker compose up -d
+
+

playbooks/roles/actors/templates/docker/docker-compose.yml

@@ -16,16 +16,16 @@ services:
         container_name: actors-mongo
         image: {{ actors_mongo_image }}
         environment:
-            - MONGO_INITDB_ROOT_USERNAME: admin
-            - MONGO_REPLICA_SET_NAME: rs0
+            MONGO_INITDB_ROOT_USERNAME: admin
+            MONGO_REPLICA_SET_NAME: rs0
         env_file:
             - {{ tapisdatadir }}/actors/env
         # volumes:
         #     - ./runtime_files/certs:/data/ssl
         #     - ./runtime_files/certs/mongo-replica-set-keyfile:/keyfiles/metakeyfile
         volumes:
             - {{ tapisdatadir }}/actors/mongo/data:/data/db
-            - {{ tapisdatadir }}/actors/mongo/keyfile:keyfile-base
+            - {{ tapisdatadir }}/actors/mongo/keyfiles:/keyfiles
         networks:
             - tapis
         ulimits:
@@ -160,7 +160,7 @@ services:
             - tapis
 
     actors-spawner:
-        contianer_name: actors-spawner
+        container_name: actors-spawner
         image: {{ actors_core_image }}
         command: "python3 -u /home/tapis/actors/spawner.py"
         volumes:

playbooks/roles/admin/templates/docker/util/parse_skexport

@@ -10,7 +10,7 @@ import json
 actors = {
     "RABBITMQ_DEFAULT_PASS":"DBCREDENTIAL_RABBITMQ_ABACO_RABBIT_MQ_HOST_RABBITMQ_ABACO_PASSWORD",
     "MONGO_INITDB_ROOT_PASSWORD": "DBCREDENTIAL_MONGO_ABACO_MONGO_DB_HOST_MONGODB_ABACO_PASSWORD",
-    "": "SERVICEPWD_ABACO_PASSWORD"
+    "SERVICEPWD_ABACO_PASSWORD": "SERVICEPWD_ABACO_PASSWORD"
     }
 
 apps = {
@@ -105,8 +105,8 @@ skadmin = {"":""}
 
 streams = {
     "influxdb_password": "DBCREDENTIAL_INFLUX_INFLUX_HOST_CHORDS_TS_PRODUCTION_ADMIN_PASSWORD",
-    "INFLUXDB_PASSWORD": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
-    "INFLUXDB_ADMIN_PW": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "INFLUXDB_PASSWORD": "DBCREDENTIAL_INFLUX_INFLUX_HOST_CHORDS_TS_PRODUCTION_ADMIN_PASSWORD",
+    "INFLUXDB_ADMIN_PW": "DBCREDENTIAL_INFLUX_INFLUX_HOST_CHORDS_TS_PRODUCTION_ADMIN_PASSWORD",
     "mysql_password": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
     "mysql-password": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
     "MYSQL_PASSWORD": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
@@ -118,7 +118,7 @@ streams = {
     "service-password": "SERVICEPWD_STREAMS_PASSWORD",
     "service_password": "SERVICEPWD_STREAMS_PASSWORD",
     "CHORDS_ADMIN_PW": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
-    "INFLUXDB_ADMIN_PW": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
+    "INFLUXDB_ADMIN_PW": "DBCREDENTIAL_INFLUX_INFLUX_HOST_CHORDS_TS_PRODUCTION_ADMIN_PASSWORD",
     "MYSQL_PASSWORD": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD",
     "CHORDS_MYSQL_PASSWORD": "DBCREDENTIAL_MYSQL_MYSQL_HOST_CHORDS_SEEDED_PRODUCTION_ADMIN_PASSWORD"
 }

playbooks/roles/apps/templates/docker/burndown

@@ -2,6 +2,4 @@
 
 echo "burndown apps:"
 
-export UID_GID="$(id -u):$(id -g)"
-
 docker compose down

playbooks/roles/apps/templates/docker/burnup

@@ -5,12 +5,6 @@ echo "burnup apps:"
 mkdir -p {{ tapisdatadir }}/apps
 mkdir -p {{ tapisdatadir }}/apps/postgres
 
-MYUID=`id -u`
-
-export UID_GID="$(id -u):$(id -g)"
-
-docker run -it --rm -v {{ tapisdatadir }}/apps:/apps {{ apps_util_image }} chown $MYUID /apps
-
 python3 ../admin/util/parse_skexport -c apps -d {{ tapisdatadir }}
 
 docker compose up -d

playbooks/roles/apps/templates/docker/docker-compose.yml

@@ -7,7 +7,6 @@ networks:
 services:
     apps-api:
         container_name: apps-api
-        user: ${UID_GID}
         image: {{ apps_api_image }}
         networks:
             - tapis

playbooks/roles/authenticator/defaults/main/images.yml

@@ -1,4 +1,4 @@
-authenticator_api_image: tapis/authenticator:1.6.1
-authenticator_migrations_image: tapis/authenticator-migrations:1.6.1
+authenticator_api_image: tapis/authenticator:1.6.2
+authenticator_migrations_image: tapis/authenticator-migrations:1.6.2
 authenticator_postgres_image: postgres:11.4
 authenticator_ldap_image: tacc/slapd:1

playbooks/roles/baseburnup/defaults/main/vars.yml

@@ -1,4 +1,4 @@
-baseburnup_tapis_deployer_version: 1.6.2
+baseburnup_tapis_deployer_version: 1.6.3
 baseburnup_service_url: "{{ global_service_url }}"
 baseburnup_vault_url: "{{ global_vault_url }}"
 

playbooks/roles/baseburnup/templates/docker/burndown

@@ -22,10 +22,22 @@ echo "Start: top-level-burndown"
 
 # actors
 
+{% if "actors" in components_to_deploy %}
+cd $mydir_absolute/actors
+./burndown
+{% endif %}
+
+
 # container-registry
 
 # globus-proxy
 
+{% if "globus-proxy" in components_to_deploy %}
+cd $mydir_absolute/globus-proxy
+./burndown
+{% endif %}
+
+
 {% if "meta" in components_to_deploy %}
 cd $mydir_absolute/meta
 ./burndown

playbooks/roles/baseburnup/templates/docker/burnup

@@ -99,12 +99,18 @@ burnup_or_exit apps
 
 ## tertiary services
 
-# actors
+{% if "actors" in components_to_deploy %}
+burnup_or_exit actors
+{% endif %}
 
 # container-registry
 
 # globus-proxy
 
+{% if "globus-proxy" in components_to_deploy %}
+burnup_or_exit globus-proxy
+{% endif %}
+
 {% if "meta" in components_to_deploy %}
 burnup_or_exit meta
 {% endif %}

playbooks/roles/files/defaults/main/images.yml

@@ -1,5 +1,5 @@
-files_api_image: tapis/tapis-files:1.6.2
-files_workers_image: tapis/tapis-files-workers:1.6.2
+files_api_image: tapis/tapis-files:1.6.3
+files_workers_image: tapis/tapis-files-workers:1.6.3
 files_postgres_image: postgres:11
 files_migrations_image: postgres:11
 files_minio_image: minio/minio

playbooks/roles/files/templates/docker/burndown

@@ -2,6 +2,4 @@
 
 echo "burndown files:"
 
-export UID_GID="$(id -u):$(id -g)"
-
 docker compose down

playbooks/roles/files/templates/docker/burnup

@@ -8,12 +8,6 @@ mkdir -p {{ tapisdatadir }}/files/postgres
 mkdir -p {{ tapisdatadir }}/files/minio
 mkdir -p {{ tapisdatadir }}/files/irods
 
-MYUID=`id -u`
-
-export UID_GID="$(id -u):$(id -g)"
-
-docker run -it --rm -v {{ tapisdatadir }}/files:/files {{ files_util_image }} chown $MYUID /files
-
 python3 ../admin/util/parse_skexport -c files -d {{ tapisdatadir }}
 
 docker compose up -d

playbooks/roles/files/templates/docker/docker-compose.yml

@@ -69,7 +69,6 @@ services:
 
   files-workers:
     container_name: files-workers
-    user: ${UID_GID}
     image: {{ files_workers_image }}
     networks:
       - tapis
@@ -97,7 +96,6 @@ services:
         condition: service_started
 
   files-api:
-    user: ${UID_GID}
     container_name: files-api
     image: {{ files_api_image }}
     networks:

playbooks/roles/globus-proxy/defaults/main/images.yml

@@ -1 +1 @@
-globus_proxy_api_image: tapis/globus-proxy:1.6.1
+globus_proxy_api_image: tapis/globus-proxy:1.6.2

playbooks/roles/jobs/defaults/main/images.yml

@@ -1,6 +1,6 @@
-jobs_api_image: tapis/jobsapi:1.6.2
-jobs_migrations_image: tapis/jobsmigrate:1.6.2
-jobs_worker_image: tapis/jobsworker:1.6.2
+jobs_api_image: tapis/jobsapi:1.6.3
+jobs_migrations_image: tapis/jobsmigrate:1.6.3
+jobs_worker_image: tapis/jobsworker:1.6.3
 jobs_postgres_image: postgres:12.4
 jobs_pgadmin_image: dpage/pgadmin4:6.20
 jobs_rabbitmq_management_image: rabbitmq:3.8.11-management

playbooks/roles/jobs/templates/docker/burndown

@@ -2,6 +2,4 @@
 
 echo "burndown jobs:"
 
-export UID_GID="$(id -u):$(id -g)"
-
 docker compose down

playbooks/roles/jobs/templates/docker/burnup

@@ -5,11 +5,6 @@ echo "burnup jobs:"
 mkdir -p {{ tapisdatadir }}/jobs
 mkdir -p {{ tapisdatadir }}/jobs/postgres
 
-export MYUID="$(id -u)"
-export UID_GID="$(id -u):$(id -g)"
-
-docker run -it --rm -v {{ tapisdatadir }}/jobs:/jobs {{ jobs_util_image }} chown $MYUID /jobs
-
 python3 ../admin/util/parse_skexport -c jobs -d {{ tapisdatadir }}
 
 docker compose up -d

playbooks/roles/jobs/templates/docker/docker-compose.yml

@@ -8,7 +8,6 @@ networks:
 services:
     jobs-api:
         container_name: jobs-api
-        user: ${UID_GID}
         networks:
           - tapis
         image: {{ jobs_api_image }}
@@ -96,7 +95,6 @@ services:
             test: rabbitmq-diagnostics -q ping
 
     jobs-altqueue:
-        user: ${UID_GID}
         container_name: jobs-altqueue
         image: {{ jobs_worker_image }}
         networks:
@@ -124,7 +122,6 @@ services:
 
     jobs-recovery:
         container_name: jobs-recovery
-        user: ${UID_GID}
         image: {{ jobs_worker_image }}
         environment:
             - TAPIS_SITE_ID={{ jobs_service_site_id }}
@@ -151,7 +148,6 @@ services:
 
     jobs-deadletter:
         container_name: jobs-deadletter
-        user: ${UID_GID}
         image: {{ jobs_worker_image }}
         environment:
             - TAPIS_SITE_ID={{ jobs_service_site_id }}
@@ -178,7 +174,6 @@ services:
 
     jobs-eventqueue:
         container_name: jobs-eventqueue
-        user: ${UID_GID}
         image: {{ jobs_worker_image }}
         environment:
             - TAPIS_SITE_ID={{ jobs_service_site_id }}
@@ -205,7 +200,6 @@ services:
 
     jobs-workers:
         container_name: jobs-workers
-        user: ${UID_GID}
         image: {{ jobs_worker_image }}
         environment:
 {% if jobs_node_name == true %}

playbooks/roles/meta/templates/docker/burnup

@@ -4,11 +4,6 @@ echo "burnup meta:"
 
 mkdir -p {{ tapisdatadir }}/meta/mongodb/data
 
-export MYUID="$(id -u)"
-export UID_GID="$(id -u):$(id -g)"
-
-docker run -it --rm -v {{ tapisdatadir }}/meta:/meta {{ meta_util_image }} chown $MYUID /meta
-
 python3 ../admin/util/parse_skexport -c meta -d {{ tapisdatadir }}
 
 docker compose up -d

playbooks/roles/monitoring/templates/docker/docker-compose.yml

@@ -6,21 +6,6 @@ networks:
 
 
 services:
-  monitoring-elasticsearch:
-    container_name: monitoring-elasticsearch
-    image: {{ monitoring_elasticsearch_image }}
-    networks: 
-      - tapis
-    env_file:
-      - {{ tapisdatadir }}/monitoring/env
-    environment:
-      - discovery.type=single-node
-      - search.max_buckets=65535
-      - ES_JAVA_OPTS=-Xmx8G -Xmx8G -Dlog4j2.formatMsgNoLookups=true
-      - xpack.security.enabled=false
-      - cluster.routing.allocation.disk.watermark.enable_for_single_data_node=true
-    volumes:
-      - {{ tapisdatadir}}/monitoring/elasticsearch/data:/usr/share/elasticsearch/data
 
   monitoring-grafana:
     container_name: monitoring-grafana
@@ -40,17 +25,6 @@ services:
       - ./prometheus-datasources.yml:/etc/grafana/provisioning/datasources/prometheus.yaml
     user: "0"
 
-  monitoring-kibana:
-    container_name: monitoring-kibana
-    image: {{ monitoring_kibana_image }}
-    env_file:
-      - {{ tapisdatadir }}/monitoring/env
-    networks: 
-      - tapis
-    environment:
-      - ELASTICSEARCH_HOSTS=http://monitoring-elasticsearch:9200
-      - ELASTICSEARCH_URL=http://monitoring-elasticsearch:9200
-
   monitoring-prometheus:
     container_name: monitoring-prometheus
     image: {{ monitoring_prometheus_image }}