Merge pull request #396 from tapis-project/staging

Staging

CHANGELOG.md

@@ -1,9 +1,30 @@
 # Changelog 
 
-## 1.6.4 
+## 1.7.0 
 
 ### Service Updates
 
+- [Apps: 1.6.4 to 1.7.0 (tapis/apps)](https://github.com/tapis-project/tapis-apps/blob/dev/CHANGELOG.md)
+- [Authenticator: 1.6.2 to 1.7.0 (tapis/authenticator, tapis/authenticator-migrations)](https://github.com/tapis-project/authenticator/blob/staging/CHANGELOG.md)
+- [Files: 1.6.4 to 1.7.0 (tapis/tapis-files, tapis/tapis-files-workers)](https://github.com/tapis-project/tapis-files/blob/dev/CHANGELOG.md)
+- [Jobs: 1.6.4 to 1.7.0 (tapis/jobsworker, jobsmigrate, jobsapi)](https://github.com/tapis-project/tapis-jobs/blob/dev/tapis-jobsapi/CHANGELOG.md)
+- [Meta: 1.6.1 to 1.7.0 (tapis/metaapi, tapis-meta-rh-server)](https://github.com/tapis-project/tapis-meta/blob/dev/CHANGELOG.md)
+- [Notifications: 1.6.2 to 1.7.0 (tapis/notifications, notifications-dispatcher)](https://github.com/tapis-project/tapis-notifications/blob/dev/CHANGELOG.md)
+- [Security: 1.6.3 to 1.7.0 (tapis/securitymigrate, securityadmin, securityapi, securityexport)](https://github.com/tapis-project/tapis-security/blob/dev/tapis-securityapi/CHANGELOG.md)
+- [Systems: 1.6.5 to 1.7.0 (tapis/systems)](https://github.com/tapis-project/tapis-systems/blob/dev/CHANGELOG.md)
+- [Globus-Proxy: 1.6.4 to 1.7.0 (tapis/systems)](https://github.com/tapis-project/tapis-systems/blob/dev/CHANGELOG.md)
+- [Workflows: 1.6.0 to 1.7.0 (tapis/workflows-api, tapis/workflows-pipelines, tapis/workflow-engine-streams)](https://github.com/tapis-project/tapis-workflows/blob/release-1.7.0/CHANGELOG.md)
+- [Pods: 1.6.0 to 1.7.0 (tapis/pods-api)](https://github.com/tapis-project/pods_service/blob/dev/CHANGELOG.md)
+- [TapisUI: 1.7.0 (tapis/tapisui)](https://github.com/tapis-project/tapis-ui/blob/dev/CHANGELOG.md)
+
+### Breaking Changes for Deployer Admins
+
+- If using the globus-proxy component, you must provide 2 variables in host_vars: `globus_client_id` and `globus_client_secret`. They correspond to the id and secret of the service client, as described here: https://docs.globus.org/guides/recipes/automate-with-service-account/ .
+
+
+## 1.6.4 
+
+### Service Updates
 
 - [Systems: 1.6.4 to 1.6.5 (tapis/systems)](Systems changes: https://github.com/tapis-project/tapis-systems/blob/1.6.5/CHANGELOG.md)
 - [Apps: 1.6.3 to 1.6.4 (tapis/apps)](https://github.com/tapis-project/tapis-apps/blob/1.6.4/CHANGELOG.md)

playbooks/roles/actors/defaults/main/images.yml

@@ -1,9 +1,9 @@
 actors_core_image: abaco/core-v3:{{ actors_service_version }}
 actors_grafana_image: grafana/grafana:9.4.7
-actors_nginx_image: abaco/nginx:1.6.0
-actors_nginxk8s_image: abaco/nginxk8s:1.6.0
+actors_nginx_image: abaco/nginx:1.7.0
+actors_nginxk8s_image: abaco/nginxk8s:1.7.0
 actors_mongo_image: mongo:4.2.6
 actors_alpine_image: alpine:3.17
-actors_mongobackup_image: tapis/mongobackup:1.6.0
+actors_mongobackup_image: tapis/mongobackup:1.7.0
 actors_rabbitmq_image: rabbitmq:3.6.12-management
-actors_util_image: tapis/ubutil2204:1.6.0
+actors_util_image: tapis/ubutil2204:1.7.0

playbooks/roles/actors/defaults/main/vars.yml

@@ -1,12 +1,11 @@
 ---
 
-
 # actors
 actors_service_url: '{{ global_service_url }}'
-actors_service_tenant_id: admin
+actors_service_tenant_id: '{{ global_service_tenant_id }}'
 actors_service_site_id: '{{ global_site_id }}'
 actors_service_name: actors
-actors_service_version: 1.6.0
+actors_service_version: 1.7.0
 actors_storage_class: '{{ global_storage_class }}'
 actors_rabbit_pvc: actors-rabbitmq-vol01
 actors_mongo_pvc: actors-mongo-vol01

playbooks/roles/admin/defaults/main/vars.yml

@@ -4,7 +4,8 @@
 admin_service_url: "{{ global_service_url }}"
 admin_devtenant_url: "{{ global_devtenant_url }}"
 admin_site_id: "{{ global_site_id }}"
-admin_service_tenant_id: admin
+admin_service_tenant_id: "{{ global_service_tenant_id }}"
+admin_service_devtenant_id: "{{ global_devtenant_id }}"
 admin_service_name: admin
 admin_service_site_id: "{{ global_site_id }}"
 admin_storage_class: "{{ global_storage_class }}"

playbooks/roles/admin/templates/docker/util/parse_skexport

@@ -28,7 +28,7 @@ apps = {
 authenticator = {
     "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_POSTGRES_AUTHENTICATOR_AUTHENTICATOR_PASSWORD",
     "postgres_password": "DBCREDENTIAL_POSTGRES_POSTGRES_AUTHENTICATOR_AUTHENTICATOR_PASSWORD",
-    "LDAP_ROOTPASS": "USER_ADMIN_AUTHENTICATOR_LDAP_TAPIS_DEV_PASSWORD",
+    "LDAP_ROOTPASS": "USER_{{ global_service_tenant_id | upper }}_AUTHENTICATOR_LDAP_TAPIS_DEV_PASSWORD",
     "service_password": "SERVICEPWD_AUTHENTICATOR_PASSWORD"
     }
 
@@ -142,12 +142,12 @@ tenants = {
     "postgres_password": "DBCREDENTIAL_POSTGRES_TENANTS_POSTGRES_TENANTS_TENANTS_PASSWORD",
     "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_TENANTS_POSTGRES_TENANTS_TENANTS_PASSWORD",
     "service_password": "SERVICEPWD_TENANTS_PASSWORD",
-    "admin_tenant_public_key": "JWTSIGNING_ADMIN_PUBLICKEY",
-    "dev_tenant_public_key": "JWTSIGNING_DEV_PUBLICKEY" 
+    "admin_tenant_public_key": "JWTSIGNING_{{ admin_service_tenant_id | upper }}_PUBLICKEY",
+    "dev_tenant_public_key": "JWTSIGNING_{{ admin_service_devtenant_id | upper }}_PUBLICKEY" 
     }
 
 tokens = {
-    "site_admin_privatekey": "JWTSIGNING_ADMIN_PRIVATEKEY"
+    "site_admin_privatekey": "JWTSIGNING_{{ admin_service_tenant_id | upper }}_PRIVATEKEY"
     }
 
 vault = {"":""}
@@ -158,7 +158,7 @@ workflows = {"":""}
 # parse args
 parser = argparse.ArgumentParser()
 parser.add_argument('-c', '--comp', help='Tapis component to export env file for', required=True)
-parser.add_argument('-d', '--dir', help='Tapis data dir as defined in the ansible', required=True)
+parser.add_argument('-d', '--dir', help='Tapis data dir as defined in the ansible host vars', required=True)
 parser.add_argument('-v', '--verbose', help='Display debug information', action='store_true')
 args = parser.parse_args()
 
@@ -219,8 +219,12 @@ if component == 'security':
     r = requests.get('http://localhost:8200/v1/auth/approle/role/sk/role-id', headers=headers)
     if args.verbose:
         print(f'getting role-id, have:: {r.json()}')
-    output_dict['TAPIS_SK_VAULT_ROLE_ID'] =  r.json()['data']['role_id']
-    output_dict['VAULT_ROLEID'] =  r.json()['data']['role_id']
+
+    try:
+        output_dict['TAPIS_SK_VAULT_ROLE_ID'] =  r.json()['data']['role_id']
+        output_dict['VAULT_ROLEID'] =  r.json()['data']['role_id']
+    except KeyError as e:
+        print(f'Encountered key error while parsing {r.json()}:: {e}')
 
     if args.verbose:
         print(f'''populating values for security, have:
@@ -246,14 +250,18 @@ with open(infile, 'r') as lines:
                     output_dict[key] = skexport_value
                     if key == 'MONGO_INITDB_ROOT_PASSWORD':
                         output_dict["MONGO_URI"] = f"mongodb://restheart:{skexport_value}@restheart-mongo:27017/?authSource=admin"
+                if "public_key" in key or "private_key" in key or "privatekey" in key:
+                        if args.verbose:
+                            print(f'Key {key} is a signing token. Need to make sure its quoted')
+                        output_dict[key] = f'"{skexport_value}"'
         except KeyError:
             pass
 
 if args.verbose:
     print(f'Completed mapping. Writing {output_dict}')
 
 for key in output_dict:
-    outfile.write(f'{key}="{output_dict[key]}"\n')
+    outfile.write(f'{key}={output_dict[key]}\n')
 
 
 outfile.close() 

playbooks/roles/apps/defaults/main/images.yml

@@ -1,4 +1,4 @@
-apps_api_image: tapis/apps:1.6.4
+apps_api_image: tapis/apps:1.7.0
 apps_postgres_image: postgres:12.4
 apps_pgadmin_image: dpage/pgadmin4:6.20
-apps_util_image: tapis/ubutil2204:1.6.0
+apps_util_image: tapis/ubutil2204:1.7.0

playbooks/roles/apps/defaults/main/vars.yml

@@ -2,7 +2,7 @@
 
 apps_service_name: apps
 apps_service_site_id: "{{ global_site_id }}" 
-apps_service_tenant_id: admin
+apps_service_tenant_id: "{{ global_service_tenant_id }}"
 apps_service_url: "{{ global_service_url }}"
 apps_storage_class: "{{ global_storage_class }}"
 apps_enable_external: false

playbooks/roles/authenticator/defaults/main/vars.yml

@@ -1,18 +1,19 @@
 ---
 
-authenticator_service_version: 1.6.3
+authenticator_service_version: 1.7.0
 authenticator_log_level: INFO
 authenticator_show_traceback: false
 authenticator_image_pull_policy: Always
 authenticator_service_url: "{{ global_service_url }}"
 authenticator_service_site_id: "{{ global_site_id }}"
-authenticator_service_tenant_id: admin
+authenticator_service_tenant_id: "{{ global_service_tenant_id }}"
 authenticator_service_name: authenticator
 authenticator_postgres_pvc: authenticator-postgres-vol01
 authenticator_ldap_pvc: authenticator-ldap-vol01
 authenticator_storage_class: "{{ global_storage_class }}"
 authenticator_dev_ldap_tenant_id: dev
 authenticator_service_tenants: ["*"]
+authenticator_primary_site_admin_tenant_base_url: "{{ global_primary_site_admin_tenant_base_url }}"
 
 
 

playbooks/roles/authenticator/templates/docker/authenticator-config.json

@@ -1,12 +1,12 @@
 {
-    "primary_site_admin_tenant_base_url": "{{authenticator_service_url}}",
+    "primary_site_admin_tenant_base_url": "{{authenticator_primary_site_admin_tenant_base_url}}",
     "service_site_id": "{{authenticator_service_site_id}}",
     "service_tenant_id": "{{authenticator_service_tenant_id}}",
     "service_name": "{{authenticator_service_name}}",
     "tenants": {{ authenticator_service_tenants | to_json }},
     "show_traceback": {{ authenticator_show_traceback | to_json }},
     "sql_db_url": "authenticator-postgres:5432",
-    "dev_ldap_tenant_id": "{{authenticator_dev_ldap_tenant_id}}",
+    "dev_ldap_tenant_id": "{{ authenticator_dev_ldap_tenant_id }}",
     "log_level": "{{authenticator_log_level}}",
     "version": "{{authenticator_service_version}}"
 }

playbooks/roles/baseburnup/defaults/main/vars.yml

@@ -1,4 +1,4 @@
-baseburnup_tapis_deployer_version: 1.6.4
+baseburnup_tapis_deployer_version: 1.7.0
 baseburnup_service_url: "{{ global_service_url }}"
 baseburnup_vault_url: "{{ global_vault_url }}"
 

playbooks/roles/files/defaults/main/images.yml

@@ -1,10 +1,10 @@
-files_api_image: tapis/tapis-files:1.6.4
-files_workers_image: tapis/tapis-files-workers:1.6.4
+files_api_image: tapis/tapis-files:1.7.0
+files_workers_image: tapis/tapis-files-workers:1.7.0
 files_postgres_image: postgres:11
 files_migrations_image: postgres:11
-files_minio_image: minio/minio
+files_minio_image: minio/minio:RELEASE.2024-09-09T16-59-28Z
 files_irods_provider_postgres_image: mjstealey/irods-provider-postgres:4.2.4
 files_pgadmin_image: dpage/pgadmin4:6.20
 files_rabbitmq_image: rabbitmq:3.8.11-management
 files_rabbitmq_management_image: rabbitmq:3-management-alpine
-files_util_image: tapis/ubutil2204:1.6.0
+files_util_image: tapis/ubutil2204:1.7.0

playbooks/roles/files/defaults/main/vars.yml

@@ -4,7 +4,7 @@ files_node_selector: null
 files_rabbitmq_hostname: files-rabbitmq
 files_service_name: files
 files_service_site_id: "{{ global_site_id }}"
-files_service_tenant_id: admin
+files_service_tenant_id: "{{ global_service_tenant_id }}" 
 files_service_url: "{{ global_service_url }}"
 files_replicas: 1
 files_files_debug: true

playbooks/roles/get_defaults/defaults/main/vars.yml

@@ -1,16 +1,36 @@
-global_service_tenant_id_default: admin
-tapisdir_default: '{{ ansible_env.HOME }}/.tapis/{{ inventory_hostname }}'
-tapisdatadir_default: '{{ ansible_env.HOME }}/.tapis-data/{{ inventory_hostname }}'
-global_vault_url_default: 'http://vault:8200'
-global_service_url_default: 'https://{{ global_service_tenant_id_default }}.{{ global_tapis_domain }}'
-global_devtenant_url_default: 'https://dev.{{ global_tapis_domain }}'
-global_primary_site_admin_tenant_base_url_default: 'https://admin.{{ global_tapis_domain }}'
-
-
-# A) There should be NO choice for primary sites, 
-# B) For associate sites, we should actually compute the field from the site record.. (or anyway, the 
-#    value needs to match what it is on the site record).
-components_to_deploy_default:
+# these should be required to be set in host_vars
+global_tapis_domain: ''
+tapisdir: ''
+tapisdatadir_default: ''
+global_site_id: ''
+proxy_nginx_cert_file: ''
+proxy_nginx_cert_key: ''
+
+
+# For primary sites, these should be left default 
+# For associate sites, these should be changed (and must be unique across the primary sites' tenant names)
+# ... AND the value needs to match what it is in the site record
+global_service_tenant_id: admin
+global_devtenant_id: dev
+
+# For primary site, this is the same as the global_tapis_domain
+# For associate site, it should be defined in host_vars
+global_service_domain: '{{ global_tapis_domain }}'
+
+# service urls
+# - should be same for primary site
+# - should be different for associate site
+
+global_primary_site_admin_tenant_base_url: 'https://{{ global_service_tenant_id }}.{{ global_service_domain }}'
+global_service_url: '{{ global_primary_site_admin_tenant_base_url }}'
+global_devtenant_url: 'https://{{ global_devtenant_id }}.{{ global_tapis_domain }}'
+
+
+### relatively safe defaults below
+
+global_vault_url: http://vault:8200
+
+components_to_deploy:
   - actors
   - admin
   - apps
@@ -35,3 +55,7 @@ components_to_deploy_default:
 #  - workflows  
 #  - test-resources
 
+
+
+
+

playbooks/roles/get_defaults/tasks/main.yml

@@ -1,19 +1,58 @@
 ---
 
-- name: Set default values for vars
-  ansible.builtin.set_fact:
-    global_service_tenant_id: '{{ global_service_tenant_id | default(global_service_tenant_id_default) }}'
-    tapisdir: '{{ tapisdir | default(tapisdir_default) }}'
-    tapisdatadir: '{{ tapisdatadir | default(tapisdatadir_default) }}'
-    global_vault_url: '{{ global_vault_url | default(global_vault_url_default) }}'
-    global_service_url: '{{ global_service_url | default(global_service_url_default) }}'
-    components_to_deploy: '{{ components_to_deploy | default(components_to_deploy_default) }}'
-    global_devtenant_url: '{{ global_devtenant_url | default(global_devtenant_url_default) }}'
-    global_primary_site_admin_tenant_base_url: '{{ global_primary_site_admin_tenant_base_url | default(global_primary_site_admin_tenant_base_url_default) }}'
-
-- name: Values being used
+# Ensure global vars are defined and checked
+
+- name: Test that important variables are present and not empty
+  assert:
+    that:
+      - tapisdir != ''
+      - tapisdatadir != ''
+      - components_to_deploy != ''
+      - global_service_tenant_id != ''
+      - global_devtenant_id != ''
+      - global_service_domain != ''
+      - global_primary_site_admin_tenant_base_url != ''
+      - global_service_url != ''
+      - global_devtenant_url != ''
+      - global_vault_url != ''
+
+- name: If associate site (site_type 2), check a few variables 
+  assert:
+    that:
+      - global_service_tenant_id != 'admin'
+      - global_devtenant_id != 'dev'
+      - global_tapis_domain != '{{ global_service_domain }}'
+  when:
+    - site_type == 2
+
+- name: Print important vars
   ansible.builtin.debug:
-    msg: 
-      - 'tapisdir: {{ tapisdir }}'
-      - 'tapisdatadir: {{ tapisdatadir }}'
-      - 'components_to_deploy: {{ components_to_deploy }}'
+    var: "{{ item }}" 
+  with_items:
+    - tapisdir
+    - tapisdatadir
+    - components_to_deploy
+    - global_service_tenant_id
+    - global_devtenant_id
+    - global_service_domain
+    - global_primary_site_admin_tenant_base_url
+    - global_service_url
+    - global_devtenant_url
+    - global_vault_url 
+
+
+# Use set_fact on global vars so the following roles can use them
+
+- name: Set global vars
+  ansible.builtin.set_fact:
+    tapisdir: '{{ tapisdir }}'
+    tapisdatadir: '{{ tapisdatadir }}'
+    components_to_deploy: '{{ components_to_deploy }}'
+    global_service_tenant_id: '{{ global_service_tenant_id }}'
+    global_devtenant_id: '{{ global_devtenant_id }}'
+    global_service_domain: '{{ global_service_domain }}'
+    global_primary_site_admin_tenant_base_url: '{{ global_primary_site_admin_tenant_base_url }}'
+    global_service_url: '{{ global_service_url }}'
+    global_devtenant_url: '{{ global_devtenant_url }}'
+    global_vault_url: '{{ global_vault_url }}'
+

playbooks/roles/jobs/defaults/main/images.yml

@@ -1,7 +1,7 @@
-jobs_api_image: tapis/jobsapi:1.6.4
-jobs_migrations_image: tapis/jobsmigrate:1.6.4
-jobs_worker_image: tapis/jobsworker:1.6.4
+jobs_api_image: tapis/jobsapi:1.7.0
+jobs_migrations_image: tapis/jobsmigrate:1.7.0
+jobs_worker_image: tapis/jobsworker:1.7.0
 jobs_postgres_image: postgres:12.4
 jobs_pgadmin_image: dpage/pgadmin4:6.20
 jobs_rabbitmq_management_image: rabbitmq:3.8.11-management
-jobs_util_image: tapis/ubutil2204:1.6.0
+jobs_util_image: tapis/ubutil2204:1.7.0

playbooks/roles/jobs/defaults/main/vars.yml

@@ -2,7 +2,7 @@
 
 jobs_service_name: jobs
 jobs_service_site_id: "{{ global_site_id }}"
-jobs_service_tenant_id: admin
+jobs_service_tenant_id: "{{ global_service_tenant_id }}"
 jobs_service_url: "{{ global_service_url }}"
 jobs_postgres_pvc: jobs-postgres-vol01
 jobs_storage_class: "{{ global_storage_class }}"

playbooks/roles/meta/defaults/main/images.yml

@@ -1,8 +1,8 @@
-meta_api_image: tapis/metaapi:1.6.1
-meta_rh_server_image: tapis/tapis-meta-rh-server:1.6.1
-meta_mongo_exporter_image: tapis/mqe:1.6.0
-meta_mongo_singlenode_image: tapis/mongo-singlenode:1.6.0
-meta_mongodb_backup_image: tapis/mongodb-backup:1.6.0
-meta_mongobackup_image: tapis/mongobackup:1.6.0
+meta_api_image: tapis/metaapi:1.7.0
+meta_rh_server_image: tapis/tapis-meta-rh-server:1.7.0
+meta_mongo_exporter_image: tapis/mqe:1.7.0
+meta_mongo_singlenode_image: tapis/mongo-singlenode:1.7.0
+meta_mongodb_backup_image: tapis/mongodb-backup:1.7.0
+meta_mongobackup_image: tapis/mongobackup:1.7.0
 meta_alpine_image: alpine:3.17
-meta_util_image: tapis/ubutil2204:1.6.0
+meta_util_image: tapis/ubutil2204:1.7.0