Merge pull request #284 from tapis-project/dev

Dev

CHANGELOG.md

@@ -2,6 +2,24 @@
 
 Notable changes between versions.
 
+## 1.4.3
+
+### Services Updated 
+
+- This release contains several bugfixes & improvements for the Docker flavor of Tapis Deployer. 
+- [ Systems: 1.4.1 to 1.4.2 (tapis/systems)](https://github.com/tapis-project/tapis-systems/blob/1.4.2/CHANGELOG.md)
+- [ Apps: 1.4.1 to 1.4.2 (tapis/apps)](https://github.com/tapis-project/tapis-apps/blob/1.4.2/CHANGELOG.md)
+- [ Notifications: 1.4.0 to 1.4.1 (tapis/notifications, notifications-dispatcher)](https://github.com/tapis-project/tapis-notifications/blob/1.4.1/CHANGELOG.md)
+- [ Files: 1.4.2 to 1.4.3 (tapis/tapis-files, tapis/tapis-files-workers)](https://github.com/tapis-project/tapis-files/blob/dev/CHANGELOG.md)
+- [ Jobs: 1.4.2 to 1.4.3 (tapis/jobsworker, jobsmigrate, jobsapi)](https://github.com/tapis-project/tapis-jobs/blob/dev/tapis-jobsapi/CHANGELOG.md)
+- [ Globus-Proxy: 1.4.2 to 1.4.3 (tapis/globus-proxy)](https://github.com/tapis-project/globus-proxy/blob/dev/CHANGELOG.md)
+
+
+### Breaking Changes for Deployer Admins
+
+- This is ONLY for Docker Tapis installs updating; it is NOT applicable to Kubernetes installs: Some components' Postgres directory volume mounts have moved within the `tapisdatadir` and may need to be moved on disk before starting the containers. Each component should now follow a similar structure, e.g. for authenticator: `tapisdatadir/authenticator/postgres/data` should contain the Postgres data, such as the `PG_VERSION` file, `pg_wal` directory, etc. 
+
+
 ## 1.4.2
 
 ### Services Updated

playbooks/generate-single-component.yml

@@ -7,6 +7,11 @@
     tapisctl_action: 'generate'
 
   tasks: 
+
+  - name: debug
+    debug:
+      var: tapisflavor 
+
   - name: Set default values for vars
     include_role: 
       name: get_defaults

playbooks/roles/actors/templates/docker/burnup

@@ -3,9 +3,9 @@
 echo "burnup actors:"
 
 mkdir -p {{ tapisdatadir }}/actors
-myuid=`id -u`
-docker run -it --rm -v {{ tapisdatadir }}/actors:/actors tapis/ubutil2204:1.4.0 chown $myuid /actors
+MYUID=`id -u`
+docker run -it --rm -v {{ tapisdatadir }}/actors:/actors tapis/ubutil2204:1.4.0 chown $MYUID /actors
 
-python3 {{ tapisdir }}/admin/util/parse_skexport -c actors -d {{ tapisdatadir }}
+python3 ../admin/util/parse_skexport -c actors -d {{ tapisdatadir }}
 
 docker compose up -d

playbooks/roles/actors/templates/docker/docker-compose.yml

@@ -66,9 +66,6 @@ services:
                 hard: 65535
         networks:
             - tapis
-        depends_on:
-            actors-mongo:
-                condition: service_started
 
     actors-nginx:
         container_name: actors-nginx
@@ -77,7 +74,7 @@ services:
             - tapis
         volumes:
 #             - ./config-local.json:/home/tapis/config.json
-          - "{{ tapisdir }}/actors/actors-config.json:/etc/nginx/sites-enabled/flask-project"
+          - "./actors-config.json:/etc/nginx/sites-enabled/flask-project"
 # #            - ./images/nginx/nginx.conf:/etc/nginx/nginx.conf
 # #            - ./images/nginx/sites-enabled:/etc/nginx/sites-enabled
         restart: always
@@ -92,7 +89,7 @@ services:
         #     - "127.0.0.1:5000:5000"
         volumes:
         #     - ./config-local.json:/home/tapis/config.json
-          - "{{ tapisdir }}/actors/actors-config.json:/home/tapis/config.json"
+          - "./actors-config.json:/home/tapis/config.json"
         #     - ./abaco.log:/home/tapis/runtime_files/logs/service.log
         #     - ./runtime_files/certs:/home/tapis/runtime_files/certs
         environment:
@@ -115,7 +112,7 @@ services:
         image: {{ actors_core_image }}
         volumes:
         #     - ./config-local.json:/home/tapis/config.json
-          - "{{ tapisdir }}/actors/actors-config.json:/home/tapis/config.json"
+          - "./actors-config.json:/home/tapis/config.json"
         #     - ./abaco.log:/home/tapis/runtime_files/logs/service.log
         #     - ./runtime_files/certs:/home/tapis/runtime_files/certs
         # ports:
@@ -141,7 +138,7 @@ services:
         image: {{ actors_core_image }}
         volumes:
         #     - ./config-local.json:/home/tapis/config.json
-          - "{{ tapisdir }}/actors/actors-config.json:/home/tapis/config.json"
+          - "./actors-config.json:/home/tapis/config.json"
         #     - ./abaco.log:/home/tapis/runtime_files/logs/service.log
         #     - ./runtime_files/certs:/home/tapis/runtime_files/certs
         ports:
@@ -168,7 +165,7 @@ services:
         command: "python3 -u /home/tapis/actors/spawner.py"
         volumes:
         #     - ./config-local.json:/home/tapis/config.json
-          - "{{ tapisdir }}/actors/actors-config.json:/home/tapis/config.json"
+          - "./actors-config.json:/home/tapis/config.json"
         #     - /var/run/docker.sock:/var/run/docker.sock
         #     - ./abaco.log:/home/tapis/runtime_files/logs/service.log
         #     - ./runtime_files/certs:/home/tapis/runtime_files/certs
@@ -193,7 +190,7 @@ services:
         volumes:
             - /:/host
         #     - ./config-local.json:/home/tapis/config.json
-            - "{{ tapisdir }}/actors/actors-config.json:/home/tapis/config.json"
+            - "./actors-config.json:/home/tapis/config.json"
             - /var/run/docker.sock:/var/run/docker.sock
         #     - ./abaco.log:/home/tapis/runtime_files/logs/service.log
         #     - ./runtime_files/certs:/home/tapis/runtime_files/certs
@@ -220,7 +217,7 @@ services:
         command: "python3 -u /home/tapis/actors/events.py"
         volumes:
         #     - ./config-local.json:/home/tapis/config.json
-          - "{{ tapisdir }}/actors/actors-config.json:/home/tapis/config.json"
+          - "./actors-config.json:/home/tapis/config.json"
         #     - /var/run/docker.sock:/var/run/docker.sock
         #     - ./abaco.log:/home/tapis/runtime_files/logs/service.log
         #     - ./runtime_files/certs:/home/tapis/runtime_files/certs
@@ -286,7 +283,7 @@ services:
             - tapis
         volumes:
         #     - ./config-local.json:/home/tapis/config.json
-          - "{{ tapisdir }}/actors/actors-config.json:/home/tapis/config.json"
+          - "./actors-config.json:/home/tapis/config.json"
         #     - ./abaco.log:/home/tapis/runtime_files/logs/service.log
         #     - ./runtime_files/certs:/home/tapis/runtime_files/certs
         # ports:

playbooks/roles/admin/templates/docker/util/parse_skexport

@@ -51,8 +51,6 @@ notifications = {"PGADMIN_DEFAULT_PASSWORD": "DBCREDENTIAL_PGADMIN_NOTIFICATIONS
                 "TAPIS_SERVICE_PASSWORD": "SERVICEPWD_NOTIFICATIONS_PASSWORD",
                 "service_password": "SERVICEPWD_NOTIFICATIONS_PASSWORD"}
 
-
-
 pgrest = {"":""}
 pods = {"":""}
 proxy = {"":""}
@@ -62,7 +60,7 @@ security = {"TAPIS_SK_VAULT_SECRET_ID": "", # these two are populated later with
             "TAPIS_PASSWORD": "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_TAPIS_PASSWORD",
             "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_POSTGRES_PASSWORD",
             "PW" : "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_POSTGRES_PASSWORD",
-            "TPW" : "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_POSTGRES_PASSWORD"}
+            "TPW" : "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_TAPIS_PASSWORD"}
 
 skadmin = {"":""}
 streams = {"":""}
@@ -98,23 +96,26 @@ args = parser.parse_args()
 # normalize component name
 component = args.comp.replace('-', '_')
 
+# expand directory vars with things like ~ and $HOME
+tapisdatadir_absolute = os.path.expanduser(os.path.expandvars(args.dir))
+
 # ensure data dir exists for component
 try:
     if args.verbose:
-        print(f"trying makedir with {os.path.join(args.dir + '/' + args.comp)}")
-    os.makedirs(os.path.join(args.dir + '/' + args.comp))
+        print(f"trying makedir with {os.path.join(tapisdatadir_absolute + '/' + args.comp)}")
+    os.makedirs(os.path.join(tapisdatadir_absolute + '/' + args.comp))
     if args.verbose:
         print(f'success')
 except FileExistsError:
     if args.verbose:
-        print(f"error making {os.path.join(args.dir + '/' + args.comp)}, already exists. Ignoring")
+        print(f"error making {os.path.join(tapisdatadir_absolute + '/' + args.comp)}, already exists. Ignoring")
     pass #ignore file exists, supposed to be created already anyway
 
 # set local vars
-infile = args.dir + '/skadmin/env'
+infile = tapisdatadir_absolute + '/skadmin/env'
 local = locals()[component]
 try:
-    outfile = open(args.dir + '/' + args.comp + '/env', 'w')
+    outfile = open((tapisdatadir_absolute + '/' + args.comp + '/env'), 'w')
     if args.verbose:
             print(f'successfuly opened {outfile}')
 except Exception as e:
@@ -131,7 +132,7 @@ if args.verbose:
 
 # populate vault values for security
 if component == 'security':
-    vault_token = open('{{ tapisdatadir }}/vault/vault-token').read()
+    vault_token = open(tapisdatadir_absolute + '/vault/vault-token').read()
     headers = {'X-Vault-Token': f'{vault_token}'}
 
     r = requests.post('http://localhost:8200/v1/auth/approle/role/sk/secret-id', headers=headers)

playbooks/roles/apps/defaults/main/images.yml

@@ -1,3 +1,3 @@
-apps_api_image: tapis/apps:1.4.1
+apps_api_image: tapis/apps:1.4.2
 apps_postgres_image: postgres:12.4
 apps_pgadmin_image: dpage/pgadmin4:6.20

playbooks/roles/apps/templates/docker/burndown

@@ -2,4 +2,6 @@
 
 echo "burndown apps:"
 
+export UID_GID="$(id -u):$(id -g)"
+
 docker compose down

playbooks/roles/apps/templates/docker/burnup

@@ -3,13 +3,15 @@
 echo "burnup apps:"
 
 mkdir -p {{ tapisdatadir }}/apps
-myuid=`id -u`
-export UID=`id -u`
-export GID=`id -g`
+mkdir -p {{ tapisdatadir }}/apps/postgres
 
-docker run -it --rm -v {{ tapisdatadir }}/apps:/apps tapis/ubutil2204:1.4.0 chown $myuid /apps
+MYUID=`id -u`
 
-python3 {{ tapisdir }}/admin/util/parse_skexport -c apps -d {{ tapisdatadir }}
+export UID_GID="$(id -u):$(id -g)"
+
+docker run -it --rm -v {{ tapisdatadir }}/apps:/apps tapis/ubutil2204:1.4.0 chown $MYUID /apps
+
+python3 ../admin/util/parse_skexport -c apps -d {{ tapisdatadir }}
 
 docker compose up -d
 

playbooks/roles/apps/templates/docker/docker-compose.yml

@@ -7,7 +7,7 @@ networks:
 services:
     apps-api:
         container_name: apps-api
-        user: ${UID}:${GID}
+        user: ${UID_GID}
         image: {{ apps_api_image }}
         networks:
             - tapis
@@ -50,7 +50,7 @@ services:
             - {{ tapisdatadir }}/apps/env
         volumes:
             - {{ tapisdatadir }}/apps/postgres:/pgdata
-            - {{ tapisdir }}/apps/apps-init-db-sh:/init-db
+            - ./apps-init-db-sh:/init-db
         command: bash -c "cp /init-db /local_initdb && echo 100 && chown $(whoami) /local_initdb && echo 200 && chmod +x /local_initdb && echo 300 && /local_initdb"
         depends_on:
             apps-postgres:

playbooks/roles/authenticator/templates/docker/burnup

@@ -1,11 +1,13 @@
 #!/bin/bash
 
 echo "burnup authenticator:"
-python3 {{ tapisdir }}/admin/util/parse_skexport -c authenticator -d {{ tapisdatadir }}
+
 mkdir -p {{ tapisdatadir }}/authenticator
 mkdir -p {{ tapisdatadir }}/authenticator/postgres
 mkdir -p {{ tapisdatadir }}/authenticator/api
 mkdir -p {{ tapisdatadir }}/authenticator/ldap
 
+python3 ../admin/util/parse_skexport -c authenticator -d {{ tapisdatadir }}
+
 docker compose up -d
 

playbooks/roles/authenticator/templates/docker/docker-compose.yml

@@ -10,9 +10,9 @@ services:
     networks:
         - tapis
     env_file:
-        - "{{ tapisdatadir }}/authenticator/env"
+        - {{ tapisdatadir }}/authenticator/env
     volumes:
-        - "{{ tapisdir }}/authenticator/authenticator-config.json:/home/tapis/config.json"
+        - ./authenticator-config.json:/home/tapis/config.json
     depends_on: 
         authenticator-postgres:
             condition: service_healthy
@@ -28,16 +28,16 @@ services:
     networks:
         - tapis
     environment:
-        # - PGDATA=/pgdata/data
+        - PGDATA=/pgdata/data
         - POSTGRES_USER=authenticator
         - POSTGRES_DB=authenticator
         - POSTGRES_HOST_AUTH_METHOD=trust
     env_file:
-        - "{{ tapisdatadir }}/authenticator/env"
+        - {{ tapisdatadir }}/authenticator/env
     volumes:
-        - "{{ tapisdatadir }}/authenticator/postgres:/var/lib/postgresql/data"
+        - {{ tapisdatadir }}/authenticator/postgres:/pgdata
     healthcheck:
-        test: pg_isready -U postgres
+        test: pg_isready -U authenticator
         interval: 5s
         timeout: 5s
         retries: 5
@@ -49,15 +49,15 @@ services:
     networks:
         - tapis
     env_file:
-        - "{{ tapisdatadir }}/authenticator/env"
+        - {{ tapisdatadir }}/authenticator/env
     depends_on:
         authenticator-postgres:
             condition: service_healthy
         authenticator-ldap:
             condition: service_started
     command: ['upgrade']
     volumes:
-        - "{{ tapisdir }}/authenticator/authenticator-config.json:/home/tapis/config.json"
+        - ./authenticator-config.json:/home/tapis/config.json
 
 
   authenticator-ldap:
@@ -69,9 +69,9 @@ services:
         - LDAP_DOMAIN=tapis
         - LDAP_ORGANISATION=Tapis
     env_file:
-        - "{{ tapisdatadir }}/authenticator/env"
+        - {{ tapisdatadir }}/authenticator/env
     volumes:
-        - "{{ tapisdatadir }}/authenticator/ldap:/data/ldap"
+        - {{ tapisdatadir }}/authenticator/ldap:/data/ldap
     depends_on:
         authenticator-postgres:
             condition: service_healthy

playbooks/roles/baseburnup/defaults/main/vars.yml

@@ -1,4 +1,4 @@
-baseburnup_tapis_deployer_version: 1.4.2
+baseburnup_tapis_deployer_version: 1.4.3
 baseburnup_service_url: "{{ global_service_url }}"
 baseburnup_vault_url: "{{ global_vault_url }}"
 

playbooks/roles/baseburnup/templates/docker/burndown

@@ -1,6 +1,5 @@
 #!/bin/bash
 
-set -e 
 
 # global Tapis burndown script
 
@@ -70,6 +69,11 @@ cd $mydir_absolute/systems
 ./burndown
 {% endif %}
 
+{% if "apps" in components_to_deploy %}
+cd $mydir_absolute/apps
+./burndown
+{% endif %}
+
 ### primary services
 
 {% if "tenants" in components_to_deploy %}