Merge pull request #206 from tapis-project/dev

Dev

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 Notable changes between versions.
 
+## 1.3.7
+
+- [Authenticator: 1.3.3 to 1.3.4 (authenticator & authenticator-migrations)](https://github.com/tapis-project/authenticator/blob/prod/CHANGELOG.md)
+
 ## 1.3.6
 
 - [Authenticator: 1.3.0 to 1.3.3 (authenticator & authenticator-migrations)](https://github.com/tapis-project/authenticator/blob/prod/CHANGELOG.md)

README.md

@@ -20,9 +20,11 @@ the [Deployment & Administration Guide](https://tapis.readthedocs.io/en/latest/d
   - Optional: Name it after your new feature or change, i.e. "refactor-tapisui"
 - Make and commit your changes to your branch
   - For example, to increase the image version for the Apps API images, edit the file `playbooks/roles/apps/defaults/main/images.yml`
-  - Include a note about your changes in the CHANGELOG.md.
+  - Include a note about your changes in the CHANGELOG.md. 
+    - Link to your component's CHANGELOG.md details for the new version. For example: 
+      - [Tapis Systems version change from 1.3.1 to 1.3.2](https://github.com/tapis-project/tapis-systems/blob/local/CHANGELOG.md#132---2023-04-25)
   - **Note** Your changes could affect other services, so please be sure to describe these issues in CHANGELOG.md, especially if there are breaking changes or if your require additional steps for upgrade.
-- Create a Pull Request from your branch against the branch you created the branch from, (probably *dev*).  
+- Create a Pull Request from your branch against the source (*dev*).  
 
 
 ### Getting Changes into a Tapis Deployer Release

playbooks/roles/actors/templates/docker/burnup

@@ -2,5 +2,7 @@
 
 echo "burnup actors:"
 
+mkdir -p {{ tapisdatadir }}/actors
+
 docker compose up -d
 

playbooks/roles/actors/templates/docker/docker-compose.yml

@@ -2,6 +2,7 @@
 networks:
     tapis:
         name: tapis
+        external: true
 
 services:
     # A note on Mongo config. You cannot declare one in this image unless you want to stop using

playbooks/roles/admin/templates/docker/util/parse_skexport

@@ -0,0 +1,123 @@
+import os 
+import argparse
+import requests
+import json
+
+
+# var mapping for all comps
+# e.g. {"{skexport_name}": "{internal_component_name}"}
+actors = {"":""}
+apps = {"":""}
+authenticator = {"POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_POSTGRES_AUTHENTICATOR_AUTHENTICATOR_PASSWORD",
+                "postgres_password": "DBCREDENTIAL_POSTGRES_POSTGRES_AUTHENTICATOR_AUTHENTICATOR_PASSWORD",
+                "LDAP_ROOTPASS":"USER_ADMIN_AUTHENTICATOR_LDAP_TAPIS_DEV_PASSWORD",
+                "service_password": "SERVICEPWD_AUTHENTICATOR_PASSWORD"}
+
+files = {"":""}
+globus_proxy = {"":""}
+jobs = {"":""}
+meta = {"":""}
+monitoring = {"":""}
+notifications = {"":""}
+pgrest = {"":""}
+pods = {"":""}
+proxy = {"":""}
+security = {"TAPIS_SK_VAULT_SECRET_ID": "",
+            "TAPIS_SK_VAULT_ROLE_ID": "",
+            "TAPIS_DB_PASSWORD" : "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_TAPIS_PASSWORD",
+            "TAPIS_PASSWORD": "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_TAPIS_PASSWORD",
+            "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_POSTGRES_PASSWORD",
+            "PW" : "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_POSTGRES_PASSWORD",
+            "TPW" : "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_TAPIS_PASSWORD"}
+
+skadmin = {"":""}
+streams = {"":""}
+systems = {"":""}
+tapisui = {"":""}
+tenants = {"postgres_password": "DBCREDENTIAL_POSTGRES_TENANTS_POSTGRES_TENANTS_TENANTS_PASSWORD",
+            "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_TENANTS_POSTGRES_TENANTS_TENANTS_PASSWORD",
+            "service_password": "SERVICEPWD_TENANTS_PASSWORD",
+            "admin_tenant_public_key": "JWTSIGNING_ADMIN_PUBLICKEY",
+            "dev_tenant_public_key": "JWTSIGNING_DEV_PUBLICKEY" }
+
+tokens = {"site_admin_privatekey": "JWTSIGNING_ADMIN_PRIVATEKEY"}
+
+vault = {"":""}
+workflows = {"":""}
+
+
+# parse args
+parser = argparse.ArgumentParser()
+parser.add_argument('-c', '--comp', help='Tapis component to export env file for', required=True)
+parser.add_argument('-d', '--dir', help='Tapis data dir as defined in the ansible', required=True)
+parser.add_argument('-v', '--verbose', help='Display debug information', action='store_true')
+args = parser.parse_args()
+
+# normalize component name
+component = args.comp.replace('-', '_')
+
+# ensure data dir exists for component
+try:
+    if args.verbose:
+        print(f"trying makedir with {os.path.join(args.dir + '/' + args.comp)}")
+    os.makedirs(os.path.join(args.dir + '/' + args.comp))
+    if args.verbose:
+        print(f'success')
+except FileExistsError:
+    if args.verbose:
+        print(f"error making {os.path.join(args.dir + '/' + args.comp)}, already exists. Ignoring")
+    pass #ignore file exists, supposed to be created already anyway
+
+# set local vars
+infile = args.dir + '/skadmin/env'
+local = locals()[component]
+outfile = open(args.dir + '/' + args.comp + '/env', 'w')
+output_dict = {}
+
+# populate vault values for security
+if component == 'security':
+    vault_token = open('{{ tapisdatadir }}/vault/vault-token').read()
+    headers = {'X-Vault-Token': f'{vault_token}'}
+
+    r = requests.post('http://localhost:8200/v1/auth/approle/role/sk/secret-id', headers=headers)
+    if args.verbose:
+        print(f'getting secret-id, have:: {r.json()}')
+    output_dict['TAPIS_SK_VAULT_SECRET_ID'] =  r.json()['data']['secret_id']
+    output_dict['VAULT_SECRETID'] =  r.json()['data']['secret_id']
+
+    r = requests.get('http://localhost:8200/v1/auth/approle/role/sk/role-id', headers=headers)
+    if args.verbose:
+        print(f'getting role-id, have:: {r.json()}')
+    output_dict['TAPIS_SK_VAULT_ROLE_ID'] =  r.json()['data']['role_id']
+    output_dict['VAULT_ROLEID'] =  r.json()['data']['role_id']
+
+    # export VAULT_ROLEID=`curl -s -X GET -H "X-Vault-Token: $VAULT_TOKEN" http://localhost:8200/v1/auth/approle/role/sk/role-id | jq -r .data.role_id`
+
+    if args.verbose:
+        print(f'''populating values for security, have:
+            vault_token: {vault_token}
+            secret_id: {output_dict['TAPIS_SK_VAULT_SECRET_ID']}
+            role_id: {output_dict['TAPIS_SK_VAULT_ROLE_ID']}
+            ''')
+
+with open(infile, 'r') as lines:
+    for line in lines:
+        if len(line) <= 1:
+            continue # ignore empty line
+        try:
+            skexport_key, skexport_value = line.split('=', 1)
+            skexport_value = skexport_value.replace('\\n', '\n').strip() # sanitize newlines on priv/pub keys, strip trailing whitespace
+            for key in local:
+                if local[key] == skexport_key:
+                    if args.verbose:
+                        print(f'writing {key}="{skexport_value}"')
+                    # outfile.write(f"{key}='{skexport_value}'\n")
+                    output_dict[key] = skexport_value
+        except KeyError:
+            pass
+
+for key in output_dict:
+    outfile.write(f'{key}="{output_dict[key]}"\n')
+
+
+outfile.close() 

playbooks/roles/admin/templates/docker/verification/authenticator-test

@@ -1,6 +1,10 @@
 #!/bin/bash
 #
 ## authenticator:
-curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{admin_service_url}}/v3/oauth2/tokens
+curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{admin_devtenant_url}}/v3/oauth2/tokens
 
 # should return 200 and JSON with token
+
+
+export pass=`grep LDAP_ROOTPASS {{ tapisdatadir }}/authenticator/env | awk -F= '{print $2}'`
+docker run --rm -it -net tapis mrghort/centosutil:1.3.1 ldapsearch -x -H ldap://authenticator-ldap:389 -Z -D "cn=admin,dc=tapis" -w "$pass" -b "ou=tenants.dev,dc=tapis"

playbooks/roles/admin/templates/docker/verification/tokens-test

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-s=`kubectl get secret tapis-tenants-secrets -o json | jq -r '.data["service-password"]' | base64 -d`
+# s=`kubectl get secret tapis-tenants-secrets -o json | jq -r '.data["service-password"]' | base64 -d`
+s=`grep SERVICEPWD_TENANTS_PASSWORD {{ tapisdatadir }}/skadmin/env | awk -F= '{print $2}' | tr -d " \t\n\r"`
 
 # tokens
 # should return 200 and JSON with token

playbooks/roles/admin/templates/docker/verification/vault-test

@@ -1,19 +1,5 @@
 #!/bin/bash
 
-#kubectl exec -it deploy/vault -- vault status
-
-
-
-. functions
-
-
-VAULT_TOKEN=`vault_token_from_secret`
-
-
-kubectl run centos -i --tty --image={{ admin_util_image }} --restart=Never --rm -- '/usr/bin/curl -H "X-Vault-Token: '$VAULT_TOKEN'" {{admin_vault_url}}/v1/sys/health | /usr/bin/jq'
-
-#
-#
-#VAULTURL="http://vault:8200"
+docker exec -it vault vault status
 
 

playbooks/roles/admin/templates/kube/verification/vault-test

@@ -1,19 +1,14 @@
 #!/bin/bash
 
-#kubectl exec -it deploy/vault -- vault status
-
-
+# similar to this if run manually 
+# kubectl exec -it deploy/vault -- vault status
 
 . functions
 
-
 VAULT_TOKEN=`vault_token_from_secret`
 
+kubectl run vault-test -i --tty --image={{ admin_centosutil_image }} --restart=Never --rm -- bash -c '/usr/bin/curl -H "X-Vault-Token: $VAULT_TOKEN" {{ admin_vault_url }}/v1/sys/health | /usr/bin/jq'
 
-kubectl run centos -i --tty --image={{ admin_centosutil_image }} --restart=Never --rm -- '/usr/bin/curl -H "X-Vault-Token: '$VAULT_TOKEN'" {{admin_vault_url}}/v1/sys/health | /usr/bin/jq'
 
-#
-#
-#VAULTURL="http://vault:8200"
 
 

playbooks/roles/apps/templates/docker/burnup

@@ -2,8 +2,7 @@
 
 echo "burnup apps:"
 
-mkdir -p {{ tapisdatadir }}/files
-mkdir -p {{ tapisdatadir }}/files/irods
+mkdir -p {{ tapisdatadir }}/apps
 
 docker compose up -d
 

playbooks/roles/apps/templates/docker/docker-compose.yml

@@ -2,6 +2,7 @@
 networks:
     tapis:
         name: tapis
+        external: true
 
 services:
     apps-api:

playbooks/roles/authenticator/defaults/main/images.yml

@@ -1,4 +1,4 @@
-authenticator_api_image: tapis/authenticator:1.3.3
-authenticator_migrations_image: tapis/authenticator-migrations:1.3.3
+authenticator_api_image: tapis/authenticator:1.3.4
+authenticator_migrations_image: tapis/authenticator-migrations:1.3.4
 authenticator_postgres_image: postgres:11.4
 authenticator_ldap_image: tacc/slapd:1

playbooks/roles/authenticator/templates/docker/authenticator-config.json

@@ -5,7 +5,7 @@
     "service_name": "{{authenticator_service_name}}",
     "tenants": {{ authenticator_service_tenants | to_json }},
     "show_traceback": {{ authenticator_show_traceback | to_json }},
-    "sql_db_url": "authenticator-postgres:5432/authenticator",
+    "sql_db_url": "authenticator-postgres:5432",
     "dev_ldap_tenant_id": "{{authenticator_dev_ldap_tenant_id}}",
     "log_level": "{{authenticator_log_level}}"
 }

playbooks/roles/authenticator/templates/docker/burnup

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 echo "burnup authenticator:"
-
+python3 {{ tapisdir }}/admin/util/parse_skexport -c authenticator -d {{ tapisdatadir }}
 mkdir -p {{ tapisdatadir }}/authenticator
 mkdir -p {{ tapisdatadir }}/authenticator/postgres
 mkdir -p {{ tapisdatadir }}/authenticator/api

playbooks/roles/authenticator/templates/docker/docker-compose.yml

@@ -1,55 +1,78 @@
-
 networks:
     tapis:
         name: tapis
+        external: true
 
 services:
   authenticator-api: 
+    container_name: authenticator-api
     image: {{ authenticator_api_image }}
-    # ports:
-    #     - "127.0.0.1:5000:5000"
     networks:
         - tapis
-    environment: # TODO
-        dev_ldap_bind_credential: "dev-ldap-bind-credential"
-        postgres_password: "postgres-password"
-        service_password: "service-password"
+    env_file:
+        - "{{ tapisdatadir }}/authenticator/env"
     volumes:
         - "{{ tapisdir }}/authenticator/authenticator-config.json:/home/tapis/config.json"
-        # - "{{ tapisdatadir }}/authenticator/api/service.log:/home/tapis/service.log"
-    container_name: authenticator-api
     depends_on: 
-        - authenticator-postgres
+        authenticator-postgres:
+            condition: service_healthy
+        authenticator-ldap:
+            condition: service_started
+        authenticator-migrations:
+            condition: service_completed_successfully
 
 
   authenticator-postgres:
+    container_name: authenticator-postgres
     image: {{ authenticator_postgres_image }}
-    # ports:
-    #     - "127.0.0.1:5432:5432"
     networks:
         - tapis
     environment:
-        PGDATA: "/pgdata/data"
-        POSTGRES_USER: "authenticator"
-        POSTGRES_DB: "authenticator"
-        POSTGRES_PASSWORD: "postgres-password" # TODO
+        # - PGDATA=/pgdata/data
+        - POSTGRES_USER=authenticator
+        - POSTGRES_DB=authenticator
+        - POSTGRES_HOST_AUTH_METHOD=trust
+    env_file:
+        - "{{ tapisdatadir }}/authenticator/env"
     volumes:
-        - {{ tapisdatadir }}/authenticator/postgres:/data/pgdata
-    container_name: authenticator-postgres
+        - "{{ tapisdatadir }}/authenticator/postgres:/var/lib/postgresql/data"
+    healthcheck:
+        test: ["CMD-SHELL", "pg_isready -U postgres"]
+        interval: 5s
+        timeout: 5s
+        retries: 5
+
+
+  authenticator-migrations:
+    container_name: authenticator-migrations
+    image: {{ authenticator_migrations_image }}
+    networks:
+        - tapis
+    env_file:
+        - "{{ tapisdatadir }}/authenticator/env"
+    depends_on:
+        authenticator-postgres:
+            condition: service_healthy
+        authenticator-ldap:
+            condition: service_started
+    command: ['upgrade']
+    volumes:
+        - "{{ tapisdir }}/authenticator/authenticator-config.json:/home/tapis/config.json"
+
 
   authenticator-ldap:
+    container_name: authenticator-ldap
     image: {{ authenticator_ldap_image }}
-    # ports:
-    #     - "127.0.0.1:389:389"
     networks:
         - tapis
-    environment: #TODO
-        LDAP_DOMAIN: "LDAP_DOMAIN"
-        LDAP_ORGANISATION: "LDAP_ORGANISATION"
-        LDAP_ROOTPASS: "dev-ldap-bind-credential"
+    environment:
+        - LDAP_DOMAIN=tapis
+        - LDAP_ORGANISATION=Tapis
+    env_file:
+        - "{{ tapisdatadir }}/authenticator/env"
     volumes:
-        - {{ tapisdatadir }}/authenticator/ldap:/data/ldap
-    container_name: authenticator-ldap
+        - "{{ tapisdatadir }}/authenticator/ldap:/data/ldap"
+    depends_on:
+        authenticator-postgres:
+            condition: service_healthy
 
-networks:
-    tapis:

playbooks/roles/baseburnup/defaults/main/vars.yml

@@ -1,4 +1,4 @@
-baseburnup_tapis_deployer_version: 1.3.6
+baseburnup_tapis_deployer_version: 1.3.7
 baseburnup_service_url: "{{ global_service_url }}"
 baseburnup_vault_url: "{{ global_vault_url }}"