Merge pull request #223 from tapis-project/dev

Dev

CHANGELOG.md

@@ -2,6 +2,23 @@
 
 Notable changes between versions.
 
+## 1.3.8
+
+- Added java heap max and min options for apps, systems, and notifications when using Docker compose.
+- [Jobs: 1.3.4 to 1.3.5 (tapis/jobsworker, jobsmigrate, jobsapi)](https://github.com/tapis-project/tapis-jobs/blob/dev/tapis-jobsapi/CHANGELOG.md)
+- [Systems: 1.3.2 to 1.3.3 (tapis/systems)](https://github.com/tapis-project/tapis-systems/blob/1.3.3/CHANGELOG.md)
+- [Files: 1.3.5 to 1.3.6 (tapis/tapis-files, tapis/tapis-files-workers)](https://github.com/tapis-project/tapis-files/blob/dev/CHANGELOG.md)
+- Docker Flavor update:  
+  - Added verification scripts for more core components
+  - Changed secrets to using a python script for parsing instead of bash scripting 
+  - Added a DB init script for files
+  - Removed hard-coded urls in proxy
+  - General cleanup & bugfixes
+
+### Breaking Changes
+
+- There is a breaking change related to how Files and Systems interact for systems of type IRODS. Please see the [CHANGELOG](https://github.com/tapis-project/tapis-files/blob/dev/CHANGELOG.md) for the Files service for more information.
+
 ## 1.3.7
 
 - [Authenticator: 1.3.3 to 1.3.4 (authenticator & authenticator-migrations)](https://github.com/tapis-project/authenticator/blob/prod/CHANGELOG.md)
@@ -19,6 +36,7 @@ Notable changes between versions.
 - [Apps: 1.3.2 to 1.3.3 (tapis/apps)](https://github.com/tapis-project/tapis-apps/blob/1.3.3/CHANGELOG.md)
 - [Notifications: 1.3.1 to 1.3.3 (tapis/notifications, notifications-dispatcher)](https://github.com/tapis-project/tapis-notifications/blob/1.3.3/CHANGELOG.md)
 
+
 ## 1.3.5
 
 **Breaking Changes**

playbooks/roles/actors/templates/docker/burnup

@@ -3,6 +3,9 @@
 echo "burnup actors:"
 
 mkdir -p {{ tapisdatadir }}/actors
+myuid=`id -u`
+docker run -it --rm -v {{ tapisdatadir }}/actors:/actors tapis/ubutil2204:1.3.0 chown $myuid /actors
 
-docker compose up -d
+python3 {{ tapisdir }}/admin/util/parse_skexport -c actors -d {{ tapisdatadir }}
 
+docker compose up -d

playbooks/roles/actors/templates/docker/docker-compose.yml

@@ -16,14 +16,16 @@ services:
         container_name: actors-mongo
         image: {{ actors_mongo_image }}
         environment:
-            MONGO_INITDB_ROOT_USERNAME: admin
-            MONGO_INITDB_ROOT_PASSWORD: admin
-            MONGO_REPLICA_SET_NAME: rs0
-        # ports:
-        #     - "127.0.0.1:27017:27017"
+            - MONGO_INITDB_ROOT_USERNAME: admin
+            - MONGO_REPLICA_SET_NAME: rs0
+        env_file:
+            - {{ tapisdatadir }}/actors/env
         # volumes:
         #     - ./runtime_files/certs:/data/ssl
         #     - ./runtime_files/certs/mongo-replica-set-keyfile:/keyfiles/metakeyfile
+        volumes:
+            - {{ tapisdatadir }}/actors/mongo/data:/data/db
+            - {{ tapisdatadir }}/actors/mongo/keyfile:keyfile-base
         networks:
             - tapis
         ulimits:
@@ -47,18 +49,16 @@ services:
         #command: "mongod --bind_ip_all --replSet rs0 --keyFile /keyfiles/keyfile-to-use --tlsMode requireTLS --tlsCertificateKeyFile "/data/ssl/server.pem" --tlsCAFile "/data/ssl/ca.pem"
 
 
-    actors-rabbit:
-        container_name: actors-rabbit
+    actors-rabbitmq:
+        container_name: actors-rabbitmq
         image: {{ actors_rabbitmq_image }}
-        # ports:
-        #     - "127.0.0.1:5672:5672"
-        #     - "127.0.0.1:15672:15672"
         environment:
             RABBITMQ_NODENAME: abaco-rabbit
             RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+A 128"
             RABBITMQ_DEFAULT_VHOST: abaco_vhost
             RABBITMQ_DEFAULT_USER: admin
-            RABBITMQ_DEFAULT_PASS: admin
+        env_file:
+            - {{ tapisdatadir }}/actors/env
         ulimits:
             nproc: 65535
             nofile:
@@ -67,23 +67,23 @@ services:
         networks:
             - tapis
         depends_on:
-            - actors-mongo
+            actors-mongo:
+                condition: service_started
 
     actors-nginx:
         container_name: actors-nginx
         image: {{ actors_nginx_image }}
+        networks:
+            - tapis
         volumes:
 #             - ./config-local.json:/home/tapis/config.json
-          - "{{ tapisdir }}/actors/actors-config.json:/home/tapis/config.json"
+          - "{{ tapisdir }}/actors/actors-config.json:/etc/nginx/sites-enabled/flask-project"
 # #            - ./images/nginx/nginx.conf:/etc/nginx/nginx.conf
 # #            - ./images/nginx/sites-enabled:/etc/nginx/sites-enabled
-#         ports:
-#             - "127.0.0.1:8000:80"
         restart: always
         depends_on:
-            - actors-mongo
-        networks:
-            - tapis
+            actors-mongo:
+                condition: service_started
 
     actors-reg:
         container_name: actors-reg

playbooks/roles/admin/templates/docker/util/parse_skexport

@@ -6,33 +6,75 @@ import json
 
 # var mapping for all comps
 # e.g. {"{skexport_name}": "{internal_component_name}"}
-actors = {"":""}
-apps = {"":""}
+actors = {"RABBITMQ_DEFAULT_PASS":"DBCREDENTIAL_RABBITMQ_ABACO_RABBIT_MQ_HOST_RABBITMQ_ABACO_PASSWORD",
+            "MONGO_INITDB_ROOT_PASSWORD": "DBCREDENTIAL_MONGO_ABACO_MONGO_DB_HOST_MONGODB_ABACO_PASSWORD",
+            "": "SERVICEPWD_ABACO_PASSWORD"}
+
+apps = {"TAPIS_DB_PASSWORD":"DBCREDENTIAL_POSTGRES_APPS_POSTGRES_TAPISAPPDB_POSTGRES_PASSWORD",
+        "TAPIS_SERVICE_PASSWORD": "SERVICEPWD_APPS_PASSWORD",
+        "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_APPS_POSTGRES_TAPISAPPDB_POSTGRES_PASSWORD",
+        "POSTGRES_USER_PASSWORD": "DBCREDENTIAL_POSTGRES_APPS_POSTGRES_TAPISAPPDB_TAPIS_APP_PASSWORD",
+        "service_password": "SERVICEPWD_APPS_PASSWORD",
+        "MONITOR_PASSWORD": "DBCREDENTIAL_POSTGRES_APPS_POSTGRES_TAPISAPPDB_TAPIS_APP_PASSWORD"}
+
 authenticator = {"POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_POSTGRES_AUTHENTICATOR_AUTHENTICATOR_PASSWORD",
                 "postgres_password": "DBCREDENTIAL_POSTGRES_POSTGRES_AUTHENTICATOR_AUTHENTICATOR_PASSWORD",
                 "LDAP_ROOTPASS":"USER_ADMIN_AUTHENTICATOR_LDAP_TAPIS_DEV_PASSWORD",
                 "service_password": "SERVICEPWD_AUTHENTICATOR_PASSWORD"}
 
-files = {"":""}
+files = {"RABBITMQ_PASSWORD":"DBCREDENTIAL_RABBITMQ_TAPISFILES_RABBITMQ_FILES_FILES_PASSWORD",
+         "RABBITMQ_DEFAULT_PASS":"DBCREDENTIAL_RABBITMQ_TAPISFILES_RABBITMQ_FILES_FILES_PASSWORD",
+         "DB_PASSWORD":"DBCREDENTIAL_POSTGRES_TAPISFILES_POSTGRES_FILES_FILES_PASSWORD",
+         "POSTGRES_PASSWORD":"DBCREDENTIAL_POSTGRES_TAPISFILES_POSTGRES_FILES_FILES_PASSWORD",
+         "SERVICE_PASSWORD":"SERVICEPWD_FILES_PASSWORD"}
+
 globus_proxy = {"":""}
-jobs = {"":""}
+jobs = {"TAPIS_SERVICE_PASSWORD": "SERVICEPWD_JOBS_PASSWORD",
+        "TAPIS_DB_PASSWORD": "DBCREDENTIAL_POSTGRES_JOBS_POSTGRES_TAPISJOBSDB_TAPIS_PASSWORD",
+        "TAPIS_QUEUE_ADMIN_PASSWORD": "DBCREDENTIAL_RABBITMQ_JOBS_RABBITMQ_JOBSHOST_TAPIS_PASSWORD",
+        "TAPIS_QUEUE_PASSWORD": "DBCREDENTIAL_RABBITMQ_JOBS_RABBITMQ_JOBSHOST_JOBS_PASSWORD",
+        "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_JOBS_POSTGRES_TAPISJOBSDB_TAPIS_PASSWORD",
+        "RABBITMQ_DEFAULT_PASS": "DBCREDENTIAL_RABBITMQ_JOBS_RABBITMQ_JOBSHOST_TAPIS_PASSWORD",
+        "TPW": "DBCREDENTIAL_POSTGRES_JOBS_POSTGRES_TAPISJOBSDB_TAPIS_PASSWORD",
+        "PW": "DBCREDENTIAL_POSTGRES_JOBS_POSTGRES_TAPISJOBSDB_TAPIS_PASSWORD"}
+
 meta = {"":""}
 monitoring = {"":""}
-notifications = {"":""}
+notifications = {"PGADMIN_DEFAULT_PASSWORD": "DBCREDENTIAL_PGADMIN_NOTIFICATIONS_POSTGRES_TAPISNTFDB_WOW_PASSWORD",
+                "TAPIS_DB_USER_PASSWORD": "DBCREDENTIAL_POSTGRES_NOTIFICATIONS_POSTGRES_TAPISNTFDB_POSTGRES_PASSWORD",
+                "POSTGRES_USER_PASSWORD": "DBCREDENTIAL_POSTGRES_NOTIFICATIONS_POSTGRES_TAPISNTFDB_POSTGRES_PASSWORD",
+                "TAPIS_DB_PASSWORD": "DBCREDENTIAL_POSTGRES_NOTIFICATIONS_POSTGRES_TAPISNTFDB_TAPIS_NTF_PASSWORD",
+                "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_NOTIFICATIONS_POSTGRES_TAPISNTFDB_TAPIS_NTF_PASSWORD",
+                "TAPIS_QUEUE_PASSWORD": "DBCREDENTIAL_RABBITMQ_NOTIFICATIONS_RABBITMQ_NOTIFICATIONSHOST_NOTIFICATIONS_PASSWORD",
+                "RABBITMQ_DEFAULT_PASS": "DBCREDENTIAL_RABBITMQ_NOTIFICATIONS_RABBITMQ_NOTIFICATIONSHOST_TAPIS_PASSWORD",
+                "TAPIS_QUEUE_ADMIN_PASSWORD": "DBCREDENTIAL_RABBITMQ_NOTIFICATIONS_RABBITMQ_NOTIFICATIONSHOST_TAPIS_PASSWORD",
+                "TAPIS_SERVICE_PASSWORD": "SERVICEPWD_NOTIFICATIONS_PASSWORD",
+                "service_password": "SERVICEPWD_NOTIFICATIONS_PASSWORD"}
+
+
+
 pgrest = {"":""}
 pods = {"":""}
 proxy = {"":""}
-security = {"TAPIS_SK_VAULT_SECRET_ID": "",
+security = {"TAPIS_SK_VAULT_SECRET_ID": "", # these two are populated later with curl cmds
             "TAPIS_SK_VAULT_ROLE_ID": "",
             "TAPIS_DB_PASSWORD" : "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_TAPIS_PASSWORD",
             "TAPIS_PASSWORD": "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_TAPIS_PASSWORD",
             "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_POSTGRES_PASSWORD",
             "PW" : "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_POSTGRES_PASSWORD",
-            "TPW" : "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_TAPIS_PASSWORD"}
+            "TPW" : "DBCREDENTIAL_POSTGRES_SK_POSTGRES_TAPISSECDB_POSTGRES_PASSWORD"}
 
 skadmin = {"":""}
 streams = {"":""}
-systems = {"":""}
+systems = {"pgadmin_password": "DBCREDENTIAL_PGADMIN_SYSTEMS_POSTGRES_TAPISSYSDB_WOW_PASSWORD",
+            "postgres_password": "DBCREDENTIAL_POSTGRES_SYSTEMS_POSTGRES_TAPISSYSDB_POSTGRES_PASSWORD",
+            "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_SYSTEMS_POSTGRES_TAPISSYSDB_POSTGRES_PASSWORD",
+            "postgres_user_password": "DBCREDENTIAL_POSTGRES_SYSTEMS_POSTGRES_TAPISSYSDB_TAPIS_SYS_PASSWORD",
+            "service_password": "SERVICEPWD_SYSTEMS_PASSWORD",
+            "TAPIS_SERVICE_PASSWORD": "SERVICEPWD_SYSTEMS_PASSWORD",
+            "TAPIS_DB_PASSWORD": "DBCREDENTIAL_POSTGRES_SYSTEMS_POSTGRES_TAPISSYSDB_POSTGRES_PASSWORD",
+            }
+
 tapisui = {"":""}
 tenants = {"postgres_password": "DBCREDENTIAL_POSTGRES_TENANTS_POSTGRES_TENANTS_TENANTS_PASSWORD",
             "POSTGRES_PASSWORD": "DBCREDENTIAL_POSTGRES_TENANTS_POSTGRES_TENANTS_TENANTS_PASSWORD",
@@ -71,9 +113,22 @@ except FileExistsError:
 # set local vars
 infile = args.dir + '/skadmin/env'
 local = locals()[component]
-outfile = open(args.dir + '/' + args.comp + '/env', 'w')
+try:
+    outfile = open(args.dir + '/' + args.comp + '/env', 'w')
+    if args.verbose:
+            print(f'successfuly opened {outfile}')
+except Exception as e:
+    print(f'''error opening outfile {outfile}::
+    {e}''')
 output_dict = {}
 
+if args.verbose:
+    print(f'''have local args:
+    component: {component}
+    infile: {infile}
+    local: {local}
+    outfile: {outfile}''')
+
 # populate vault values for security
 if component == 'security':
     vault_token = open('{{ tapisdatadir }}/vault/vault-token').read()
@@ -91,8 +146,6 @@ if component == 'security':
     output_dict['TAPIS_SK_VAULT_ROLE_ID'] =  r.json()['data']['role_id']
     output_dict['VAULT_ROLEID'] =  r.json()['data']['role_id']
 
-    # export VAULT_ROLEID=`curl -s -X GET -H "X-Vault-Token: $VAULT_TOKEN" http://localhost:8200/v1/auth/approle/role/sk/role-id | jq -r .data.role_id`
-
     if args.verbose:
         print(f'''populating values for security, have:
             vault_token: {vault_token}
@@ -101,6 +154,8 @@ if component == 'security':
             ''')
 
 with open(infile, 'r') as lines:
+    if args.verbose:
+        print(f'successfuly opened {infile}')
     for line in lines:
         if len(line) <= 1:
             continue # ignore empty line

playbooks/roles/admin/templates/docker/verification/README.md

@@ -3,8 +3,14 @@
 
 order:
 
+- vault-test
 - sk-test
 - tenants-test
 - tokens-test
 - authenticator-test
+- systems-test
+- files-test
+- jobs-test
+- notifications-test
+- streams-test
 

playbooks/roles/admin/templates/docker/verification/authenticator-test

@@ -5,6 +5,5 @@ curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "passw
 
 # should return 200 and JSON with token
 
-
-export pass=`grep LDAP_ROOTPASS {{ tapisdatadir }}/authenticator/env | awk -F= '{print $2}'`
-docker run --rm -it -net tapis mrghort/centosutil:1.3.1 ldapsearch -x -H ldap://authenticator-ldap:389 -Z -D "cn=admin,dc=tapis" -w "$pass" -b "ou=tenants.dev,dc=tapis"
+export pass=`grep ^LDAP_ROOTPASS {{ tapisdatadir }}/authenticator/env | awk -F= '{print $2}' | tr -d '"'`
+docker run --rm -it --network tapis tapis/centosutil:1.3.1 ldapsearch -x -H ldap://authenticator-ldap:389 -Z -D "cn=admin,dc=tapis" -w "$pass" -b "ou=tenants.dev,dc=tapis"

playbooks/roles/admin/templates/docker/verification/files-test

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+TOK=`curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{ admin_service_url }}/v3/oauth2/tokens | jq -r '.result.access_token.access_token'`
+
+curl {{ admin_devtenant_url }}/v3/files/healthcheck | jq 
+

playbooks/roles/admin/templates/docker/verification/jobs-test

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+TOK=`curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{ admin_service_url }}/v3/oauth2/tokens | jq -r '.result.access_token.access_token'`
+
+curl {{ admin_devtenant_url }}/v3/jobs/healthcheck | jq 
+

playbooks/roles/admin/templates/docker/verification/notifications-test

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+TOK=`curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{ admin_service_url }}/v3/oauth2/tokens | jq -r '.result.access_token.access_token'`
+
+curl  {{ admin_devtenant_url }}/v3/notifications/healthcheck | jq

playbooks/roles/admin/templates/docker/verification/sk-test

@@ -3,13 +3,13 @@
 
 
 echo "hello"
-docker run --net tapis --rm -it {{ admin_util_image }} curl http://security-api:8000/v3/security/hello
+curl -s {{ admin_devtenant_url }}/v3/security/hello | jq .status
 echo
 
 echo "ready"
-docker run --net tapis --rm -it {{ admin_util_image }} curl http://security-api:8000/v3/security/ready
+curl -s {{ admin_devtenant_url }}/v3/security/ready | jq .status
 echo
 
 echo "healthcheck"
-docker run --net tapis --rm -it {{ admin_util_image }} curl http://security-api:8000/v3/security/healthcheck
+curl -s {{ admin_devtenant_url }}/v3/security/healthcheck | jq .status
 echo

playbooks/roles/admin/templates/docker/verification/streams-test

@@ -1,9 +1,9 @@
 #!/bin/bash
 
-curl {{admin_service_url}}/v3/streams/hello 
+curl {{admin_devtenant_url}}/v3/streams/hello 
 
 
-curl {{admin_service_url}}/v3/streams/hello
+curl {{admin_devtenant_url}}/v3/streams/hello
 
 
 

playbooks/roles/admin/templates/docker/verification/systems-test

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+TOK=`curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{ admin_service_url }}/v3/oauth2/tokens | jq -r '.result.access_token.access_token'`
+
+curl {{ admin_devtenant_url }}/v3/systems/healthcheck | jq 
+

playbooks/roles/admin/templates/docker/verification/tenants-test

@@ -3,5 +3,5 @@
 # tenants
 # should return 200 and JSON with tenants list.
 
-curl {{admin_service_url}}/v3/tenants | jq
+curl {{admin_devtenant_url}}/v3/tenants | jq
 

playbooks/roles/admin/templates/docker/verification/tokens-test

@@ -6,5 +6,5 @@ s=`grep SERVICEPWD_TENANTS_PASSWORD {{ tapisdatadir }}/skadmin/env | awk -F= '{p
 # tokens
 # should return 200 and JSON with token
 
-curl -u "tenants:$s" -H "Content-type: application/json" -d '{"token_tenant_id": "admin", "account_type": "service", "token_username": "tenants", "target_site_id": "{{ admin_service_site_id }}"}'  {{ admin_service_url }}/v3/tokens
+curl -u "tenants:$s" -H "Content-type: application/json" -d '{"token_tenant_id": "admin", "account_type": "service", "token_username": "tenants", "target_site_id": "{{ admin_service_site_id }}"}'  {{ admin_devtenant_url }}/v3/tokens
 

playbooks/roles/apps/defaults/main/vars.yml

@@ -11,4 +11,6 @@ apps_postgres_pvc: apps-postgres-vol01
 apps_service_password: null
 apps_postgres_password: null
 apps_pgadmin_password: null
+apps_heap_min: 1g
+apps_heap_max: 4g