Merge pull request #372 from tapis-project/staging

Staging

playbooks/roles/admin/templates/docker/verification/authenticator-test

@@ -7,4 +7,4 @@ gettoken
 # should return 200 and JSON with token
 
 export pass=`grep ^LDAP_ROOTPASS {{ tapisdatadir }}/authenticator/env | awk -F= '{print $2}' | tr -d '"'`
-docker run --rm -it --network tapis tapis/centosutil:1.4.0 ldapsearch -x -H ldap://authenticator-ldap:389 -Z -D "cn=admin,dc=tapis" -w "$pass" -b "ou=tenants.dev,dc=tapis"
+docker run --rm -it --network tapis tapis/centosutil:1.4.0 ldapsearch -x -H ldap://authenticator-ldap:389 -D "cn=admin,dc=tapis" -w "$pass" -b "ou=tenants.dev,dc=tapis"

playbooks/roles/admin/templates/docker/verification/functions

@@ -10,7 +10,7 @@ SERVICEURL="{{admin_service_url}}"
 ### util
 
 gettoken(){
-  tok=$(curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{ admin_service_url }}/v3/oauth2/tokens | jq -r .result.access_token.access_token)
+  tok=$(curl -d '{"username": "testuser2", "password": "testuser2", "grant_type": "password"}' -H "Content-type: application/json"  {{ admin_devtenant_url }}/v3/oauth2/tokens | jq -r .result.access_token.access_token)
 }
 
 ### checks

playbooks/roles/admin/templates/docker/verification/tenants-test

@@ -1,8 +1,7 @@
 #!/bin/bash
 
 # tenants
-source functions
-gettoken
+
 # should return 200 and JSON with tenants list.
 
 curl {{admin_devtenant_url}}/v3/tenants | jq

playbooks/roles/admin/templates/kube/verification/streams-test

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-curl {{admin_service_url}}/v3/streams/hello 
+curl {{admin_service_url}}/v3/streams/ready 
 
 
 curl {{admin_service_url}}/v3/streams/hello

playbooks/roles/admin/templates/kube/verification/systems-test

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+curl {{admin_service_url}}/v3/systems/ready 
+
+
+curl {{admin_service_url}}/v3/systems/hello
+
+
+

playbooks/roles/apps/defaults/main/vars.yml

@@ -13,4 +13,4 @@ apps_postgres_password: null
 apps_pgadmin_password: null
 apps_heap_min: 1g
 apps_heap_max: 4g
-
+apps_port: 8081

playbooks/roles/authenticator/templates/docker/docker-compose.yml

@@ -72,6 +72,10 @@ services:
     depends_on:
         authenticator-postgres:
             condition: service_healthy
+    ulimits:
+        nofile:
+            soft: 1024
+            hard: 4096
     healthcheck:
         test: slapcat
 

playbooks/roles/baseburnup/templates/docker/burndown

@@ -26,7 +26,10 @@ echo "Start: top-level-burndown"
 
 # globus-proxy
 
-# meta
+{% if "meta" in components_to_deploy %}
+cd $mydir_absolute/meta
+./burndown
+{% endif %}
 
 # monitoring
 
@@ -43,7 +46,10 @@ cd $mydir_absolute/notifications
 
 # pods
 
-# streams
+{% if "streams" in components_to_deploy %}
+cd $mydir_absolute/streams
+./burndown streams
+{% endif %}
 
 {% if "tapisui" in components_to_deploy %}
 cd $mydir_absolute/tapisui

playbooks/roles/jobs/defaults/main/vars.yml

@@ -9,3 +9,4 @@ jobs_storage_class: "{{ global_storage_class }}"
 jobs_rabbitmq_pvc: jobs-rabbitmq-vol01
 jobs_node_selector: null
 jobs_node_name: true
+jobs_port: 8082

playbooks/roles/meta/templates/docker/burnup

@@ -9,7 +9,7 @@ export UID_GID="$(id -u):$(id -g)"
 
 docker run -it --rm -v {{ tapisdatadir }}/meta:/meta {{ meta_util_image }} chown $MYUID /meta
 
-python3 {{ tapisdir }}/admin/util/parse_skexport -c meta -d {{ tapisdatadir }}
+python3 ../admin/util/parse_skexport -c meta -d {{ tapisdatadir }}
 
 docker compose up -d
 

playbooks/roles/monitoring/templates/docker/burnup

@@ -11,6 +11,6 @@ export UID_GID="$(id -u):$(id -g)"
 
 docker run -it --rm -v {{ tapisdatadir }}/monitoring:/monitoring {{ monitoring_util_image }} chown $MYUID /monitoring
 
-python3 {{ tapisdir }}/admin/util/parse_skexport -c monitoring -d {{ tapisdatadir }}
+python3 ../admin/util/parse_skexport -c monitoring -d {{ tapisdatadir }}
 
-docker compose up -d
+docker compose up -d

playbooks/roles/notifications/defaults/main/vars.yml

@@ -13,4 +13,5 @@ notifications_mail_port: 25
 notifications_heap_min: 1g
 notifications_heap_max: 4g
 
+notifications_port: 8083
 

playbooks/roles/notifications/templates/docker/docker-compose.yml

@@ -29,10 +29,10 @@ services:
       notifications-rabbitmq:
         condition: service_healthy
     command: java -Xms{{ notifications_heap_min }} -Xmx{{ notifications_heap_max }} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.time=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED -cp "notifications.jar:dependencies/*" edu.utexas.tacc.tapis.notifications.api.NotificationsApplication
-    {% if notifications_port is not none %}
+{% if notifications_port is not none %}
     ports: 
-            - "{{notifications_port}}:8080"
-    {% endif %}
+      - "{{notifications_port}}:8080"
+{% endif %}
 
   notifications-init-db:
     container_name: notifications-init-db

playbooks/roles/proxy/templates/docker/docker-compose.yml

@@ -14,9 +14,9 @@ services:
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf:ro
       - ./locations:/etc/nginx/conf.d/locations:ro
-      - {{proxy_nginx_cert_file}}:/tmp/ssl/tls.crt:ro
-      - {{proxy_nginx_cert_key}}:/tmp/ssl/tls.key:ro
-      - {{tapisdir}}proxy/token-revoked.json:/etc/nginx/html/token-revoked.json
+      - {{ proxy_nginx_cert_file }}:/tmp/ssl/tls.crt:ro
+      - {{ proxy_nginx_cert_key }}:/tmp/ssl/tls.key:ro
+      - ./token-revoked.json:/etc/nginx/html/token-revoked.json
     networks:
       - tapis
     restart: always
@@ -27,7 +27,7 @@ services:
     ports:
       - "127.0.0.1:8000:8000"
     volumes:
-      - {{ tapisdir }}/proxy/site-router-config.json:/home/tapis/config.json:ro
+      - ./site-router-config.json:/home/tapis/config.json:ro
     networks:
       - tapis
     restart: always

playbooks/roles/proxy/templates/docker/locations/notifications.conf

@@ -6,7 +6,7 @@ location /v3/notifications
 
     resolver 127.0.0.11;
     # set $upstream "http://notifications-api:8080";
-    set $upstream {{ notifications_host | default("http://notifications-api:;")}}:{{ notifications_port | default("8080")}};
+    set $upstream {{ notifications_host | default("http://notifications-api")}}:{{ notifications_port | default("8080")}};
     proxy_pass $upstream;
 
 

playbooks/roles/proxy/templates/docker/token-revoked.json

@@ -0,0 +1,2 @@
+{"status": "error", "message": "Token is revoked.", "version": "{{ proxy_site_router_api_version }}", "result": null }
+

playbooks/roles/security/templates/docker/burnup

@@ -8,7 +8,7 @@ mkdir -p {{ tapisdatadir }}/security/postgres
 python3 ../admin/util/parse_skexport -c security -d {{ tapisdatadir }}
 
 export UID_GID="$(id -u):$(id -g)"
-docker run -it --rm -v {{ tapisdatadir }}/security:/security {{ meta_util_image }} chown $UID_GID /security
+docker run -it --rm -v {{ tapisdatadir }}/security:/security {{ security_util_image }} chown $UID_GID /security
 
 docker compose up -d
 

playbooks/roles/streams/templates/docker/burnup

@@ -11,6 +11,6 @@ export UID_GID="$(id -u):$(id -g)"
 
 docker run -it --rm -v {{ tapisdatadir }}/streams:/streams {{ streams_util_image }} chown $MYUID /streams
 
-python3 {{ tapisdir }}/admin/util/parse_skexport -c streams -d {{ tapisdatadir }}
+python3 ../admin/util/parse_skexport -c streams -d {{ tapisdatadir }}
 
-docker compose up -d
+docker compose up -d

playbooks/roles/streams/templates/docker/docker-compose.yml

@@ -24,7 +24,7 @@ services:
     networks:
       - tapis
     volumes:
-      - {{tapisdatadir}}/streams/mysql/data:/var/lib/mysql
+      - {{ tapisdatadir }}/streams/mysql/data:/var/lib/mysql
       - /var/run/docker.sock:/var/run/docker.sock
       - ./chords/tapis_chords_start.sh:/chords/chords_start.sh
       - ./chords/influxdb.yml:/chords/config/influxdb.yml

playbooks/roles/systems/defaults/main/vars.yml

@@ -10,3 +10,4 @@ systems_postgres_pvc: systems-postgres-vol01
 systems_globus_client_id: null
 systems_heap_max: 3G
 systems_heap_min: 1G
+systems_port: 8084

playbooks/roles/tokens/templates/docker/docker-compose.yml

@@ -11,7 +11,6 @@ services:
         image: {{ tokens_api_image }}
         volumes:
             - ./tokens-config.json:/home/tapis/config.json
-            - ../admin/verification/tokens-test:/home/tapis/healthcheck
         networks:
             - tapis
         env_file: 

playbooks/roles/vault/templates/docker/burnup

@@ -26,13 +26,18 @@ then
 fi 
 
 # check if vault is unsealed already
-sleep 5
-if [ `docker exec -it vault vault status -format=json | jq -r .sealed` == "false" ]
+#sleep 5
+if VAULTSTAT=`docker exec -it vault vault status -format=json`
 then
-  echo "Vault is up and unsealed." 
-  exit 0
+  echo $VAULTSTAT
+  if [ `echo $VAULTSTAT | jq -r .sealed` == "false" ]
+  then
+    echo "Vault is up and unsealed."
+    exit 0
+  fi
 fi
 
+
 # TODO: should we do an initialized check? requires more logic in further checks
 # check if vault is initialized
 #if [ `docker exec -it vault vault status -format=json | jq -r .initialized` == "false" ]
@@ -96,7 +101,7 @@ then
 fi
 
 # give vault some time to get fully booted up before the following services try to use it
-sleep 30
+#sleep 30
 
 exit 0
 

playbooks/roles/vault/templates/docker/docker-compose.yml

@@ -20,4 +20,10 @@ services:
       - {{ vault_data_dir }}/certs:/vault/certs
     networks:
       - tapis
+    healthcheck:
+      test: [ "CMD", "wget", "--spider", "--proxy", "off", "http://127.0.0.1:8200/v1/sys/health?standbyok=true" ]
+      interval: 10s
+      timeout: 3s
+      retries: 10
+      start_period: 5s