diff --git a/CHANGELOG.md b/CHANGELOG.md index 62a02296..b10ad3f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ Notable changes between versions. +## 1.3.9 + +- [Pods: 1.3.0 to 1.3.1 (tapis/pods-api)](https://github.com/tapis-project/pods_service/blob/prod/CHANGELOG.md#131---2023-06-06) +- [Abaco: 1.3.0 to 1.3.1 (abaco/core-v3)](https://github.com/TACC/abaco/blob/prod-v3/CHANGELOG.md#131---2023-06-06) +- Refactored deployment scripts for files and added a script to create the files db if it doesn't exist + ## 1.3.8 - Added java heap max and min options for apps, systems, and notifications when using Docker compose. @@ -14,7 +20,7 @@ Notable changes between versions. - Added a DB init script for files - Removed hard-coded urls in proxy - General cleanup & bugfixes - + ### Breaking Changes - There is a breaking change related to how Files and Systems interact for systems of type IRODS. Please see the [CHANGELOG](https://github.com/tapis-project/tapis-files/blob/dev/CHANGELOG.md) for the Files service for more information. diff --git a/playbooks/roles/actors/defaults/main/vars.yml b/playbooks/roles/actors/defaults/main/vars.yml index 1742f151..480c3885 100644 --- a/playbooks/roles/actors/defaults/main/vars.yml +++ b/playbooks/roles/actors/defaults/main/vars.yml @@ -6,7 +6,7 @@ actors_service_url: '{{ global_service_url }}' actors_service_tenant_id: admin actors_service_site_id: '{{ global_site_id }}' actors_service_name: actors -actors_service_version: 1.3.0 +actors_service_version: 1.3.1 actors_storage_class: '{{ global_storage_class }}' actors_rabbit_pvc: actors-rabbitmq-vol01 actors_mongo_pvc: actors-mongo-vol01 diff --git a/playbooks/roles/actors/templates/kube/api/burndown b/playbooks/roles/actors/templates/kube/api/burndown index 61b0036c..85a499bf 100755 --- a/playbooks/roles/actors/templates/kube/api/burndown +++ b/playbooks/roles/actors/templates/kube/api/burndown @@ -3,7 +3,6 @@ # configs kubectl delete -f ../actors-config.yml kubectl delete -f nginx-config.yml -kubectl delete -f prometheus-config.yml kubectl delete -f configmap.yml # apps @@ -11,8 +10,6 @@ kubectl delete -f reg.yml kubectl delete -f mes.yml kubectl delete -f admin.yml kubectl delete -f events.yml -kubectl delete -f prometheus.yml -kubectl delete -f grafana.yml kubectl delete -f metrics.yml kubectl delete -f nginx.yml diff --git a/playbooks/roles/actors/templates/kube/api/burnup b/playbooks/roles/actors/templates/kube/api/burnup index 910d3302..3a3ef87c 100755 --- a/playbooks/roles/actors/templates/kube/api/burnup +++ b/playbooks/roles/actors/templates/kube/api/burnup @@ -6,7 +6,6 @@ kubectl apply -f security.yml # configs kubectl apply -f ../actors-config.yml kubectl apply -f nginx-config.yml -kubectl apply -f prometheus-config.yml kubectl apply -f configmap.yml # pvcs @@ -32,6 +31,4 @@ kubectl apply -f reg.yml kubectl apply -f mes.yml kubectl apply -f admin.yml kubectl apply -f events.yml -kubectl apply -f prometheus.yml -kubectl apply -f grafana.yml kubectl apply -f metrics.yml diff --git a/playbooks/roles/actors/templates/kube/api/grafana.yml b/playbooks/roles/actors/templates/kube/api/grafana.yml deleted file mode 100644 index 2b564591..00000000 --- a/playbooks/roles/actors/templates/kube/api/grafana.yml +++ /dev/null @@ -1,23 +0,0 @@ - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: actors-grafana -spec: - selector: - matchLabels: - app: actors-grafana - template: - metadata: - labels: - app: actors-grafana - spec: - containers: - - name: actors-grafana - image: {{ actors_grafana_image }} - ports: - - name: grafana - containerPort: 3000 - -# TODO config - diff --git a/playbooks/roles/actors/templates/kube/api/metrics.yml b/playbooks/roles/actors/templates/kube/api/metrics.yml index b6e3d2b7..fb3bd4f3 100644 --- a/playbooks/roles/actors/templates/kube/api/metrics.yml +++ b/playbooks/roles/actors/templates/kube/api/metrics.yml @@ -16,6 +16,7 @@ spec: - name: actors-metrics image: {{ actors_core_image }} imagePullPolicy: {{actors_image_pull_policy}} + command: ["/home/tapis/actors/metrics_cron_check.sh"] ports: - name: flask containerPort: 5000 diff --git a/playbooks/roles/actors/templates/kube/api/prometheus-config.yml b/playbooks/roles/actors/templates/kube/api/prometheus-config.yml deleted file mode 100644 index bd5706e2..00000000 --- a/playbooks/roles/actors/templates/kube/api/prometheus-config.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Datasources -apiVersion: v1 -kind: ConfigMap -metadata: - name: actors-prometheus-config -data: - prometheus: |- - global: - scrape_interval: 15s # By default, scrape targets every 15 seconds. - evaluation_interval: 15s # By default, scrape targets every 15 seconds. - # scrape_timeout is set to the global default (10s). - - # Attach these labels to any time series or alerts when communicating with - # external systems (federation, remote storage, Alertmanager). - external_labels: - monitor: 'abaco-monitor' - - # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. - rule_files: - - 'alert.rules.yml' - - - # A scrape configuration containing exactly one endpoint to scrape: - # Here it's Prometheus itself. - scrape_configs: - # The job name is added as a label `job=` to any timeseries scraped from this config. - - job_name: 'prometheus' - - # Override the global default and scrape targets from this job every 5 seconds. - scrape_interval: 5s - - # metrics_path defaults to '/metrics' - # scheme defaults to 'http'. - static_configs: - - targets: ['prometheus:9090'] - - - job_name: 'abaco' - scrape_interval: 5s - metrics_path: '/metrics' - static_configs: - - targets: ['actors-metrics:5000'] - labels: - group: 'abaco' - - - job_name: 'cron' - scrape_interval: 50s - metrics_path: '/cron' - static_configs: - - targets: ['actors-metrics:5000'] - labels: - group: 'abaco' diff --git a/playbooks/roles/actors/templates/kube/api/prometheus.yml b/playbooks/roles/actors/templates/kube/api/prometheus.yml deleted file mode 100644 index c833e73d..00000000 --- a/playbooks/roles/actors/templates/kube/api/prometheus.yml +++ /dev/null @@ -1,31 +0,0 @@ - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: actors-prometheus -spec: - selector: - matchLabels: - app: actors-prometheus - template: - metadata: - labels: - app: actors-prometheus - spec: - containers: - - name: actors-prometheus - image: {{ actors_prometheus_image }} - ports: - - name: prometheus - containerPort: 9090 - volumeMounts: - - name: actors-prometheus-config - mountPath: /etc/prometheus/prometheus.yml - subPath: prometheus - volumes: - - name: actors-prometheus-config - configMap: - name: actors-prometheus-config - items: - - key: prometheus - path: prometheus diff --git a/playbooks/roles/actors/templates/kube/api/service.yml b/playbooks/roles/actors/templates/kube/api/service.yml index 3f72807d..6a7fc8e4 100644 --- a/playbooks/roles/actors/templates/kube/api/service.yml +++ b/playbooks/roles/actors/templates/kube/api/service.yml @@ -92,19 +92,6 @@ spec: port: 5000 targetPort: 5000 ---- -apiVersion: v1 -kind: Service -metadata: - name: actors-metrics -spec: - selector: - app: actors-metrics - ports: - - name: flask - port: 5000 - targetPort: 5000 - --- apiVersion: v1 kind: Service @@ -117,30 +104,3 @@ spec: - name: flask port: 5000 targetPort: 5000 - ---- -apiVersion: v1 -kind: Service -metadata: - name: actors-grafana -spec: - selector: - app: actors-grafana - ports: - - name: grafana - port: 3000 - targetPort: 3000 - ---- -apiVersion: v1 -kind: Service -metadata: - name: actors-prometheus -spec: - selector: - app: actors-prometheus - ports: - - name: prometheus - port: 9090 - targetPort: 9090 - diff --git a/playbooks/roles/baseburnup/defaults/main/vars.yml b/playbooks/roles/baseburnup/defaults/main/vars.yml index 18e4f857..886592a3 100644 --- a/playbooks/roles/baseburnup/defaults/main/vars.yml +++ b/playbooks/roles/baseburnup/defaults/main/vars.yml @@ -1,4 +1,4 @@ -baseburnup_tapis_deployer_version: 1.3.8 +baseburnup_tapis_deployer_version: 1.3.9 baseburnup_service_url: "{{ global_service_url }}" baseburnup_vault_url: "{{ global_vault_url }}" diff --git a/playbooks/roles/files/templates/docker/docker-compose.yml b/playbooks/roles/files/templates/docker/docker-compose.yml index 080ce5d9..6a523c52 100644 --- a/playbooks/roles/files/templates/docker/docker-compose.yml +++ b/playbooks/roles/files/templates/docker/docker-compose.yml @@ -91,6 +91,16 @@ services: - TAPIS_SITE_ID={{ files_service_site_id }} - TOKENS_SERVICE_URL={{ files_service_url }} - TENANTS_SERVICE_URL={{ files_service_url}} + - DB_HOST=files-postgres + - DB_NAME=tapisfiles + - DB_USERNAME=tapisfiles + - POSTGRES_DB=tapisfiles + - POSTGRES_USER=tapisfiles + - RABBITMQ_DEFAULT_USER=tapisfiles + - RABBITMQ_DEFAULT_VHOST=tapisfiles + - RABBITMQ_HOSTNAME=files-rabbitmq + - RABBITMQ_USERNAME=tapisfiles + - RABBITMQ_VHOST=tapisfiles command: ["java", "-Xmx3g", "-cp", "target/tapis-files.jar:target/dependencies/*", "edu.utexas.tacc.tapis.files.lib.transfers.TransfersApp"] depends_on: files-api: diff --git a/playbooks/roles/files/templates/kube/api/burndown b/playbooks/roles/files/templates/kube/api/burndown index 90367979..c522159c 100755 --- a/playbooks/roles/files/templates/kube/api/burndown +++ b/playbooks/roles/files/templates/kube/api/burndown @@ -1,11 +1,3 @@ #!/bin/bash -kubectl delete -f ../api.yml -kubectl delete -f ../configmap.yml -kubectl delete -f ../workers.yml - -if [ "$1" = "allofit" ] -then -kubectl delete -f ../postgres.yml -kubectl delete -f ../rabbitmq.yml -fi +kubectl delete -f deploy.yml diff --git a/playbooks/roles/files/templates/kube/api/burnup b/playbooks/roles/files/templates/kube/api/burnup index edd8ed15..c973ab27 100755 --- a/playbooks/roles/files/templates/kube/api/burnup +++ b/playbooks/roles/files/templates/kube/api/burnup @@ -1,13 +1,4 @@ #!/bin/bash -kubectl apply -f ../files-config.yml -kubectl apply -f ../service.yml -kubectl apply -f ../configmap.yml -kubectl apply -f ../pvc.yml -kubectl apply -f ../rabbitmq-pvc.yml -kubectl apply -f ../postgres.yml -kubectl wait --for=condition=available deploy/files-postgres -kubectl apply -f ../rabbitmq.yml -kubectl wait --for=condition=available deploy/files-rabbitmq -kubectl apply -f ../api.yml -kubectl apply -f ../workers.yml +kubectl apply -f service.yml +kubectl apply -f deploy.yml diff --git a/playbooks/roles/files/templates/kube/api.yml b/playbooks/roles/files/templates/kube/api/deploy.yml similarity index 90% rename from playbooks/roles/files/templates/kube/api.yml rename to playbooks/roles/files/templates/kube/api/deploy.yml index 3d9bbdaa..2596c262 100644 --- a/playbooks/roles/files/templates/kube/api.yml +++ b/playbooks/roles/files/templates/kube/api/deploy.yml @@ -74,5 +74,13 @@ spec: secretKeyRef: name: tapis-files-secrets key: service-password +{% if systems_globus_client_id is defined %} + - name: TAPIS_GLOBUS_CLIENT_ID + valueFrom: + configMapKeyRef: + name: files-config + key: globus_client_id +{% endif %} + hostname: files-api restartPolicy: Always diff --git a/playbooks/roles/files/templates/kube/debug.service.yaml b/playbooks/roles/files/templates/kube/api/service.yml similarity index 70% rename from playbooks/roles/files/templates/kube/debug.service.yaml rename to playbooks/roles/files/templates/kube/api/service.yml index d36e0a2e..42126fa3 100644 --- a/playbooks/roles/files/templates/kube/debug.service.yaml +++ b/playbooks/roles/files/templates/kube/api/service.yml @@ -8,8 +8,7 @@ metadata: spec: type: NodePort ports: - - port: 8000 - targetPort: 8000 - nodePort: 32766 + - port: 8080 + targetPort: 8080 selector: app: files-api diff --git a/playbooks/roles/files/templates/kube/burndown b/playbooks/roles/files/templates/kube/burndown index 0a6c4b35..b001fc88 100755 --- a/playbooks/roles/files/templates/kube/burndown +++ b/playbooks/roles/files/templates/kube/burndown @@ -2,10 +2,25 @@ echo "files:" -kubectl delete -f api.yml -kubectl delete -f configmap.yml -kubectl delete -f workers.yml -kubectl delete -f rabbitmq.yml -kubectl delete -f postgres.yml -kubectl delete -f files-config.yml +pushd ./worker +./burndown +popd + +pushd ./api +./burndown +popd + +pushd ./rabbitmq +./burndown +popd +pushd ./pgadmin +./burndown +popd + +pushd ./postgres +./burndown +popd + +kubectl delete -f files-config.yml +kubectl delete -f configmap.yml diff --git a/playbooks/roles/files/templates/kube/burnup b/playbooks/roles/files/templates/kube/burnup index ce59aa27..2a9e8eab 100755 --- a/playbooks/roles/files/templates/kube/burnup +++ b/playbooks/roles/files/templates/kube/burnup @@ -7,16 +7,26 @@ mkdir -p {{ tapisdatadir }}/files/rabbitmq mkdir -p {{ tapisdatadir }}/files/postgres mkdir -p {{ tapisdatadir }}/files/minio mkdir -p {{ tapisdatadir }}/files/irods - -kubectl apply -f files-config.yml -kubectl apply -f service.yml kubectl apply -f configmap.yml -kubectl apply -f pvc.yml -kubectl apply -f rabbitmq-pvc.yml -kubectl apply -f postgres.yml -kubectl wait --for=condition=available deploy/files-postgres -kubectl apply -f rabbitmq.yml -kubectl wait --for=condition=available deploy/files-rabbitmq -kubectl apply -f api.yml -kubectl apply -f workers.yml +kubectl apply -f files-config.yml + +pushd ./postgres +./burnup +popd + +pushd ./pgadmin +./burnup +popd + +pushd ./rabbitmq +./burnup +popd + +pushd ./api +./burnup +popd + +pushd ./worker +./burnup +popd diff --git a/playbooks/roles/files/templates/kube/files-config.yml b/playbooks/roles/files/templates/kube/files-config.yml index ae544d84..4cd16918 100644 --- a/playbooks/roles/files/templates/kube/files-config.yml +++ b/playbooks/roles/files/templates/kube/files-config.yml @@ -6,4 +6,8 @@ data: "service_site_url": "{{files_service_url}}" "service_tenant_id": "{{files_service_tenant_id}}" "site_id": "{{files_service_site_id}}" - "service_name": "{{files_service_name}}" \ No newline at end of file + "service_name": "{{files_service_name}}" +{% if systems_globus_client_id is defined %} + "globus_client_id": "{{systems_globus_client_id}}" +{% endif %} + diff --git a/playbooks/roles/files/templates/kube/files-debbuger-service.yml b/playbooks/roles/files/templates/kube/files-debbuger-service.yml deleted file mode 100644 index e14ee80d..00000000 --- a/playbooks/roles/files/templates/kube/files-debbuger-service.yml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: files-debug -spec: - type: NodePort - selector: - app: files-api - ports: - - name: files-debug - port: 8000 - targetPort: 8000 diff --git a/playbooks/roles/files/templates/kube/pgadmin/burndown b/playbooks/roles/files/templates/kube/pgadmin/burndown new file mode 100755 index 00000000..c522159c --- /dev/null +++ b/playbooks/roles/files/templates/kube/pgadmin/burndown @@ -0,0 +1,3 @@ +#!/bin/bash + +kubectl delete -f deploy.yml diff --git a/playbooks/roles/files/templates/kube/pgadmin/burnup b/playbooks/roles/files/templates/kube/pgadmin/burnup new file mode 100755 index 00000000..cca78f9f --- /dev/null +++ b/playbooks/roles/files/templates/kube/pgadmin/burnup @@ -0,0 +1,5 @@ +#!/bin/bash + +kubectl apply -f service.yml +kubectl apply -f deploy.yml + diff --git a/playbooks/roles/files/templates/kube/pgadmin.yml b/playbooks/roles/files/templates/kube/pgadmin/deploy.yml similarity index 100% rename from playbooks/roles/files/templates/kube/pgadmin.yml rename to playbooks/roles/files/templates/kube/pgadmin/deploy.yml diff --git a/playbooks/roles/files/templates/kube/pgadmin_svc.yml b/playbooks/roles/files/templates/kube/pgadmin/service.yml similarity index 100% rename from playbooks/roles/files/templates/kube/pgadmin_svc.yml rename to playbooks/roles/files/templates/kube/pgadmin/service.yml diff --git a/playbooks/roles/files/templates/kube/postgres/burndown b/playbooks/roles/files/templates/kube/postgres/burndown new file mode 100755 index 00000000..9a85eadf --- /dev/null +++ b/playbooks/roles/files/templates/kube/postgres/burndown @@ -0,0 +1,8 @@ +#!/bin/bash + +#remove init script and config map +kubectl delete -f files-init-db.yml +kubectl delete configmap files-init-db-configmap + +#delete postgres pod +kubectl delete -f deploy.yml diff --git a/playbooks/roles/files/templates/kube/postgres/burnup b/playbooks/roles/files/templates/kube/postgres/burnup new file mode 100755 index 00000000..87bb493c --- /dev/null +++ b/playbooks/roles/files/templates/kube/postgres/burnup @@ -0,0 +1,11 @@ +#!/bin/bash + +kubectl apply -f service.yml +kubectl apply -f pvc.yml +kubectl apply -f deploy.yml +kubectl wait --for=condition=available deploy/files-postgres + +# Initialize the db +kubectl create configmap files-init-db-configmap --from-file files-init-db-sh +kubectl create -f files-init-db.yml +kubectl wait --timeout=120s --for=condition=complete job/files-init-db diff --git a/playbooks/roles/files/templates/kube/postgres.yml b/playbooks/roles/files/templates/kube/postgres/deploy.yml similarity index 100% rename from playbooks/roles/files/templates/kube/postgres.yml rename to playbooks/roles/files/templates/kube/postgres/deploy.yml diff --git a/playbooks/roles/files/templates/kube/postgres/files-init-db-sh b/playbooks/roles/files/templates/kube/postgres/files-init-db-sh new file mode 100755 index 00000000..96684ebe --- /dev/null +++ b/playbooks/roles/files/templates/kube/postgres/files-init-db-sh @@ -0,0 +1,127 @@ +#!/bin/bash +# Script to initialize Files service DB using psql +# Create database, user and schema +# Postgres password must be set in env var PG_PASSWORD + +PG_HOST=${DB_HOST} +PG_USER=${DB_USERNAME} +PG_DATABASE=${POSTGRES_DB} +PG_PORT=${DB_PORT} +PG_PASSWORD=${POSTGRES_PASSWORD} +PG_ADMIN=${POSTGRES_USER} + +function usage() { + echo "$0 [-p|--pg_port ] [-u|--pg_user ] [-w|pg_password ] [-d|--pg_database ] [-a|--pg_admin] [-h|--pg_host]" + + echo "OPTIONS:" + echo " -p --port" + echo " The port to run postgres on" + echo + echo " -u --pg_user" + echo " The postgres user for the service" + echo + echo " -w --pg_pass" + echo " The postgres password for the service" + echo + echo " -d --pg_db" + echo " The postgres database name for the service" + echo + echo " -a --pg_admin" + echo " The postgres user to use for creating the new database and user" + echo + echo " -h --pg_host" + echo " The hostname of the postgres server" + echo + exit 1 +} + +while [[ $# -gt 0 ]]; do + case $1 in + -p|--pg_port) + PG_PORT="$2" + shift # past argument + shift # past value + ;; + -u|--pg_user) + PG_USER="$2" + shift # past argument + shift # past value + ;; + -w|--pg_password) + PG_PASSWORD="$2" + shift # past argument + shift # past value + ;; + -d|--pg_database) + PG_DATABASE="$2" + shift # past argument + shift # past value + ;; + -a|--pg_admin) + PG_ADMIN="$2" + shift # past argument + shift # past value + ;; + -h|--pg_host) + PG_HOST="$2" + shift # past argument + shift # past value + ;; + -*|--*) + echo "Unknown option $1" + usage + ;; + *) + echo "Unknown positional arguement $1" + usage + esac +done + +if [[ -z $PG_HOST ]]; then + PG_HOST=files-postgres +fi + +if [[ -z $PG_USER ]]; then + PG_USER=tapis_files +fi + +if [[ -z $PG_DATABASE ]]; then + PG_DATABASE=tapisfilesdb +fi + +if [[ -z $PG_PORT ]]; then + PG_PORT="5432" +fi + +if [[ -z $PG_ADMIN ]]; then + PG_ADMIN="postgres" +fi + +if [ -z "${PG_PASSWORD}" ]; then + echo "Please set env var PG_PASSWORD before running this script" + usage +fi + +# Put PGPASSWORD in environment for psql to pick up +export PGPASSWORD=${POSTGRES_PASSWORD} + +# Run psql command to create database if it does not exist +echo "SELECT 'CREATE DATABASE ${PG_DATABASE} ENCODING=\"UTF8\" LC_COLLATE=\"en_US.utf8\" LC_CTYPE=\"en_US.utf8\" ' \ + WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${PG_DATABASE}')\gexec" \ + | psql --host=${PG_HOST} --username=${PG_ADMIN} + + +# Run sql to create user and schema if they do not exist +psql --host=${PG_HOST} --username=${PG_ADMIN} --dbname=${PG_DATABASE} -q << EOB +-- Create user if it does not exist +DO \$\$ +BEGIN + CREATE ROLE ${PG_USER} WITH LOGIN; + EXCEPTION WHEN DUPLICATE_OBJECT THEN + RAISE NOTICE 'User already exists. User name: "${PG_USER}"'; +END +\$\$; +ALTER USER ${PG_USER} WITH ENCRYPTED PASSWORD '${PG_PASSWORD}'; +GRANT ALL PRIVILEGES ON DATABASE ${PG_DATABASE} TO ${PG_USER}; +ALTER USER ${PG_USER} SUPERUSER; +EOB diff --git a/playbooks/roles/files/templates/kube/postgres/files-init-db.yml b/playbooks/roles/files/templates/kube/postgres/files-init-db.yml new file mode 100644 index 00000000..f5dd00e8 --- /dev/null +++ b/playbooks/roles/files/templates/kube/postgres/files-init-db.yml @@ -0,0 +1,33 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: files-init-db +spec: + ttlSecondsAfterFinished: 60 + template: + spec: + restartPolicy: Never + containers: + - name: files-init-db + image: {{ files_postgres_image }} + imagePullPolicy: Always + command: ["/tmp/files-init-db-sh"] + volumeMounts: + - name: files-init-db-vol + mountPath: /tmp + envFrom: + - configMapRef: + name: files-environment-vars + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: tapis-files-secrets + key: postgres-password + volumes: + - name: files-init-db-vol + configMap: + name: files-init-db-configmap + defaultMode: 0777 + backoffLimit: 4 + diff --git a/playbooks/roles/files/templates/kube/pvc.yml b/playbooks/roles/files/templates/kube/postgres/pvc.yml similarity index 100% rename from playbooks/roles/files/templates/kube/pvc.yml rename to playbooks/roles/files/templates/kube/postgres/pvc.yml diff --git a/playbooks/roles/files/templates/kube/postgres/service.yml b/playbooks/roles/files/templates/kube/postgres/service.yml new file mode 100644 index 00000000..98e0a0dd --- /dev/null +++ b/playbooks/roles/files/templates/kube/postgres/service.yml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: files + name: files-postgres +spec: + ports: + - port: 5432 + targetPort: 5432 + selector: + app: files-postgres diff --git a/playbooks/roles/files/templates/kube/rabbitmq/burndown b/playbooks/roles/files/templates/kube/rabbitmq/burndown new file mode 100755 index 00000000..c522159c --- /dev/null +++ b/playbooks/roles/files/templates/kube/rabbitmq/burndown @@ -0,0 +1,3 @@ +#!/bin/bash + +kubectl delete -f deploy.yml diff --git a/playbooks/roles/files/templates/kube/rabbitmq/burnup b/playbooks/roles/files/templates/kube/rabbitmq/burnup new file mode 100755 index 00000000..81727513 --- /dev/null +++ b/playbooks/roles/files/templates/kube/rabbitmq/burnup @@ -0,0 +1,6 @@ +#!/bin/bash + +kubectl apply -f service.yml +kubectl apply -f pvc.yml +kubectl apply -f deploy.yml +kubectl wait --for=condition=available deploy/files-rabbitmq diff --git a/playbooks/roles/files/templates/kube/rabbitmq.yml b/playbooks/roles/files/templates/kube/rabbitmq/deploy.yml similarity index 100% rename from playbooks/roles/files/templates/kube/rabbitmq.yml rename to playbooks/roles/files/templates/kube/rabbitmq/deploy.yml diff --git a/playbooks/roles/files/templates/kube/rabbitmq-pvc.yml b/playbooks/roles/files/templates/kube/rabbitmq/pvc.yml similarity index 100% rename from playbooks/roles/files/templates/kube/rabbitmq-pvc.yml rename to playbooks/roles/files/templates/kube/rabbitmq/pvc.yml diff --git a/playbooks/roles/files/templates/kube/rabbitmq/service.yml b/playbooks/roles/files/templates/kube/rabbitmq/service.yml new file mode 100644 index 00000000..4f3b26c3 --- /dev/null +++ b/playbooks/roles/files/templates/kube/rabbitmq/service.yml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: files + name: files-rabbitmq +spec: + ports: + - port: 5672 + targetPort: 5672 + selector: + app: files-rabbitmq diff --git a/playbooks/roles/files/templates/kube/service.yml b/playbooks/roles/files/templates/kube/service.yml deleted file mode 100644 index 3e82ec69..00000000 --- a/playbooks/roles/files/templates/kube/service.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: files - name: files-rabbitmq -spec: - ports: - - port: 5672 - targetPort: 5672 - selector: - app: files-rabbitmq ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: files - name: files-postgres -spec: - ports: - - port: 5432 - targetPort: 5432 - selector: - app: files-postgres ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: files - name: files-api -spec: - type: NodePort - ports: - - port: 8080 - targetPort: 8080 - selector: - app: files-api diff --git a/playbooks/roles/files/templates/kube/worker/burndown b/playbooks/roles/files/templates/kube/worker/burndown new file mode 100755 index 00000000..c522159c --- /dev/null +++ b/playbooks/roles/files/templates/kube/worker/burndown @@ -0,0 +1,3 @@ +#!/bin/bash + +kubectl delete -f deploy.yml diff --git a/playbooks/roles/files/templates/kube/worker/burnup b/playbooks/roles/files/templates/kube/worker/burnup new file mode 100755 index 00000000..5a10c4ad --- /dev/null +++ b/playbooks/roles/files/templates/kube/worker/burnup @@ -0,0 +1,3 @@ +#!/bin/bash + +kubectl apply -f deploy.yml diff --git a/playbooks/roles/files/templates/kube/workers.yml b/playbooks/roles/files/templates/kube/worker/deploy.yml similarity index 89% rename from playbooks/roles/files/templates/kube/workers.yml rename to playbooks/roles/files/templates/kube/worker/deploy.yml index 90f8d916..f8700abb 100644 --- a/playbooks/roles/files/templates/kube/workers.yml +++ b/playbooks/roles/files/templates/kube/worker/deploy.yml @@ -62,6 +62,14 @@ spec: secretKeyRef: name: tapis-files-secrets key: service-password +{% if systems_globus_client_id is defined %} + - name: TAPIS_GLOBUS_CLIENT_ID + valueFrom: + configMapKeyRef: + name: files-config + key: globus_client_id +{% endif %} + image: {{ files_workers_image }} imagePullPolicy: Always name: files-workers diff --git a/playbooks/roles/pods/defaults/main/vars.yml b/playbooks/roles/pods/defaults/main/vars.yml index 1b4c76b1..42c0d2df 100644 --- a/playbooks/roles/pods/defaults/main/vars.yml +++ b/playbooks/roles/pods/defaults/main/vars.yml @@ -1,7 +1,7 @@ --- pods_service_name: pods -pods_image_version: 1.3.0 +pods_image_version: 1.3.1 pods_service_site_id: "{{ global_site_id }}" pods_service_tenant_id: admin pods_primary_site_admin_tenant_base_url: "{{ global_primary_site_admin_tenant_base_url }}" diff --git a/playbooks/roles/pods/templates/kube/api.yml b/playbooks/roles/pods/templates/kube/api.yml index 38b4f8bb..efee8252 100644 --- a/playbooks/roles/pods/templates/kube/api.yml +++ b/playbooks/roles/pods/templates/kube/api.yml @@ -27,8 +27,10 @@ spec: - name: fastapi containerPort: 8000 env: - - name: api + - name: PODS_COMPONENT value: api + - name: DEBUG_SLEEP_LOOP + value: "false" - name: SERVICE_PASSWORD valueFrom: secretKeyRef: diff --git a/playbooks/roles/pods/templates/kube/burndown b/playbooks/roles/pods/templates/kube/burndown index 6aa1893c..7d8424fa 100755 --- a/playbooks/roles/pods/templates/kube/burndown +++ b/playbooks/roles/pods/templates/kube/burndown @@ -20,6 +20,8 @@ kubectl delete -f traefik-proxy.yml # Storage kubectl delete -f postgres.yml kubectl delete -f rabbitmq.yml +kubectl delete -f nfs.yml # PVC - Don't burn these down silly. #kubectl delete -f postgres-pvc.yml +#kubectl delete -f nfs-pvc.yml \ No newline at end of file diff --git a/playbooks/roles/pods/templates/kube/burnup b/playbooks/roles/pods/templates/kube/burnup index 089dacb3..572d2c00 100755 --- a/playbooks/roles/pods/templates/kube/burnup +++ b/playbooks/roles/pods/templates/kube/burnup @@ -22,13 +22,16 @@ kubectl create configmap pods-traefik-conf --from-file=traefik.yml # PVC kubectl apply -f postgres-pvc.yml kubectl wait --for=condition=complete job/chown-pods-postgres-pvc +kubectl apply -f nfs-pvc.yml +kubectl wait --for=condition=complete job/pods-nfs-mkdirs # Storage kubectl apply -f postgres.yml kubectl apply -f rabbitmq.yml +kubectl apply -f nfs.yml # App kubectl apply -f api.yml kubectl apply -f spawner.yml kubectl apply -f health.yml -kubectl apply -f traefik-proxy.yml +kubectl apply -f traefik-proxy.yml \ No newline at end of file diff --git a/playbooks/roles/pods/templates/kube/config.json b/playbooks/roles/pods/templates/kube/config.json index 52ae6354..d7a64313 100644 --- a/playbooks/roles/pods/templates/kube/config.json +++ b/playbooks/roles/pods/templates/kube/config.json @@ -16,6 +16,11 @@ "rabbitmq_dash_host": "pods-rabbitmq-dash", "rabbitmq_user": "pods-admin", "rabbitmq_pass": "$env{RABBITMQ_PASSWORD}", + "nfs_tapis_system_id": "pods-nfs-server", + "nfs_pvc_name": "pods-nfs-vol", + "nfs_base_path": "/podsnfs/", + "nfs_pods_user_password": "defaultpass", + "nfs_develop_mode": false, "image_allow_list": {{ pods_image_allow_list | to_json }}, "pvc_storage_class_name": "{{ pods_pvc_storage_class_name }}", "spawner_abaco_conf_host_path": "$env{abaco_host_path}/config-local.json", @@ -42,4 +47,3 @@ "{{ pods_service_site_id }}_tenant_object": { } } - diff --git a/playbooks/roles/pods/templates/kube/health.yml b/playbooks/roles/pods/templates/kube/health.yml index 41ecbd72..d6448993 100644 --- a/playbooks/roles/pods/templates/kube/health.yml +++ b/playbooks/roles/pods/templates/kube/health.yml @@ -16,7 +16,6 @@ spec: - name: pods-health image: {{ pods_api_image }} imagePullPolicy: {{pods_image_pull_policy}} - command: ["python3", "-u", "/home/tapis/service/health.py"] resources: requests: cpu: "500m" @@ -25,8 +24,10 @@ spec: cpu: "2" memory: "2G" env: - - name: api - value: api + - name: PODS_COMPONENT + value: health + - name: DEBUG_SLEEP_LOOP + value: 'false' - name: SERVICE_PASSWORD valueFrom: secretKeyRef: diff --git a/playbooks/roles/pods/templates/kube/nfs-pvc.yml b/playbooks/roles/pods/templates/kube/nfs-pvc.yml new file mode 100644 index 00000000..3787c56e --- /dev/null +++ b/playbooks/roles/pods/templates/kube/nfs-pvc.yml @@ -0,0 +1,39 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: pods-nfs-vol +spec: + accessModes: + - ReadWriteOnce + storageClassName: rbd-new + resources: + requests: + storage: 5Gi + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: pods-nfs-mkdirs +spec: + ttlSecondsAfterFinished: 60 + template: + spec: + restartPolicy: Never + containers: + - name: pods-nfs-vol + image: alpine:latest + command: + - sh + - -c + - | + mkdir -p /podsnfs/volumes + mkdir -p /podsnfs/snapshots + chmod 777 /podsnfs/volumes /podsnfs/snapshots + volumeMounts: + - name: pods-nfs-data + mountPath: /podsnfs + volumes: + - name: pods-nfs-data + persistentVolumeClaim: + claimName: pods-nfs-vol diff --git a/playbooks/roles/pods/templates/kube/nfs.yml b/playbooks/roles/pods/templates/kube/nfs.yml new file mode 100644 index 00000000..2fb5af16 --- /dev/null +++ b/playbooks/roles/pods/templates/kube/nfs.yml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pods-nfs +spec: + selector: + matchLabels: + app: pods-nfs + template: + metadata: + labels: + app: pods-nfs + spec: + containers: + - name: pods-nfs + image: itsthenetwork/nfs-server-alpine:12 + # This is an NFS image which we deploy SSH in along with a lot of permissioning/keys + # We create a pods user using adduser + # We then "unlock" the user in Alpine by removing the ! in /etc/shadow for the user. User's are locked by default. + # You cannot ssh via pubkey to a locked user in Alpine. I could not way to do this during the adduser step. + # We install openssh/openrc/acl for ssh, service management, and permissioning later + # We enable the sshd service + # We touch /run/openrc/softlevel so that openrc will run on a host it wasn't started up on + # we create some prerequisite folders + # Create our neccessary keys for the pods user, and the public to authorized_keys for ssh access + # ssh-keyget -A creates host keys that sshd requires + # chown the pods user's ssh folder + # Turn on PubKeyAuthentication in the ssd_config + # Restart sshd + # Run the nfsd.sh script which is the entrypoint for the nfs-server-alpine image, it starts the NFS process. + command: + - /bin/sh + - -c + - | + adduser -D -g "Pods service user used by Files API to manage NFS folder." pods &&\ + sed -i 's/pods:!:/pods::/g' /etc/shadow &&\ + apk add --no-cache openssh openrc acl &&\ + rc-status &&\ + rc-update add sshd &&\ + touch /run/openrc/softlevel &&\ + setfacl -R -m u:pods:rwx /podsnfs &&\ + mkdir /home/pods/.ssh &&\ + ssh-keygen -f /home/pods/.ssh/podskey -m PEM -q -N '' &&\ + cp /home/pods/.ssh/podskey.pub /home/pods/.ssh/authorized_keys &&\ + ssh-keygen -A &&\ + chown pods:pods /home/pods/.ssh/* &&\ + sed -i 's/#PubkeyAuthentication/PubkeyAuthentication/g' /etc/ssh/sshd_config &&\ + rc-service sshd restart &&\ + /usr/bin/nfsd.sh + securityContext: + privileged: true + env: + - name: SHARED_DIRECTORY + value: "/podsnfs" + ports: + - name: pods-nfs + containerPort: 2049 + - name: pods-nfs-ssh + containerPort: 22 + volumeMounts: + - name: pods-nfs-vol + mountPath: "/podsnfs" + + volumes: + - name: pods-nfs-vol + persistentVolumeClaim: + claimName: pods-nfs-vol diff --git a/playbooks/roles/pods/templates/kube/security.yml b/playbooks/roles/pods/templates/kube/security.yml index 3973314f..b2c8c5cc 100644 --- a/playbooks/roles/pods/templates/kube/security.yml +++ b/playbooks/roles/pods/templates/kube/security.yml @@ -16,7 +16,7 @@ metadata: name: pods-role rules: - apiGroups: [""] - resources: ["pods", "services"] + resources: ["pods", "services", "pods/exec"] verbs: ["list", "create", "get", "watch", "delete"] - apiGroups: [""] resources: ["pods/log"] @@ -40,4 +40,3 @@ roleRef: kind: Role name: pods-role apiGroup: rbac.authorization.k8s.io - diff --git a/playbooks/roles/pods/templates/kube/services.yml b/playbooks/roles/pods/templates/kube/services.yml index e9f66f2d..8f35de72 100644 --- a/playbooks/roles/pods/templates/kube/services.yml +++ b/playbooks/roles/pods/templates/kube/services.yml @@ -79,4 +79,28 @@ spec: port: 8080 targetPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: pods-nfs +spec: + selector: + app: pods-nfs + ports: + - name: pods-nfs + port: 2049 + targetPort: 2049 +--- +apiVersion: v1 +kind: Service +metadata: + name: pods-nfs-ssh +spec: + selector: + app: pods-nfs + ports: + - name: pods-nfs-ssh + port: 22 + targetPort: 22 diff --git a/playbooks/roles/pods/templates/kube/spawner.yml b/playbooks/roles/pods/templates/kube/spawner.yml index 46465b7a..5b4b8560 100644 --- a/playbooks/roles/pods/templates/kube/spawner.yml +++ b/playbooks/roles/pods/templates/kube/spawner.yml @@ -16,7 +16,6 @@ spec: - name: pods-spawner image: {{ pods_api_image }} imagePullPolicy: {{pods_image_pull_policy}} - command: ["python3", "-u", "/home/tapis/service/spawner.py"] resources: requests: cpu: "1" @@ -25,8 +24,10 @@ spec: cpu: "2" memory: "2G" env: - - name: api - value: api + - name: PODS_COMPONENT + value: spawner + - name: DEBUG_SLEEP_LOOP + value: "false" - name: SERVICE_PASSWORD valueFrom: secretKeyRef: diff --git a/playbooks/roles/security/templates/kube/api/api.yml b/playbooks/roles/security/templates/kube/api/api.yml index b14fdb26..8d3ef4e0 100644 --- a/playbooks/roles/security/templates/kube/api/api.yml +++ b/playbooks/roles/security/templates/kube/api/api.yml @@ -11,6 +11,9 @@ spec: labels: app: sk-api spec: +{% if skadmin_sk_privileged_sa is defined and not none %} + serviceAccountName: {{ skadmin_sk_privileged_sa }} +{% endif %} {% if security_renew_sk_script == true %} initContainers: - name: renew-sk-secret