Merge branch 'dev' into prod

CHANGELOG.md

@@ -1,13 +1,52 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
-## 1.4.0 - 2023-07-06
+## 1.5.3 - 2023-12-01
+- `1.5.1` and `1.5.2`: No Changes. Jumping to `1.5.3` to match deployer version.
+
+### Breaking Changes:
+- Implemented direct access to NFS server instead of routing through Files for volumes/snapshots.
+    - This solves occassional networking hiccups causing troubles when Files couldn't be accessed.
+    - This solves deployment across multiple namespaces as Files access was a stickler.
+- Health is now split into health and health-central.
+    - `health-central` deploys with the main stack.
+        - It takes care of metrics, traefik management, and NFS management.
+    - `health` deploys with computer (health and spawner) in whatever namespace.
+        - Takes care of Kubernetes health and management in a particular namespace.
+- New deployment files for the above features along with deleting no longer used files.
+    - Works locally as well.
+- Fix in traefik to properly throw a 500 so proxy backup in nginx works properly when no location is matched.
+
+### New features:
+- Added health deployment that doesn't restart, allowing for easier debugging.
+- Improvements to NFS permissions
+
+### Bug fixes:
+- Improvements for health regarding processes when new tenants are created while already running.
+
+
+## 1.5.0 - 2023-10-24
 
 ### Breaking Changes:
 - No change.
 
 ### New features:
+- `action_logs` added to pod object along with logs endpoint for detailed audit of actions done on pod_id.
+
+### Bug fixes:
+- Fixed some user nested validation errors not showing proper error messages.
+- Ensure pods always save logs
+- Fixed migrations for action_logs and how it works if logs are empty.
+- Better normalized paths.
+
+
+## 1.4.0 - 2023-07-06
+
+### Breaking Changes:
 - No change.
+
+### New features:
+- Better certs.
 
 ### Bug fixes:
 - No change.

Dockerfile

@@ -37,7 +37,9 @@ COPY configschema.json entry.sh alembic.ini /home/tapis/
 RUN chmod +x /home/tapis/entry.sh
 
 # Permission finalization
-#RUN chown -R tapis:tapis /home/tapis
+RUN chown -R tapis:tapis /home/tapis
 
+# Run everything as tapis user (uid 4872)
+USER tapis
 
 CMD ["/home/tapis/entry.sh"]

Makefile

@@ -55,6 +55,9 @@ export SERVICE_NAME := pods
 # SERVICE_PASS to use throughout. Must be filled.
 export SERVICE_PASS := password
 
+# STATIC_NFS_IP to use throughout. Must be filled.
+export STATIC_NFS_IP := 10.96.175.175
+
 # DEV_TOOLS bool. Whether or not to start jupyter + mount pods/service folder in pods (main).
 # options: "false" | "true"
 # default: "false"
@@ -76,12 +79,13 @@ help:
 up: vars build
 	@echo "Makefile: $(GREEN)up$(NC)"
 	@echo "  🔍 : Looking to run ./burnup in deployment folder."
-	rm -rf deployment; mkdir deployment; cp -r deployment-template/* deployment;
+	rm -rf deployment; mkdir deployment; cp -r deploymentTemplate/* deployment;
 	cd deployment
 	@echo "  🔨 : Created deployment folder with templates."
 	@sed -i 's/"version".*/"version": "$(TAG)",/g' config.json
 	@sed -i 's/MAKEFILE_SERVICE_NAME/$(SERVICE_NAME)/g' *
 	@sed -i 's/MAKEFILE_SERVICE_PASS/$(SERVICE_PASS)/g' *
+	@sed -i 's/MAKEFILE_STATIC_NFS_IP/$(STATIC_NFS_IP)/g' *
 	@sed -i 's/MAKEFILE_TAG/$(TAG)/g' *
 	@echo "  🔥 : Running burnup."
 ifeq ($(DEV_TOOLS),true)
@@ -101,8 +105,8 @@ else
 	@echo "  🔗 : Jupyter Lab URL: dev_tools is set to 'false'"
 endif
 	@echo "  🔗 : API URL: $(LCYAN)http://$$(minikube ip):$$(kubectl get service pods-traefik | grep -o -P '(?<= 80:)\d+(?=/TCP)')$(NC)/v3"
-	@echo "  🔗 : Docs URL: $(LCYAN)http://$$(minikube ip):$$(kubectl get service pods-api-nodeport | grep -o -P '(?<=8000:)\d+(?=/TCP)')$(NC)/docs"
-	@echo "  🔗 : Spec URL: $(LCYAN)http://$$(minikube ip):$$(kubectl get service pods-api-nodeport | grep -o -P '(?<=8000:)\d+(?=/TCP)')$(NC)/openapi.json"
+	@echo "  🔗 : Docs URL: $(LCYAN)http://$$(minikube ip):$$(kubectl get service pods-api | grep -o -P '(?<=8000:)\d+(?=/TCP)')$(NC)/docs"
+	@echo "  🔗 : Spec URL: $(LCYAN)http://$$(minikube ip):$$(kubectl get service pods-api | grep -o -P '(?<=8000:)\d+(?=/TCP)')$(NC)/openapi.json"
 	@echo "  🔗 : Traefik Dash URL: $(LCYAN)http://$$(minikube ip):$$(kubectl get service pods-traefik | grep -o -P '(?<=8080:)\d+(?=/TCP)')$(NC)/dashboard"
 	@echo ""
 

README.md

@@ -36,7 +36,7 @@ Generally devs will use `make clean up` over and over again. This will clean up
 
 ### Explanation of what's happening during `up`.
 The Makefile `up` target is the most complex. This is a light explainer.
-`up` takes the deployment-template directory, copies it, replaces (with sed) variables using the Makefile variables (such as image tag, k8 namespace, service_password).
+`up` takes the deploymentTemplate directory, copies it, replaces (with sed) variables using the Makefile variables (such as image tag, k8 namespace, service_password).
 Once the new deployment directory is created. We then run `./burnup`, which starts the following pods: `api`, `health`, `spawner`, `postgres`, `traefik`, `rabbitmq`.
 The `api` pod contains the server and also initializes the postgres database (using an alembic migration) and rabbitmq (using rabbitmqadmin script).
 

configschema.json

@@ -11,8 +11,6 @@
       "global_tenant_object",
       "global_site_object",
       "spawner_host_id",
-      "nfs_tapis_system_id",
-      "nfs_develop_mode",
       "nfs_base_path"
     ],
     "properties": {
@@ -92,34 +90,10 @@
           "type": "string"
         }
       },
-      "nfs_tapis_system_id": {
-        "type": "string",
-        "description": "system_id to create/use to deal with pods-nfs files API."
-      },
       "nfs_base_path": {
         "type": "string",
         "description": "Base path for nfs system root_dir. Should be /{base_path}/{tenant}/, tenant is added at runtime."
       },
-      "nfs_develop_mode": {
-        "type": "boolean",
-        "description": "While developing locally we expect nfs does not work. This stops nfs specific code from running when not needed."
-      },
-      "nfs_develop_remote_url": {
-        "type": "string",
-        "description": "If service not actually deployed, you can specify the nfs_url systems should use for initial configuration."
-      },
-      "nfs_develop_private_key": {
-        "type": "string",
-        "description": "nfs private key for 'pods' user to create Systems credential. Derived during runtime if not provided (works when Files ands Pods are deployed in the same cluster)"
-      },
-      "nfs_develop_public_key": {
-        "type": "string",
-        "description": "nfs public key for 'pods' user to create Systems credential. Derived during runtime if not provided (works when Files ands Pods are deployed in the same cluster)"
-      },
-      "nfs_pods_user_password": {
-        "type": "string",
-        "description": "nfs pod password for 'pods' user to create Systems credential. Archived in favor of keys."
-      },
       "global_tenant_object": {
         "type": "object",
         "description": "Object containing global parameters which tenants may overwrite at times.",

deploymentNamespace2/burndown

@@ -0,0 +1,11 @@
+# Deploy
+kubectl delete -n testing -f spawner.yml
+kubectl delete -n testing -f health.yml
+
+# Security/Role
+kubectl delete -f security.yml
+
+# Configs
+kubectl delete configmap pods-config
+
+sleep 3 # let k8 catch up

deploymentNamespace2/burnup

@@ -0,0 +1,9 @@
+# Security/Role
+kubectl apply -n testing -f security.yml
+
+# Configs
+kubectl create configmap pods-config -n testing --from-file=config.json
+
+# Deploy
+kubectl apply -n testing -f spawner.yml
+kubectl apply -n testing -f health.yml

deploymentNamespace2/config.json

@@ -0,0 +1,54 @@
+{
+    "server": "dev",
+    "primary_site_admin_tenant_base_url": "https://admin.develop.tapis.io",
+    "service_tenant_id": "admin",
+    "service_site_id": "tacc",
+    "service_name": "pods",
+    "service_password": "SERVICE_PASSWORD",
+    "tenants": [
+        "tacc",
+        "dev",
+        "admin"
+    ],
+    "version": "dev",
+    "log_file": "",
+    "log_level": "DEBUG",
+    "log_filing_strategy": "split",
+    "tapisservice.auth_log_file": "/home/tapis/tapisservice.log",
+    "tapisservice.logs_log_file": "/home/tapis/tapisservice.log",
+    "tapisservice.tenants_log_file": "/home/tapis/tapisservice.log",
+    "postgres_user": "pods-admin",
+    "postgres_pass": "adminadmin",
+    "postgres_host": "minikubeIp:nodePort",
+    "rabbitmq_uri": "amqp://minikubeIp:nodePort",
+    "rabbitmq_dash_host": "notNeededOutsideOfCentralSpace",
+    "rabbitmq_user": "pods-admin",
+    "rabbitmq_pass": "adminadmin",
+    "image_allow_list": [
+        "jstubbs/abaco_test",
+        "tuyamei/smartfoodshed-server",
+        "ssquid/kapp",
+        "av11/sd-webui-auto-bulk",
+        "jolibrain/deepdetect_cpu",
+        "notchristiangarcia/vaapi-v1-f",
+        "notchristiangarcia/vaapi-v1-b"
+    ],
+    "nfs_pvc_name": "notNeededOutsideOfCentralSpace",
+    "nfs_base_path": "/podsnfs",
+    "pvc_storage_class_name": "standard",
+    "spawner_host_id": 0,
+    "spawner_host_queues": [
+        "tacc"
+    ],
+    "show_traceback": false,
+    "site_id": "tacc",
+    "global_site_object": {
+        "site_rabbitmq_pass": "defaultpass"
+    },
+    "global_tenant_object": {
+        "log_ex_limit": 86400,
+        "log_ex": 43200
+    },
+    "tacc_tenant_object": {
+    }  
+}