Data Lifecycle Operations (#921)

* Add internal endpoint to perform data lifecycle operations
* Update helm chart to deploy data lifecycle job
* Delete minioCleanup service

.github/workflows/deploy-config.json

@@ -53,10 +53,5 @@
     "dir_name": "code_generator_funcadl_xAOD",
     "image_name": "servicex_code_gen_atlas_xaod",
     "test_required": true
-  },
-  {
-    "dir_name": "minio_cleanup",
-    "image_name": "servicex_minio_cleanup",
-    "test_required": false
   }
 ]

helm/servicex/templates/data-lifecycle/cronjob.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.dataLifecycle.enabled -}}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ .Release.Name }}-data-lifecycle-job
+spec:
+  schedule: {{ .Values.dataLifecycle.schedule | default  "* */8 * * *" | quote  }}
+  concurrencyPolicy: "Forbid"
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: {{ .Release.Name }}-data-lifecycle-job
+        spec:
+          containers:
+          - name: {{ .Release.Name }}-data-lifecycle-job
+            image: {{ .Values.dataLifecycle.image }}:{{ .Values.dataLifecycle.tag }}
+            imagePullPolicy: {{ .Values.dataLifecycle.pullPolicy }}
+            env:
+              - name: RETENTION_PERIOD
+                value: {{ .Values.dataLifecycle.retention }}
+
+            command:
+            - /bin/sh
+            - -c
+            - |
+              CUTOFF_TIMESTAMP=$(date -d "$RETENTION_PERIOD" +%Y-%m-%dT%H:%M:%S) &&
+              echo "Request data lifecycle with cutoff timestamp: $CUTOFF_TIMESTAMP" &&
+              curl --request POST "http://{{ .Release.Name }}-servicex-app:8000/servicex/internal/data-lifecycle?cutoff_timestamp=$CUTOFF_TIMESTAMP"
+          restartPolicy: OnFailure
+{{- end }}

helm/servicex/values.yaml

@@ -188,14 +188,18 @@ x509Secrets:
   pullPolicy: Always
   tag: develop
   vomsOrg: atlas
-minioCleanup:
+
+dataLifecycle:
   enabled: false
-  image: sslhep/servicex_minio_cleanup
-  threads: "6"
-  logLevel: "INFO"
+  # This image should be a fully functional linux image with gnu date installed
+  # ideally it should be an image we have already pulled
+  image: python
+  tag: "3.10"
   pullPolicy: Always
-  tag: develop
   schedule: "* */8 * * *"
-  maxAge: 30
-  maxSize: "1G"
-  normSize: "700M"
+
+  # We will archive any transforms older than this. Use the gnu date command
+  # --date argument.
+  # See https://www.geeksforgeeks.org/date-command-linux-examples/#4-how-to-display-past-dates
+  # for examples
+  retention: "7 days ago"