Add script to convert kapel output records to Gratia records

.github/workflows/build-k8s-container.yaml

@@ -0,0 +1,69 @@
+---
+name: Build and Push Kubernetes Probe Docker image
+
+on:
+  push:
+    branches: [ 2.x ]
+    paths:
+      - '.github/workflows/build-k8s-container.yml'
+      - 'gratia-output-kapel/**'
+      - 'common/**'
+  repository_dispatch:
+    types:
+      - dispatch-build
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: startsWith(github.repository, 'opensciencegrid/')
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: make date tag
+      id: mkdatetag
+      run: echo "dtag=$(date +%Y%m%d-%H%M)" >> $GITHUB_OUTPUT
+
+    - name: make tags
+      id: mktags
+      env:
+        DTAG=${{ steps.mkdatetag.outputs.dtag }}
+      run: |
+        reponame="osg-htc/gratia-probe-k8s"
+        tags=()
+        for registry in docker.io hub.opensciencegrid.org; do
+            tags+=("$registry/$reponame:release")
+            tags+=("$registry/$reponame:release-$DTAG)
+        done
+
+        IFS=,
+        echo "image_tags=${tags[*]}" >> $GITHUB_OUTPUT
+        echo "ts_image=hub.opensciencegrid.org/$reponame:release-$DTAG"
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2.7.0
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@v2.2.0
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+
+    - name: Log in to OSG Harbor
+      uses: docker/login-action@v2.2.0
+      with:
+        registry: hub.opensciencegrid.org
+        username: ${{ secrets.OSG_HARBOR_ROBOT_USER }}
+        password: ${{ secrets.OSG_HARBOR_ROBOT_PASSWORD }}
+
+    - name: Build and push Docker images
+      uses: docker/build-push-action@v4
+      with:
+        context: .
+        file: osg-pilot-container/Dockerfile
+        push: true
+        build-args: |
+          TIMESTAMP_IMAGE=${{ steps.mktags.outputs.ts_image }}
+        tags: "${{ steps.mktags.outputs.image_tags }}"

gratia-output-kapel/Dockerfile → kubernetes/Dockerfile

@@ -1,14 +1,17 @@
 FROM almalinux:9
 
+ARG TIMESTAMP_IMAGE=gratia-probe-k8s:release-$(date +%Y%m%d-%H%M)
+ENV GRATIA_PROBE_VERSION=$TIMESTAMP_IMAGE
+
 # Install EPEL, the OSG software base repo, and gratia-probe-common
 RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
     yum install -y https://repo.opensciencegrid.org/osg/23-main/osg-23-main-el9-release-latest.rpm && \
     yum install -y gratia-probe-common python3-pip
 
-WORKDIR /etc/kapel-gratia-output
-COPY requirements.txt /etc/kapel-gratia-output/
+WORKDIR /gratia
+COPY requirements.txt /gratia
 RUN pip install -r requirements.txt
-COPY gratia-config /etc/kapel-gratia-output/gratia-config
-COPY *.py /etc/kapel-gratia-output/
+COPY ProbeConfig /etc/gratia/kubernetes
+COPY *.py /gratia
 
-CMD python3 gratia_output.py
+CMD python3 kubernetes_meter.py

gratia-output-kapel/gratia-config → kubernetes/ProbeConfig

@@ -12,8 +12,8 @@
 
     Title2="Probe information and functional configuration"
 
-    ProbeName="osg-pilot-container:localhost" 
-    SiteName="OSG Pilot Container Probe"
+    ProbeName="k8s:localhost"
+    SiteName="Kubernetes Probe"
     Grid="OSG"
     SuppressUnknownVORecords="0"
     SuppressNoDNRecords="0"

gratia-output-kapel/gratia_output_config.py → kubernetes/gratia_k8s_config.py

@@ -4,7 +4,7 @@
 
 # Read config settings from environment variables (and a named env file in CWD if specified),
 # do input validation, and return a config object. Note, if a '.env' file exists in CWD it will be used by default.
-class GratiaOutputConfig:
+class GratiaK8sConfig:
     def __init__(self, envFile=None):
         env = Env()
         # Read a .env file if one is specified, otherwise only environment variables will be used.
@@ -15,7 +15,7 @@ def __init__(self, envFile=None):
 
         # infrastructure info
         self.infrastructure_type = env.str("INFRASTRUCTURE_TYPE", "grid")
-        self.infrastructure_description = env.str("INFRASTRUCTURE_DESCRIPTION", "APEL-KUBERNETES")
+        self.infrastructure_description = env.str("INFRASTRUCTURE_DESCRIPTION", "GRATIA-KUBERNETES")
 
         # optionally define number of nodes and processors. Should not be necessary to
         # set a default of 0 here but see https://github.com/apel/apel/issues/241
@@ -26,7 +26,4 @@ def __init__(self, envFile=None):
         # Gratia config
         self.gratia_config_path = env.str("GRATIA_CONFIG_PATH", None)
 
-        self.gratia_reporter = env.str("GRATIA_REPORTER", None)
-        self.gratia_service = env.str("GRATIA_SERVICE", None)
-        self.gratia_probe_manager = env.str("GRATIA_PROBE_MANAGER", None)
         self.gratia_probe_version = env.str("GRATIA_PROBE_VERSION", None)

gratia-output-kapel/gratia_output.py → kubernetes/kubernetes_meter.py

@@ -1,7 +1,7 @@
 import argparse
 import datetime
 
-from gratia_output_config import GratiaOutputConfig
+from gratia_output_config import GratiaK8sConfig
 from dirq.QueueSimple import QueueSimple
 import os
 from datetime import datetime, timezone 
@@ -113,7 +113,7 @@ def send_gratia_records(records: list[ApelRecordConverter]):
 
 
 
-def setup_gratia(config: GratiaOutputConfig):
+def setup_gratia(config: GratiaK8sConfig):
 
     if not config.gratia_config_path or not os.path.exists(config.gratia_config_path):
         raise Exception("No valid gratia config path given")
@@ -123,9 +123,9 @@ def setup_gratia(config: GratiaOutputConfig):
     GratiaWrapper.ExclusiveLock()
 
     # Register gratia
-    GratiaCore.RegisterReporter(config.gratia_reporter)
-    GratiaCore.RegisterService(config.gratia_service, config.gratia_probe_version)
-    GratiaCore.setProbeBatchManager(config.gratia_probe_manager)
+    GratiaCore.RegisterReporter("kubernetes_meter")
+    GratiaCore.RegisterService("Kubernetes", config.gratia_probe_version)
+    GratiaCore.setProbeBatchManager("kubernetes")
 
     GratiaCore.Initialize(config.gratia_config_path)
 
@@ -143,7 +143,7 @@ def batch_dirq(queue, batch_size):
 
 def main(envFile: str):
     print(f'Starting Gratia post-processor: {__file__} with envFile {envFile} at {datetime.now(tz=timezone.utc).isoformat()}')
-    cfg = GratiaOutputConfig(envFile)
+    cfg = GratiaK8sConfig(envFile)
 
     setup_gratia(cfg)
 

kubernetes/sample.env

@@ -0,0 +1,7 @@
+NAMESPACE=OSG
+SITE_NAME=OSG
+SUBMIT_HOST=OSG
+BENCHMARK_VALUE=5
+VO_NAME=OSG
+
+GRATIA_CONFIG_PATH=/etc/gratia/kubernetes/ProbeConfig