feat(dd-apm): Added initial config for dd apm (#1617)

* feat(dd-apm): Added initial config for dd apm * feat(dd-apm): Added datadog apm integration to core python services * fix expected rendered fence k8s yaml * remove duplicate label * fix labels * fix labels again * fix expected file for sheepdog yaml Co-authored-by: Edward Malinowski <[email protected]> Co-authored-by: Marcelo Costa <[email protected]>
uc-cdis · Jun 9, 2021 · 8f47591 · 8f47591
1 parent de9c2af
commit 8f47591
Show file tree

Hide file tree

Showing 10 changed files with 266 additions and 4 deletions.
diff --git a/Docker/python-nginx/python3.6-alpine3.7/dockerrun.sh b/Docker/python-nginx/python3.6-alpine3.7/dockerrun.sh
@@ -86,9 +86,17 @@ if [ -f ./wsgi.py ] && [ "$GEN3_DEBUG" = "True" ]; then
   echo -e "\napplication.debug=True\n" >> ./wsgi.py
 fi
 
+if [[ -z $DD_ENABLED ]]; then
 (
   run uwsgi --ini /etc/uwsgi/uwsgi.ini
 ) &
+else
+pip install ddtrace
+echo "import=ddtrace.bootstrap.sitecustomize" >> /etc/uwsgi/uwsgi.ini
+(
+  ddtrace-run uwsgi --enable-threads --ini /etc/uwsgi/uwsgi.ini
+) &
+fi
 
 if [[ $GEN3_DRYRUN == "False" ]]; then
   (

diff --git a/gen3/lib/g3k_manifest.sh b/gen3/lib/g3k_manifest.sh
@@ -239,7 +239,13 @@ g3k_manifest_filter() {
     # zsh friendly upper case
     kvKey=$(echo "GEN3_${key}_IMAGE" | tr '[:lower:]' '[:upper:]')
     kvList+=("$kvKey" "image: $value")
+    kvLabelKey=$(echo "GEN3_${key}_VERSION" | tr '[:lower:]' '[:upper:]')
+    version=$(echo $value | rev | cut -d ':' -f 1 | rev)
+    kvList+=("$kvLabelKey" "tags.datadoghq.com/version: '$version'")
   done
+  environment="$(g3k_config_lookup ".global.environment" "$manifestPath")"
+  kvEnvKey=$(echo "GEN3_ENV_LABEL" | tr '[:lower:]' '[:upper:]')
+  kvList+=("$kvEnvKey" "tags.datadoghq.com/env: $environment")
   for key in $(g3k_config_lookup '. | keys[]' "$manifestPath"); do
     gen3_log_debug "harvesting key $key"
     for key2 in $(g3k_config_lookup ".[\"${key}\"] "' | to_entries | map(select((.value|type != "array") and (.value|type != "object"))) | map(.key)[]' "$manifestPath" | grep '^[a-zA-Z]'); do

diff --git a/gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml b/gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml
@@ -30,6 +30,9 @@ spec:
         netnolimit: "yes"
         public: "yes"
         userhelper: "yes"
+        tags.datadoghq.com/service: "fence"
+        tags.datadoghq.com/env: null
+        tags.datadoghq.com/version: 'master'
         date: "1579711361"
     spec:
       affinity:
@@ -99,12 +102,40 @@ spec:
       - name: fence
         image: quay.io/cdis/fence:master
         env:
+        - name: DD_ENABLED
+          valueFrom:
+            configMapKeyRef:
+              name: manifest-global
+              key: dd_enabled
+              optional: true
+        - name: DD_ENV
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.labels['tags.datadoghq.com/env']
+        - name: DD_SERVICE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.labels['tags.datadoghq.com/service']
+        - name: DD_VERSION
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.labels['tags.datadoghq.com/version']
+        - name: DD_LOGS_INJECTION
+          value: "true"
+        - name: DD_PROFILING_ENABLED
+          value: "true"
+        - name: DD_TRACE_SAMPLE_RATE
+          value: "1"
         - name: GEN3_UWSGI_TIMEOUT
           valueFrom:
             configMapKeyRef:
               name: manifest-global
               key: uwsgi-timeout
               optional: true
+        - name: DD_AGENT_HOST
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
         - name: AWS_STS_REGIONAL_ENDPOINTS
           value: regional
         - name: PYTHONPATH
@@ -211,7 +242,7 @@ spec:
           - |
             echo "${FENCE_PUBLIC_CONFIG:-""}" > "/var/www/fence/fence-config-public.yaml"
             python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml
-            bash /fence/dockerrun.bash && if [[ -f /dockerrun.sh ]]; then /dockerrun.sh; fi
+            bash /fence/dockerrun.bash && if [[ -f /dockerrun.sh ]]; then bash /dockerrun.sh; fi
       - name: nginx-prometheus-exporter-wrapper
         image: quay.io/cdis/nginx-prometheus-exporter-wrapper:pybase3-1.4.0
         command: ["/bin/bash"]

diff --git a/gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml b/gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml
@@ -22,6 +22,9 @@ spec:
         public: "yes"
         # to download dictionary
         s3: "yes"
+        tags.datadoghq.com/service: "sheepdog"
+        tags.datadoghq.com/env: null
+        tags.datadoghq.com/version: 'master'
         date: "1522344212"
     spec:
       affinity:
@@ -74,6 +77,34 @@ spec:
           - containerPort: 80
           - containerPort: 443
           env:
+          - name: DD_ENABLED
+            valueFrom:
+              configMapKeyRef:
+                name: manifest-global
+                key: dd_enabled
+                optional: true
+          - name: DD_ENV
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/env']
+          - name: DD_SERVICE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/service']
+          - name: DD_VERSION
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/version']
+          - name: DD_LOGS_INJECTION
+            value: "true"
+          - name: DD_PROFILING_ENABLED
+            value: "true"
+          - name: DD_TRACE_SAMPLE_RATE
+            value: "1"
+          - name: DD_AGENT_HOST
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
           - name: DICTIONARY_URL
             valueFrom:
               configMapKeyRef:
@@ -152,4 +183,4 @@ spec:
               memory: 1024Mi
             limits:
               cpu: 2
-              memory: 2048Mi
+              memory: 2048Mi
diff --git a/kube/services/fence/fence-deploy.yaml b/kube/services/fence/fence-deploy.yaml
@@ -30,6 +30,9 @@ spec:
         netnolimit: "yes"
         public: "yes"
         userhelper: "yes"
+        tags.datadoghq.com/service: "fence"
+        GEN3_ENV_LABEL
+        GEN3_FENCE_VERSION
         GEN3_DATE_LABEL
     spec:
       affinity:
@@ -99,12 +102,40 @@ spec:
       - name: fence
         GEN3_FENCE_IMAGE
         env:
+        - name: DD_ENABLED
+          valueFrom:
+            configMapKeyRef:
+              name: manifest-global
+              key: dd_enabled
+              optional: true
+        - name: DD_ENV
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.labels['tags.datadoghq.com/env']
+        - name: DD_SERVICE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.labels['tags.datadoghq.com/service']
+        - name: DD_VERSION
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.labels['tags.datadoghq.com/version']
+        - name: DD_LOGS_INJECTION
+          value: "true"
+        - name: DD_PROFILING_ENABLED
+          value: "true"
+        - name: DD_TRACE_SAMPLE_RATE
+          value: "1"
         - name: GEN3_UWSGI_TIMEOUT
           valueFrom:
             configMapKeyRef:
               name: manifest-global
               key: uwsgi-timeout
               optional: true
+        - name: DD_AGENT_HOST
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
         - name: AWS_STS_REGIONAL_ENDPOINTS
           value: regional
         - name: PYTHONPATH
@@ -211,7 +242,7 @@ spec:
           - |
             echo "${FENCE_PUBLIC_CONFIG:-""}" > "/var/www/fence/fence-config-public.yaml"
             python /var/www/fence/yaml_merge.py /var/www/fence/fence-config-public.yaml /var/www/fence/fence-config-secret.yaml > /var/www/fence/fence-config.yaml
-            bash /fence/dockerrun.bash && if [[ -f /dockerrun.sh ]]; then /dockerrun.sh; fi
+            bash /fence/dockerrun.bash && if [[ -f /dockerrun.sh ]]; then bash /dockerrun.sh; fi
       - name: nginx-prometheus-exporter-wrapper
         GEN3_NGINX_PROMETHEUS_EXPORTER_WRAPPER_IMAGE|-image: quay.io/cdis/nginx-prometheus-exporter-wrapper:pybase3-1.4.0-|
         command: ["/bin/bash"]

diff --git a/kube/services/indexd/indexd-deploy.yaml b/kube/services/indexd/indexd-deploy.yaml
@@ -26,6 +26,9 @@ spec:
         app: indexd
         release: production
         public: "yes"
+        tags.datadoghq.com/service: "indexd"
+        GEN3_ENV_LABEL
+        GEN3_INDEXD_VERSION
         GEN3_DATE_LABEL
     spec:
       affinity:
@@ -63,6 +66,34 @@ spec:
       - name: indexd
         GEN3_INDEXD_IMAGE
         env:
+          - name: DD_ENABLED
+            valueFrom:
+              configMapKeyRef:
+                name: manifest-global
+                key: dd_enabled
+                optional: true
+          - name: DD_ENV
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/env']
+          - name: DD_SERVICE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/service']
+          - name: DD_VERSION
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/version']
+          - name: DD_LOGS_INJECTION
+            value: "true"
+          - name: DD_PROFILING_ENABLED
+            value: "true"
+          - name: DD_TRACE_SAMPLE_RATE
+            value: "1"
+          - name: DD_AGENT_HOST
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
           - name: GEN3_DEBUG
             GEN3_DEBUG_FLAG|-value: "False"-|
           - name: DIST

diff --git a/kube/services/peregrine/peregrine-deploy.yaml b/kube/services/peregrine/peregrine-deploy.yaml
@@ -24,6 +24,9 @@ spec:
         public: "yes"
         # to download dictionary
         s3: "yes"
+        tags.datadoghq.com/service: "peregrine"
+        GEN3_ENV_LABEL
+        GEN3_PEREGRINE_VERSION
         GEN3_DATE_LABEL
     spec:
       affinity:
@@ -64,6 +67,34 @@ spec:
           - containerPort: 80
           - containerPort: 443
           env:
+          - name: DD_ENABLED
+            valueFrom:
+              configMapKeyRef:
+                name: manifest-global
+                key: dd_enabled
+                optional: true
+          - name: DD_ENV
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/env']
+          - name: DD_SERVICE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/service']
+          - name: DD_VERSION
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/version']
+          - name: DD_LOGS_INJECTION
+            value: "true"
+          - name: DD_PROFILING_ENABLED
+            value: "true"
+          - name: DD_TRACE_SAMPLE_RATE
+            value: "1"
+          - name: DD_AGENT_HOST
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
           - name: DICTIONARY_URL
             valueFrom:
               configMapKeyRef:

diff --git a/kube/services/pidgin/pidgin-deploy.yaml b/kube/services/pidgin/pidgin-deploy.yaml
@@ -18,6 +18,9 @@ spec:
       labels:
         app: pidgin
         public: "yes"
+        tags.datadoghq.com/service: "pidgin"
+        GEN3_ENV_LABEL
+        GEN3_PIDGIN_VERSION
         GEN3_DATE_LABEL
     spec:
       affinity:
@@ -44,6 +47,34 @@ spec:
         - name: pidgin
           GEN3_PIDGIN_IMAGE
           env:
+          - name: DD_ENABLED
+            valueFrom:
+              configMapKeyRef:
+                name: manifest-global
+                key: dd_enabled
+                optional: true
+          - name: DD_ENV
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/env']
+          - name: DD_SERVICE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/service']
+          - name: DD_VERSION
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['tags.datadoghq.com/version']
+          - name: DD_LOGS_INJECTION
+            value: "true"
+          - name: DD_PROFILING_ENABLED
+            value: "true"
+          - name: DD_TRACE_SAMPLE_RATE
+            value: "1"
+          - name: DD_AGENT_HOST
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
           - name: GEN3_DEBUG
             GEN3_DEBUG_FLAG|-value: "False"-|
           livenessProbe:

diff --git a/kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml b/kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml
@@ -30,6 +30,9 @@ spec:
         netnolimit: "yes"
         public: "yes"
         userhelper: "yes"
+        tags.datadoghq.com/service: "presigned-url-fence"
+        GEN3_ENV_LABEL
+        GEN3_FENCE_VERSION
         GEN3_DATE_LABEL
     spec:
       affinity:
@@ -99,6 +102,34 @@ spec:
       - name: fence
         GEN3_FENCE_IMAGE
         env:
+        - name: DD_ENABLED
+          valueFrom:
+            configMapKeyRef:
+              name: manifest-global
+              key: dd_enabled
+              optional: true
+        - name: DD_ENV
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.labels['tags.datadoghq.com/env']
+        - name: DD_SERVICE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.labels['tags.datadoghq.com/service']
+        - name: DD_VERSION
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.labels['tags.datadoghq.com/version']
+        - name: DD_LOGS_INJECTION
+          value: "true"
+        - name: DD_PROFILING_ENABLED
+          value: "true"
+        - name: DD_TRACE_SAMPLE_RATE
+          value: "1"
+        - name: DD_AGENT_HOST
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
         - name: NGINX_RATE_LIMIT
           value: "6"
         - name: PYTHONPATH
@@ -213,7 +244,7 @@ spec:
               nginx_limit=$(cat /fence/fence/config-default.yaml | sed -n -e 's/^.*OVERRIDE_NGINX_RATE_LIMIT: //p')
             fi
             export OVERRIDE_NGINX_RATE_LIMIT="$nginx_limit"
-            ([[ ! -f /entrypoint.sh ]] || bash /entrypoint.sh) && /fence/dockerrun.bash && if [[ -f /dockerrun.sh ]]; then /dockerrun.sh; fi
+            ([[ ! -f /entrypoint.sh ]] || bash /entrypoint.sh) && /fence/dockerrun.bash && if [[ -f /dockerrun.sh ]]; then bash /dockerrun.sh; fi
       - name: nginx-prometheus-exporter-wrapper
         GEN3_NGINX_PROMETHEUS_EXPORTER_WRAPPER_IMAGE|-image: quay.io/cdis/nginx-prometheus-exporter-wrapper:pybase3-1.4.0-|
         command: ["/bin/bash"]