From 1d594ce2d55842c2f295912858a6f3b5f8db09e8 Mon Sep 17 00:00:00 2001
From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com>
Date: Fri, 19 Jan 2024 15:42:55 -0700
Subject: [PATCH 01/67] Update web_whitelist (add elastic.co) (#2442)

---
 files/squid_whitelist/web_whitelist | 1 +
 1 file changed, 1 insertion(+)

diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist
index 0cbc0a6bc..83070d335 100644
--- a/files/squid_whitelist/web_whitelist
+++ b/files/squid_whitelist/web_whitelist
@@ -77,6 +77,7 @@ golang.org
 gopkg.in
 grafana.com
 grafana.github.io
+helm.elastic.co
 http.us.debian.org
 ifconfig.io
 ingress.coralogix.us

From 87704ae2a846d03903f58cfdc869dbd76e5d8aae Mon Sep 17 00:00:00 2001
From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com>
Date: Mon, 22 Jan 2024 14:39:51 -0700
Subject: [PATCH 02/67] VPC CNI Migration From Calico (#2440)

* Adding "internet: yes" label

This will allow the "netpolicy-external-egress" networkpolicy to apply to revproxy and allow for all egress traffic.

* updating the aws vpc cni version in our "kube-setup-system-services" script

* adding "sudo mount -t bpf bpffs /sys/fs/bpf" to Karpenter node templates as AWS VPC CNI addon requires it in order to manage networkpolicies
---
 gen3/bin/kube-setup-system-services.sh            | 2 +-
 kube/services/karpenter/nodeTemplateDefault.yaml  | 1 +
 kube/services/karpenter/nodeTemplateGPU.yaml      | 1 +
 kube/services/karpenter/nodeTemplateJupyter.yaml  | 1 +
 kube/services/karpenter/nodeTemplateWorkflow.yaml | 1 +
 kube/services/revproxy/revproxy-deploy.yaml       | 1 +
 6 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/gen3/bin/kube-setup-system-services.sh b/gen3/bin/kube-setup-system-services.sh
index 609ee01c7..0afa7d586 100644
--- a/gen3/bin/kube-setup-system-services.sh
+++ b/gen3/bin/kube-setup-system-services.sh
@@ -19,7 +19,7 @@ gen3_load "gen3/gen3setup"
 kubeproxy=${kubeproxy:-1.24.7}
 coredns=${coredns:-1.8.7}
 kubednsautoscaler=${kubednsautoscaler:-1.8.6}
-cni=${cni:-1.12.2}
+cni=${cni:-1.14.1}
 calico=${calico:-1.7.8}
 
 
diff --git a/kube/services/karpenter/nodeTemplateDefault.yaml b/kube/services/karpenter/nodeTemplateDefault.yaml
index a3dbf6480..107c5e6cc 100644
--- a/kube/services/karpenter/nodeTemplateDefault.yaml
+++ b/kube/services/karpenter/nodeTemplateDefault.yaml
@@ -37,6 +37,7 @@ spec:
     sudo  dracut -f
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
+    sudo mount -t bpf bpffs /sys/fs/bpf
 
     --BOUNDARY
     Content-Type: text/cloud-config; charset="us-ascii"
diff --git a/kube/services/karpenter/nodeTemplateGPU.yaml b/kube/services/karpenter/nodeTemplateGPU.yaml
index 5270b697f..c4fd535d7 100644
--- a/kube/services/karpenter/nodeTemplateGPU.yaml
+++ b/kube/services/karpenter/nodeTemplateGPU.yaml
@@ -37,6 +37,7 @@ spec:
     sudo  dracut -f
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
+    sudo mount -t bpf bpffs /sys/fs/bpf
 
     --BOUNDARY
     Content-Type: text/cloud-config; charset="us-ascii"
diff --git a/kube/services/karpenter/nodeTemplateJupyter.yaml b/kube/services/karpenter/nodeTemplateJupyter.yaml
index 74f24926a..bca4436d1 100644
--- a/kube/services/karpenter/nodeTemplateJupyter.yaml
+++ b/kube/services/karpenter/nodeTemplateJupyter.yaml
@@ -37,6 +37,7 @@ spec:
     sudo  dracut -f
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
+    sudo mount -t bpf bpffs /sys/fs/bpf
 
     --BOUNDARY
     Content-Type: text/cloud-config; charset="us-ascii"
diff --git a/kube/services/karpenter/nodeTemplateWorkflow.yaml b/kube/services/karpenter/nodeTemplateWorkflow.yaml
index ec2b81a60..22c95aba1 100644
--- a/kube/services/karpenter/nodeTemplateWorkflow.yaml
+++ b/kube/services/karpenter/nodeTemplateWorkflow.yaml
@@ -37,6 +37,7 @@ spec:
     sudo  dracut -f
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
+    sudo mount -t bpf bpffs /sys/fs/bpf
 
     --BOUNDARY
     Content-Type: text/cloud-config; charset="us-ascii"
diff --git a/kube/services/revproxy/revproxy-deploy.yaml b/kube/services/revproxy/revproxy-deploy.yaml
index 9d5caab1b..9f10ce90b 100644
--- a/kube/services/revproxy/revproxy-deploy.yaml
+++ b/kube/services/revproxy/revproxy-deploy.yaml
@@ -21,6 +21,7 @@ spec:
         app: revproxy
         # allow access from workspaces
         userhelper: "yes"
+        internet: "yes"
         GEN3_DATE_LABEL
     spec:
       affinity:

From 013ce31b4bb8eb49bbff2674f889d0ac28b6cc82 Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Fri, 26 Jan 2024 12:50:15 -0500
Subject: [PATCH 03/67] This was still set to 5 for some reason (#2445)

---
 kube/services/argo/values.yaml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/kube/services/argo/values.yaml b/kube/services/argo/values.yaml
index 67fa05a09..473f7041e 100644
--- a/kube/services/argo/values.yaml
+++ b/kube/services/argo/values.yaml
@@ -1,5 +1,5 @@
 controller:
-  parallelism: 5
+  parallelism: 3
   metricsConfig:
     # -- Enables prometheus metrics server
     enabled: true
@@ -28,11 +28,11 @@ controller:
             }
           ]
         }
-      }  
+      }
 
   resourceRateLimit:
     limit: 40
-    burst: 4    
+    burst: 4
 
   # -- enable persistence using postgres
   persistence:
@@ -49,7 +49,7 @@ controller:
       port: 5432
       database: GEN3_ARGO_DB_NAME
       tableName: argo_workflows
-  #   # the database secrets must be in the same namespace of the controller
+      #   # the database secrets must be in the same namespace of the controller
       userNameSecret:
         name: argo-db-creds
         key: db_username
@@ -58,7 +58,7 @@ controller:
         key: db_password
     nodeStatusOffLoad: true
 
-  workflowDefaults: 
+  workflowDefaults:
     spec:
       archiveLogs: true
 
@@ -77,11 +77,11 @@ server:
   baseHref: "/argo/"
   # -- Extra arguments to provide to the Argo server binary, such as for disabling authentication.
   extraArgs:
-  - --auth-mode=server 
-  - --auth-mode=client
+    - --auth-mode=server
+    - --auth-mode=client
   extraEnv:
-  - name: ARGO_HTTP1
-    value: "true"
+    - name: ARGO_HTTP1
+      value: "true"
   resources:
     requests:
       memory: 8Gi

From 368eeb0f3046ae90f4c7ee01658fe30752daf9c9 Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Wed, 31 Jan 2024 10:31:27 -0600
Subject: [PATCH 04/67] chore(karpenter-upgrade): Updraded karpenter to a
 supported version when running eks 1.25+ (#2450)

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
---
 gen3/bin/kube-setup-karpenter.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gen3/bin/kube-setup-karpenter.sh b/gen3/bin/kube-setup-karpenter.sh
index 8ba8ed9d9..4dba4eb40 100644
--- a/gen3/bin/kube-setup-karpenter.sh
+++ b/gen3/bin/kube-setup-karpenter.sh
@@ -24,7 +24,9 @@ gen3_deploy_karpenter() {
         karpenter=$(g3k_config_lookup .global.karpenter_version)
       fi
       export clusterversion=`kubectl version --short -o json | jq -r .serverVersion.minor`
-      if [ "${clusterversion}" = "24+" ]; then
+      if [ "${clusterversion}" = "25+" ]; then
+        karpenter=${karpenter:-v0.27.0}
+      elif [ "${clusterversion}" = "24+" ]; then
         karpenter=${karpenter:-v0.24.0}
       else
         karpenter=${karpenter:-v0.22.0}

From 12baa786e1771e0d4f5533844fee978fd80a51a8 Mon Sep 17 00:00:00 2001
From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com>
Date: Wed, 31 Jan 2024 12:32:54 -0700
Subject: [PATCH 05/67] adding "migrate to vpc cni" script to cloud-auto
 (#2446)

* adding "migrate to vpc cni" script to cloud-auto

* updating the migrate to vpc script

* changing to master branch

* removing uneeded comments

* Updated karpenter templates

* adding changes

---------

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
Co-authored-by: emalinowski <emalinowski@uchicago.edu>
---
 gen3/bin/migrate-to-vpc-cni.sh                | 138 ++++++++++++++++++
 .../karpenter/nodeTemplateDefault.yaml        |   8 +-
 kube/services/karpenter/nodeTemplateGPU.yaml  |   7 +-
 .../karpenter/nodeTemplateJupyter.yaml        |   7 +-
 .../karpenter/nodeTemplateWorkflow.yaml       |   7 +-
 5 files changed, 163 insertions(+), 4 deletions(-)
 create mode 100644 gen3/bin/migrate-to-vpc-cni.sh

diff --git a/gen3/bin/migrate-to-vpc-cni.sh b/gen3/bin/migrate-to-vpc-cni.sh
new file mode 100644
index 000000000..510d9ebef
--- /dev/null
+++ b/gen3/bin/migrate-to-vpc-cni.sh
@@ -0,0 +1,138 @@
+#!/bin/bash
+
+source "${GEN3_HOME}/gen3/lib/utils.sh"
+gen3_load "gen3/gen3setup"
+
+#Get the K8s NS
+ctx="$(g3kubectl config current-context)"
+ctxNamespace="$(g3kubectl config view -ojson | jq -r ".contexts | map(select(.name==\"$ctx\")) | .[0] | .context.namespace")"
+
+# Set the cluster name variable
+CLUSTER_NAME=`gen3 api environment`
+
+# Check if in default ns
+if [[ ("$ctxNamespace" != "default" && "$ctxNamespace" != "null") ]]; then
+    gen3_log_err "Namespace must be default"
+    exit 1
+fi
+
+# Cd into Cloud-automation repo and pull the latest from master
+gen3_log_info "Pulling the latest from Cloud-Auto"
+cd /home/$CLUSTER_NAME/cloud-automation || { gen3_log_err "Cloud-automation repo not found"; exit 1; }
+#### Change to master
+git checkout master || { gen3_log_err "Failed to checkout master branch"; exit 1; }
+git pull || { gen3_log_err "Failed to pull from the repository"; exit 1; }
+
+# Update the Karpenter Node Template
+gen3_log_info "Apply new Karpenter Node Template"
+if [[ -d $(g3k_manifest_init)/$(g3k_hostname)/manifests/karpenter ]]; then
+    gen3_log_info "Karpenter setup in manifest. Open a cdismanifest PR and add this line to aws node templates: https://github.com/uc-cdis/cloud-automation/blob/master/kube/services/karpenter/nodeTemplateDefault.yaml#L40"
+    while true; do
+        read -p "Have you updated your manifest? (yes/no): " yn
+        case $yn in
+            [Yy]* ) 
+                gen3_log_info "Proceeding with Karpenter deployment..."
+                gen3 kube-setup-karpenter deploy --force || { gen3_log_err "kube-setup-karpenter failed"; exit 1; }
+                break
+                ;;
+            [Nn]* ) 
+                gen3_log_info "Please update the cdismanifest before proceeding."
+                exit 1
+                ;;
+            * ) 
+                gen3_log_info "Please answer yes or no."
+                ;;
+        esac
+    done
+else
+    gen3 kube-setup-karpenter deploy --force || { gen3_log_err "kube-setup-karpenter failed"; exit 1; }
+fi
+
+# Cordon all the nodes before running gen3 roll all"
+gen3_log_info "Cordoning all nodes"
+kubectl get nodes --no-headers -o custom-columns=":metadata.name" | grep -v '^fargate' | xargs -I{} kubectl cordon {}
+
+# Run a "gen3 roll all" so all nodes use the new mounted BPF File System
+gen3_log_info "Cycling all the nodes by running gen3 roll all"
+gen3 roll all --fast || exit 1
+
+# Confirm that all nodes have been rotated
+while true; do
+    read -p "Roll all complete. Have all cordoned nodes been rotated? (yes/no): " yn
+    case $yn in
+        [Yy]* ) 
+            gen3_log_info "Continuing with script..."
+            break
+            ;;
+        [Nn]* ) 
+            gen3_log_info "Please drain any remaining nodes with 'kubectl drain <node_name> --ignore-daemonsets --delete-emptydir-data'"
+            ;;
+        * ) 
+            gen3_log_info "Please answer yes or no."
+            ;;
+    esac
+done
+
+
+# Delete all existing network policies
+gen3_log_info "Deleting networkpolicies"
+kubectl delete networkpolicies --all
+
+# Delete all Calico related resources from the “kube-system” namespace
+gen3_log_info "Deleting all Calico related resources"
+kubectl get deployments -n kube-system | grep calico | awk '{print $1}' | xargs kubectl delete deployment -n kube-system
+kubectl get daemonsets -n kube-system | grep calico | awk '{print $1}' | xargs kubectl delete daemonset -n kube-system
+kubectl get services -n kube-system | grep calico | awk '{print $1}' | xargs kubectl delete service -n kube-system
+kubectl get replicasets -n kube-system | grep calico | awk '{print $1}' | xargs kubectl delete replicaset -n kube-system
+
+# Backup the current VPC CNI configuration in case of rollback
+gen3_log_info "Backing up current VPC CNI Configuration..."
+kubectl get daemonset aws-node -n kube-system -o yaml > aws-k8s-cni-old.yaml || { gen3_log_err "Error backig up VPC CNI configuration"; exit 1; }
+
+# Check to ensure we are not using an AWS plugin to manage the VPC CNI Plugin
+if aws eks describe-addon --cluster-name "$CLUSTER_NAME" --addon-name vpc-cni --query addon.addonVersion --output text 2>/dev/null; then
+    gen3_log_err "Error: VPC CNI Plugin is managed by AWS. Please log into the AWS UI and delete the VPC CNI Plugin in Amazon EKS, then re-run this script."
+    exit 1
+else
+    gen3_log_info "No managed VPC CNI Plugin found, proceeding with the script."
+fi
+
+# Apply the new VPC CNI Version
+gen3_log_info "Applying new version of VPC CNI"
+g3kubectl apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.14.1/config/master/aws-k8s-cni.yaml || { gen3_log_err "Failed to apply new VPC CNI version"; exit 1; }
+
+# Check the version to make sure it updated
+NEW_VERSION=$(kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3)
+gen3_log_info "Current version of aws-k8s-cni is: $NEW_VERSION"
+if [ "$NEW_VERSION" != "v1.14.1" ]; then
+    gen3_log_info "The version of aws-k8s-cni has not been updated correctly."
+    exit 1
+fi
+
+# Edit the amazon-vpc-cni configmap to enable network policy controller
+gen3_log_info "Enabling NetworkPolicies in VPC CNI Configmap"
+kubectl patch configmap -n kube-system amazon-vpc-cni --type merge -p '{"data":{"enable-network-policy-controller":"true"}}' || { gen3_log_err "Configmap patch failed"; exit 1; }
+
+# Edit the aws-node daemonset
+gen3_log_info "Enabling NetworkPolicies in aws-node Daemonset"
+kubectl patch daemonset aws-node -n kube-system --type=json -p='[{"op": "add", "path": "/spec/template/spec/containers/1/args", "value": ["--enable-network-policy=true", "--enable-ipv6=false", "--enable-cloudwatch-logs=false", "--metrics-bind-addr=:8162", "--health-probe-bind-addr=:8163"]}]' || { gen3_log_err "Daemonset edit failed"; exit 1; }
+
+# Ensure all the aws-nodes are running
+kubectl get pods -n kube-system | grep aws
+while true; do
+    read -p "Do all the aws-node pods in the kube-system ns have 2/2 containers running? (yes/no): " yn
+    case $yn in
+        [Yy]* ) 
+            gen3_log_info "Running kube-setup-networkpolicy..."
+            gen3 kube-setup-networkpolicy || exit 1
+            break
+            ;;
+        [Nn]* ) 
+            gen3_log_err "Look at aws-node logs to figure out what went wrong. View this document for more details: https://docs.google.com/document/d/1fcBTciQSSwjvHktEnO_7EObY-xR_EvJ2NtgUa70wvL8"
+            gen3_log_info "Rollback instructions are also available in the above document"
+            ;;
+        * ) 
+            gen3_log_info "Please answer yes or no."
+            ;;
+    esac
+done
\ No newline at end of file
diff --git a/kube/services/karpenter/nodeTemplateDefault.yaml b/kube/services/karpenter/nodeTemplateDefault.yaml
index 107c5e6cc..114de5aba 100644
--- a/kube/services/karpenter/nodeTemplateDefault.yaml
+++ b/kube/services/karpenter/nodeTemplateDefault.yaml
@@ -37,11 +37,17 @@ spec:
     sudo  dracut -f
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
-    sudo mount -t bpf bpffs /sys/fs/bpf
 
     --BOUNDARY
     Content-Type: text/cloud-config; charset="us-ascii"
 
+    mounts:
+      - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
+
+    --BOUNDARY
+
+    Content-Type: text/cloud-config; charset="us-ascii"
+
     power_state:
       delay: now
       mode: reboot
diff --git a/kube/services/karpenter/nodeTemplateGPU.yaml b/kube/services/karpenter/nodeTemplateGPU.yaml
index c4fd535d7..cd3eb7386 100644
--- a/kube/services/karpenter/nodeTemplateGPU.yaml
+++ b/kube/services/karpenter/nodeTemplateGPU.yaml
@@ -37,7 +37,12 @@ spec:
     sudo  dracut -f
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
-    sudo mount -t bpf bpffs /sys/fs/bpf
+
+    --BOUNDARY
+    Content-Type: text/cloud-config; charset="us-ascii"
+
+    mounts:
+      - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
 
     --BOUNDARY
     Content-Type: text/cloud-config; charset="us-ascii"
diff --git a/kube/services/karpenter/nodeTemplateJupyter.yaml b/kube/services/karpenter/nodeTemplateJupyter.yaml
index bca4436d1..af2da8436 100644
--- a/kube/services/karpenter/nodeTemplateJupyter.yaml
+++ b/kube/services/karpenter/nodeTemplateJupyter.yaml
@@ -37,7 +37,12 @@ spec:
     sudo  dracut -f
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
-    sudo mount -t bpf bpffs /sys/fs/bpf
+
+    --BOUNDARY
+    Content-Type: text/cloud-config; charset="us-ascii"
+
+    mounts:
+      - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
 
     --BOUNDARY
     Content-Type: text/cloud-config; charset="us-ascii"
diff --git a/kube/services/karpenter/nodeTemplateWorkflow.yaml b/kube/services/karpenter/nodeTemplateWorkflow.yaml
index 22c95aba1..8609d0746 100644
--- a/kube/services/karpenter/nodeTemplateWorkflow.yaml
+++ b/kube/services/karpenter/nodeTemplateWorkflow.yaml
@@ -37,7 +37,12 @@ spec:
     sudo  dracut -f
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
-    sudo mount -t bpf bpffs /sys/fs/bpf
+
+    --BOUNDARY
+    Content-Type: text/cloud-config; charset="us-ascii"
+
+    mounts:
+      - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
 
     --BOUNDARY
     Content-Type: text/cloud-config; charset="us-ascii"

From c6358c90bf58ca2b38eeb290ef721f0ed40ae2dc Mon Sep 17 00:00:00 2001
From: Hara Prasad <haraprasadj@uchicago.edu>
Date: Thu, 1 Feb 2024 12:59:21 -0800
Subject: [PATCH 06/67] Update jenkins version (#2455)

---
 Docker/jenkins/Jenkins/Dockerfile  | 2 +-
 Docker/jenkins/Jenkins2/Dockerfile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Docker/jenkins/Jenkins/Dockerfile b/Docker/jenkins/Jenkins/Dockerfile
index 7cce68b58..04ebe5864 100644
--- a/Docker/jenkins/Jenkins/Dockerfile
+++ b/Docker/jenkins/Jenkins/Dockerfile
@@ -1,4 +1,4 @@
-FROM jenkins/jenkins:2.439-jdk21
+FROM jenkins/jenkins:2.426.3-lts-jdk21
 
 USER root
 
diff --git a/Docker/jenkins/Jenkins2/Dockerfile b/Docker/jenkins/Jenkins2/Dockerfile
index 9976a07c2..e6b73bc76 100644
--- a/Docker/jenkins/Jenkins2/Dockerfile
+++ b/Docker/jenkins/Jenkins2/Dockerfile
@@ -1,4 +1,4 @@
-FROM jenkins/jenkins:2.415-jdk11
+FROM jenkins/jenkins:2.426.3-lts-jdk21
 
 USER root
 

From 916d1ca7f7a47103eee518574c16a8aa2b2c1f7c Mon Sep 17 00:00:00 2001
From: Mingfei Shao <2475897+mfshao@users.noreply.github.com>
Date: Fri, 2 Feb 2024 12:35:42 -0600
Subject: [PATCH 07/67] update script (#2454)

Co-authored-by: Michael Lukowski <lukowski9@gmail.com>
---
 files/scripts/healdata/heal-cedar-data-ingest.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py
index 4a7d88c3c..71575e3c5 100644
--- a/files/scripts/healdata/heal-cedar-data-ingest.py
+++ b/files/scripts/healdata/heal-cedar-data-ingest.py
@@ -24,11 +24,14 @@
     "Questionnaire/Survey/Assessment - unvalidated instrument": "Questionnaire/Survey/Assessment",
     "Cis Male": "Male",
     "Cis Female": "Female",
-    "Trans Male": "Female-to-male transsexual",
-    "Trans Female": "Male-to-female transsexual",
-    "Agender, Non-binary, gender non-conforming": "Other",
-    "Gender Queer": "Other",
-    "Intersex": "Intersexed",
+    "Trans Male": "Transgender man/trans man/female-to-male (FTM)",
+    "Female-to-male transsexual": "Transgender man/trans man/female-to-male (FTM)",
+    "Trans Female": "Transgender woman/trans woman/male-to-female (MTF)",
+    "Male-to-female transsexual": "Transgender woman/trans woman/male-to-female (MTF)",
+    "Agender, Non-binary, gender non-conforming": "Genderqueer/gender nonconforming/neither exclusively male nor female",
+    "Gender Queer": "Genderqueer/gender nonconforming/neither exclusively male nor female",
+    "Intersex": "Genderqueer/gender nonconforming/neither exclusively male nor female",
+    "Intersexed": "Genderqueer/gender nonconforming/neither exclusively male nor female",
     "Buisness Development": "Business Development"
 }
 

From f6ded7a98e76135657ef8b2e2c0e0455e571414d Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Mon, 5 Feb 2024 13:46:57 -0600
Subject: [PATCH 08/67] Chore/squid al23 (#2456)

* chore(squid-al23): Updated squid bootstrap script to work with al23

* chore(squid-al23): Updated squid bootstrap script to work with al23

* chore(squid-al23): Updated squid bootstrap script to work with al23

* chore(squid-al23): Updated squid bootstrap script to work with al23

---------

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
---
 flavors/squid_auto/squid_running_on_docker.sh | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh
index 05607f304..7504cc35e 100644
--- a/flavors/squid_auto/squid_running_on_docker.sh
+++ b/flavors/squid_auto/squid_running_on_docker.sh
@@ -8,6 +8,9 @@ DISTRO=$(awk -F '[="]*' '/^NAME/ { print $2 }' < /etc/os-release)
 WORK_USER="ubuntu"
 if [[ $DISTRO == "Amazon Linux" ]]; then
   WORK_USER="ec2-user"
+  if [[ $(awk -F '[="]*' '/^VERSION_ID/ { print $2 }' < /etc/os-release) == "2023" ]]; then
+    DISTRO="al2023"
+  fi
 fi
 HOME_FOLDER="/home/${WORK_USER}"
 SUB_FOLDER="${HOME_FOLDER}/cloud-automation"
@@ -201,8 +204,10 @@ function install_awslogs {
   if [[ $DISTRO == "Ubuntu" ]]; then
     wget ${AWSLOGS_DOWNLOAD_URL} -O amazon-cloudwatch-agent.deb
     dpkg -i -E ./amazon-cloudwatch-agent.deb
-  else
+  elif [[ $DISTRO == "Amazon Linux" ]]; then
     sudo yum install amazon-cloudwatch-agent nc -y
+  elif [[ $DISTRO == "al2023" ]]; then
+    sudo dnf install amazon-cloudwatch-agent nc -y
   fi
   
   # Configure the AWS logs
@@ -292,6 +297,19 @@ function main(){
       --volume ${SQUID_CACHE_DIR}:${SQUID_CACHE_DIR} \
       --volume ${SQUID_CONFIG_DIR}:${SQUID_CONFIG_DIR}:ro \
        quay.io/cdis/squid:${SQUID_IMAGE_TAG}
+
+  max_attempts=3
+  attempt_counter=0
+  while [ $attempt_counter -lt $max_attempts ]; do
+    sleep 10
+    if [[ -z "$(sudo lsof -i:3128)" ]]; then
+      echo "Squid not healthy, restarting."
+      docker restart squid
+    else
+      echo "Squid healthy"
+      break
+    fi
+  done
 }
 
 main

From f24a0dacc4053058d5bb949d5e21c938e68b82eb Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Tue, 6 Feb 2024 10:45:03 -0600
Subject: [PATCH 09/67] chore(squid-al23): Updated squid bootstrap script to
 work with al23 (#2457)

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
---
 flavors/squid_auto/squid_running_on_docker.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh
index 7504cc35e..2b0f07e45 100644
--- a/flavors/squid_auto/squid_running_on_docker.sh
+++ b/flavors/squid_auto/squid_running_on_docker.sh
@@ -298,7 +298,7 @@ function main(){
       --volume ${SQUID_CONFIG_DIR}:${SQUID_CONFIG_DIR}:ro \
        quay.io/cdis/squid:${SQUID_IMAGE_TAG}
 
-  max_attempts=3
+  max_attempts=10
   attempt_counter=0
   while [ $attempt_counter -lt $max_attempts ]; do
     sleep 10
@@ -307,7 +307,6 @@ function main(){
       docker restart squid
     else
       echo "Squid healthy"
-      break
     fi
   done
 }

From 57120c9e9fea0b79c411f3d88844fca8987fb11f Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Tue, 6 Feb 2024 12:26:06 -0600
Subject: [PATCH 10/67] chore(tigera-helm): Updated squid to support calico
 chart repo (#2458)

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
---
 files/squid_whitelist/web_wildcard_whitelist | 1 +
 1 file changed, 1 insertion(+)

diff --git a/files/squid_whitelist/web_wildcard_whitelist b/files/squid_whitelist/web_wildcard_whitelist
index 44f468097..3dca3946a 100644
--- a/files/squid_whitelist/web_wildcard_whitelist
+++ b/files/squid_whitelist/web_wildcard_whitelist
@@ -100,6 +100,7 @@
 .sourceforge.net
 .southsideweekly.com
 .theanvil.io
+.tigera.io
 .twistlock.com
 .ubuntu.com
 .ucsc.edu

From 8c00a2306cba07ff3894aa4688ac14cbdfc6724e Mon Sep 17 00:00:00 2001
From: "J. Q" <55899496+jawadqur@users.noreply.github.com>
Date: Tue, 6 Feb 2024 15:53:14 -0600
Subject: [PATCH 11/67] Update web_whitelist

---
 files/squid_whitelist/web_whitelist | 1 +
 1 file changed, 1 insertion(+)

diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist
index 83070d335..625c20b29 100644
--- a/files/squid_whitelist/web_whitelist
+++ b/files/squid_whitelist/web_whitelist
@@ -34,6 +34,7 @@ cernvm.cern.ch
 charts.bitnami.com
 charts.helm.sh
 cloud.r-project.org
+coredns.github.io
 coreos.com
 covidstoplight.org
 cpan.mirrors.tds.net

From 36ea4d58f7f16cb5a05a844dd6db545036994f4d Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Wed, 7 Feb 2024 10:09:09 -0600
Subject: [PATCH 12/67] fix(alb-policy): policy fix (#2459)

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
---
 gen3/bin/kube-setup-ingress.sh | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/gen3/bin/kube-setup-ingress.sh b/gen3/bin/kube-setup-ingress.sh
index d0bcff9a4..b75470f73 100644
--- a/gen3/bin/kube-setup-ingress.sh
+++ b/gen3/bin/kube-setup-ingress.sh
@@ -232,6 +232,28 @@ gen3_ingress_setup_role() {
                 }
             }
         },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:AddTags"
+            ],
+            "Resource": [
+                "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
+                "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
+                "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
+            ],
+            "Condition": {
+                "StringEquals": {
+                    "elasticloadbalancing:CreateAction": [
+                        "CreateTargetGroup",
+                        "CreateLoadBalancer"
+                    ]
+                },
+                "Null": {
+                    "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
         {
             "Effect": "Allow",
             "Action": [
@@ -329,4 +351,4 @@ g3kubectl apply -f "${GEN3_HOME}/kube/services/revproxy/revproxy-service.yaml"
 envsubst <$scriptDir/ingress.yaml | g3kubectl apply -f -
 if [ "$deployWaf" = true ]; then
   gen3_ingress_setup_waf
-fi
\ No newline at end of file
+fi

From 301744b69a069cfddccd8a67fc77aa5362955dd0 Mon Sep 17 00:00:00 2001
From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com>
Date: Wed, 7 Feb 2024 10:51:02 -0700
Subject: [PATCH 13/67] removing "--short" flag as it is now deprecated (#2462)

---
 gen3/bin/kube-setup-karpenter.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gen3/bin/kube-setup-karpenter.sh b/gen3/bin/kube-setup-karpenter.sh
index 4dba4eb40..c8762c2e5 100644
--- a/gen3/bin/kube-setup-karpenter.sh
+++ b/gen3/bin/kube-setup-karpenter.sh
@@ -23,7 +23,7 @@ gen3_deploy_karpenter() {
       if g3k_config_lookup .global.karpenter_version; then
         karpenter=$(g3k_config_lookup .global.karpenter_version)
       fi
-      export clusterversion=`kubectl version --short -o json | jq -r .serverVersion.minor`
+      export clusterversion=`kubectl version -o json | jq -r .serverVersion.minor`
       if [ "${clusterversion}" = "25+" ]; then
         karpenter=${karpenter:-v0.27.0}
       elif [ "${clusterversion}" = "24+" ]; then

From d01c0fa1fd8b612ab7b03c1d9a408e5e8bf36656 Mon Sep 17 00:00:00 2001
From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com>
Date: Wed, 7 Feb 2024 10:53:02 -0700
Subject: [PATCH 14/67] reverting the BPF (#2461)

---
 kube/services/karpenter/nodeTemplateDefault.yaml  | 8 ++++----
 kube/services/karpenter/nodeTemplateGPU.yaml      | 8 ++++----
 kube/services/karpenter/nodeTemplateJupyter.yaml  | 8 ++++----
 kube/services/karpenter/nodeTemplateWorkflow.yaml | 8 ++++----
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/kube/services/karpenter/nodeTemplateDefault.yaml b/kube/services/karpenter/nodeTemplateDefault.yaml
index 114de5aba..6ba8b3a0f 100644
--- a/kube/services/karpenter/nodeTemplateDefault.yaml
+++ b/kube/services/karpenter/nodeTemplateDefault.yaml
@@ -38,11 +38,11 @@ spec:
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
 
-    --BOUNDARY
-    Content-Type: text/cloud-config; charset="us-ascii"
+    # --BOUNDARY
+    # Content-Type: text/cloud-config; charset="us-ascii"
 
-    mounts:
-      - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
+    # mounts:
+    #   - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
 
     --BOUNDARY
 
diff --git a/kube/services/karpenter/nodeTemplateGPU.yaml b/kube/services/karpenter/nodeTemplateGPU.yaml
index cd3eb7386..925e7a9a0 100644
--- a/kube/services/karpenter/nodeTemplateGPU.yaml
+++ b/kube/services/karpenter/nodeTemplateGPU.yaml
@@ -38,11 +38,11 @@ spec:
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
 
-    --BOUNDARY
-    Content-Type: text/cloud-config; charset="us-ascii"
+    # --BOUNDARY
+    # Content-Type: text/cloud-config; charset="us-ascii"
 
-    mounts:
-      - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
+    # mounts:
+    #   - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
 
     --BOUNDARY
     Content-Type: text/cloud-config; charset="us-ascii"
diff --git a/kube/services/karpenter/nodeTemplateJupyter.yaml b/kube/services/karpenter/nodeTemplateJupyter.yaml
index af2da8436..1c8970ad6 100644
--- a/kube/services/karpenter/nodeTemplateJupyter.yaml
+++ b/kube/services/karpenter/nodeTemplateJupyter.yaml
@@ -38,11 +38,11 @@ spec:
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
 
-    --BOUNDARY
-    Content-Type: text/cloud-config; charset="us-ascii"
+    # --BOUNDARY
+    # Content-Type: text/cloud-config; charset="us-ascii"
 
-    mounts:
-      - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
+    # mounts:
+    #   - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
 
     --BOUNDARY
     Content-Type: text/cloud-config; charset="us-ascii"
diff --git a/kube/services/karpenter/nodeTemplateWorkflow.yaml b/kube/services/karpenter/nodeTemplateWorkflow.yaml
index 8609d0746..6e47b22f9 100644
--- a/kube/services/karpenter/nodeTemplateWorkflow.yaml
+++ b/kube/services/karpenter/nodeTemplateWorkflow.yaml
@@ -38,11 +38,11 @@ spec:
     # configure grub
     sudo /sbin/grubby --update-kernel=ALL --args="fips=1"
 
-    --BOUNDARY
-    Content-Type: text/cloud-config; charset="us-ascii"
+    # --BOUNDARY
+    # Content-Type: text/cloud-config; charset="us-ascii"
 
-    mounts:
-      - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
+    # mounts:
+    #   - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime']
 
     --BOUNDARY
     Content-Type: text/cloud-config; charset="us-ascii"

From 21000363e7c3d562de45dfa7d07ffd24260a300d Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Wed, 7 Feb 2024 12:54:00 -0600
Subject: [PATCH 15/67] chore(remove-coredns-autoscaler): Removed dns
 autoscaler deployment (#2460)

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
---
 gen3/bin/kube-roll-all.sh              | 2 +-
 gen3/bin/kube-setup-system-services.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/gen3/bin/kube-roll-all.sh b/gen3/bin/kube-roll-all.sh
index c9cec5a25..6a67f2bdd 100644
--- a/gen3/bin/kube-roll-all.sh
+++ b/gen3/bin/kube-roll-all.sh
@@ -274,7 +274,7 @@ if [[ "$GEN3_ROLL_FAST" != "true" ]]; then
   else
     gen3 kube-setup-autoscaler &
   fi
-  gen3 kube-setup-kube-dns-autoscaler &
+  #gen3 kube-setup-kube-dns-autoscaler &
   gen3 kube-setup-metrics deploy || true
   gen3 kube-setup-tiller || true
   #
diff --git a/gen3/bin/kube-setup-system-services.sh b/gen3/bin/kube-setup-system-services.sh
index 0afa7d586..c26a04cb5 100644
--- a/gen3/bin/kube-setup-system-services.sh
+++ b/gen3/bin/kube-setup-system-services.sh
@@ -39,7 +39,7 @@ calico_yaml="https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${calico}
 
 g3kubectl set image daemonset.apps/kube-proxy -n kube-system kube-proxy=${kube_proxy_image}
 g3kubectl set image --namespace kube-system deployment.apps/coredns coredns=${coredns_image}
-g3k_kv_filter "${GEN3_HOME}/kube/services/kube-dns-autoscaler/dns-horizontal-autoscaler.yaml" SERVICE "coredns" IMAGE "$kubednsautoscaler_image" | g3kubectl apply -f -
+#g3k_kv_filter "${GEN3_HOME}/kube/services/kube-dns-autoscaler/dns-horizontal-autoscaler.yaml" SERVICE "coredns" IMAGE "$kubednsautoscaler_image" | g3kubectl apply -f -
 g3kubectl apply -f ${cni_image}
 g3kubectl apply -f ${calico_yaml}
 

From 5f7aed08fa60ce1364c8016ff95ad9df6d34bfa1 Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Wed, 7 Feb 2024 12:55:03 -0600
Subject: [PATCH 16/67] fix(squid-cron): Added crontab to al23 squid (#2463)

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
---
 flavors/squid_auto/squid_running_on_docker.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh
index 2b0f07e45..557809b69 100644
--- a/flavors/squid_auto/squid_running_on_docker.sh
+++ b/flavors/squid_auto/squid_running_on_docker.sh
@@ -63,6 +63,8 @@ fi
 function install_basics(){
   if [[ $DISTRO == "Ubuntu" ]]; then
     apt -y install atop
+  elif [[ $DISTRO == "al2023" ]]; then
+    sudo dnf install cronie nc -y
   fi
 }
 
@@ -207,7 +209,7 @@ function install_awslogs {
   elif [[ $DISTRO == "Amazon Linux" ]]; then
     sudo yum install amazon-cloudwatch-agent nc -y
   elif [[ $DISTRO == "al2023" ]]; then
-    sudo dnf install amazon-cloudwatch-agent nc -y
+    sudo dnf install amazon-cloudwatch-agent -y
   fi
   
   # Configure the AWS logs

From 0ebd73040e0c3eba67de10b85061a60c969890c9 Mon Sep 17 00:00:00 2001
From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com>
Date: Fri, 9 Feb 2024 09:39:38 -0700
Subject: [PATCH 17/67] Update ingress.yaml to use newer fips based tls policy
 (#2447)

---
 kube/services/ingress/ingress.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kube/services/ingress/ingress.yaml b/kube/services/ingress/ingress.yaml
index 65916679a..3f1f31259 100644
--- a/kube/services/ingress/ingress.yaml
+++ b/kube/services/ingress/ingress.yaml
@@ -11,7 +11,7 @@ metadata:
     alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
     alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=600
     alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}'
-    alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-2021-06
+    alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04
 spec:
   ingressClassName: alb
   rules:

From 1bf159a57d59dac52544c65fc01f057f53e0b0a7 Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Fri, 9 Feb 2024 14:38:55 -0600
Subject: [PATCH 18/67] fix syntax (#2464)

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
---
 flavors/squid_auto/squid_running_on_docker.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh
index 557809b69..94fe22122 100644
--- a/flavors/squid_auto/squid_running_on_docker.sh
+++ b/flavors/squid_auto/squid_running_on_docker.sh
@@ -303,6 +303,7 @@ function main(){
   max_attempts=10
   attempt_counter=0
   while [ $attempt_counter -lt $max_attempts ]; do
+    ((attempt_counter++))
     sleep 10
     if [[ -z "$(sudo lsof -i:3128)" ]]; then
       echo "Squid not healthy, restarting."

From 0f98195cf8b794b044c063b19281a39aca43b3ee Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Fri, 9 Feb 2024 15:32:21 -0600
Subject: [PATCH 19/67] prevent reboot (#2465)

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
---
 flavors/squid_auto/squid_running_on_docker.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh
index 94fe22122..2d7cf8e68 100644
--- a/flavors/squid_auto/squid_running_on_docker.sh
+++ b/flavors/squid_auto/squid_running_on_docker.sh
@@ -303,7 +303,7 @@ function main(){
   max_attempts=10
   attempt_counter=0
   while [ $attempt_counter -lt $max_attempts ]; do
-    ((attempt_counter++))
+    #((attempt_counter++))
     sleep 10
     if [[ -z "$(sudo lsof -i:3128)" ]]; then
       echo "Squid not healthy, restarting."

From 8e02b097b822927ae1c2768fa93b828e2150d732 Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Mon, 12 Feb 2024 16:07:45 -0500
Subject: [PATCH 20/67] Feat/argo va testing revproxy modifications (#2466)

* It can't be that easy

* It wasn't that easy. What we're going to do is use the manifest to tell us what namespace to look in, and then look there

* Dumb typo

* Yet another dumb typo

* Moving to a different setup, where specify the argo server URL in the manifest. This is more flexible

* I'm blaming the last commit on Monday brain

* It's a .conf file, not a yaml file

* Need to update in both places
---
 gen3/bin/kube-setup-revproxy.sh                 | 17 ++++++++---------
 ...o-workflows-server.conf => argo-server.conf} |  2 +-
 2 files changed, 9 insertions(+), 10 deletions(-)
 rename kube/services/revproxy/gen3.nginx.conf/{argo-argo-workflows-server.conf => argo-server.conf} (86%)

diff --git a/gen3/bin/kube-setup-revproxy.sh b/gen3/bin/kube-setup-revproxy.sh
index fcc2ef3b7..5db9850a1 100644
--- a/gen3/bin/kube-setup-revproxy.sh
+++ b/gen3/bin/kube-setup-revproxy.sh
@@ -111,15 +111,14 @@ for name in $(g3kubectl get services -o json | jq -r '.items[] | .metadata.name'
   fi
 done
 
-if g3kubectl get namespace argo > /dev/null 2>&1;
-then
-  for argo in $(g3kubectl get services -n argo -o jsonpath='{.items[*].metadata.name}');
-  do
-    filePath="$scriptDir/gen3.nginx.conf/${argo}.conf"
-    if [[ -f "$filePath" ]]; then
-      confFileList+=("--from-file" "$filePath")
-    fi
-  done
+
+if g3k_manifest_lookup .argo.argo_server_service_url 2> /dev/null; then
+  argo_server_service_url=$(g3k_manifest_lookup .argo.argo_server_service_url)
+  g3k_kv_filter "${scriptDir}/gen3.nginx.conf/argo-server.conf" SERVICE_URL "${argo_server_service_url}" > /tmp/argo-server-with-url.conf
+  filePath="/tmp/argo-server-with-url.conf"
+  if [[ -f "$filePath" ]]; then
+    confFileList+=("--from-file" "$filePath")
+  fi 
 fi
 
 if g3kubectl get namespace argocd > /dev/null 2>&1;
diff --git a/kube/services/revproxy/gen3.nginx.conf/argo-argo-workflows-server.conf b/kube/services/revproxy/gen3.nginx.conf/argo-server.conf
similarity index 86%
rename from kube/services/revproxy/gen3.nginx.conf/argo-argo-workflows-server.conf
rename to kube/services/revproxy/gen3.nginx.conf/argo-server.conf
index cb8def3aa..1cdd4608c 100644
--- a/kube/services/revproxy/gen3.nginx.conf/argo-argo-workflows-server.conf
+++ b/kube/services/revproxy/gen3.nginx.conf/argo-server.conf
@@ -7,7 +7,7 @@
               auth_request /gen3-authz;
 
               set $proxy_service  "argo";
-              set $upstream http://argo-argo-workflows-server.argo.svc.cluster.local:2746;
+              set $upstream SERVICE_URL;
 
               rewrite ^/argo/(.*) /$1 break;
 

From e081b2385cffdfb832063fbd7c3930f0310ed738 Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Mon, 12 Feb 2024 17:02:16 -0600
Subject: [PATCH 21/67] fix(karpenter-policy): Added kms permissions for
 karpenter (#2467)

* fix(karpenter-policy): Added kms permissions for karpenter

* fix(karpenter-policy): Added kms permissions for karpenter

* fix(karpenter-policy): Added kms permissions for karpenter

---------

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
Co-authored-by: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com>
---
 gen3/bin/kube-setup-karpenter.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/gen3/bin/kube-setup-karpenter.sh b/gen3/bin/kube-setup-karpenter.sh
index c8762c2e5..2737ed6ee 100644
--- a/gen3/bin/kube-setup-karpenter.sh
+++ b/gen3/bin/kube-setup-karpenter.sh
@@ -79,6 +79,14 @@ gen3_deploy_karpenter() {
                   "Effect": "Allow",
                   "Resource": "*",
                   "Sid": "ConditionalEC2Termination"
+              },
+              {
+                  "Sid": "VisualEditor0",
+                  "Effect": "Allow",
+                  "Action": [
+                      "kms:*"
+                  ],
+                  "Resource": "*"
               }
           ],
           "Version": "2012-10-17"

From 11b94fb4e86ac779bf855a451bd16c7e7ecbe5d1 Mon Sep 17 00:00:00 2001
From: Andrew Prokhorenkov <aprokh@uchicago.edu>
Date: Tue, 13 Feb 2024 09:47:57 -0600
Subject: [PATCH 22/67] Feat/argo wrapper namespace config (#2468)

* feat: argo-wrapper-namespace-config update

* feat: some extra updates

* fix: name for file

* feat: updates
---
 gen3/bin/kube-setup-argo-wrapper.sh                 | 13 ++++++++++++-
 kube/services/argo-wrapper/argo-wrapper-deploy.yaml |  9 ++++++++-
 kube/services/argo-wrapper/config.ini               |  4 ++++
 3 files changed, 24 insertions(+), 2 deletions(-)
 create mode 100644 kube/services/argo-wrapper/config.ini

diff --git a/gen3/bin/kube-setup-argo-wrapper.sh b/gen3/bin/kube-setup-argo-wrapper.sh
index 5727a703e..306050b12 100644
--- a/gen3/bin/kube-setup-argo-wrapper.sh
+++ b/gen3/bin/kube-setup-argo-wrapper.sh
@@ -19,5 +19,16 @@ if [[ -z "$GEN3_SOURCE_ONLY" ]]; then
   gen3 roll argo-wrapper
   g3kubectl apply -f "${GEN3_HOME}/kube/services/argo-wrapper/argo-wrapper-service.yaml"
 
+  if g3k_manifest_lookup .argo.argo_server_service_url 2> /dev/null; then
+    argo_server_service_url=$(g3k_manifest_lookup .argo.argo_server_service_url)
+
+    export ARGO_HOST=${argo_server_service_url}
+    export ARGO_NAMESPACE=argo-$(gen3 db namespace)
+    envsubst <"${GEN3_HOME}/kube/services/argo-wrapper/config.ini" > /tmp/config.ini
+    
+    g3kubectl delete configmap argo-wrapper-namespace-config
+    g3kubectl create configmap argo-wrapper-namespace-config --from-file /tmp/config.ini
+  fi
+
   gen3_log_info "the argo-wrapper service has been deployed onto the kubernetes cluster"
-fi
\ No newline at end of file
+fi
diff --git a/kube/services/argo-wrapper/argo-wrapper-deploy.yaml b/kube/services/argo-wrapper/argo-wrapper-deploy.yaml
index 65f68d98a..00d118746 100644
--- a/kube/services/argo-wrapper/argo-wrapper-deploy.yaml
+++ b/kube/services/argo-wrapper/argo-wrapper-deploy.yaml
@@ -58,7 +58,10 @@ spec:
           configMap:
             name: manifest-argo
             optional: true
-          
+        - name: argo-wrapper-namespace-config
+          configMap:
+            name: argo-wrapper-namespace-config
+
       containers:
       - name: argo-wrapper
         GEN3_ARGO-WRAPPER_IMAGE
@@ -70,3 +73,7 @@ spec:
             readOnly: true
             mountPath: /argo.json
             subPath: argo.json
+          - name: argo-wrapper-namespace-config
+            readOnly: true
+            mountPath: /src/config.ini
+            subPath: config.ini
diff --git a/kube/services/argo-wrapper/config.ini b/kube/services/argo-wrapper/config.ini
new file mode 100644
index 000000000..334438ac2
--- /dev/null
+++ b/kube/services/argo-wrapper/config.ini
@@ -0,0 +1,4 @@
+[DEFAULT]
+ARGO_ACCESS_METHOD = access
+ARGO_HOST = $ARGO_HOST
+ARGO_NAMESPACE = $ARGO_NAMESPACE

From 464f6ebc68f1481c83597920ae411226cdff7ee0 Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Wed, 14 Feb 2024 12:22:23 -0600
Subject: [PATCH 23/67] Update squid_running_on_docker.sh (#2469)

---
 flavors/squid_auto/squid_running_on_docker.sh | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh
index 2d7cf8e68..812a9f738 100644
--- a/flavors/squid_auto/squid_running_on_docker.sh
+++ b/flavors/squid_auto/squid_running_on_docker.sh
@@ -74,10 +74,18 @@ function install_docker(){
   # Docker
   ###############################################################
   # Install docker from sources
-  curl -fsSL ${DOCKER_DOWNLOAD_URL}/gpg | sudo apt-key add -
-  add-apt-repository "deb [arch=amd64] ${DOCKER_DOWNLOAD_URL} $(lsb_release -cs) stable"
-  apt update
-  apt install -y docker-ce
+  if [[ $DISTRO == "Ubuntu" ]]; then
+    curl -fsSL ${DOCKER_DOWNLOAD_URL}/gpg | sudo apt-key add -
+    add-apt-repository "deb [arch=amd64] ${DOCKER_DOWNLOAD_URL} $(lsb_release -cs) stable"
+    apt update
+    apt install -y docker-ce
+  else
+    sudo yum update -y
+    sudo yum install -y docker   
+    # Start and enable Docker service
+    sudo systemctl start docker
+    sudo systemctl enable docker
+  fi
   mkdir -p /etc/docker
   cp ${SUB_FOLDER}/flavors/squid_auto/startup_configs/docker-daemon.json /etc/docker/daemon.json
   chmod -R 0644 /etc/docker

From 982e7b6fb1d7bf8fb1ba167e1adf2de7ba995a8b Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Thu, 15 Feb 2024 10:57:46 -0500
Subject: [PATCH 24/67] Lowering parallelism to 1 for VA workflows (#2470)

* Lowering parallelism to 1 for VA workflows

* Caught changes we don't want
---
 kube/services/argo/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kube/services/argo/values.yaml b/kube/services/argo/values.yaml
index 473f7041e..7c2a04531 100644
--- a/kube/services/argo/values.yaml
+++ b/kube/services/argo/values.yaml
@@ -1,5 +1,5 @@
 controller:
-  parallelism: 3
+  parallelism: 1
   metricsConfig:
     # -- Enables prometheus metrics server
     enabled: true

From 0015d1c83ba3e9d0f9cbda1d7eb774e8a57ee68a Mon Sep 17 00:00:00 2001
From: vzpgb <45467497+vzpgb@users.noreply.github.com>
Date: Fri, 16 Feb 2024 10:19:27 -0600
Subject: [PATCH 25/67] Increases VA prod parallel workflows to 3 (#2473)

---
 kube/services/argo/values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kube/services/argo/values.yaml b/kube/services/argo/values.yaml
index 7c2a04531..473f7041e 100644
--- a/kube/services/argo/values.yaml
+++ b/kube/services/argo/values.yaml
@@ -1,5 +1,5 @@
 controller:
-  parallelism: 1
+  parallelism: 3
   metricsConfig:
     # -- Enables prometheus metrics server
     enabled: true

From f4e99c4c17f28bc4e4ad0bdd476f6b65b94ab183 Mon Sep 17 00:00:00 2001
From: burtonk <117617405+k-burt-uch@users.noreply.github.com>
Date: Mon, 19 Feb 2024 15:08:18 -0600
Subject: [PATCH 26/67] Add azure blob storage to web_wildcard_whitelist
 (#2475)

---
 packer/configs/web_wildcard_whitelist | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packer/configs/web_wildcard_whitelist b/packer/configs/web_wildcard_whitelist
index c58eeefe8..621dec3d5 100644
--- a/packer/configs/web_wildcard_whitelist
+++ b/packer/configs/web_wildcard_whitelist
@@ -44,4 +44,5 @@
 .yahooapis.com
 .cloudfront.net
 .docker.io
+.blob.core.windows.net
 .googleapis.com

From 1f8632ff493afad17e14d610eb35c01e1b81e359 Mon Sep 17 00:00:00 2001
From: "J. Q" <55899496+jawadqur@users.noreply.github.com>
Date: Mon, 19 Feb 2024 16:12:31 -0600
Subject: [PATCH 27/67] Add whitelist for snap to work (#2476)

---
 files/squid_whitelist/web_whitelist          | 1 -
 files/squid_whitelist/web_wildcard_whitelist | 2 ++
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist
index 625c20b29..c191b2e8c 100644
--- a/files/squid_whitelist/web_whitelist
+++ b/files/squid_whitelist/web_whitelist
@@ -15,7 +15,6 @@ ctds-planx.atlassian.net
 data.cityofchicago.org
 dataguids.org
 api.login.yahoo.com
-api.snapcraft.io
 apt.kubernetes.io
 argoproj.github.io
 archive.cloudera.com
diff --git a/files/squid_whitelist/web_wildcard_whitelist b/files/squid_whitelist/web_wildcard_whitelist
index 3dca3946a..b71ee76c2 100644
--- a/files/squid_whitelist/web_wildcard_whitelist
+++ b/files/squid_whitelist/web_wildcard_whitelist
@@ -97,6 +97,8 @@
 .sks-keyservers.net
 .slack.com
 .slack-msgs.com
+.snapcraft.io
+.snapcraftcontent.com
 .sourceforge.net
 .southsideweekly.com
 .theanvil.io

From fe5b5502603ec4158c9f077fad83eb421402763d Mon Sep 17 00:00:00 2001
From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com>
Date: Tue, 20 Feb 2024 14:56:05 -0600
Subject: [PATCH 28/67] gen3 logs snapshot: also get initContainers logs
 (#2478)

---
 gen3/lib/logs/snapshot.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/gen3/lib/logs/snapshot.sh b/gen3/lib/logs/snapshot.sh
index 31cb80283..d3d3b2c6c 100644
--- a/gen3/lib/logs/snapshot.sh
+++ b/gen3/lib/logs/snapshot.sh
@@ -36,10 +36,11 @@ gen3_logs_snapshot_container() {
 # Snapshot all the pods
 #
 gen3_logs_snapshot_all() {
+  # For each pod for which we can list the containers, get the pod name and get its list of containers
+  # (container names + initContainers names). Diplay them as lines of "<pod name>  <container name>".
   g3kubectl get pods -o json | \
-    jq -r '.items | map(select(.status.phase != "Pending" and .status.phase != "Unknown")) | map( {pod: .metadata.name, containers: .spec.containers | map(.name) } ) | map( .pod as $pod | .containers | map( { pod: $pod, cont: .})[]) | map(select(.cont != "pause" and .cont != "jupyterhub"))[] | .pod + "  " + .cont' | \
+    jq -r '.items | map(select(.status.phase != "Pending" and .status.phase != "Unknown")) | map( {pod: .metadata.name, containers: [(.spec.containers | select(.!=null) | map(.name)), (.spec.initContainers | select(.!=null) | map(.name)) | add ] } ) | map( .pod as $pod | .containers | map( { pod: $pod, cont: .})[]) | map(select(.cont != "pause" and .cont != "jupyterhub"))[] | .pod + "  " + .cont' | \
     while read -r line; do
       gen3_logs_snapshot_container $line
     done
 }
-

From ff88b7b9ab6898587c96c78bb49ae6456cddabad Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Wed, 21 Feb 2024 11:46:29 -0500
Subject: [PATCH 29/67] Adding namespace and overall parallelism so we can run
 5 workflows in preprod, and 3 in prod at the same time (#2479)

---
 kube/services/argo/values.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/kube/services/argo/values.yaml b/kube/services/argo/values.yaml
index 473f7041e..2b46ced0f 100644
--- a/kube/services/argo/values.yaml
+++ b/kube/services/argo/values.yaml
@@ -1,5 +1,6 @@
 controller:
-  parallelism: 3
+  parallelism: 8
+  namespaceParallelism: 3
   metricsConfig:
     # -- Enables prometheus metrics server
     enabled: true

From d8fd2813a1e13e40bdb58d33abd2a6101f398bba Mon Sep 17 00:00:00 2001
From: Andrew Prokhorenkov <aprokh@uchicago.edu>
Date: Thu, 22 Feb 2024 10:47:16 -0600
Subject: [PATCH 30/67] feat(argo-wrapper): update mountpath (#2481)

---
 kube/services/argo-wrapper/argo-wrapper-deploy.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kube/services/argo-wrapper/argo-wrapper-deploy.yaml b/kube/services/argo-wrapper/argo-wrapper-deploy.yaml
index 00d118746..89ec29ecc 100644
--- a/kube/services/argo-wrapper/argo-wrapper-deploy.yaml
+++ b/kube/services/argo-wrapper/argo-wrapper-deploy.yaml
@@ -75,5 +75,5 @@ spec:
             subPath: argo.json
           - name: argo-wrapper-namespace-config
             readOnly: true
-            mountPath: /src/config.ini
+            mountPath: /argowrapper/config.ini
             subPath: config.ini

From 2df523111e44e3e1eb3ccc5563066ea1c13f4505 Mon Sep 17 00:00:00 2001
From: Andrew Prokhorenkov <aprokh@uchicago.edu>
Date: Thu, 22 Feb 2024 11:03:28 -0600
Subject: [PATCH 31/67] feat(argo-wrapper): update config.ini, missed option
 (#2482)

---
 kube/services/argo-wrapper/config.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/kube/services/argo-wrapper/config.ini b/kube/services/argo-wrapper/config.ini
index 334438ac2..1a1f20c29 100644
--- a/kube/services/argo-wrapper/config.ini
+++ b/kube/services/argo-wrapper/config.ini
@@ -2,3 +2,4 @@
 ARGO_ACCESS_METHOD = access
 ARGO_HOST = $ARGO_HOST
 ARGO_NAMESPACE = $ARGO_NAMESPACE
+COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = http://cohort-middleware-service/cohortdefinition-stats/by-source-id/{}/by-team-project?team-project={}

From e51ed100e8b3b338b7029777a8daadf8b0e22002 Mon Sep 17 00:00:00 2001
From: Andrew Prokhorenkov <aprokh@uchicago.edu>
Date: Thu, 22 Feb 2024 11:21:56 -0600
Subject: [PATCH 32/67] fix(argo-wrapper): fix typo (#2483)

---
 kube/services/argo-wrapper/config.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kube/services/argo-wrapper/config.ini b/kube/services/argo-wrapper/config.ini
index 1a1f20c29..40ac392fd 100644
--- a/kube/services/argo-wrapper/config.ini
+++ b/kube/services/argo-wrapper/config.ini
@@ -2,4 +2,4 @@
 ARGO_ACCESS_METHOD = access
 ARGO_HOST = $ARGO_HOST
 ARGO_NAMESPACE = $ARGO_NAMESPACE
-COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = http://cohort-middleware-service/cohortdefinition-stats/by-source-id/{}/by-team-project?team-project={}
+COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = http://cohort-middleware-service/cohortdefinition-stats/by-source-id/{}/by-team-project?team-project={}

From c5c54865cc2261bfd91b0d83cdbc50b8381254ef Mon Sep 17 00:00:00 2001
From: Hara Prasad <haraprasadj@uchicago.edu>
Date: Fri, 23 Feb 2024 08:17:29 -0800
Subject: [PATCH 33/67] Prevent eviction of ETL job pod while running (#2484)

---
 kube/services/jobs/etl-job.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/kube/services/jobs/etl-job.yaml b/kube/services/jobs/etl-job.yaml
index fa201c99a..6b9b887ec 100644
--- a/kube/services/jobs/etl-job.yaml
+++ b/kube/services/jobs/etl-job.yaml
@@ -2,6 +2,8 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
+  annotations:
+    karpenter.sh/do-not-evict: "true"
   name: etl
 spec:
   backoffLimit: 0

From 13fb2397dfaac8165c7a71e1fce769bda08ee760 Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Tue, 27 Feb 2024 11:51:18 -0500
Subject: [PATCH 34/67] Fix/argo wrapper default url (#2486)

* Adding a secret to contain all values for any workflow templates

* Adding an override namespace argument to kube-setup-argo

* Fixing a typo

* You'd think I'd never written Bash before

* Adding the last part of the namespace override logic

* Debug prints

* Trying to figure out why the override namespace logic doesn't work

* Just checking

* Desperate

* I think we can't use echo in gen3 scripts

* Sanity check

* Just a commit

* Checking if we get the argo_namespace correctly

* Making the print clearer

* It's debug print time

* Fixing references

* Adding some stuff

* Removing debugs, and adding --overwrite

* Adding double quotes

* Fixing kube-setup-argo-wrapper to have default values for argo-service URL and argo namespace

* Fixing /tmp/config.ini collisions

* Turns out that wasn't the way to go, instead we'll just delete /tmp/config.ini to account for multiple environments

* Removed some extra stuff that snuck in
---
 .secrets.baseline                   |  2 +-
 gen3/bin/kube-setup-argo-wrapper.sh | 23 ++++++++++++++++-------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/.secrets.baseline b/.secrets.baseline
index 0a8fe9cc9..936e306e5 100644
--- a/.secrets.baseline
+++ b/.secrets.baseline
@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2023-10-26T21:32:44Z",
+  "generated_at": "2024-02-23T20:30:41Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
diff --git a/gen3/bin/kube-setup-argo-wrapper.sh b/gen3/bin/kube-setup-argo-wrapper.sh
index 306050b12..9f7cc52ce 100644
--- a/gen3/bin/kube-setup-argo-wrapper.sh
+++ b/gen3/bin/kube-setup-argo-wrapper.sh
@@ -18,17 +18,26 @@ if [[ -z "$GEN3_SOURCE_ONLY" ]]; then
 
   gen3 roll argo-wrapper
   g3kubectl apply -f "${GEN3_HOME}/kube/services/argo-wrapper/argo-wrapper-service.yaml"
+ 
 
   if g3k_manifest_lookup .argo.argo_server_service_url 2> /dev/null; then
-    argo_server_service_url=$(g3k_manifest_lookup .argo.argo_server_service_url)
+    export ARGO_HOST=$(g3k_manifest_lookup .argo.argo_server_service_url)
+  else
+    export ARGO_HOST="http://argo-argo-workflows-server.argo.svc.cluster.local:2746"
+  fi 
+
+  if g3k_config_lookup '.argo_namespace' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json 2> /dev/null; then
+    export ARGO_NAMESPACE=$(g3k_config_lookup '.argo_namespace' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json)
+  else
+    export ARGO_NAMESPACE="argo"
+  fi
 
-    export ARGO_HOST=${argo_server_service_url}
-    export ARGO_NAMESPACE=argo-$(gen3 db namespace)
-    envsubst <"${GEN3_HOME}/kube/services/argo-wrapper/config.ini" > /tmp/config.ini
+  envsubst <"${GEN3_HOME}/kube/services/argo-wrapper/config.ini" > /tmp/config.ini
     
-    g3kubectl delete configmap argo-wrapper-namespace-config
-    g3kubectl create configmap argo-wrapper-namespace-config --from-file /tmp/config.ini
-  fi
+  g3kubectl delete configmap argo-wrapper-namespace-config
+  g3kubectl create configmap argo-wrapper-namespace-config --from-file /tmp/config.ini
+
+  rm /tmp/config.ini
 
   gen3_log_info "the argo-wrapper service has been deployed onto the kubernetes cluster"
 fi

From 1a44bef0712ca5ffb4262615999787ef536644be Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Wed, 28 Feb 2024 06:20:07 -0600
Subject: [PATCH 35/67] fix(jenkins-zone): Added zone (#2487)

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
Co-authored-by: Ajo Augustine <ajoaugustine@gmail.com>
---
 kube/services/jenkins/jenkins-deploy.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/kube/services/jenkins/jenkins-deploy.yaml b/kube/services/jenkins/jenkins-deploy.yaml
index c0eae2040..954e996f2 100644
--- a/kube/services/jenkins/jenkins-deploy.yaml
+++ b/kube/services/jenkins/jenkins-deploy.yaml
@@ -38,6 +38,10 @@ spec:
                 operator: In
                 values:
                 - on-demand                
+              - key: topology.kubernetes.io/zone
+                operator: In
+                values:
+                - us-east-1a
       serviceAccountName: jenkins-service
       securityContext:
         runAsUser: 1000

From 6c27fc9a7fe6cd3affd7a108db3c80b51c664ad1 Mon Sep 17 00:00:00 2001
From: Michael Lukowski <lukowski9@gmail.com>
Date: Thu, 29 Feb 2024 09:59:10 -0600
Subject: [PATCH 36/67] updating the cedar data ingest (#2472)

* updating the cedar data ingest

* fix unsanitized request

* address some comments
---
 .../healdata/heal-cedar-data-ingest.py        | 96 +++++++++++++++++++
 1 file changed, 96 insertions(+)

diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py
index 71575e3c5..d3ff246c3 100644
--- a/files/scripts/healdata/heal-cedar-data-ingest.py
+++ b/files/scripts/healdata/heal-cedar-data-ingest.py
@@ -35,6 +35,16 @@
     "Buisness Development": "Business Development"
 }
 
+# repository links
+REPOSITORY_STUDY_ID_LINK_TEMPLATE = {
+    "NIDDK Central": "https://repository.niddk.nih.gov/studies/<STUDY_ID>/",
+    "NIDA Data Share": "https://datashare.nida.nih.gov/study/<STUDY_ID>",
+    "NICHD DASH": "https://dash.nichd.nih.gov/study/<STUDY_ID>",
+    "ICPSR": "https://www.icpsr.umich.edu/web/ICPSR/studies/<STUDY_ID>",
+    "BioSystics-AP": "https://biosystics-ap.com/assays/assaystudy/<STUDY_ID>/",
+}
+
+
 # Defines field that we don't want to include in the filters
 OMITTED_VALUES_MAPPING = {
     "study_metadata.human_subject_applicability.gender_applicability": "Not applicable"
@@ -114,6 +124,31 @@ def get_client_token(client_id: str, client_secret: str):
     return token
 
 
+def get_related_studies(serial_num, hostname):
+    related_study_result = []
+
+    if serial_num:
+        mds = requests.get(f"https://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000")
+        if mds.status_code == 200:
+            related_study_metadata = mds.json()
+
+            for (
+                related_study_metadata_key,
+                related_study_metadata_value,
+            ) in related_study_metadata.items():
+                title = (
+                    related_study_metadata_value.get(
+                        "gen3_discovery", {}
+                    )
+                    .get("study_metadata", {})
+                    .get("minimal_info", {})
+                    .get("study_name", "")
+                )
+                link = f"https://{hostname}/portal/discovery/{related_study_metadata_key}/"
+                related_study_result.append({"title": title, "link": link})
+    return related_study_result
+
+
 parser = argparse.ArgumentParser()
 
 parser.add_argument("--directory", help="CEDAR Directory ID for registering ")
@@ -214,6 +249,67 @@ def get_client_token(client_id: str, client_secret: str):
                 mds_res["gen3_discovery"]["study_metadata"].update(cedar_record)
                 mds_res["gen3_discovery"]["study_metadata"]["metadata_location"]["other_study_websites"] = cedar_record_other_study_websites
 
+                # setup citations
+                doi_citation = mds_res["gen3_discovery"]["study_metadata"].get("doi_citation", "")
+                mds_res["gen3_discovery"]["study_metadata"]["citation"]["heal_platform_citation"] = doi_citation
+
+
+                # setup repository_study_link
+                data_repositories = (
+                    mds_res.get("study_metadata", {})
+                    .get("metadata_location", {})
+                    .get("data_repositories", [])
+                )
+                repository_citation = "Users must also include a citation to the data as specified by the local repository."
+                repository_citation_additional_text = ' The link to the study page at the local repository can be found in the "Data" tab.'
+                for repository in data_repositories:
+                    if (
+                        repository["repository_name"]
+                        and repository["repository_name"]
+                        in REPOSITORY_STUDY_ID_LINK_TEMPLATE
+                        and repository["repository_study_ID"]
+                    ):
+                        repository_study_link = REPOSITORY_STUDY_ID_LINK_TEMPLATE[
+                            repository["repository_name"]
+                        ].replace("<STUDY_ID>", repository["repository_study_ID"])
+                        repository.update({"repository_study_link": repository_study_link})
+                        if repository_citation_additional_text not in repository_citation:
+                            repository_citation += repository_citation_additional_text
+                if len(data_repositories):
+                    data_repositories[0] = {
+                        **data_repositories[0],
+                        "repository_citation": repository_citation,
+                    }
+                mds_res["gen3_discovery"]["study_metadata"][
+                    "metadata_location"
+                ]["data_repositories"] = data_repositories
+
+
+
+                # set up related studies
+                serial_num = None
+                try:
+                    serial_num = (
+                        mds_res
+                        .get("nih_reporter", {})
+                        .get("project_num_split", {})
+                        .get("serial_num", None)
+                    )
+                except Exception:
+                    print(f"Unable to get serial number for study")
+
+                if serial_num == None:
+                    print(f"Unable to get serial number for study")
+
+                related_study_result = get_related_studies(serial_num, hostname)
+                existing_related_study_result = mds_res.get("related_studies", [])
+                for related_study in related_study_result:
+                    if related_study not in existing_related_study_result:
+                        existing_related_study_result.append(copy.deepcopy(related_study))
+                mds_res["gen3_discovery"][
+                    "related_studies"
+                ] = copy.deepcopy(existing_related_study_result)
+
                 # merge data from cedar that is not study level metadata into a level higher
                 deleted_keys = []
                 for key, value in mds_res["gen3_discovery"]["study_metadata"].items():

From 072279d59732ff0109e9ca18bb05479b3bdcd577 Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Thu, 29 Feb 2024 11:16:29 -0500
Subject: [PATCH 37/67] Feat/s3 bucket template (#2485)

* Adding a secret to contain all values for any workflow templates

* Adding an override namespace argument to kube-setup-argo

* Fixing a typo

* You'd think I'd never written Bash before

* Adding the last part of the namespace override logic

* Debug prints

* Trying to figure out why the override namespace logic doesn't work

* Just checking

* Desperate

* I think we can't use echo in gen3 scripts

* Sanity check

* Just a commit

* Checking if we get the argo_namespace correctly

* Making the print clearer

* It's debug print time

* Fixing references

* Adding some stuff

* Removing debugs, and adding --overwrite

* Adding double quotes
---
 .secrets.baseline           |  4 +--
 gen3/bin/kube-setup-argo.sh | 66 ++++++++++++++++++++++++++-----------
 2 files changed, 49 insertions(+), 21 deletions(-)

diff --git a/.secrets.baseline b/.secrets.baseline
index 936e306e5..b7e06622d 100644
--- a/.secrets.baseline
+++ b/.secrets.baseline
@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-02-23T20:30:41Z",
+  "generated_at": "2024-02-23T20:30:41Z"
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -342,7 +342,7 @@
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 191,
+        "line_number": 206,
         "type": "Secret Keyword"
       }
     ],
diff --git a/gen3/bin/kube-setup-argo.sh b/gen3/bin/kube-setup-argo.sh
index ff2438833..20676145b 100644
--- a/gen3/bin/kube-setup-argo.sh
+++ b/gen3/bin/kube-setup-argo.sh
@@ -5,10 +5,25 @@ source "${GEN3_HOME}/gen3/lib/utils.sh"
 gen3_load "gen3/gen3setup"
 gen3_load "gen3/lib/kube-setup-init"
 
+override_namespace=false
+force=false
+
+for arg in "${@}"; do
+  if [ "$arg" == "--override-namespace" ]; then
+    override_namespace=true
+  elif [ "$arg" == "--force" ]; then
+    force=true
+  else 
+    #Print usage info and exit
+    gen3_log_info "Usage: gen3 kube-setup-argo [--override-namespace] [--force]"
+    exit 1
+  fi
+done
 
 ctx="$(g3kubectl config current-context)"
 ctxNamespace="$(g3kubectl config view -ojson | jq -r ".contexts | map(select(.name==\"$ctx\")) | .[0] | .context.namespace")"
 
+argo_namespace=$(g3k_config_lookup '.argo_namespace' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json)
 
 function setup_argo_buckets {
   local accountNumber
@@ -32,13 +47,13 @@ function setup_argo_buckets {
   roleName="gen3-argo-${environment//_/-}-role"
   bucketPolicy="argo-bucket-policy-${nameSpace}"
   internalBucketPolicy="argo-internal-bucket-policy-${nameSpace}"
-  if [[ ! -z $(g3k_config_lookup '."s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) || ! -z $(g3k_config_lookup '.argo."s3-bucket"') ]]; then
-    if [[ ! -z $(g3k_config_lookup '."s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) ]]; then
+  if [[ ! -z $(g3k_config_lookup '."downloadable-s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) || ! -z $(g3k_config_lookup '.argo."downloadable-s3-bucket"') ]]; then
+    if [[ ! -z $(g3k_config_lookup '."downloadable-s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) ]]; then
       gen3_log_info "Using S3 bucket found in manifest: ${bucketName}"
-      bucketName=$(g3k_config_lookup '."s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json)
+      bucketName=$(g3k_config_lookup '."downloadable-s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json)
     else
       gen3_log_info "Using S3 bucket found in manifest: ${bucketName}"
-      bucketName=$(g3k_config_lookup '.argo."s3-bucket"')
+      bucketName=$(g3k_config_lookup '.argo."downloadable-s3-bucket"')
     fi
   fi
   if [[ ! -z $(g3k_config_lookup '."internal-s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) || ! -z $(g3k_config_lookup '.argo."internal-s3-bucket"') ]]; then
@@ -131,19 +146,19 @@ EOF
     g3kubectl create namespace argo || true
     g3kubectl label namespace argo app=argo || true
     # Grant admin access within the argo namespace to the default SA in the argo namespace
-    g3kubectl create rolebinding argo-admin --clusterrole=admin --serviceaccount=argo:default -n argo || true
+    g3kubectl create rolebinding argo-admin --clusterrole=admin --serviceaccount=argo:default -n $argo_namespace || true
   fi
   gen3_log_info "Creating IAM role ${roleName}"
   if aws iam get-role --role-name "${roleName}" > /dev/null 2>&1; then
       gen3_log_info "IAM role ${roleName} already exists.."
       roleArn=$(aws iam get-role --role-name "${roleName}" --query 'Role.Arn' --output text)
       gen3_log_info "Role annotate"
-      g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} -n argo
-      g3kubectl annotate serviceaccount argo eks.amazonaws.com/role-arn=${roleArn} -n $nameSpace
+      g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} --overwrite -n $argo_namespace
+      g3kubectl annotate serviceaccount argo eks.amazonaws.com/role-arn=${roleArn} --overwrite -n $nameSpace
   else
         gen3 awsrole create $roleName argo $nameSpace -f all_namespaces
         roleArn=$(aws iam get-role --role-name "${roleName}" --query 'Role.Arn' --output text)
-        g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} -n argo
+        g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} -n $argo_namespace
   fi
 
   # Grant admin access within the current namespace to the argo SA in the current namespace
@@ -177,34 +192,47 @@ EOF
     for serviceName in indexd; do
       secretName="${serviceName}-creds"
       # Only delete if secret is found to prevent early exits
-      if [[ ! -z $(g3kubectl get secrets -n argo | grep $secretName) ]]; then
-        g3kubectl delete secret "$secretName" -n argo > /dev/null 2>&1
+      if [[ ! -z $(g3kubectl get secrets -n $argo_namespace | grep $secretName) ]]; then
+        g3kubectl delete secret "$secretName" -n $argo_namespace > /dev/null 2>&1
       fi
     done
     sleep 1  # I think delete is async - give backend a second to finish
     indexdFencePassword=$(cat $(gen3_secrets_folder)/creds.json | jq -r .indexd.user_db.$indexd_admin_user)
-    g3kubectl create secret generic "indexd-creds" --from-literal=user=$indexd_admin_user --from-literal=password=$indexdFencePassword -n argo
+    g3kubectl create secret generic "indexd-creds" --from-literal=user=$indexd_admin_user --from-literal=password=$indexdFencePassword -n $argo_namespace
   fi
 }
 
 function setup_argo_db() {
-  if ! secret="$(g3kubectl get secret argo-db-creds -n argo 2> /dev/null)"; then
+  if ! secret="$(g3kubectl get secret argo-db-creds -n $argo_namespace 2> /dev/null)"; then
     gen3_log_info "Setting up argo db persistence"
     gen3 db setup argo || true
     dbCreds=$(gen3 secrets decode argo-g3auto dbcreds.json)
-    g3kubectl create secret -n argo generic argo-db-creds --from-literal=db_host=$(echo $dbCreds  | jq -r .db_host) --from-literal=db_username=$(echo $dbCreds  | jq -r .db_username) --from-literal=db_password=$(echo $dbCreds  | jq -r .db_password) --from-literal=db_database=$(echo $dbCreds  | jq -r .db_database)
+    g3kubectl create secret -n $argo_namespace generic argo-db-creds --from-literal=db_host=$(echo $dbCreds  | jq -r .db_host) --from-literal=db_username=$(echo $dbCreds  | jq -r .db_username) --from-literal=db_password=$(echo $dbCreds  | jq -r .db_password) --from-literal=db_database=$(echo $dbCreds  | jq -r .db_database)
   else
     gen3_log_info "Argo DB setup already completed"
   fi
 }
 
-  setup_argo_buckets
+function setup_argo_template_secret() {
+  gen3_log_info "Started the template secret process"
+  downloadable_bucket_name=$(g3k_config_lookup '."downloadable-s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json)
+  # Check if the secret already exists
+    if [[ ! -z $(g3kubectl get secret argo-template-values-secret -n $argo_namespace) ]]; then
+      gen3_log_info "Argo template values secret already exists, assuming it's stale and deleting"
+      g3kubectl delete secret argo-template-values-secret -n $argo_namespace
+    fi
+  gen3_log_info "Creating argo template values secret"
+  g3kubectl create secret generic argo-template-values-secret --from-literal=DOWNLOADABLE_BUCKET=$downloadable_bucket_name -n $argo_namespace 
+}
+
+setup_argo_buckets
 # only do this if we are running in the default namespace
-if [[ "$ctxNamespace" == "default" || "$ctxNamespace" == "null" ]]; then
+if [[ "$ctxNamespace" == "default" || "$ctxNamespace" == "null" || "$override_namespace" == true ]]; then
   setup_argo_db
-  if (! helm status argo -n argo > /dev/null 2>&1 )  || [[ "$1" == "--force" ]]; then
-    DBHOST=$(kubectl get secrets -n argo argo-db-creds -o json | jq -r .data.db_host | base64 -d)
-    DBNAME=$(kubectl get secrets -n argo argo-db-creds -o json | jq -r .data.db_database | base64 -d)
+  setup_argo_template_secret 
+  if (! helm status argo -n $argo_namespace > /dev/null 2>&1 )  || [[ "$force" == true ]]; then
+    DBHOST=$(kubectl get secrets -n $argo_namespace argo-db-creds -o json | jq -r .data.db_host | base64 -d)
+    DBNAME=$(kubectl get secrets -n $argo_namespace argo-db-creds -o json | jq -r .data.db_database | base64 -d)
     if [[ -z $internalBucketName ]]; then
       BUCKET=$bucketName
     else
@@ -218,7 +246,7 @@ if [[ "$ctxNamespace" == "default" || "$ctxNamespace" == "null" ]]; then
 
     helm repo add argo https://argoproj.github.io/argo-helm --force-update 2> >(grep -v 'This is insecure' >&2)
     helm repo update 2> >(grep -v 'This is insecure' >&2)
-    helm upgrade --install argo argo/argo-workflows -n argo -f ${valuesFile} --version 0.29.1
+    helm upgrade --install argo argo/argo-workflows -n $argo_namespace -f ${valuesFile} --version 0.29.1
   else
     gen3_log_info "kube-setup-argo exiting - argo already deployed, use --force to redeploy"
   fi

From 90b66091c0d1d8b89e7182ad84201e99903d8e44 Mon Sep 17 00:00:00 2001
From: Mingfei Shao <2475897+mfshao@users.noreply.github.com>
Date: Thu, 29 Feb 2024 13:29:05 -0600
Subject: [PATCH 38/67] Update heal-cedar-data-ingest.py (#2490)

---
 files/scripts/healdata/heal-cedar-data-ingest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py
index d3ff246c3..1da4ac2d5 100644
--- a/files/scripts/healdata/heal-cedar-data-ingest.py
+++ b/files/scripts/healdata/heal-cedar-data-ingest.py
@@ -128,7 +128,7 @@ def get_related_studies(serial_num, hostname):
     related_study_result = []
 
     if serial_num:
-        mds = requests.get(f"https://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000")
+        mds = requests.get(f"http://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000")
         if mds.status_code == 200:
             related_study_metadata = mds.json()
 

From bd6bc767c0461f8fc1f1d90fa8a755be1e7fc381 Mon Sep 17 00:00:00 2001
From: Mingfei Shao <2475897+mfshao@users.noreply.github.com>
Date: Thu, 29 Feb 2024 13:49:47 -0600
Subject: [PATCH 39/67] fix missed import (#2491)

---
 .secrets.baseline                             | 346 ++++--------------
 .../healdata/heal-cedar-data-ingest.py        |   2 +-
 2 files changed, 73 insertions(+), 275 deletions(-)

diff --git a/.secrets.baseline b/.secrets.baseline
index b7e06622d..2583e269f 100644
--- a/.secrets.baseline
+++ b/.secrets.baseline
@@ -1,9 +1,9 @@
 {
   "exclude": {
-    "files": "^.secrets.baseline$",
+    "files": null,
     "lines": null
   },
-  "generated_at": "2024-02-23T20:30:41Z"
+  "generated_at": "2024-02-29T19:38:46Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -61,14 +61,12 @@
     "Chef/repo/data_bags/README.md": [
       {
         "hashed_secret": "8a9250639e092d90f164792e35073a9395bff366",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 45,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "6367c48dd193d56ea7b0baad25b19455e529f5ee",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 51,
         "type": "Secret Keyword"
@@ -77,25 +75,22 @@
     "Docker/jenkins/Jenkins-CI-Worker/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 121,
+        "line_number": 124,
         "type": "Secret Keyword"
       }
     ],
     "Docker/jenkins/Jenkins-Worker/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 143,
+        "line_number": 139,
         "type": "Secret Keyword"
       }
     ],
     "Docker/jenkins/Jenkins/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 107,
         "type": "Secret Keyword"
@@ -104,7 +99,6 @@
     "Docker/jenkins/Jenkins2/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 108,
         "type": "Secret Keyword"
@@ -113,7 +107,6 @@
     "Docker/sidecar/service.key": [
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1,
         "type": "Private Key"
@@ -122,7 +115,6 @@
     "Jenkins/Stacks/Jenkins/jenkins.env.sample": [
       {
         "hashed_secret": "eecee33686ac5861c2a7edc8b46bd0e5432bfddd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Secret Keyword"
@@ -131,7 +123,6 @@
     "ansible/roles/awslogs/defaults/main.yaml": [
       {
         "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Basic Auth Credentials"
@@ -140,14 +131,12 @@
     "ansible/roles/slurm/README.md": [
       {
         "hashed_secret": "4acfde1ff9c353ba2ef0dbe0df73bda2743cba42",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 86,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "579649582303921502d9e6d3f8755f13fdd2b476",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 86,
         "type": "Secret Keyword"
@@ -156,7 +145,6 @@
     "apis_configs/config_helper.py": [
       {
         "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 66,
         "type": "Basic Auth Credentials"
@@ -165,7 +153,6 @@
     "apis_configs/fence_credentials.json": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 23,
         "type": "Secret Keyword"
@@ -174,21 +161,18 @@
     "apis_configs/fence_settings.py": [
       {
         "hashed_secret": "3ef0fb8a603abdc0b6caac44a23fdc6792f77ddf",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Basic Auth Credentials"
       },
       {
         "hashed_secret": "b60d121b438a380c343d5ec3c2037564b82ffef3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 58,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 80,
         "type": "Basic Auth Credentials"
@@ -197,7 +181,6 @@
     "apis_configs/indexd_settings.py": [
       {
         "hashed_secret": "0a0d18c85e096611b5685b62bc60ec534d19bacc",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 59,
         "type": "Basic Auth Credentials"
@@ -206,7 +189,6 @@
     "apis_configs/peregrine_settings.py": [
       {
         "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 46,
         "type": "Basic Auth Credentials"
@@ -215,7 +197,6 @@
     "apis_configs/sheepdog_settings.py": [
       {
         "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 46,
         "type": "Basic Auth Credentials"
@@ -224,7 +205,6 @@
     "doc/Gen3-data-upload.md": [
       {
         "hashed_secret": "b8bd20d4a2701dc3aba0efbbf325f1359392d93e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
@@ -233,7 +213,6 @@
     "doc/api.md": [
       {
         "hashed_secret": "625de83a7517422051911680cc803921ff99db90",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 47,
         "type": "Hex High Entropy String"
@@ -242,28 +221,24 @@
     "doc/gen3OnK8s.md": [
       {
         "hashed_secret": "2db6d21d365f544f7ca3bcfb443ac96898a7a069",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 113,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "ff9ee043d85595eb255c05dfe32ece02a53efbb2",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 143,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "70374248fd7129088fef42b8f568443f6dce3a48",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 170,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "bcf22dfc6fb76b7366b1f1675baf2332a0e6a7ce",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 189,
         "type": "Secret Keyword"
@@ -272,7 +247,6 @@
     "doc/kube-setup-data-ingestion-job.md": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Secret Keyword"
@@ -281,7 +255,6 @@
     "doc/logs.md": [
       {
         "hashed_secret": "9addbf544119efa4a64223b649750a510f0d463f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Secret Keyword"
@@ -290,7 +263,6 @@
     "doc/slurm_cluster.md": [
       {
         "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 184,
         "type": "Secret Keyword"
@@ -299,14 +271,12 @@
     "files/dashboard/usage-reports/package-lock.json": [
       {
         "hashed_secret": "e095101882f706c4de95e0f75c5bcb9666e3f448",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 10,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "5422e4f96964d5739998b25ac214520c1b113e5b",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 15,
         "type": "Base64 High Entropy String"
@@ -315,14 +285,12 @@
     "gen3/bin/api.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 407,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 477,
         "type": "Secret Keyword"
@@ -331,7 +299,6 @@
     "gen3/bin/kube-dev-namespace.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 135,
         "type": "Secret Keyword"
@@ -340,7 +307,6 @@
     "gen3/bin/kube-setup-argo.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 206,
         "type": "Secret Keyword"
@@ -349,7 +315,6 @@
     "gen3/bin/kube-setup-aurora-monitoring.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 59,
         "type": "Secret Keyword"
@@ -358,7 +323,6 @@
     "gen3/bin/kube-setup-certs.sh": [
       {
         "hashed_secret": "2e9ee120fd25e31048598693aca91d5473898a99",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 50,
         "type": "Secret Keyword"
@@ -367,14 +331,12 @@
     "gen3/bin/kube-setup-dashboard.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 40,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 41,
         "type": "Secret Keyword"
@@ -383,14 +345,12 @@
     "gen3/bin/kube-setup-data-ingestion-job.sh": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 37,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "8695a632956b1b0ea7b66993dcc98732da39148c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 102,
         "type": "Secret Keyword"
@@ -399,7 +359,6 @@
     "gen3/bin/kube-setup-dicom-server.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 43,
         "type": "Secret Keyword"
@@ -408,7 +367,6 @@
     "gen3/bin/kube-setup-dicom.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 78,
         "type": "Secret Keyword"
@@ -417,14 +375,26 @@
     "gen3/bin/kube-setup-jenkins.sh": [
       {
         "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
+        "is_verified": false,
+        "line_number": 22,
+        "type": "Secret Keyword"
+      }
+    ],
+    "gen3/bin/kube-setup-jenkins2.sh": [
+      {
+        "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f",
+        "is_verified": false,
+        "line_number": 18,
+        "type": "Secret Keyword"
+      },
+      {
+        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
         "is_verified": false,
         "line_number": 22,
         "type": "Secret Keyword"
@@ -433,7 +403,6 @@
     "gen3/bin/kube-setup-metadata.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 35,
         "type": "Secret Keyword"
@@ -442,21 +411,18 @@
     "gen3/bin/kube-setup-revproxy.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 38,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 55,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 57,
         "type": "Secret Keyword"
@@ -465,21 +431,18 @@
     "gen3/bin/kube-setup-secrets.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 79,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 82,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "6f7531b95bbc99ac25a5cc82edb825f319c5dee8",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 95,
         "type": "Secret Keyword"
@@ -488,14 +451,12 @@
     "gen3/bin/kube-setup-sftp.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 36,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "83d11e3aec005a3b9a2077c6800683e202a95af4",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 51,
         "type": "Secret Keyword"
@@ -504,7 +465,6 @@
     "gen3/bin/kube-setup-sheepdog.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 33,
         "type": "Secret Keyword"
@@ -513,28 +473,24 @@
     "gen3/bin/kube-setup-sower-jobs.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 25,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 120,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 122,
         "type": "Secret Keyword"
@@ -543,21 +499,18 @@
     "gen3/bin/kube-setup-ssjdispatcher.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 117,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "7992309146efaa8da936e34b0bd33242cd0e9f93",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 184,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 197,
         "type": "Secret Keyword"
@@ -566,14 +519,12 @@
     "gen3/lib/aws.sh": [
       {
         "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 640,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "5b4b6c62d3d99d202f095c38c664eded8f640ce8",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 660,
         "type": "Secret Keyword"
@@ -582,14 +533,12 @@
     "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml": [
       {
         "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 33,
         "type": "Basic Auth Credentials"
       },
       {
         "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 286,
         "type": "Secret Keyword"
@@ -598,7 +547,6 @@
     "gen3/lib/bootstrap/templates/Gen3Secrets/creds.json": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
@@ -607,7 +555,6 @@
     "gen3/lib/bootstrap/templates/Gen3Secrets/g3auto/dbfarm/servers.json": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Secret Keyword"
@@ -616,7 +563,6 @@
     "gen3/lib/logs/utils.sh": [
       {
         "hashed_secret": "76143b4ffc8aa2a53f9700ce229f904e69f1e8b5",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 3,
         "type": "Secret Keyword"
@@ -625,7 +571,6 @@
     "gen3/lib/manifestDefaults/hatchery/hatchery.json": [
       {
         "hashed_secret": "0da0e0005ca04acb407af2681d0bede6d9406039",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 78,
         "type": "Secret Keyword"
@@ -634,14 +579,12 @@
     "gen3/lib/onprem.sh": [
       {
         "hashed_secret": "29e52a9bac8f274fa41c51fce9c98eba0dd99cb3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 68,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "50f013532a9770a2c2cfdc38b7581dd01df69b70",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 84,
         "type": "Secret Keyword"
@@ -650,14 +593,12 @@
     "gen3/lib/secrets/rotate-postgres.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 162,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 250,
         "type": "Secret Keyword"
@@ -666,49 +607,42 @@
     "gen3/lib/testData/etlconvert/expected2.yaml": [
       {
         "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 10,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 13,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 16,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 33,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 35,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 36,
         "type": "Base64 High Entropy String"
@@ -717,7 +651,6 @@
     "gen3/test/secretsTest.sh": [
       {
         "hashed_secret": "c2c715092ef59cba22520f109f041efca84b8938",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 25,
         "type": "Secret Keyword"
@@ -726,28 +659,24 @@
     "gen3/test/terraformTest.sh": [
       {
         "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 156,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 172,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 175,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "329b7cd8191942bedd337107934d365c43a86e6c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 175,
         "type": "Secret Keyword"
@@ -756,21 +685,18 @@
     "kube/services/argocd/values.yaml": [
       {
         "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 360,
         "type": "Private Key"
       },
       {
         "hashed_secret": "edbd5e119f94badb9f99a67ac6ff4c7a5204ad61",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 379,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "91dfd9ddb4198affc5c194cd8ce6d338fde470e2",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 412,
         "type": "Secret Keyword"
@@ -779,7 +705,6 @@
     "kube/services/datadog/values.yaml": [
       {
         "hashed_secret": "4a8ce7ae6a8a7f2624e232b61b18c2ac9789c44b",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 23,
         "type": "Secret Keyword"
@@ -788,401 +713,362 @@
     "kube/services/fenceshib/fenceshib-configmap.yaml": [
       {
         "hashed_secret": "a985e14b9d6744a2d04f29347693b55c116e478c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 375,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "adc747bc5eb82ef4b017f5c3759dcee5aa28c36f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 376,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "59b1702ff0eaf92c9271cbd12f587de97df7e13b",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 377,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "b4a748bbfbbca8925d932a47ab3dcb970d34caf5",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 378,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "af646701a84f7dd9f0e87753f54def881326e78a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 379,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "20c15ad9742124dc06e1612282c49bb443ebcbd9",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 380,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "9caded71b967a11b7a6cd0f20db91f06f3517d12",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 381,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "8f19501bc9241b71f7b6db929fb35ab12635dcd7",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 382,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "d6220f6a55df1ed11c4250f42ab07bb9da20541a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 383,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "dadd9b96636f9529f2547d05d754dc310ceba0c3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 384,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "3074bc66584550e20c3697a28f67a0762394943c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 385,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "823131319b4c4b4688f44d3e832bfa9696f16b52",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 386,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "015b780cbfb76988caf52de8ac974a6781e53110",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 387,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "5c8fac33207d74d667680ade09447ea8f43b76d7",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 388,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "c0c4bb09d8394e8f001e337bd27ccac355433d9e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 389,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f95631bcbbbc56e18487dcb242cfb1b3e74b16a1",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 390,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "01a692ab6232e0882a313d148981bab58ab98f53",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 391,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "658060a680d415ce6690ad2c3b622ddb33ddd50a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 392,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "80915b0bd9daa5e1f95cad573892980b1b5a2294",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 393,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "cc55977b293d8cdca8a2c19dfea6874e70057c41",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 394,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "e400ed02add75dd5f3a8c212857acf12027437d1",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 395,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "2e819c8baa3b0508a32b77de258655b3f3a6f7cb",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 396,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "546ed926d58ea5492ab6adb8be94a67aa44ac433",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 397,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f056f2deceed268e7af6dbdaf2577079c76e006a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 398,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "d75efee28f4798c3a9c6f44b78a8500513ef28b2",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 399,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "7803ae08cdc22a5e0b025eff3c9ef0628eedc165",
-        "is_secret": false,
+        "hashed_secret": "fbad0bc8f7792b03f89cd3780eb7cf79f284c525",
         "is_verified": false,
         "line_number": 419,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "b8b61e87f5b58b0eeb597b2122ea0cea2ccab3d9",
-        "is_secret": false,
+        "hashed_secret": "3f6480956a775dacb44e2c39aa3d4722a347f7ab",
         "is_verified": false,
         "line_number": 420,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "787745fc904c3bd7eddc3d1aab683a376c13890f",
-        "is_secret": false,
+        "hashed_secret": "17f32ae55b14d708ca121722c2cae37189f19daf",
         "is_verified": false,
         "line_number": 423,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "81361d672f238f505a6246ef9b655ee2f48d67e7",
-        "is_secret": false,
+        "hashed_secret": "08a74689ca077515d406093720a7e5675fb42bb8",
         "is_verified": false,
         "line_number": 424,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "7c98bff76ac3f273d15ed9bc3dd5294d323ab577",
-        "is_secret": false,
+        "hashed_secret": "fa577bb3b2600d2d522dcfea8f1e34896760fcf2",
         "is_verified": false,
         "line_number": 425,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "46038fc88daceed8dd46817ca45c72ae0270fdd4",
-        "is_secret": false,
+        "hashed_secret": "37254f15cca211a1bd5f7ceb23de2b3eb8fb33aa",
         "is_verified": false,
         "line_number": 426,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "acad0c57b4f5cbed1b4863ed06d02784180a9f92",
-        "is_secret": false,
+        "hashed_secret": "86865593e038509467b91c2d5f36ccc09c3f422b",
         "is_verified": false,
         "line_number": 427,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "1b57f49a6ee337c16ecd6aabfc0dff3b3821cd09",
-        "is_secret": false,
+        "hashed_secret": "a899a8d9e114b2a8e108f90e6a72c056db22489f",
         "is_verified": false,
         "line_number": 428,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "5b688158be36e8b3f265a462ed599dcf69290084",
-        "is_secret": false,
+        "hashed_secret": "756b4825f886afd83c25563ac9d45f318d695c48",
         "is_verified": false,
         "line_number": 429,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "965996e12c8b50b3c325d96003e8984a4ece658a",
-        "is_secret": false,
+        "hashed_secret": "89882eeb0aca97717a7e4afcf4bc08d077813c7f",
         "is_verified": false,
         "line_number": 430,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "584f0c58e764e948af1a35c9e60447aa0f84c6f5",
-        "is_secret": false,
+        "hashed_secret": "347140d7b7ceb4e501c3c9c2ea4f29338e2f145e",
         "is_verified": false,
         "line_number": 431,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "bcaf897786d060a675ee9d654a84ae8baf96e9d0",
-        "is_secret": false,
+        "hashed_secret": "61dbf70eb10d609e60c7b87faf8f755ff48abc46",
         "is_verified": false,
         "line_number": 432,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "0c09277fa183e06d32065f9386a3b4190b445df3",
-        "is_secret": false,
+        "hashed_secret": "24cd54c4b2f58378bba008cb2df68ac663fba7c8",
         "is_verified": false,
         "line_number": 433,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "5a51be06b305d6664e4afd25f21869b0f8b5039b",
-        "is_secret": false,
+        "hashed_secret": "fa4f9626ae4b98f4b61203c5bafb6f21c9c31e5d",
         "is_verified": false,
         "line_number": 434,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "b38404f8853d734e3d03577b2c1084b4540c8708",
-        "is_secret": false,
+        "hashed_secret": "b1370003d9cc1e346c83dba33e0418c7775a0c15",
         "is_verified": false,
         "line_number": 435,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "126ccc602cffcb8292beb57137f7f6719e317b72",
-        "is_secret": false,
+        "hashed_secret": "c66526e195e423a7ba7d68ac661cdcd8600dcd1f",
         "is_verified": false,
         "line_number": 436,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "6681c1d7e1d327642a32cb8864ad51e4b8f981e5",
-        "is_secret": false,
+        "hashed_secret": "d29d7044f0944eb30e02cf445f6998e3343dd811",
         "is_verified": false,
         "line_number": 437,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "7f7b1f316ece195e5f584fe2faf6f9edc6942c6f",
-        "is_secret": false,
+        "hashed_secret": "80a869460f33722387d8d58e7d9d2e1bbd5d1fe1",
+        "is_verified": false,
+        "line_number": 438,
+        "type": "Base64 High Entropy String"
+      },
+      {
+        "hashed_secret": "4a06e2a02cbc665adccb4162dc57836895da65b8",
         "is_verified": false,
         "line_number": 439,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "bb908c7bc655057f2edc42815c5dff82e9dea529",
-        "is_secret": false,
+        "hashed_secret": "ba2549f35835dfa101d3f660f7604dc78e3e226f",
         "is_verified": false,
         "line_number": 440,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "bc2a0d18e3dd142df7b34e95342d47bf8aadabcb",
-        "is_secret": false,
+        "hashed_secret": "f354d4ee5fdb94ad29c7b3600264467f45b80eaa",
         "is_verified": false,
         "line_number": 441,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "d60f0bcea109bb6edb6e45fd387f5f2c86e49e1a",
-        "is_secret": false,
+        "hashed_secret": "bf17b587868ba7c3db9865b114261b5b8f1df870",
         "is_verified": false,
         "line_number": 442,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "e549dd40a741557cc1c4e377df0a141354e22688",
-        "is_secret": false,
+        "hashed_secret": "de1fd7a0d32cba528b4d80818c6601f2588d5383",
         "is_verified": false,
         "line_number": 443,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "2dd2486dae84cad50387c20bf687b6fbc6162b58",
-        "is_secret": false,
+        "hashed_secret": "bcad65055f6de654541db2bf27d4e27bd54d94c7",
         "is_verified": false,
         "line_number": 444,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "71622010fc7eb09d9273f59c548bde6a5da5dc0e",
-        "is_secret": false,
+        "hashed_secret": "f2e16f2dd532f65f79341342fdf57a093fc408d8",
         "is_verified": false,
         "line_number": 445,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "6f0115cf53bd49ec990c562ac6cbfc452c83cd46",
-        "is_secret": false,
+        "hashed_secret": "bb036a679a7d2df9fd2ca57068a446bf7f7dd106",
         "is_verified": false,
         "line_number": 446,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "70dddd534b2f9bb70871fefe0845b79c3b69363f",
-        "is_secret": false,
+        "hashed_secret": "5aa6568b1e8185578a6e964f5c322783ad349554",
+        "is_verified": false,
+        "line_number": 447,
+        "type": "Base64 High Entropy String"
+      },
+      {
+        "hashed_secret": "4d14835ff0b0bf5aad480296cb705c74ac65f413",
         "is_verified": false,
         "line_number": 448,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "acf3536b0416aa99608b0be17e87655370ece829",
-        "is_secret": false,
+        "hashed_secret": "3f23f77dcf454ad73c4d61c44fd9aa584ef946c1",
+        "is_verified": false,
+        "line_number": 451,
+        "type": "Base64 High Entropy String"
+      },
+      {
+        "hashed_secret": "1739fe5e5dfcf851b64f8b7b11538f1de29ce0b5",
         "is_verified": false,
-        "line_number": 449,
+        "line_number": 452,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "1d13ee35c7279c1fae1c6474ed47611994273e41",
-        "is_secret": false,
+        "hashed_secret": "8129db302110714fc735e3494bd82a65690e0963",
         "is_verified": false,
-        "line_number": 450,
+        "line_number": 453,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "d38cf89b25bd7378cdb4e00b4b59293001dd500b",
-        "is_secret": false,
+        "hashed_secret": "b48bfc62091164086a703115a0e68bdb09212591",
         "is_verified": false,
-        "line_number": 451,
+        "line_number": 454,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "1648f34ce2f1b563a8ed1c6d5d55b5e76a395903",
-        "is_secret": false,
+        "hashed_secret": "a10284feaf27f84081073a3267e3dce24ca7b911",
         "is_verified": false,
-        "line_number": 452,
+        "line_number": 455,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "9bf63f6f49fb01ff80959bc5a60c8688df92cc02",
-        "is_secret": false,
+        "hashed_secret": "3fd80f31de4be8dde9d2b421e832c7d4043fd49a",
         "is_verified": false,
-        "line_number": 453,
+        "line_number": 456,
         "type": "Base64 High Entropy String"
       }
     ],
     "kube/services/jobs/indexd-authz-job.yaml": [
       {
         "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 87,
         "type": "Basic Auth Credentials"
@@ -1191,14 +1077,12 @@
     "kube/services/monitoring/grafana-values.yaml": [
       {
         "hashed_secret": "2ae868079d293e0a185c671c7bcdac51df36e385",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 162,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "7a64ff8446b06d38dc271019994f13823a2cbcf4",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 166,
         "type": "Secret Keyword"
@@ -1207,7 +1091,6 @@
     "kube/services/revproxy/helpers.js": [
       {
         "hashed_secret": "1d278d3c888d1a2fa7eed622bfc02927ce4049af",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 10,
         "type": "Base64 High Entropy String"
@@ -1216,7 +1099,6 @@
     "kube/services/revproxy/helpersTest.js": [
       {
         "hashed_secret": "e029d4904cc728879d70030572bf37d4510367cb",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 22,
         "type": "JSON Web Token"
@@ -1225,7 +1107,6 @@
     "kube/services/superset/superset-deploy.yaml": [
       {
         "hashed_secret": "96e4aceb7cf284be363aa248a32a7cc89785a9f7",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 38,
         "type": "Secret Keyword"
@@ -1234,14 +1115,12 @@
     "kube/services/superset/superset-redis.yaml": [
       {
         "hashed_secret": "4af3596275edcb7cd5cc6c3c38bc10479902a08f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 165,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9fe1c31809da38c55b2b64bfab47b92bc5f6b7b9",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 265,
         "type": "Secret Keyword"
@@ -1250,35 +1129,30 @@
     "kube/services/superset/values.yaml": [
       {
         "hashed_secret": "6f803b24314c39062efe38d0c1da8c472f47eab3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 54,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "6eae3a5b062c6d0d79f070c26e6d62486b40cb46",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 86,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "3eb416223e9e69e6bb8ee19793911ad1ad2027d8",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 212,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "ff55435345834a3fe224936776c2aa15f6ed5358",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 396,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "98a84a63e5633d17e3b27b69695f87aa7189e9dc",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 503,
         "type": "Secret Keyword"
@@ -1287,280 +1161,240 @@
     "package-lock.json": [
       {
         "hashed_secret": "0656ad0df3af4633dc369f13d5e8806973c5fd9d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1481,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "00091d875d922437c5fc9e6067a08e78c2482e87",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1489,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "c4e5cc37e115bf7d86e76e3d799705bf691e4d00",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1521,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "0512e37fbedf1d16828680a038a241b4780a5c04",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1547,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "01868fd50edbfe6eb91e5b01209b543adc6857af",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1611,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a6f48bf1e398deffc7fd31da17c3506b46c97a93",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1640,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "85ce358dbdec0996cf3ccd2bf1c6602af68c181e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1648,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "6f9bfb49cb818d2fe07592515e4c3f7a0bbd7e0e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1664,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "7098a3e6d6d2ec0a40f04fe12509c5c6f4c49c0e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1683,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "1664ad175bba1795a7ecad572bae7e0740b94f56",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1733,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "1ec4ce2eb945ce2f816dcb6ebdd1e10247f439a3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1742,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a7af5768a6d936e36f28e1030d7f894d7aaf555e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1755,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "6fbc7dd864586173160874f2a86ca7d2d552cb85",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1769,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "81a961f2c89c6209328b74a8768e30fd76c3ac72",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1855,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "797d4751c536c421cb82b9f62e0a804af30d78f5",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1889,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "0d55babfa89f240142c0adfc7b560500a1d3ae7c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1894,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "e9fdc3025cd10bd8aa4508611e6b7b7a9d650a2c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1921,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "4cf9419259c0ce8eee84b468af3c72db8b001620",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1950,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "24816e3eb4308e247bde7c1d09ffb7b79c519b71",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1983,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "e9adfe8a333d45f4776fe0eab31608be5d7b6a7d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2004,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "03d6fb388dd1b185129b14221f7127715822ece6",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2013,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "ee161bb3f899720f95cee50a5f9ef9c9ed96278b",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2046,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "ebeb5b574fa1ed24a40248275e6136759e766466",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2078,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a6a555a428522ccf439fd516ce7c7e269274363f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2083,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f7f85d9f7c87f1e576dcaf4cf50f35728f9a3265",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2111,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "3f1646b60abe74297d2f37a1eee5dc771ad834fc",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2138,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "fd933c71e82d5519ae0cb0779b370d02f6935759",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2143,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "7090aa59cb52ad1f1810b08c4ac1ddf5c8fce523",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2150,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "756444bea4ea3d67844d8ddf58ad32356e9c2430",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2188,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f74135fdd6b8dafdfb01ebbc61c5e5c24ee27cf8",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2291,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "56fbae787f4aed7d0632e95840d71bd378d3a36f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2303,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "81cb6be182eb79444202c4563080aee75296a672",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2308,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f0f3f7bce32184893046ac5f8cc80da56c3ca539",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2317,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "097893233346336f4003acfb6eb173ee59e648f0",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2327,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "bb14c3b4ef4a9f2e86ffdd44b88d9b6729419671",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2332,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "71344a35cff67ef081920095d1406601fb5e9b97",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2340,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "eb3db6990fd43477a35dfeffc90b3f1ffa83c7bd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2349,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "266288bdc14807b538d1e48a5891e361fa9b4a14",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2357,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "800477261175fd21f23e7321923e1fba6ae55471",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2369,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "3f0c251b9c2c21454445a98fde6915ceacde2136",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2387,
         "type": "Base64 High Entropy String"
@@ -1569,7 +1403,6 @@
     "tf_files/aws/cognito/README.md": [
       {
         "hashed_secret": "f6920f370a30262b7dd70e97293c73ec89739b70",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 106,
         "type": "Secret Keyword"
@@ -1578,14 +1411,12 @@
     "tf_files/aws/commons/README.md": [
       {
         "hashed_secret": "d02e53411e8cb4cd709778f173f7bc9a3455f8ed",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 60,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9dc0da3613af850c5a018b0a88a5626fb8888e4e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 78,
         "type": "Secret Keyword"
@@ -1594,7 +1425,6 @@
     "tf_files/aws/eks/sample.tfvars": [
       {
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 107,
         "type": "Hex High Entropy String"
@@ -1603,7 +1433,6 @@
     "tf_files/aws/eks/variables.tf": [
       {
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 133,
         "type": "Hex High Entropy String"
@@ -1612,14 +1441,12 @@
     "tf_files/aws/modules/common-logging/README.md": [
       {
         "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 57,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 59,
         "type": "Hex High Entropy String"
@@ -1628,28 +1455,24 @@
     "tf_files/aws/modules/common-logging/lambda_function.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Hex High Entropy String"
@@ -1658,21 +1481,18 @@
     "tf_files/aws/modules/common-logging/testLambda.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
@@ -1681,7 +1501,6 @@
     "tf_files/aws/modules/eks/variables.tf": [
       {
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 113,
         "type": "Hex High Entropy String"
@@ -1690,14 +1509,12 @@
     "tf_files/aws/modules/management-logs/README.md": [
       {
         "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 54,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 56,
         "type": "Hex High Entropy String"
@@ -1706,28 +1523,24 @@
     "tf_files/aws/modules/management-logs/lambda_function.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Hex High Entropy String"
@@ -1736,42 +1549,36 @@
     "tf_files/aws/modules/management-logs/testLambda.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "3cf8eb4e9254e1d6cc523da01f8b798b9a83101a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "51118900cd675df1b44f254057398f3e52902a5d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "60a6dfc8d43cd2f5c6292899fc2f94f2d4fc32c4",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Hex High Entropy String"
@@ -1780,7 +1587,6 @@
     "tf_files/aws/slurm/README.md": [
       {
         "hashed_secret": "fd85d792fa56981cf6a8d2a5c0857c74af86e99d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 83,
         "type": "Secret Keyword"
@@ -1789,7 +1595,6 @@
     "tf_files/azure/cloud.tf": [
       {
         "hashed_secret": "7c1a4b52b64e4106041971c345a1f3eab58fb2a4",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 424,
         "type": "Secret Keyword"
@@ -1798,7 +1603,6 @@
     "tf_files/gcp-bwg/roots/commons_setup/variables/answerfile-commons_setup-001.template.tfvars": [
       {
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 231,
         "type": "Secret Keyword"
@@ -1807,7 +1611,6 @@
     "tf_files/gcp-bwg/roots/templates/answerfile-commons_setup-001.template.tfvars": [
       {
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 231,
         "type": "Secret Keyword"
@@ -1816,7 +1619,6 @@
     "tf_files/gcp-bwg/roots/templates/answerfile-env-tenant.user.tfvars_NO_APP_SETUP": [
       {
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 262,
         "type": "Secret Keyword"
@@ -1825,21 +1627,18 @@
     "tf_files/gcp/commons/sample.tfvars": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 11,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "253c7b5e7c83a86346fc4501495b130813f08105",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 37,
         "type": "Secret Keyword"
@@ -1848,7 +1647,6 @@
     "tf_files/shared/modules/k8s_configs/creds.tpl": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 8,
         "type": "Secret Keyword"
diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py
index 1da4ac2d5..730a3b36e 100644
--- a/files/scripts/healdata/heal-cedar-data-ingest.py
+++ b/files/scripts/healdata/heal-cedar-data-ingest.py
@@ -1,5 +1,5 @@
 import argparse
-import json
+import copy
 import sys
 import requests
 import pydash

From 8103e82b1044daca4290831db63653b40b24b97c Mon Sep 17 00:00:00 2001
From: Hara Prasad <haraprasadj@uchicago.edu>
Date: Thu, 29 Feb 2024 12:58:32 -0800
Subject: [PATCH 40/67] Add jenkins-dcp to the env pool reset script (#2489)

---
 files/scripts/ci-env-pool-reset.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/files/scripts/ci-env-pool-reset.sh b/files/scripts/ci-env-pool-reset.sh
index c0c1f67c6..362cfbfd5 100644
--- a/files/scripts/ci-env-pool-reset.sh
+++ b/files/scripts/ci-env-pool-reset.sh
@@ -29,6 +29,7 @@ source "${GEN3_HOME}/gen3/gen3setup.sh"
 cat - > jenkins-envs-services.txt <<EOF
 jenkins-blood
 jenkins-brain
+jenkins-dcp
 jenkins-genomel
 jenkins-niaid
 EOF

From b9ab8af811ec2b4c5a7bf341e1f1051f3dfa840b Mon Sep 17 00:00:00 2001
From: George Thomas <98996322+george42-ctds@users.noreply.github.com>
Date: Thu, 29 Feb 2024 15:20:15 -0800
Subject: [PATCH 41/67] (HP-572) create dynamodb table for gen3-license
 workspace (#2441)

* (HP-572) create dynamodb table for gen3-license workspace

* (HP-572) get GSI from hatchery config

* (HP-572) remove extra quotes from 'GSI'

* (HP-572) move distribute-license for backwards compatibility

---------

Co-authored-by: Mingfei Shao <2475897+mfshao@users.noreply.github.com>
---
 gen3/bin/kube-setup-hatchery.sh | 57 ++++++++++++++++++++++++++++++---
 1 file changed, 53 insertions(+), 4 deletions(-)

diff --git a/gen3/bin/kube-setup-hatchery.sh b/gen3/bin/kube-setup-hatchery.sh
index 691fb354a..5454d1e24 100644
--- a/gen3/bin/kube-setup-hatchery.sh
+++ b/gen3/bin/kube-setup-hatchery.sh
@@ -20,11 +20,60 @@ gen3 jupyter j-namespace setup
 #
 (g3k_kv_filter ${GEN3_HOME}/kube/services/hatchery/serviceaccount.yaml BINDING_ONE "name: hatchery-binding1-$namespace" BINDING_TWO "name: hatchery-binding2-$namespace" CURRENT_NAMESPACE "namespace: $namespace" | g3kubectl apply -f -) || true
 
+function exists_or_create_gen3_license_table() {
+    # Create dynamodb table for gen3-license if it does not exist.
+    TARGET_TABLE="$1"
+    echo "Checking for dynamoDB table: ${TARGET_TABLE}"
 
-# cron job to distribute licenses if using Stata workspaces
-if [ "$(g3kubectl get configmaps/manifest-hatchery -o yaml | grep "\"image\": .*stata.*")" ];
-then
-    gen3 job cron distribute-licenses '* * * * *'
+    FOUND_TABLE=`aws dynamodb list-tables | jq -r .TableNames | jq -c -r '.[]' | grep $TARGET_TABLE`
+    if [ -n "$FOUND_TABLE" ]; then
+        echo "Target table already exists in dynamoDB: $FOUND_TABLE"
+    else
+        echo "Creating table ${TARGET_TABLE}"
+        GSI=`g3kubectl get configmaps/manifest-hatchery -o json | jq -r '.data."license-user-maps-global-secondary-index"'`
+        if [[ -z "$GSI" || "$GSI" == "null" ]]; then
+            echo "Error: No global-secondary-index in configuration"
+            return 0
+        fi
+        aws dynamodb create-table \
+            --no-cli-pager \
+            --table-name "$TARGET_TABLE" \
+            --attribute-definitions AttributeName=itemId,AttributeType=S \
+                AttributeName=environment,AttributeType=S \
+                AttributeName=isActive,AttributeType=S \
+            --key-schema AttributeName=itemId,KeyType=HASH \
+                AttributeName=environment,KeyType=RANGE \
+            --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5 \
+            --global-secondary-indexes \
+                "[
+                    {
+                        \"IndexName\": \"$GSI\",
+                        \"KeySchema\": [{\"AttributeName\":\"environment\",\"KeyType\":\"HASH\"},
+                            {\"AttributeName\":\"isActive\",\"KeyType\":\"RANGE\"}],
+                        \"Projection\":{
+                            \"ProjectionType\":\"INCLUDE\",
+                            \"NonKeyAttributes\":[\"itemId\",\"userId\",\"licenseId\",\"licenseType\"]
+                        },
+                        \"ProvisionedThroughput\": {
+                            \"ReadCapacityUnits\": 5,
+                            \"WriteCapacityUnits\": 3
+                        }
+                    }
+                ]"
+    fi
+}
+
+TARGET_TABLE=`g3kubectl get configmaps/manifest-hatchery -o json | jq -r '.data."license-user-maps-dynamodb-table"'`
+if [[ -z "$TARGET_TABLE" || "$TARGET_TABLE" == "null" ]]; then
+    echo "No gen3-license table in configuration"
+    # cron job to distribute licenses if using Stata workspaces but not using dynamoDB
+    if [ "$(g3kubectl get configmaps/manifest-hatchery -o yaml | grep "\"image\": .*stata.*")" ];
+    then
+        gen3 job cron distribute-licenses '* * * * *'
+    fi
+else
+    echo "Found gen3-license table in configuration: $TARGET_TABLE"
+    exists_or_create_gen3_license_table "$TARGET_TABLE"
 fi
 
 policy=$( cat <<EOM

From fae0b676d78b25e637e086122d060c3b962b1772 Mon Sep 17 00:00:00 2001
From: Mingfei Shao <2475897+mfshao@users.noreply.github.com>
Date: Fri, 1 Mar 2024 15:46:10 -0600
Subject: [PATCH 42/67] Fix/revert (#2493)

* Revert "fix missed import (#2491)"

This reverts commit bd6bc767c0461f8fc1f1d90fa8a755be1e7fc381.

* Revert "Update heal-cedar-data-ingest.py (#2490)"

This reverts commit 90b66091c0d1d8b89e7182ad84201e99903d8e44.

* Revert "updating the cedar data ingest (#2472)"

This reverts commit 6c27fc9a7fe6cd3affd7a108db3c80b51c664ad1.
---
 .secrets.baseline                             | 346 ++++++++++++++----
 .../healdata/heal-cedar-data-ingest.py        |  98 +----
 2 files changed, 275 insertions(+), 169 deletions(-)

diff --git a/.secrets.baseline b/.secrets.baseline
index 2583e269f..b7e06622d 100644
--- a/.secrets.baseline
+++ b/.secrets.baseline
@@ -1,9 +1,9 @@
 {
   "exclude": {
-    "files": null,
+    "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-02-29T19:38:46Z",
+  "generated_at": "2024-02-23T20:30:41Z"
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -61,12 +61,14 @@
     "Chef/repo/data_bags/README.md": [
       {
         "hashed_secret": "8a9250639e092d90f164792e35073a9395bff366",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 45,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "6367c48dd193d56ea7b0baad25b19455e529f5ee",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 51,
         "type": "Secret Keyword"
@@ -75,22 +77,25 @@
     "Docker/jenkins/Jenkins-CI-Worker/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
+        "is_secret": false,
         "is_verified": false,
-        "line_number": 124,
+        "line_number": 121,
         "type": "Secret Keyword"
       }
     ],
     "Docker/jenkins/Jenkins-Worker/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
+        "is_secret": false,
         "is_verified": false,
-        "line_number": 139,
+        "line_number": 143,
         "type": "Secret Keyword"
       }
     ],
     "Docker/jenkins/Jenkins/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 107,
         "type": "Secret Keyword"
@@ -99,6 +104,7 @@
     "Docker/jenkins/Jenkins2/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 108,
         "type": "Secret Keyword"
@@ -107,6 +113,7 @@
     "Docker/sidecar/service.key": [
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1,
         "type": "Private Key"
@@ -115,6 +122,7 @@
     "Jenkins/Stacks/Jenkins/jenkins.env.sample": [
       {
         "hashed_secret": "eecee33686ac5861c2a7edc8b46bd0e5432bfddd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Secret Keyword"
@@ -123,6 +131,7 @@
     "ansible/roles/awslogs/defaults/main.yaml": [
       {
         "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Basic Auth Credentials"
@@ -131,12 +140,14 @@
     "ansible/roles/slurm/README.md": [
       {
         "hashed_secret": "4acfde1ff9c353ba2ef0dbe0df73bda2743cba42",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 86,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "579649582303921502d9e6d3f8755f13fdd2b476",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 86,
         "type": "Secret Keyword"
@@ -145,6 +156,7 @@
     "apis_configs/config_helper.py": [
       {
         "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 66,
         "type": "Basic Auth Credentials"
@@ -153,6 +165,7 @@
     "apis_configs/fence_credentials.json": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 23,
         "type": "Secret Keyword"
@@ -161,18 +174,21 @@
     "apis_configs/fence_settings.py": [
       {
         "hashed_secret": "3ef0fb8a603abdc0b6caac44a23fdc6792f77ddf",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Basic Auth Credentials"
       },
       {
         "hashed_secret": "b60d121b438a380c343d5ec3c2037564b82ffef3",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 58,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 80,
         "type": "Basic Auth Credentials"
@@ -181,6 +197,7 @@
     "apis_configs/indexd_settings.py": [
       {
         "hashed_secret": "0a0d18c85e096611b5685b62bc60ec534d19bacc",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 59,
         "type": "Basic Auth Credentials"
@@ -189,6 +206,7 @@
     "apis_configs/peregrine_settings.py": [
       {
         "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 46,
         "type": "Basic Auth Credentials"
@@ -197,6 +215,7 @@
     "apis_configs/sheepdog_settings.py": [
       {
         "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 46,
         "type": "Basic Auth Credentials"
@@ -205,6 +224,7 @@
     "doc/Gen3-data-upload.md": [
       {
         "hashed_secret": "b8bd20d4a2701dc3aba0efbbf325f1359392d93e",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
@@ -213,6 +233,7 @@
     "doc/api.md": [
       {
         "hashed_secret": "625de83a7517422051911680cc803921ff99db90",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 47,
         "type": "Hex High Entropy String"
@@ -221,24 +242,28 @@
     "doc/gen3OnK8s.md": [
       {
         "hashed_secret": "2db6d21d365f544f7ca3bcfb443ac96898a7a069",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 113,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "ff9ee043d85595eb255c05dfe32ece02a53efbb2",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 143,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "70374248fd7129088fef42b8f568443f6dce3a48",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 170,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "bcf22dfc6fb76b7366b1f1675baf2332a0e6a7ce",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 189,
         "type": "Secret Keyword"
@@ -247,6 +272,7 @@
     "doc/kube-setup-data-ingestion-job.md": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Secret Keyword"
@@ -255,6 +281,7 @@
     "doc/logs.md": [
       {
         "hashed_secret": "9addbf544119efa4a64223b649750a510f0d463f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Secret Keyword"
@@ -263,6 +290,7 @@
     "doc/slurm_cluster.md": [
       {
         "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 184,
         "type": "Secret Keyword"
@@ -271,12 +299,14 @@
     "files/dashboard/usage-reports/package-lock.json": [
       {
         "hashed_secret": "e095101882f706c4de95e0f75c5bcb9666e3f448",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 10,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "5422e4f96964d5739998b25ac214520c1b113e5b",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 15,
         "type": "Base64 High Entropy String"
@@ -285,12 +315,14 @@
     "gen3/bin/api.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 407,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 477,
         "type": "Secret Keyword"
@@ -299,6 +331,7 @@
     "gen3/bin/kube-dev-namespace.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 135,
         "type": "Secret Keyword"
@@ -307,6 +340,7 @@
     "gen3/bin/kube-setup-argo.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 206,
         "type": "Secret Keyword"
@@ -315,6 +349,7 @@
     "gen3/bin/kube-setup-aurora-monitoring.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 59,
         "type": "Secret Keyword"
@@ -323,6 +358,7 @@
     "gen3/bin/kube-setup-certs.sh": [
       {
         "hashed_secret": "2e9ee120fd25e31048598693aca91d5473898a99",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 50,
         "type": "Secret Keyword"
@@ -331,12 +367,14 @@
     "gen3/bin/kube-setup-dashboard.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 40,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 41,
         "type": "Secret Keyword"
@@ -345,12 +383,14 @@
     "gen3/bin/kube-setup-data-ingestion-job.sh": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 37,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "8695a632956b1b0ea7b66993dcc98732da39148c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 102,
         "type": "Secret Keyword"
@@ -359,6 +399,7 @@
     "gen3/bin/kube-setup-dicom-server.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 43,
         "type": "Secret Keyword"
@@ -367,6 +408,7 @@
     "gen3/bin/kube-setup-dicom.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 78,
         "type": "Secret Keyword"
@@ -375,26 +417,14 @@
     "gen3/bin/kube-setup-jenkins.sh": [
       {
         "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_verified": false,
-        "line_number": 22,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/bin/kube-setup-jenkins2.sh": [
-      {
-        "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f",
-        "is_verified": false,
-        "line_number": 18,
-        "type": "Secret Keyword"
-      },
-      {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 22,
         "type": "Secret Keyword"
@@ -403,6 +433,7 @@
     "gen3/bin/kube-setup-metadata.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 35,
         "type": "Secret Keyword"
@@ -411,18 +442,21 @@
     "gen3/bin/kube-setup-revproxy.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 38,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 55,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 57,
         "type": "Secret Keyword"
@@ -431,18 +465,21 @@
     "gen3/bin/kube-setup-secrets.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 79,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 82,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "6f7531b95bbc99ac25a5cc82edb825f319c5dee8",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 95,
         "type": "Secret Keyword"
@@ -451,12 +488,14 @@
     "gen3/bin/kube-setup-sftp.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 36,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "83d11e3aec005a3b9a2077c6800683e202a95af4",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 51,
         "type": "Secret Keyword"
@@ -465,6 +504,7 @@
     "gen3/bin/kube-setup-sheepdog.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 33,
         "type": "Secret Keyword"
@@ -473,24 +513,28 @@
     "gen3/bin/kube-setup-sower-jobs.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 25,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 120,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 122,
         "type": "Secret Keyword"
@@ -499,18 +543,21 @@
     "gen3/bin/kube-setup-ssjdispatcher.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 117,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "7992309146efaa8da936e34b0bd33242cd0e9f93",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 184,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 197,
         "type": "Secret Keyword"
@@ -519,12 +566,14 @@
     "gen3/lib/aws.sh": [
       {
         "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 640,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "5b4b6c62d3d99d202f095c38c664eded8f640ce8",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 660,
         "type": "Secret Keyword"
@@ -533,12 +582,14 @@
     "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml": [
       {
         "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 33,
         "type": "Basic Auth Credentials"
       },
       {
         "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 286,
         "type": "Secret Keyword"
@@ -547,6 +598,7 @@
     "gen3/lib/bootstrap/templates/Gen3Secrets/creds.json": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
@@ -555,6 +607,7 @@
     "gen3/lib/bootstrap/templates/Gen3Secrets/g3auto/dbfarm/servers.json": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Secret Keyword"
@@ -563,6 +616,7 @@
     "gen3/lib/logs/utils.sh": [
       {
         "hashed_secret": "76143b4ffc8aa2a53f9700ce229f904e69f1e8b5",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 3,
         "type": "Secret Keyword"
@@ -571,6 +625,7 @@
     "gen3/lib/manifestDefaults/hatchery/hatchery.json": [
       {
         "hashed_secret": "0da0e0005ca04acb407af2681d0bede6d9406039",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 78,
         "type": "Secret Keyword"
@@ -579,12 +634,14 @@
     "gen3/lib/onprem.sh": [
       {
         "hashed_secret": "29e52a9bac8f274fa41c51fce9c98eba0dd99cb3",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 68,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "50f013532a9770a2c2cfdc38b7581dd01df69b70",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 84,
         "type": "Secret Keyword"
@@ -593,12 +650,14 @@
     "gen3/lib/secrets/rotate-postgres.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 162,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 250,
         "type": "Secret Keyword"
@@ -607,42 +666,49 @@
     "gen3/lib/testData/etlconvert/expected2.yaml": [
       {
         "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 10,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 13,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 16,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 33,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 35,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 36,
         "type": "Base64 High Entropy String"
@@ -651,6 +717,7 @@
     "gen3/test/secretsTest.sh": [
       {
         "hashed_secret": "c2c715092ef59cba22520f109f041efca84b8938",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 25,
         "type": "Secret Keyword"
@@ -659,24 +726,28 @@
     "gen3/test/terraformTest.sh": [
       {
         "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 156,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 172,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 175,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "329b7cd8191942bedd337107934d365c43a86e6c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 175,
         "type": "Secret Keyword"
@@ -685,18 +756,21 @@
     "kube/services/argocd/values.yaml": [
       {
         "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 360,
         "type": "Private Key"
       },
       {
         "hashed_secret": "edbd5e119f94badb9f99a67ac6ff4c7a5204ad61",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 379,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "91dfd9ddb4198affc5c194cd8ce6d338fde470e2",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 412,
         "type": "Secret Keyword"
@@ -705,6 +779,7 @@
     "kube/services/datadog/values.yaml": [
       {
         "hashed_secret": "4a8ce7ae6a8a7f2624e232b61b18c2ac9789c44b",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 23,
         "type": "Secret Keyword"
@@ -713,362 +788,401 @@
     "kube/services/fenceshib/fenceshib-configmap.yaml": [
       {
         "hashed_secret": "a985e14b9d6744a2d04f29347693b55c116e478c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 375,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "adc747bc5eb82ef4b017f5c3759dcee5aa28c36f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 376,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "59b1702ff0eaf92c9271cbd12f587de97df7e13b",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 377,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "b4a748bbfbbca8925d932a47ab3dcb970d34caf5",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 378,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "af646701a84f7dd9f0e87753f54def881326e78a",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 379,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "20c15ad9742124dc06e1612282c49bb443ebcbd9",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 380,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "9caded71b967a11b7a6cd0f20db91f06f3517d12",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 381,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "8f19501bc9241b71f7b6db929fb35ab12635dcd7",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 382,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "d6220f6a55df1ed11c4250f42ab07bb9da20541a",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 383,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "dadd9b96636f9529f2547d05d754dc310ceba0c3",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 384,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "3074bc66584550e20c3697a28f67a0762394943c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 385,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "823131319b4c4b4688f44d3e832bfa9696f16b52",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 386,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "015b780cbfb76988caf52de8ac974a6781e53110",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 387,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "5c8fac33207d74d667680ade09447ea8f43b76d7",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 388,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "c0c4bb09d8394e8f001e337bd27ccac355433d9e",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 389,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f95631bcbbbc56e18487dcb242cfb1b3e74b16a1",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 390,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "01a692ab6232e0882a313d148981bab58ab98f53",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 391,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "658060a680d415ce6690ad2c3b622ddb33ddd50a",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 392,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "80915b0bd9daa5e1f95cad573892980b1b5a2294",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 393,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "cc55977b293d8cdca8a2c19dfea6874e70057c41",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 394,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "e400ed02add75dd5f3a8c212857acf12027437d1",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 395,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "2e819c8baa3b0508a32b77de258655b3f3a6f7cb",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 396,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "546ed926d58ea5492ab6adb8be94a67aa44ac433",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 397,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f056f2deceed268e7af6dbdaf2577079c76e006a",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 398,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "d75efee28f4798c3a9c6f44b78a8500513ef28b2",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 399,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "fbad0bc8f7792b03f89cd3780eb7cf79f284c525",
+        "hashed_secret": "7803ae08cdc22a5e0b025eff3c9ef0628eedc165",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 419,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "3f6480956a775dacb44e2c39aa3d4722a347f7ab",
+        "hashed_secret": "b8b61e87f5b58b0eeb597b2122ea0cea2ccab3d9",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 420,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "17f32ae55b14d708ca121722c2cae37189f19daf",
+        "hashed_secret": "787745fc904c3bd7eddc3d1aab683a376c13890f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 423,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "08a74689ca077515d406093720a7e5675fb42bb8",
+        "hashed_secret": "81361d672f238f505a6246ef9b655ee2f48d67e7",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 424,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "fa577bb3b2600d2d522dcfea8f1e34896760fcf2",
+        "hashed_secret": "7c98bff76ac3f273d15ed9bc3dd5294d323ab577",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 425,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "37254f15cca211a1bd5f7ceb23de2b3eb8fb33aa",
+        "hashed_secret": "46038fc88daceed8dd46817ca45c72ae0270fdd4",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 426,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "86865593e038509467b91c2d5f36ccc09c3f422b",
+        "hashed_secret": "acad0c57b4f5cbed1b4863ed06d02784180a9f92",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 427,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "a899a8d9e114b2a8e108f90e6a72c056db22489f",
+        "hashed_secret": "1b57f49a6ee337c16ecd6aabfc0dff3b3821cd09",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 428,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "756b4825f886afd83c25563ac9d45f318d695c48",
+        "hashed_secret": "5b688158be36e8b3f265a462ed599dcf69290084",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 429,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "89882eeb0aca97717a7e4afcf4bc08d077813c7f",
+        "hashed_secret": "965996e12c8b50b3c325d96003e8984a4ece658a",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 430,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "347140d7b7ceb4e501c3c9c2ea4f29338e2f145e",
+        "hashed_secret": "584f0c58e764e948af1a35c9e60447aa0f84c6f5",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 431,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "61dbf70eb10d609e60c7b87faf8f755ff48abc46",
+        "hashed_secret": "bcaf897786d060a675ee9d654a84ae8baf96e9d0",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 432,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "24cd54c4b2f58378bba008cb2df68ac663fba7c8",
+        "hashed_secret": "0c09277fa183e06d32065f9386a3b4190b445df3",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 433,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "fa4f9626ae4b98f4b61203c5bafb6f21c9c31e5d",
+        "hashed_secret": "5a51be06b305d6664e4afd25f21869b0f8b5039b",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 434,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "b1370003d9cc1e346c83dba33e0418c7775a0c15",
+        "hashed_secret": "b38404f8853d734e3d03577b2c1084b4540c8708",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 435,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "c66526e195e423a7ba7d68ac661cdcd8600dcd1f",
+        "hashed_secret": "126ccc602cffcb8292beb57137f7f6719e317b72",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 436,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "d29d7044f0944eb30e02cf445f6998e3343dd811",
+        "hashed_secret": "6681c1d7e1d327642a32cb8864ad51e4b8f981e5",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 437,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "80a869460f33722387d8d58e7d9d2e1bbd5d1fe1",
-        "is_verified": false,
-        "line_number": 438,
-        "type": "Base64 High Entropy String"
-      },
-      {
-        "hashed_secret": "4a06e2a02cbc665adccb4162dc57836895da65b8",
+        "hashed_secret": "7f7b1f316ece195e5f584fe2faf6f9edc6942c6f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 439,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "ba2549f35835dfa101d3f660f7604dc78e3e226f",
+        "hashed_secret": "bb908c7bc655057f2edc42815c5dff82e9dea529",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 440,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "f354d4ee5fdb94ad29c7b3600264467f45b80eaa",
+        "hashed_secret": "bc2a0d18e3dd142df7b34e95342d47bf8aadabcb",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 441,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "bf17b587868ba7c3db9865b114261b5b8f1df870",
+        "hashed_secret": "d60f0bcea109bb6edb6e45fd387f5f2c86e49e1a",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 442,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "de1fd7a0d32cba528b4d80818c6601f2588d5383",
+        "hashed_secret": "e549dd40a741557cc1c4e377df0a141354e22688",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 443,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "bcad65055f6de654541db2bf27d4e27bd54d94c7",
+        "hashed_secret": "2dd2486dae84cad50387c20bf687b6fbc6162b58",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 444,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "f2e16f2dd532f65f79341342fdf57a093fc408d8",
+        "hashed_secret": "71622010fc7eb09d9273f59c548bde6a5da5dc0e",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 445,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "bb036a679a7d2df9fd2ca57068a446bf7f7dd106",
+        "hashed_secret": "6f0115cf53bd49ec990c562ac6cbfc452c83cd46",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 446,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "5aa6568b1e8185578a6e964f5c322783ad349554",
-        "is_verified": false,
-        "line_number": 447,
-        "type": "Base64 High Entropy String"
-      },
-      {
-        "hashed_secret": "4d14835ff0b0bf5aad480296cb705c74ac65f413",
+        "hashed_secret": "70dddd534b2f9bb70871fefe0845b79c3b69363f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 448,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "3f23f77dcf454ad73c4d61c44fd9aa584ef946c1",
-        "is_verified": false,
-        "line_number": 451,
-        "type": "Base64 High Entropy String"
-      },
-      {
-        "hashed_secret": "1739fe5e5dfcf851b64f8b7b11538f1de29ce0b5",
+        "hashed_secret": "acf3536b0416aa99608b0be17e87655370ece829",
+        "is_secret": false,
         "is_verified": false,
-        "line_number": 452,
+        "line_number": 449,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "8129db302110714fc735e3494bd82a65690e0963",
+        "hashed_secret": "1d13ee35c7279c1fae1c6474ed47611994273e41",
+        "is_secret": false,
         "is_verified": false,
-        "line_number": 453,
+        "line_number": 450,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "b48bfc62091164086a703115a0e68bdb09212591",
+        "hashed_secret": "d38cf89b25bd7378cdb4e00b4b59293001dd500b",
+        "is_secret": false,
         "is_verified": false,
-        "line_number": 454,
+        "line_number": 451,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "a10284feaf27f84081073a3267e3dce24ca7b911",
+        "hashed_secret": "1648f34ce2f1b563a8ed1c6d5d55b5e76a395903",
+        "is_secret": false,
         "is_verified": false,
-        "line_number": 455,
+        "line_number": 452,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "3fd80f31de4be8dde9d2b421e832c7d4043fd49a",
+        "hashed_secret": "9bf63f6f49fb01ff80959bc5a60c8688df92cc02",
+        "is_secret": false,
         "is_verified": false,
-        "line_number": 456,
+        "line_number": 453,
         "type": "Base64 High Entropy String"
       }
     ],
     "kube/services/jobs/indexd-authz-job.yaml": [
       {
         "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 87,
         "type": "Basic Auth Credentials"
@@ -1077,12 +1191,14 @@
     "kube/services/monitoring/grafana-values.yaml": [
       {
         "hashed_secret": "2ae868079d293e0a185c671c7bcdac51df36e385",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 162,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "7a64ff8446b06d38dc271019994f13823a2cbcf4",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 166,
         "type": "Secret Keyword"
@@ -1091,6 +1207,7 @@
     "kube/services/revproxy/helpers.js": [
       {
         "hashed_secret": "1d278d3c888d1a2fa7eed622bfc02927ce4049af",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 10,
         "type": "Base64 High Entropy String"
@@ -1099,6 +1216,7 @@
     "kube/services/revproxy/helpersTest.js": [
       {
         "hashed_secret": "e029d4904cc728879d70030572bf37d4510367cb",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 22,
         "type": "JSON Web Token"
@@ -1107,6 +1225,7 @@
     "kube/services/superset/superset-deploy.yaml": [
       {
         "hashed_secret": "96e4aceb7cf284be363aa248a32a7cc89785a9f7",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 38,
         "type": "Secret Keyword"
@@ -1115,12 +1234,14 @@
     "kube/services/superset/superset-redis.yaml": [
       {
         "hashed_secret": "4af3596275edcb7cd5cc6c3c38bc10479902a08f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 165,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9fe1c31809da38c55b2b64bfab47b92bc5f6b7b9",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 265,
         "type": "Secret Keyword"
@@ -1129,30 +1250,35 @@
     "kube/services/superset/values.yaml": [
       {
         "hashed_secret": "6f803b24314c39062efe38d0c1da8c472f47eab3",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 54,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "6eae3a5b062c6d0d79f070c26e6d62486b40cb46",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 86,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "3eb416223e9e69e6bb8ee19793911ad1ad2027d8",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 212,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "ff55435345834a3fe224936776c2aa15f6ed5358",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 396,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "98a84a63e5633d17e3b27b69695f87aa7189e9dc",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 503,
         "type": "Secret Keyword"
@@ -1161,240 +1287,280 @@
     "package-lock.json": [
       {
         "hashed_secret": "0656ad0df3af4633dc369f13d5e8806973c5fd9d",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1481,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "00091d875d922437c5fc9e6067a08e78c2482e87",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1489,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "c4e5cc37e115bf7d86e76e3d799705bf691e4d00",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1521,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "0512e37fbedf1d16828680a038a241b4780a5c04",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1547,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "01868fd50edbfe6eb91e5b01209b543adc6857af",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1611,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a6f48bf1e398deffc7fd31da17c3506b46c97a93",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1640,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "85ce358dbdec0996cf3ccd2bf1c6602af68c181e",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1648,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "6f9bfb49cb818d2fe07592515e4c3f7a0bbd7e0e",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1664,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "7098a3e6d6d2ec0a40f04fe12509c5c6f4c49c0e",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1683,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "1664ad175bba1795a7ecad572bae7e0740b94f56",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1733,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "1ec4ce2eb945ce2f816dcb6ebdd1e10247f439a3",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1742,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a7af5768a6d936e36f28e1030d7f894d7aaf555e",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1755,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "6fbc7dd864586173160874f2a86ca7d2d552cb85",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1769,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "81a961f2c89c6209328b74a8768e30fd76c3ac72",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1855,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "797d4751c536c421cb82b9f62e0a804af30d78f5",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1889,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "0d55babfa89f240142c0adfc7b560500a1d3ae7c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1894,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "e9fdc3025cd10bd8aa4508611e6b7b7a9d650a2c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1921,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "4cf9419259c0ce8eee84b468af3c72db8b001620",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1950,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "24816e3eb4308e247bde7c1d09ffb7b79c519b71",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 1983,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "e9adfe8a333d45f4776fe0eab31608be5d7b6a7d",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2004,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "03d6fb388dd1b185129b14221f7127715822ece6",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2013,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "ee161bb3f899720f95cee50a5f9ef9c9ed96278b",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2046,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "ebeb5b574fa1ed24a40248275e6136759e766466",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2078,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a6a555a428522ccf439fd516ce7c7e269274363f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2083,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f7f85d9f7c87f1e576dcaf4cf50f35728f9a3265",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2111,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "3f1646b60abe74297d2f37a1eee5dc771ad834fc",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2138,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "fd933c71e82d5519ae0cb0779b370d02f6935759",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2143,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "7090aa59cb52ad1f1810b08c4ac1ddf5c8fce523",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2150,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "756444bea4ea3d67844d8ddf58ad32356e9c2430",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2188,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f74135fdd6b8dafdfb01ebbc61c5e5c24ee27cf8",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2291,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "56fbae787f4aed7d0632e95840d71bd378d3a36f",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2303,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "81cb6be182eb79444202c4563080aee75296a672",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2308,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f0f3f7bce32184893046ac5f8cc80da56c3ca539",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2317,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "097893233346336f4003acfb6eb173ee59e648f0",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2327,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "bb14c3b4ef4a9f2e86ffdd44b88d9b6729419671",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2332,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "71344a35cff67ef081920095d1406601fb5e9b97",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2340,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "eb3db6990fd43477a35dfeffc90b3f1ffa83c7bd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2349,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "266288bdc14807b538d1e48a5891e361fa9b4a14",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2357,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "800477261175fd21f23e7321923e1fba6ae55471",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2369,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "3f0c251b9c2c21454445a98fde6915ceacde2136",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 2387,
         "type": "Base64 High Entropy String"
@@ -1403,6 +1569,7 @@
     "tf_files/aws/cognito/README.md": [
       {
         "hashed_secret": "f6920f370a30262b7dd70e97293c73ec89739b70",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 106,
         "type": "Secret Keyword"
@@ -1411,12 +1578,14 @@
     "tf_files/aws/commons/README.md": [
       {
         "hashed_secret": "d02e53411e8cb4cd709778f173f7bc9a3455f8ed",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 60,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9dc0da3613af850c5a018b0a88a5626fb8888e4e",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 78,
         "type": "Secret Keyword"
@@ -1425,6 +1594,7 @@
     "tf_files/aws/eks/sample.tfvars": [
       {
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 107,
         "type": "Hex High Entropy String"
@@ -1433,6 +1603,7 @@
     "tf_files/aws/eks/variables.tf": [
       {
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 133,
         "type": "Hex High Entropy String"
@@ -1441,12 +1612,14 @@
     "tf_files/aws/modules/common-logging/README.md": [
       {
         "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 57,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 59,
         "type": "Hex High Entropy String"
@@ -1455,24 +1628,28 @@
     "tf_files/aws/modules/common-logging/lambda_function.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Hex High Entropy String"
@@ -1481,18 +1658,21 @@
     "tf_files/aws/modules/common-logging/testLambda.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
@@ -1501,6 +1681,7 @@
     "tf_files/aws/modules/eks/variables.tf": [
       {
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 113,
         "type": "Hex High Entropy String"
@@ -1509,12 +1690,14 @@
     "tf_files/aws/modules/management-logs/README.md": [
       {
         "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 54,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 56,
         "type": "Hex High Entropy String"
@@ -1523,24 +1706,28 @@
     "tf_files/aws/modules/management-logs/lambda_function.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Hex High Entropy String"
@@ -1549,36 +1736,42 @@
     "tf_files/aws/modules/management-logs/testLambda.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "3cf8eb4e9254e1d6cc523da01f8b798b9a83101a",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "51118900cd675df1b44f254057398f3e52902a5d",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "60a6dfc8d43cd2f5c6292899fc2f94f2d4fc32c4",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Hex High Entropy String"
@@ -1587,6 +1780,7 @@
     "tf_files/aws/slurm/README.md": [
       {
         "hashed_secret": "fd85d792fa56981cf6a8d2a5c0857c74af86e99d",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 83,
         "type": "Secret Keyword"
@@ -1595,6 +1789,7 @@
     "tf_files/azure/cloud.tf": [
       {
         "hashed_secret": "7c1a4b52b64e4106041971c345a1f3eab58fb2a4",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 424,
         "type": "Secret Keyword"
@@ -1603,6 +1798,7 @@
     "tf_files/gcp-bwg/roots/commons_setup/variables/answerfile-commons_setup-001.template.tfvars": [
       {
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 231,
         "type": "Secret Keyword"
@@ -1611,6 +1807,7 @@
     "tf_files/gcp-bwg/roots/templates/answerfile-commons_setup-001.template.tfvars": [
       {
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 231,
         "type": "Secret Keyword"
@@ -1619,6 +1816,7 @@
     "tf_files/gcp-bwg/roots/templates/answerfile-env-tenant.user.tfvars_NO_APP_SETUP": [
       {
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 262,
         "type": "Secret Keyword"
@@ -1627,18 +1825,21 @@
     "tf_files/gcp/commons/sample.tfvars": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 11,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "253c7b5e7c83a86346fc4501495b130813f08105",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 37,
         "type": "Secret Keyword"
@@ -1647,6 +1848,7 @@
     "tf_files/shared/modules/k8s_configs/creds.tpl": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "is_secret": false,
         "is_verified": false,
         "line_number": 8,
         "type": "Secret Keyword"
diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py
index 730a3b36e..71575e3c5 100644
--- a/files/scripts/healdata/heal-cedar-data-ingest.py
+++ b/files/scripts/healdata/heal-cedar-data-ingest.py
@@ -1,5 +1,5 @@
 import argparse
-import copy
+import json
 import sys
 import requests
 import pydash
@@ -35,16 +35,6 @@
     "Buisness Development": "Business Development"
 }
 
-# repository links
-REPOSITORY_STUDY_ID_LINK_TEMPLATE = {
-    "NIDDK Central": "https://repository.niddk.nih.gov/studies/<STUDY_ID>/",
-    "NIDA Data Share": "https://datashare.nida.nih.gov/study/<STUDY_ID>",
-    "NICHD DASH": "https://dash.nichd.nih.gov/study/<STUDY_ID>",
-    "ICPSR": "https://www.icpsr.umich.edu/web/ICPSR/studies/<STUDY_ID>",
-    "BioSystics-AP": "https://biosystics-ap.com/assays/assaystudy/<STUDY_ID>/",
-}
-
-
 # Defines field that we don't want to include in the filters
 OMITTED_VALUES_MAPPING = {
     "study_metadata.human_subject_applicability.gender_applicability": "Not applicable"
@@ -124,31 +114,6 @@ def get_client_token(client_id: str, client_secret: str):
     return token
 
 
-def get_related_studies(serial_num, hostname):
-    related_study_result = []
-
-    if serial_num:
-        mds = requests.get(f"http://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000")
-        if mds.status_code == 200:
-            related_study_metadata = mds.json()
-
-            for (
-                related_study_metadata_key,
-                related_study_metadata_value,
-            ) in related_study_metadata.items():
-                title = (
-                    related_study_metadata_value.get(
-                        "gen3_discovery", {}
-                    )
-                    .get("study_metadata", {})
-                    .get("minimal_info", {})
-                    .get("study_name", "")
-                )
-                link = f"https://{hostname}/portal/discovery/{related_study_metadata_key}/"
-                related_study_result.append({"title": title, "link": link})
-    return related_study_result
-
-
 parser = argparse.ArgumentParser()
 
 parser.add_argument("--directory", help="CEDAR Directory ID for registering ")
@@ -249,67 +214,6 @@ def get_related_studies(serial_num, hostname):
                 mds_res["gen3_discovery"]["study_metadata"].update(cedar_record)
                 mds_res["gen3_discovery"]["study_metadata"]["metadata_location"]["other_study_websites"] = cedar_record_other_study_websites
 
-                # setup citations
-                doi_citation = mds_res["gen3_discovery"]["study_metadata"].get("doi_citation", "")
-                mds_res["gen3_discovery"]["study_metadata"]["citation"]["heal_platform_citation"] = doi_citation
-
-
-                # setup repository_study_link
-                data_repositories = (
-                    mds_res.get("study_metadata", {})
-                    .get("metadata_location", {})
-                    .get("data_repositories", [])
-                )
-                repository_citation = "Users must also include a citation to the data as specified by the local repository."
-                repository_citation_additional_text = ' The link to the study page at the local repository can be found in the "Data" tab.'
-                for repository in data_repositories:
-                    if (
-                        repository["repository_name"]
-                        and repository["repository_name"]
-                        in REPOSITORY_STUDY_ID_LINK_TEMPLATE
-                        and repository["repository_study_ID"]
-                    ):
-                        repository_study_link = REPOSITORY_STUDY_ID_LINK_TEMPLATE[
-                            repository["repository_name"]
-                        ].replace("<STUDY_ID>", repository["repository_study_ID"])
-                        repository.update({"repository_study_link": repository_study_link})
-                        if repository_citation_additional_text not in repository_citation:
-                            repository_citation += repository_citation_additional_text
-                if len(data_repositories):
-                    data_repositories[0] = {
-                        **data_repositories[0],
-                        "repository_citation": repository_citation,
-                    }
-                mds_res["gen3_discovery"]["study_metadata"][
-                    "metadata_location"
-                ]["data_repositories"] = data_repositories
-
-
-
-                # set up related studies
-                serial_num = None
-                try:
-                    serial_num = (
-                        mds_res
-                        .get("nih_reporter", {})
-                        .get("project_num_split", {})
-                        .get("serial_num", None)
-                    )
-                except Exception:
-                    print(f"Unable to get serial number for study")
-
-                if serial_num == None:
-                    print(f"Unable to get serial number for study")
-
-                related_study_result = get_related_studies(serial_num, hostname)
-                existing_related_study_result = mds_res.get("related_studies", [])
-                for related_study in related_study_result:
-                    if related_study not in existing_related_study_result:
-                        existing_related_study_result.append(copy.deepcopy(related_study))
-                mds_res["gen3_discovery"][
-                    "related_studies"
-                ] = copy.deepcopy(existing_related_study_result)
-
                 # merge data from cedar that is not study level metadata into a level higher
                 deleted_keys = []
                 for key, value in mds_res["gen3_discovery"]["study_metadata"].items():

From 6ee728ad3e375b1a0e521ae256ef13ca29529067 Mon Sep 17 00:00:00 2001
From: Alexander VanTol <Avantol13@users.noreply.github.com>
Date: Mon, 4 Mar 2024 11:04:31 -0600
Subject: [PATCH 43/67] Automation for Gen3 Discovery AI Service (#2396)

* feat(gen3-openai): rough initial testing, no automation for rolling

* feat(gen3-discovery-ai): initial deployment

* fix(ai): fix setup jq escaping

* fix(ai): fix file name

* fix(ai): fix deployment configmap

* fix(ai): fix configmap

* fix(ai): env path

* feat(image): use from manifest

* chore(ai): better naming

* fix(ai): fix mount path for cfg

* fix(ai): first attempt to fix issue of needing write volume for chromadb persistance

* fix(ai): k8s deploy command

* fix(ai): fix duplicate name

* chore(ai): don't sent telemetry data

* chore(ai): more logging in init

* chore(logs): more

* fix(ai): mv instead of cp

* fix(ai): back to cp, can't mv b/c of readonly

* feat(ai): use s3 and service account + role to handle persisted vectorstore data instead of configmap

* fix(ai): fix setup

* fix(ai): fix setup

* fix(ai): fix automation

* fix(ai): automation

* fix(ai): fix logic for setup

* fix(ai): mount storage config and don't use gen3/jq since they're not available

* fix(ai): fix wrong path

* fix(ai): quotes

* fix(ai): quoting

* fix(ai): use awshelper for access to aws commands

* fix(ai): move files to correct location

* fix(ai): only get folder

* fix(ai): fix sync

* fix(ai): clear folder before syncing

* fix(ai): update bucket contents every roll for updates

* feat(ai): support TSV loading from manifest config

* fix(ai): fix init so aws syncing is done with awshelper image and loading into vectorstore is with service image

* fix(ai): fix loading

* fix(ai): fix loading

* fix(ai): sync all files

* feat(ai): add google secret loading and mounting

* fix(ai): mount to container, not inits

* fix(mount): don't create another dir

* fix(mount): don't create another dir

* fix(mounts): fix paths

* fix(mounts): mount all secrets

* fix(secrets): allow .env file to be a secret

* fix(secrets): revert failed attempt to support .env

* chore(ai): cd to dir with pyproject.toml

* chore(ai): try to fix issue with pyproject.toml

* fix(ai): actually we need to poetry run

* chore(ai): debug lines

* chore(ai): debug lines

* chore(ai): debug lines

* chore(ai): debug lines

* chore(ai): debug lines

* fix(mount): don't overwrite whole dir

* fix(ai): mounts

* chore(ai): remove debug lines

* fix(ai): remove debug

* chore(debug): debug line

* chore(debug): remove debug line

* feat(ai): add to roll all, fix port in service yaml

* fix(ai): fix nginx conf file name

* fix(nginx): fix routing for AI service to add trailing slash after "ai"

* Update web_whitelist

* Update kube-setup-gen3-discovery-ai.sh

* Update README.md

* Update gen3-discovery-ai-deploy.yaml

* Update gen3-discovery-ai-deploy.yaml

* Update gen3-discovery-ai-service.yaml

* Update kube-setup-gen3-discovery-ai.sh

* feat(discovery): update to data load commands and strategy to support markdown
---
 gen3/bin/kube-roll-all.sh                     |   6 +
 gen3/bin/kube-setup-gen3-discovery-ai.sh      | 154 +++++++++++++++
 kube/services/gen3-discovery-ai/README.md     |  42 ++++
 .../gen3-discovery-ai-deploy.yaml             | 181 ++++++++++++++++++
 .../gen3-discovery-ai-service.yaml            |  21 ++
 .../gen3-discovery-ai-service.conf            |  12 ++
 6 files changed, 416 insertions(+)
 create mode 100644 gen3/bin/kube-setup-gen3-discovery-ai.sh
 create mode 100644 kube/services/gen3-discovery-ai/README.md
 create mode 100644 kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml
 create mode 100644 kube/services/gen3-discovery-ai/gen3-discovery-ai-service.yaml
 create mode 100644 kube/services/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf

diff --git a/gen3/bin/kube-roll-all.sh b/gen3/bin/kube-roll-all.sh
index 6a67f2bdd..1dca87c68 100644
--- a/gen3/bin/kube-roll-all.sh
+++ b/gen3/bin/kube-roll-all.sh
@@ -243,6 +243,12 @@ else
   gen3_log_info "not deploying dicom-viewer - no manifest entry for '.versions[\"dicom-viewer\"]'"
 fi
 
+if g3k_manifest_lookup '.versions["gen3-discovery-ai"]' 2> /dev/null; then
+  gen3 kube-setup-gen3-discovery-ai &
+else
+  gen3_log_info "not deploying gen3-discovery-ai - no manifest entry for '.versions[\"gen3-discovery-ai\"]'"
+fi
+
 if g3k_manifest_lookup '.versions["ohdsi-atlas"]' && g3k_manifest_lookup '.versions["ohdsi-webapi"]' 2> /dev/null; then
   gen3 kube-setup-ohdsi &
 else
diff --git a/gen3/bin/kube-setup-gen3-discovery-ai.sh b/gen3/bin/kube-setup-gen3-discovery-ai.sh
new file mode 100644
index 000000000..44a472a74
--- /dev/null
+++ b/gen3/bin/kube-setup-gen3-discovery-ai.sh
@@ -0,0 +1,154 @@
+#!/bin/bash
+#
+# Deploy the gen3-discovery-ai service
+#
+
+source "${GEN3_HOME}/gen3/lib/utils.sh"
+gen3_load "gen3/gen3setup"
+
+# NOTE: no db for this service yet, but we'll likely need it in the future
+setup_database() {
+  gen3_log_info "setting up gen3-discovery-ai service ..."
+
+  if g3kubectl describe secret gen3-discovery-ai-g3auto > /dev/null 2>&1; then
+    gen3_log_info "gen3-discovery-ai-g3auto secret already configured"
+    return 0
+  fi
+  if [[ -n "$JENKINS_HOME" || ! -f "$(gen3_secrets_folder)/creds.json" ]]; then
+    gen3_log_err "skipping db setup in non-adminvm environment"
+    return 0
+  fi
+  # Setup .env file that gen3-discovery-ai service consumes
+  if [[ ! -f "$secretsFolder/gen3-discovery-ai.env" || ! -f "$secretsFolder/base64Authz.txt" ]]; then
+    local secretsFolder="$(gen3_secrets_folder)/g3auto/gen3-discovery-ai"
+
+    if [[ ! -f "$secretsFolder/dbcreds.json" ]]; then
+      if ! gen3 db setup gen3-discovery-ai; then
+        gen3_log_err "Failed setting up database for gen3-discovery-ai service"
+        return 1
+      fi
+    fi
+    if [[ ! -f "$secretsFolder/dbcreds.json" ]]; then
+      gen3_log_err "dbcreds not present in Gen3Secrets/"
+      return 1
+    fi
+
+    # go ahead and rotate the password whenever we regen this file
+    local password="$(gen3 random)"
+    cat - > "$secretsFolder/gen3-discovery-ai.env" <<EOM
+DEBUG=0
+DB_HOST=$(jq -r .db_host < "$secretsFolder/dbcreds.json")
+DB_USER=$(jq -r .db_username < "$secretsFolder/dbcreds.json")
+DB_PASSWORD=$(jq -r .db_password < "$secretsFolder/dbcreds.json")
+DB_DATABASE=$(jq -r .db_database < "$secretsFolder/dbcreds.json")
+ADMIN_LOGINS=gateway:$password
+EOM
+    # make it easy for nginx to get the Authorization header ...
+    echo -n "gateway:$password" | base64 > "$secretsFolder/base64Authz.txt"
+  fi
+  gen3 secrets sync 'setup gen3-discovery-ai-g3auto secrets'
+}
+
+if ! g3k_manifest_lookup '.versions."gen3-discovery-ai"' 2> /dev/null; then
+  gen3_log_info "kube-setup-gen3-discovery-ai exiting - gen3-discovery-ai service not in manifest"
+  exit 0
+fi
+
+# There's no db for this service *yet* 
+#
+# if ! setup_database; then
+#   gen3_log_err "kube-setup-gen3-discovery-ai bailing out - database failed setup"
+#   exit 1
+# fi
+
+setup_storage() {
+  local saName="gen3-discovery-ai-sa"
+  g3kubectl create sa "$saName" > /dev/null 2>&1 || true
+
+  local secret
+  local secretsFolder="$(gen3_secrets_folder)/g3auto/gen3-discovery-ai"
+
+  secret="$(g3kubectl get secret gen3-discovery-ai-g3auto -o json 2> /dev/null)"
+  local hasStorageCfg
+  hasStorageCfg=$(jq -r '.data | has("storage_config.json")' <<< "$secret")
+
+  if [ "$hasStorageCfg" = "false" ]; then
+    gen3_log_info "setting up storage for gen3-discovery-ai service"
+    #
+    # gen3-discovery-ai-g3auto secret still does not exist
+    # we need to setup an S3 bucket and IAM creds
+    # let's avoid creating multiple buckets for different
+    # deployments to the same k8s cluster (dev, etc)
+    #
+    local bucketName
+    local accountNumber
+    local environment
+
+    if ! accountNumber="$(aws sts get-caller-identity --output text --query 'Account')"; then
+      gen3_log_err "could not determine account numer"
+      return 1
+    fi
+
+    gen3_log_info "accountNumber: ${accountNumber}"
+
+    if ! environment="$(g3kubectl get configmap manifest-global -o json | jq -r .data.environment)"; then
+      gen3_log_err "could not determine environment from manifest-global - bailing out of gen3-discovery-ai setup"
+      return 1
+    fi
+
+    gen3_log_info "environment: ${environment}"
+
+    # try to come up with a unique but composable bucket name
+    bucketName="gen3-discovery-ai-${accountNumber}-${environment//_/-}"
+
+    gen3_log_info "bucketName: ${bucketName}"
+
+    if aws s3 ls --page-size 1 "s3://${bucketName}" > /dev/null 2>&1; then
+      gen3_log_info "${bucketName} s3 bucket already exists - probably in use by another namespace - copy the creds from there to $(gen3_secrets_folder)/g3auto/gen3-discovery-ai"
+      # continue on ...
+    elif ! gen3 s3 create "${bucketName}"; then
+      gen3_log_err "maybe failed to create bucket ${bucketName}, but maybe not, because the terraform script is flaky"
+    fi
+
+    local hostname
+    hostname="$(gen3 api hostname)"
+    jq -r -n --arg bucket "${bucketName}" --arg hostname "${hostname}" '.bucket=$bucket | .prefix=$hostname' > "${secretsFolder}/storage_config.json"
+    gen3 secrets sync 'setup gen3-discovery-ai credentials'
+
+    local roleName
+    roleName="$(gen3 api safe-name gen3-discovery-ai)" || return 1
+      
+    if ! gen3 awsrole info "$roleName" > /dev/null; then # setup role
+      bucketName="$( (gen3 secrets decode 'gen3-discovery-ai-g3auto' 'storage_config.json' || echo ERROR) | jq -r .bucket)" || return 1
+      gen3 awsrole create "$roleName" "$saName" || return 1
+      gen3 s3 attach-bucket-policy "$bucketName" --read-write --role-name "${roleName}"
+      # try to give the gitops role read/write permissions on the bucket
+      local gitopsRoleName
+      gitopsRoleName="$(gen3 api safe-name gitops)"
+      gen3 s3 attach-bucket-policy "$bucketName" --read-write --role-name "${gitopsRoleName}"
+    fi
+  fi
+
+  return 0
+}
+
+if ! setup_storage; then
+  gen3_log_err "kube-setup-gen3-discovery-ai bailing out - storage failed setup"
+  exit 1
+fi
+
+gen3_log_info "Setup complete, syncing configuration to bucket"
+
+bucketName="$( (gen3 secrets decode 'gen3-discovery-ai-g3auto' 'storage_config.json' || echo ERROR) | jq -r .bucket)" || exit 1
+aws s3 sync "$(dirname $(g3k_manifest_path))/gen3-discovery-ai/knowledge" "s3://$bucketName" --delete
+
+gen3 roll gen3-discovery-ai
+g3kubectl apply -f "${GEN3_HOME}/kube/services/gen3-discovery-ai/gen3-discovery-ai-service.yaml"
+
+if [[ -z "$GEN3_ROLL_ALL" ]]; then
+  gen3 kube-setup-networkpolicy
+  gen3 kube-setup-revproxy
+fi
+
+gen3_log_info "The gen3-discovery-ai service has been deployed onto the kubernetes cluster"
+gen3_log_info "test with: curl https://commons-host/ai"
diff --git a/kube/services/gen3-discovery-ai/README.md b/kube/services/gen3-discovery-ai/README.md
new file mode 100644
index 000000000..4c20678e0
--- /dev/null
+++ b/kube/services/gen3-discovery-ai/README.md
@@ -0,0 +1,42 @@
+# Gen3 Discovery AI Configuration
+
+Expects data in a `gen3-discovery-ai` folder relative to 
+where the `manifest.json` is. 
+
+Basic setup:
+
+`{{dir where manifest.json is}}/gen3-discovery-ai/knowledge/`
+
+- `tsvs` folder
+    - tsvs with topic_name at beginning of file
+- `markdown` folder
+    - {{topic_name_1}}
+        - markdown file(s)
+    - {{topic_name_2}}
+        - markdown file(s)
+
+The `kube-setup-gen3-discovery-ai` script syncs the above `/knowledge` folder to
+an S3 bucket. The service configuration then pulls from the S3 bucket and runs load commands 
+to get the data into chromadb.
+
+> Note: See the `gen3-discovery-ai` service repo docs and README for more details on data load capabilities.
+
+Check the `gen3-discovery-ai-deploy.yaml` for what commands are being run in the automation.
+
+Expects secrets setup in `g3auto/gen3-discovery-ai` folder
+ - `credentials.json`: Google service account key if using a topic with Google Vertex AI
+ - `env`: .env file contents for service configuration (see service repo for a default one)
+
+## Populating Disk for In-Memory Vectordb Chromadb
+
+In order to setup pre-configured topics, we need to load a bunch of data 
+into Chromadb (which is an in-mem vectordb with an option to persist to disk).
+
+To load topics consistently, we setup an S3 bucket to house the persisted 
+data for the vectordb.
+
+### Getting data from S3 in mem
+
+We specify a path for Chromadb to use for persisted data and when it sees 
+data there, it loads it in. So the deployment automation: 1. aws syncs the bucket
+and then 2. calls a script to load the files into the in-mem vectorstore from there. 
diff --git a/kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml b/kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml
new file mode 100644
index 000000000..dcfe03248
--- /dev/null
+++ b/kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml
@@ -0,0 +1,181 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gen3-discovery-ai-deployment
+spec:
+  selector:
+    # Only select pods based on the 'app' label
+    matchLabels:
+      app: gen3-discovery-ai
+      release: production
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  template:
+    metadata:
+      labels:
+        app: gen3-discovery-ai
+        release: production
+        GEN3_DATE_LABEL
+    spec:
+      serviceAccountName: gen3-discovery-ai-sa
+      volumes:
+        - name: gen3-discovery-ai-g3auto-volume
+          secret:
+            secretName: gen3-discovery-ai-g3auto
+        - name: gen3-discovery-ai-knowledge-library-volume
+          emptyDir: {}
+      initContainers:
+      # chromadb's persisted disk support requires the ability to write. We don't technically need this ability
+      # since we're populating the entirety of the database from configured files (no live updates).
+      #
+      # Solution: utilize emptyDir as a writable space.
+      #
+      # Procedure: in init containers, copy files from s3 to writable
+      #            temporary space in emptyDir, use files from writable space
+      #            to load into knowledge libary, move final knowledge library
+      #            files into top-level emptyDir and make available in final container
+      - name: gen3-discovery-ai-aws-init
+        GEN3_AWSHELPER_IMAGE|-image: quay.io/cdis/awshelper:master-|
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+        env:
+          - name: GEN3_DEBUG
+            GEN3_DEBUG_FLAG|-value: "False"-|
+        volumeMounts:
+          - name: gen3-discovery-ai-g3auto-volume
+            readOnly: true
+            mountPath: /gen3discoveryai/.env
+            subPath: env
+          - name: gen3-discovery-ai-g3auto-volume
+            readOnly: true
+            mountPath: /gen3discoveryai/credentials.json
+            subPath: credentials.json
+          - name: gen3-discovery-ai-g3auto-volume
+            readOnly: true
+            mountPath: /gen3discoveryai/storage_config.json
+            subPath: storage_config.json
+          - name: gen3-discovery-ai-knowledge-library-volume
+            mountPath: /gen3discoveryai/knowledge
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: 1
+          limits:
+            cpu: 2
+            memory: 512Mi
+        command: ["/bin/bash"]
+        args:
+          - "-c"
+          - |
+            bucketName=$(grep -o "\"bucket\": *\"[^\"]*\"" /gen3discoveryai/storage_config.json | awk -F'"' '{print $4}')
+            echo BUCKET: "$bucketName"
+            echo
+            echo BEFORE /gen3discoveryai/knowledge  
+            ls -Ra /gen3discoveryai/knowledge 
+            echo
+            echo syncing from s3
+            aws s3 sync "s3://${bucketName}" "/gen3discoveryai/knowledge/tmp"
+            echo
+            echo AFTER /gen3discoveryai/knowledge 
+            ls -Ra /gen3discoveryai/knowledge
+      - name: gen3-discovery-ai-knowledge-init
+        GEN3_GEN3-DISCOVERY-AI_IMAGE
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+        env:
+          - name: GEN3_DEBUG
+            GEN3_DEBUG_FLAG|-value: "False"-|
+          - name: ANONYMIZED_TELEMETRY
+            value: "False"
+          - name: GOOGLE_APPLICATION_CREDENTIALS
+            value: /gen3discoveryai/credentials.json
+        volumeMounts:
+          - name: gen3-discovery-ai-g3auto-volume
+            readOnly: true
+            mountPath: /gen3discoveryai/.env
+            subPath: env
+          - name: gen3-discovery-ai-g3auto-volume
+            readOnly: true
+            mountPath: /gen3discoveryai/credentials.json
+            subPath: credentials.json
+          - name: gen3-discovery-ai-g3auto-volume
+            readOnly: true
+            mountPath: /gen3discoveryai/storage_config.json
+            subPath: storage_config.json
+          - name: gen3-discovery-ai-knowledge-library-volume
+            mountPath: /gen3discoveryai/knowledge
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: 1
+          limits:
+            cpu: 2
+            memory: 512Mi
+        command: ["/bin/bash"]
+        args:
+          - "-c"
+          - |
+            echo
+            echo BEFORE /gen3discoveryai/knowledge  
+            ls -Ra /gen3discoveryai/knowledge 
+            echo running load_into_knowledge_store.py
+            poetry run python /gen3discoveryai/bin/load_into_knowledge_store.py tsvs /gen3discoveryai/knowledge/tmp/tsvs
+            
+            if [ -d "/gen3discoveryai/knowledge/tmp/markdown" ]; then
+                for dir in "/gen3discoveryai/knowledge/tmp/markdown"/*; do
+                    if [ -d "$dir" ]; then
+                        dir_name=$(basename "$dir")
+
+                        echo "Processing directory: $dir_name. Full path: $dir"
+                        poetry run python /gen3discoveryai/bin/load_into_knowledge_store.py markdown --topic $dir_name $dir
+                    fi
+                done
+            else
+                echo "Not syncing markdown, directory not found: /gen3discoveryai/knowledge/tmp/markdown"
+            fi
+
+            rm -r /gen3discoveryai/knowledge/tmp/
+            echo
+            echo AFTER /gen3discoveryai/knowledge 
+            ls -Ra /gen3discoveryai/knowledge
+      containers:
+      - name: gen3-discovery-ai
+        GEN3_GEN3-DISCOVERY-AI_IMAGE
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+        env:
+          - name: GEN3_DEBUG
+            GEN3_DEBUG_FLAG|-value: "False"-|
+          - name: ANONYMIZED_TELEMETRY
+            value: "False"
+          - name: GOOGLE_APPLICATION_CREDENTIALS
+            value: /gen3discoveryai/credentials.json
+        volumeMounts:
+          - name: gen3-discovery-ai-g3auto-volume
+            readOnly: true
+            mountPath: /gen3discoveryai/.env
+            subPath: env
+          - name: gen3-discovery-ai-g3auto-volume
+            readOnly: true
+            mountPath: /gen3discoveryai/credentials.json
+            subPath: credentials.json
+          - name: gen3-discovery-ai-g3auto-volume
+            readOnly: true
+            mountPath: /gen3discoveryai/storage_config.json
+            subPath: storage_config.json
+          - name: gen3-discovery-ai-knowledge-library-volume
+            mountPath: /gen3discoveryai/knowledge
+        imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: 1
+          limits:
+            cpu: 2
+            # NOTE: If the configured data for the knowledge library (vector database) is large, you may need to bump this
+            memory: 512Mi
diff --git a/kube/services/gen3-discovery-ai/gen3-discovery-ai-service.yaml b/kube/services/gen3-discovery-ai/gen3-discovery-ai-service.yaml
new file mode 100644
index 000000000..b4734c3b8
--- /dev/null
+++ b/kube/services/gen3-discovery-ai/gen3-discovery-ai-service.yaml
@@ -0,0 +1,21 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: gen3-discovery-ai-service
+spec:
+  selector:
+    app: gen3-discovery-ai
+    release: production
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8089
+      name: http
+      nodePort: null
+    - protocol: TCP
+      port: 443
+      targetPort: 443
+      name: https
+      nodePort: null
+  type: ClusterIP
+
diff --git a/kube/services/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf b/kube/services/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf
new file mode 100644
index 000000000..42e9a3758
--- /dev/null
+++ b/kube/services/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf
@@ -0,0 +1,12 @@
+          location /ai {
+              if ($csrf_check !~ ^ok-\S.+$) {
+                return 403 "failed csrf check";
+              }
+
+              set $proxy_service  "gen3-discovery-ai-service";
+              set $upstream http://gen3-discovery-ai-service$des_domain;
+              rewrite ^/ai/(.*) /$1 break;
+              proxy_pass $upstream;
+              proxy_redirect http://$host/ https://$host/ai/;
+              client_max_body_size 0;
+          }

From 077e475463098416001ebd3ced41ec06d1ee631b Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Wed, 6 Mar 2024 15:21:16 -0500
Subject: [PATCH 44/67] Raising requests for Jenkins pods (#2495)

---
 .secrets.baseline | 2 +-
 Jenkinsfile       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.secrets.baseline b/.secrets.baseline
index b7e06622d..fbed122fd 100644
--- a/.secrets.baseline
+++ b/.secrets.baseline
@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-02-23T20:30:41Z"
+  "generated_at": "2024-02-23T20:30:41Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
diff --git a/Jenkinsfile b/Jenkinsfile
index 4e3470ded..9c70a2e37 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -99,8 +99,8 @@ spec:
     resources:
       requests:
         cpu: 0.2
-        memory: 200Mi
-        ephemeral-storage: 200Mi
+        memory: 400Mi
+        ephemeral-storage: 1Gi
     env:
     - name: AWS_DEFAULT_REGION
       value: us-east-1

From 62894388c1ac907587916462d641788c87226e4a Mon Sep 17 00:00:00 2001
From: Michael Lukowski <lukowski9@gmail.com>
Date: Thu, 7 Mar 2024 10:42:36 -0600
Subject: [PATCH 45/67] fix cedar ingestion problems (#2494)

* fix cedar ingestion problems

* add default falue to get statement

---------

Co-authored-by: Mingfei Shao <2475897+mfshao@users.noreply.github.com>
---
 .secrets.baseline                             | 360 +++++-------------
 .../healdata/heal-cedar-data-ingest.py        | 101 +++++
 2 files changed, 187 insertions(+), 274 deletions(-)

diff --git a/.secrets.baseline b/.secrets.baseline
index fbed122fd..200b69841 100644
--- a/.secrets.baseline
+++ b/.secrets.baseline
@@ -1,9 +1,9 @@
 {
   "exclude": {
-    "files": "^.secrets.baseline$",
+    "files": null,
     "lines": null
   },
-  "generated_at": "2024-02-23T20:30:41Z",
+  "generated_at": "2024-03-04T21:42:56Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -61,14 +61,12 @@
     "Chef/repo/data_bags/README.md": [
       {
         "hashed_secret": "8a9250639e092d90f164792e35073a9395bff366",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 45,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "6367c48dd193d56ea7b0baad25b19455e529f5ee",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 51,
         "type": "Secret Keyword"
@@ -77,25 +75,22 @@
     "Docker/jenkins/Jenkins-CI-Worker/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 121,
+        "line_number": 124,
         "type": "Secret Keyword"
       }
     ],
     "Docker/jenkins/Jenkins-Worker/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
-        "is_secret": false,
         "is_verified": false,
-        "line_number": 143,
+        "line_number": 139,
         "type": "Secret Keyword"
       }
     ],
     "Docker/jenkins/Jenkins/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 107,
         "type": "Secret Keyword"
@@ -104,7 +99,6 @@
     "Docker/jenkins/Jenkins2/Dockerfile": [
       {
         "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 108,
         "type": "Secret Keyword"
@@ -113,7 +107,6 @@
     "Docker/sidecar/service.key": [
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1,
         "type": "Private Key"
@@ -122,7 +115,6 @@
     "Jenkins/Stacks/Jenkins/jenkins.env.sample": [
       {
         "hashed_secret": "eecee33686ac5861c2a7edc8b46bd0e5432bfddd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Secret Keyword"
@@ -131,7 +123,6 @@
     "ansible/roles/awslogs/defaults/main.yaml": [
       {
         "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Basic Auth Credentials"
@@ -140,14 +131,12 @@
     "ansible/roles/slurm/README.md": [
       {
         "hashed_secret": "4acfde1ff9c353ba2ef0dbe0df73bda2743cba42",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 86,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "579649582303921502d9e6d3f8755f13fdd2b476",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 86,
         "type": "Secret Keyword"
@@ -156,7 +145,6 @@
     "apis_configs/config_helper.py": [
       {
         "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 66,
         "type": "Basic Auth Credentials"
@@ -165,7 +153,6 @@
     "apis_configs/fence_credentials.json": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 23,
         "type": "Secret Keyword"
@@ -174,21 +161,18 @@
     "apis_configs/fence_settings.py": [
       {
         "hashed_secret": "3ef0fb8a603abdc0b6caac44a23fdc6792f77ddf",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Basic Auth Credentials"
       },
       {
         "hashed_secret": "b60d121b438a380c343d5ec3c2037564b82ffef3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 58,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 80,
         "type": "Basic Auth Credentials"
@@ -197,7 +181,6 @@
     "apis_configs/indexd_settings.py": [
       {
         "hashed_secret": "0a0d18c85e096611b5685b62bc60ec534d19bacc",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 59,
         "type": "Basic Auth Credentials"
@@ -206,7 +189,6 @@
     "apis_configs/peregrine_settings.py": [
       {
         "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 46,
         "type": "Basic Auth Credentials"
@@ -215,7 +197,6 @@
     "apis_configs/sheepdog_settings.py": [
       {
         "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 46,
         "type": "Basic Auth Credentials"
@@ -224,7 +205,6 @@
     "doc/Gen3-data-upload.md": [
       {
         "hashed_secret": "b8bd20d4a2701dc3aba0efbbf325f1359392d93e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
@@ -233,7 +213,6 @@
     "doc/api.md": [
       {
         "hashed_secret": "625de83a7517422051911680cc803921ff99db90",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 47,
         "type": "Hex High Entropy String"
@@ -242,28 +221,24 @@
     "doc/gen3OnK8s.md": [
       {
         "hashed_secret": "2db6d21d365f544f7ca3bcfb443ac96898a7a069",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 113,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "ff9ee043d85595eb255c05dfe32ece02a53efbb2",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 143,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "70374248fd7129088fef42b8f568443f6dce3a48",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 170,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "bcf22dfc6fb76b7366b1f1675baf2332a0e6a7ce",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 189,
         "type": "Secret Keyword"
@@ -272,7 +247,6 @@
     "doc/kube-setup-data-ingestion-job.md": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Secret Keyword"
@@ -281,7 +255,6 @@
     "doc/logs.md": [
       {
         "hashed_secret": "9addbf544119efa4a64223b649750a510f0d463f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Secret Keyword"
@@ -290,7 +263,6 @@
     "doc/slurm_cluster.md": [
       {
         "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 184,
         "type": "Secret Keyword"
@@ -299,14 +271,12 @@
     "files/dashboard/usage-reports/package-lock.json": [
       {
         "hashed_secret": "e095101882f706c4de95e0f75c5bcb9666e3f448",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 10,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "5422e4f96964d5739998b25ac214520c1b113e5b",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 15,
         "type": "Base64 High Entropy String"
@@ -315,14 +285,12 @@
     "gen3/bin/api.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 407,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 477,
         "type": "Secret Keyword"
@@ -331,7 +299,6 @@
     "gen3/bin/kube-dev-namespace.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 135,
         "type": "Secret Keyword"
@@ -340,7 +307,6 @@
     "gen3/bin/kube-setup-argo.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 206,
         "type": "Secret Keyword"
@@ -349,7 +315,6 @@
     "gen3/bin/kube-setup-aurora-monitoring.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 59,
         "type": "Secret Keyword"
@@ -358,7 +323,6 @@
     "gen3/bin/kube-setup-certs.sh": [
       {
         "hashed_secret": "2e9ee120fd25e31048598693aca91d5473898a99",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 50,
         "type": "Secret Keyword"
@@ -367,14 +331,12 @@
     "gen3/bin/kube-setup-dashboard.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 40,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 41,
         "type": "Secret Keyword"
@@ -383,14 +345,12 @@
     "gen3/bin/kube-setup-data-ingestion-job.sh": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 37,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "8695a632956b1b0ea7b66993dcc98732da39148c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 102,
         "type": "Secret Keyword"
@@ -399,7 +359,6 @@
     "gen3/bin/kube-setup-dicom-server.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 43,
         "type": "Secret Keyword"
@@ -408,23 +367,48 @@
     "gen3/bin/kube-setup-dicom.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 78,
         "type": "Secret Keyword"
       }
     ],
+    "gen3/bin/kube-setup-gen3-discovery-ai.sh": [
+      {
+        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "is_verified": false,
+        "line_number": 37,
+        "type": "Secret Keyword"
+      },
+      {
+        "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
+        "is_verified": false,
+        "line_number": 71,
+        "type": "Secret Keyword"
+      }
+    ],
     "gen3/bin/kube-setup-jenkins.sh": [
       {
         "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
+        "is_verified": false,
+        "line_number": 22,
+        "type": "Secret Keyword"
+      }
+    ],
+    "gen3/bin/kube-setup-jenkins2.sh": [
+      {
+        "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f",
+        "is_verified": false,
+        "line_number": 18,
+        "type": "Secret Keyword"
+      },
+      {
+        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
         "is_verified": false,
         "line_number": 22,
         "type": "Secret Keyword"
@@ -433,7 +417,6 @@
     "gen3/bin/kube-setup-metadata.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 35,
         "type": "Secret Keyword"
@@ -442,21 +425,18 @@
     "gen3/bin/kube-setup-revproxy.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 38,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 55,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 57,
         "type": "Secret Keyword"
@@ -465,21 +445,18 @@
     "gen3/bin/kube-setup-secrets.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 79,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 82,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "6f7531b95bbc99ac25a5cc82edb825f319c5dee8",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 95,
         "type": "Secret Keyword"
@@ -488,14 +465,12 @@
     "gen3/bin/kube-setup-sftp.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 36,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "83d11e3aec005a3b9a2077c6800683e202a95af4",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 51,
         "type": "Secret Keyword"
@@ -504,7 +479,6 @@
     "gen3/bin/kube-setup-sheepdog.sh": [
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 33,
         "type": "Secret Keyword"
@@ -513,28 +487,24 @@
     "gen3/bin/kube-setup-sower-jobs.sh": [
       {
         "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 25,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 120,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 122,
         "type": "Secret Keyword"
@@ -543,21 +513,18 @@
     "gen3/bin/kube-setup-ssjdispatcher.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 117,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "7992309146efaa8da936e34b0bd33242cd0e9f93",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 184,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 197,
         "type": "Secret Keyword"
@@ -566,14 +533,12 @@
     "gen3/lib/aws.sh": [
       {
         "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 640,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "5b4b6c62d3d99d202f095c38c664eded8f640ce8",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 660,
         "type": "Secret Keyword"
@@ -582,14 +547,12 @@
     "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml": [
       {
         "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 33,
         "type": "Basic Auth Credentials"
       },
       {
         "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 286,
         "type": "Secret Keyword"
@@ -598,7 +561,6 @@
     "gen3/lib/bootstrap/templates/Gen3Secrets/creds.json": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
@@ -607,7 +569,6 @@
     "gen3/lib/bootstrap/templates/Gen3Secrets/g3auto/dbfarm/servers.json": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Secret Keyword"
@@ -616,7 +577,6 @@
     "gen3/lib/logs/utils.sh": [
       {
         "hashed_secret": "76143b4ffc8aa2a53f9700ce229f904e69f1e8b5",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 3,
         "type": "Secret Keyword"
@@ -625,7 +585,6 @@
     "gen3/lib/manifestDefaults/hatchery/hatchery.json": [
       {
         "hashed_secret": "0da0e0005ca04acb407af2681d0bede6d9406039",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 78,
         "type": "Secret Keyword"
@@ -634,14 +593,12 @@
     "gen3/lib/onprem.sh": [
       {
         "hashed_secret": "29e52a9bac8f274fa41c51fce9c98eba0dd99cb3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 68,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "50f013532a9770a2c2cfdc38b7581dd01df69b70",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 84,
         "type": "Secret Keyword"
@@ -650,14 +607,12 @@
     "gen3/lib/secrets/rotate-postgres.sh": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 162,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 250,
         "type": "Secret Keyword"
@@ -666,49 +621,42 @@
     "gen3/lib/testData/etlconvert/expected2.yaml": [
       {
         "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 10,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 13,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 16,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 33,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 35,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 36,
         "type": "Base64 High Entropy String"
@@ -717,7 +665,6 @@
     "gen3/test/secretsTest.sh": [
       {
         "hashed_secret": "c2c715092ef59cba22520f109f041efca84b8938",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 25,
         "type": "Secret Keyword"
@@ -726,28 +673,24 @@
     "gen3/test/terraformTest.sh": [
       {
         "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 156,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 172,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 175,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "329b7cd8191942bedd337107934d365c43a86e6c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 175,
         "type": "Secret Keyword"
@@ -756,21 +699,18 @@
     "kube/services/argocd/values.yaml": [
       {
         "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 360,
         "type": "Private Key"
       },
       {
         "hashed_secret": "edbd5e119f94badb9f99a67ac6ff4c7a5204ad61",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 379,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "91dfd9ddb4198affc5c194cd8ce6d338fde470e2",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 412,
         "type": "Secret Keyword"
@@ -779,7 +719,6 @@
     "kube/services/datadog/values.yaml": [
       {
         "hashed_secret": "4a8ce7ae6a8a7f2624e232b61b18c2ac9789c44b",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 23,
         "type": "Secret Keyword"
@@ -788,401 +727,362 @@
     "kube/services/fenceshib/fenceshib-configmap.yaml": [
       {
         "hashed_secret": "a985e14b9d6744a2d04f29347693b55c116e478c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 375,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "adc747bc5eb82ef4b017f5c3759dcee5aa28c36f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 376,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "59b1702ff0eaf92c9271cbd12f587de97df7e13b",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 377,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "b4a748bbfbbca8925d932a47ab3dcb970d34caf5",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 378,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "af646701a84f7dd9f0e87753f54def881326e78a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 379,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "20c15ad9742124dc06e1612282c49bb443ebcbd9",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 380,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "9caded71b967a11b7a6cd0f20db91f06f3517d12",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 381,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "8f19501bc9241b71f7b6db929fb35ab12635dcd7",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 382,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "d6220f6a55df1ed11c4250f42ab07bb9da20541a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 383,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "dadd9b96636f9529f2547d05d754dc310ceba0c3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 384,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "3074bc66584550e20c3697a28f67a0762394943c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 385,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "823131319b4c4b4688f44d3e832bfa9696f16b52",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 386,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "015b780cbfb76988caf52de8ac974a6781e53110",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 387,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "5c8fac33207d74d667680ade09447ea8f43b76d7",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 388,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "c0c4bb09d8394e8f001e337bd27ccac355433d9e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 389,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f95631bcbbbc56e18487dcb242cfb1b3e74b16a1",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 390,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "01a692ab6232e0882a313d148981bab58ab98f53",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 391,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "658060a680d415ce6690ad2c3b622ddb33ddd50a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 392,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "80915b0bd9daa5e1f95cad573892980b1b5a2294",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 393,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "cc55977b293d8cdca8a2c19dfea6874e70057c41",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 394,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "e400ed02add75dd5f3a8c212857acf12027437d1",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 395,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "2e819c8baa3b0508a32b77de258655b3f3a6f7cb",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 396,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "546ed926d58ea5492ab6adb8be94a67aa44ac433",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 397,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f056f2deceed268e7af6dbdaf2577079c76e006a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 398,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "d75efee28f4798c3a9c6f44b78a8500513ef28b2",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 399,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "7803ae08cdc22a5e0b025eff3c9ef0628eedc165",
-        "is_secret": false,
+        "hashed_secret": "fbad0bc8f7792b03f89cd3780eb7cf79f284c525",
         "is_verified": false,
         "line_number": 419,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "b8b61e87f5b58b0eeb597b2122ea0cea2ccab3d9",
-        "is_secret": false,
+        "hashed_secret": "3f6480956a775dacb44e2c39aa3d4722a347f7ab",
         "is_verified": false,
         "line_number": 420,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "787745fc904c3bd7eddc3d1aab683a376c13890f",
-        "is_secret": false,
+        "hashed_secret": "17f32ae55b14d708ca121722c2cae37189f19daf",
         "is_verified": false,
         "line_number": 423,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "81361d672f238f505a6246ef9b655ee2f48d67e7",
-        "is_secret": false,
+        "hashed_secret": "08a74689ca077515d406093720a7e5675fb42bb8",
         "is_verified": false,
         "line_number": 424,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "7c98bff76ac3f273d15ed9bc3dd5294d323ab577",
-        "is_secret": false,
+        "hashed_secret": "fa577bb3b2600d2d522dcfea8f1e34896760fcf2",
         "is_verified": false,
         "line_number": 425,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "46038fc88daceed8dd46817ca45c72ae0270fdd4",
-        "is_secret": false,
+        "hashed_secret": "37254f15cca211a1bd5f7ceb23de2b3eb8fb33aa",
         "is_verified": false,
         "line_number": 426,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "acad0c57b4f5cbed1b4863ed06d02784180a9f92",
-        "is_secret": false,
+        "hashed_secret": "86865593e038509467b91c2d5f36ccc09c3f422b",
         "is_verified": false,
         "line_number": 427,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "1b57f49a6ee337c16ecd6aabfc0dff3b3821cd09",
-        "is_secret": false,
+        "hashed_secret": "a899a8d9e114b2a8e108f90e6a72c056db22489f",
         "is_verified": false,
         "line_number": 428,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "5b688158be36e8b3f265a462ed599dcf69290084",
-        "is_secret": false,
+        "hashed_secret": "756b4825f886afd83c25563ac9d45f318d695c48",
         "is_verified": false,
         "line_number": 429,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "965996e12c8b50b3c325d96003e8984a4ece658a",
-        "is_secret": false,
+        "hashed_secret": "89882eeb0aca97717a7e4afcf4bc08d077813c7f",
         "is_verified": false,
         "line_number": 430,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "584f0c58e764e948af1a35c9e60447aa0f84c6f5",
-        "is_secret": false,
+        "hashed_secret": "347140d7b7ceb4e501c3c9c2ea4f29338e2f145e",
         "is_verified": false,
         "line_number": 431,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "bcaf897786d060a675ee9d654a84ae8baf96e9d0",
-        "is_secret": false,
+        "hashed_secret": "61dbf70eb10d609e60c7b87faf8f755ff48abc46",
         "is_verified": false,
         "line_number": 432,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "0c09277fa183e06d32065f9386a3b4190b445df3",
-        "is_secret": false,
+        "hashed_secret": "24cd54c4b2f58378bba008cb2df68ac663fba7c8",
         "is_verified": false,
         "line_number": 433,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "5a51be06b305d6664e4afd25f21869b0f8b5039b",
-        "is_secret": false,
+        "hashed_secret": "fa4f9626ae4b98f4b61203c5bafb6f21c9c31e5d",
         "is_verified": false,
         "line_number": 434,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "b38404f8853d734e3d03577b2c1084b4540c8708",
-        "is_secret": false,
+        "hashed_secret": "b1370003d9cc1e346c83dba33e0418c7775a0c15",
         "is_verified": false,
         "line_number": 435,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "126ccc602cffcb8292beb57137f7f6719e317b72",
-        "is_secret": false,
+        "hashed_secret": "c66526e195e423a7ba7d68ac661cdcd8600dcd1f",
         "is_verified": false,
         "line_number": 436,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "6681c1d7e1d327642a32cb8864ad51e4b8f981e5",
-        "is_secret": false,
+        "hashed_secret": "d29d7044f0944eb30e02cf445f6998e3343dd811",
         "is_verified": false,
         "line_number": 437,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "7f7b1f316ece195e5f584fe2faf6f9edc6942c6f",
-        "is_secret": false,
+        "hashed_secret": "80a869460f33722387d8d58e7d9d2e1bbd5d1fe1",
+        "is_verified": false,
+        "line_number": 438,
+        "type": "Base64 High Entropy String"
+      },
+      {
+        "hashed_secret": "4a06e2a02cbc665adccb4162dc57836895da65b8",
         "is_verified": false,
         "line_number": 439,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "bb908c7bc655057f2edc42815c5dff82e9dea529",
-        "is_secret": false,
+        "hashed_secret": "ba2549f35835dfa101d3f660f7604dc78e3e226f",
         "is_verified": false,
         "line_number": 440,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "bc2a0d18e3dd142df7b34e95342d47bf8aadabcb",
-        "is_secret": false,
+        "hashed_secret": "f354d4ee5fdb94ad29c7b3600264467f45b80eaa",
         "is_verified": false,
         "line_number": 441,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "d60f0bcea109bb6edb6e45fd387f5f2c86e49e1a",
-        "is_secret": false,
+        "hashed_secret": "bf17b587868ba7c3db9865b114261b5b8f1df870",
         "is_verified": false,
         "line_number": 442,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "e549dd40a741557cc1c4e377df0a141354e22688",
-        "is_secret": false,
+        "hashed_secret": "de1fd7a0d32cba528b4d80818c6601f2588d5383",
         "is_verified": false,
         "line_number": 443,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "2dd2486dae84cad50387c20bf687b6fbc6162b58",
-        "is_secret": false,
+        "hashed_secret": "bcad65055f6de654541db2bf27d4e27bd54d94c7",
         "is_verified": false,
         "line_number": 444,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "71622010fc7eb09d9273f59c548bde6a5da5dc0e",
-        "is_secret": false,
+        "hashed_secret": "f2e16f2dd532f65f79341342fdf57a093fc408d8",
         "is_verified": false,
         "line_number": 445,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "6f0115cf53bd49ec990c562ac6cbfc452c83cd46",
-        "is_secret": false,
+        "hashed_secret": "bb036a679a7d2df9fd2ca57068a446bf7f7dd106",
         "is_verified": false,
         "line_number": 446,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "70dddd534b2f9bb70871fefe0845b79c3b69363f",
-        "is_secret": false,
+        "hashed_secret": "5aa6568b1e8185578a6e964f5c322783ad349554",
+        "is_verified": false,
+        "line_number": 447,
+        "type": "Base64 High Entropy String"
+      },
+      {
+        "hashed_secret": "4d14835ff0b0bf5aad480296cb705c74ac65f413",
         "is_verified": false,
         "line_number": 448,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "acf3536b0416aa99608b0be17e87655370ece829",
-        "is_secret": false,
+        "hashed_secret": "3f23f77dcf454ad73c4d61c44fd9aa584ef946c1",
         "is_verified": false,
-        "line_number": 449,
+        "line_number": 451,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "1d13ee35c7279c1fae1c6474ed47611994273e41",
-        "is_secret": false,
+        "hashed_secret": "1739fe5e5dfcf851b64f8b7b11538f1de29ce0b5",
         "is_verified": false,
-        "line_number": 450,
+        "line_number": 452,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "d38cf89b25bd7378cdb4e00b4b59293001dd500b",
-        "is_secret": false,
+        "hashed_secret": "8129db302110714fc735e3494bd82a65690e0963",
         "is_verified": false,
-        "line_number": 451,
+        "line_number": 453,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "1648f34ce2f1b563a8ed1c6d5d55b5e76a395903",
-        "is_secret": false,
+        "hashed_secret": "b48bfc62091164086a703115a0e68bdb09212591",
         "is_verified": false,
-        "line_number": 452,
+        "line_number": 454,
         "type": "Base64 High Entropy String"
       },
       {
-        "hashed_secret": "9bf63f6f49fb01ff80959bc5a60c8688df92cc02",
-        "is_secret": false,
+        "hashed_secret": "a10284feaf27f84081073a3267e3dce24ca7b911",
         "is_verified": false,
-        "line_number": 453,
+        "line_number": 455,
+        "type": "Base64 High Entropy String"
+      },
+      {
+        "hashed_secret": "3fd80f31de4be8dde9d2b421e832c7d4043fd49a",
+        "is_verified": false,
+        "line_number": 456,
         "type": "Base64 High Entropy String"
       }
     ],
     "kube/services/jobs/indexd-authz-job.yaml": [
       {
         "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 87,
         "type": "Basic Auth Credentials"
@@ -1191,14 +1091,12 @@
     "kube/services/monitoring/grafana-values.yaml": [
       {
         "hashed_secret": "2ae868079d293e0a185c671c7bcdac51df36e385",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 162,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "7a64ff8446b06d38dc271019994f13823a2cbcf4",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 166,
         "type": "Secret Keyword"
@@ -1207,7 +1105,6 @@
     "kube/services/revproxy/helpers.js": [
       {
         "hashed_secret": "1d278d3c888d1a2fa7eed622bfc02927ce4049af",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 10,
         "type": "Base64 High Entropy String"
@@ -1216,7 +1113,6 @@
     "kube/services/revproxy/helpersTest.js": [
       {
         "hashed_secret": "e029d4904cc728879d70030572bf37d4510367cb",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 22,
         "type": "JSON Web Token"
@@ -1225,7 +1121,6 @@
     "kube/services/superset/superset-deploy.yaml": [
       {
         "hashed_secret": "96e4aceb7cf284be363aa248a32a7cc89785a9f7",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 38,
         "type": "Secret Keyword"
@@ -1234,14 +1129,12 @@
     "kube/services/superset/superset-redis.yaml": [
       {
         "hashed_secret": "4af3596275edcb7cd5cc6c3c38bc10479902a08f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 165,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9fe1c31809da38c55b2b64bfab47b92bc5f6b7b9",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 265,
         "type": "Secret Keyword"
@@ -1250,35 +1143,30 @@
     "kube/services/superset/values.yaml": [
       {
         "hashed_secret": "6f803b24314c39062efe38d0c1da8c472f47eab3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 54,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "6eae3a5b062c6d0d79f070c26e6d62486b40cb46",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 86,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "3eb416223e9e69e6bb8ee19793911ad1ad2027d8",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 212,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "ff55435345834a3fe224936776c2aa15f6ed5358",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 396,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "98a84a63e5633d17e3b27b69695f87aa7189e9dc",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 503,
         "type": "Secret Keyword"
@@ -1287,280 +1175,240 @@
     "package-lock.json": [
       {
         "hashed_secret": "0656ad0df3af4633dc369f13d5e8806973c5fd9d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1481,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "00091d875d922437c5fc9e6067a08e78c2482e87",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1489,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "c4e5cc37e115bf7d86e76e3d799705bf691e4d00",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1521,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "0512e37fbedf1d16828680a038a241b4780a5c04",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1547,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "01868fd50edbfe6eb91e5b01209b543adc6857af",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1611,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a6f48bf1e398deffc7fd31da17c3506b46c97a93",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1640,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "85ce358dbdec0996cf3ccd2bf1c6602af68c181e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1648,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "6f9bfb49cb818d2fe07592515e4c3f7a0bbd7e0e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1664,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "7098a3e6d6d2ec0a40f04fe12509c5c6f4c49c0e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1683,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "1664ad175bba1795a7ecad572bae7e0740b94f56",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1733,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "1ec4ce2eb945ce2f816dcb6ebdd1e10247f439a3",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1742,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a7af5768a6d936e36f28e1030d7f894d7aaf555e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1755,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "6fbc7dd864586173160874f2a86ca7d2d552cb85",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1769,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "81a961f2c89c6209328b74a8768e30fd76c3ac72",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1855,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "797d4751c536c421cb82b9f62e0a804af30d78f5",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1889,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "0d55babfa89f240142c0adfc7b560500a1d3ae7c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1894,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "e9fdc3025cd10bd8aa4508611e6b7b7a9d650a2c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1921,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "4cf9419259c0ce8eee84b468af3c72db8b001620",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1950,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "24816e3eb4308e247bde7c1d09ffb7b79c519b71",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 1983,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "e9adfe8a333d45f4776fe0eab31608be5d7b6a7d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2004,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "03d6fb388dd1b185129b14221f7127715822ece6",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2013,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "ee161bb3f899720f95cee50a5f9ef9c9ed96278b",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2046,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "ebeb5b574fa1ed24a40248275e6136759e766466",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2078,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a6a555a428522ccf439fd516ce7c7e269274363f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2083,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f7f85d9f7c87f1e576dcaf4cf50f35728f9a3265",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2111,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "3f1646b60abe74297d2f37a1eee5dc771ad834fc",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2138,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "fd933c71e82d5519ae0cb0779b370d02f6935759",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2143,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "7090aa59cb52ad1f1810b08c4ac1ddf5c8fce523",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2150,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "756444bea4ea3d67844d8ddf58ad32356e9c2430",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2188,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f74135fdd6b8dafdfb01ebbc61c5e5c24ee27cf8",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2291,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "56fbae787f4aed7d0632e95840d71bd378d3a36f",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2303,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "81cb6be182eb79444202c4563080aee75296a672",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2308,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "f0f3f7bce32184893046ac5f8cc80da56c3ca539",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2317,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "097893233346336f4003acfb6eb173ee59e648f0",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2327,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "bb14c3b4ef4a9f2e86ffdd44b88d9b6729419671",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2332,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "71344a35cff67ef081920095d1406601fb5e9b97",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2340,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "eb3db6990fd43477a35dfeffc90b3f1ffa83c7bd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2349,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "266288bdc14807b538d1e48a5891e361fa9b4a14",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2357,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "800477261175fd21f23e7321923e1fba6ae55471",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2369,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "3f0c251b9c2c21454445a98fde6915ceacde2136",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 2387,
         "type": "Base64 High Entropy String"
@@ -1569,7 +1417,6 @@
     "tf_files/aws/cognito/README.md": [
       {
         "hashed_secret": "f6920f370a30262b7dd70e97293c73ec89739b70",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 106,
         "type": "Secret Keyword"
@@ -1578,14 +1425,12 @@
     "tf_files/aws/commons/README.md": [
       {
         "hashed_secret": "d02e53411e8cb4cd709778f173f7bc9a3455f8ed",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 60,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "9dc0da3613af850c5a018b0a88a5626fb8888e4e",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 78,
         "type": "Secret Keyword"
@@ -1594,7 +1439,6 @@
     "tf_files/aws/eks/sample.tfvars": [
       {
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 107,
         "type": "Hex High Entropy String"
@@ -1603,7 +1447,6 @@
     "tf_files/aws/eks/variables.tf": [
       {
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 133,
         "type": "Hex High Entropy String"
@@ -1612,14 +1455,12 @@
     "tf_files/aws/modules/common-logging/README.md": [
       {
         "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 57,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 59,
         "type": "Hex High Entropy String"
@@ -1628,28 +1469,24 @@
     "tf_files/aws/modules/common-logging/lambda_function.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Hex High Entropy String"
@@ -1658,21 +1495,18 @@
     "tf_files/aws/modules/common-logging/testLambda.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
@@ -1681,7 +1515,6 @@
     "tf_files/aws/modules/eks/variables.tf": [
       {
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 113,
         "type": "Hex High Entropy String"
@@ -1690,14 +1523,12 @@
     "tf_files/aws/modules/management-logs/README.md": [
       {
         "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 54,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 56,
         "type": "Hex High Entropy String"
@@ -1706,28 +1537,24 @@
     "tf_files/aws/modules/management-logs/lambda_function.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 18,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 30,
         "type": "Hex High Entropy String"
@@ -1736,42 +1563,36 @@
     "tf_files/aws/modules/management-logs/testLambda.py": [
       {
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 5,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "3cf8eb4e9254e1d6cc523da01f8b798b9a83101a",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "51118900cd675df1b44f254057398f3e52902a5d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "60a6dfc8d43cd2f5c6292899fc2f94f2d4fc32c4",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 6,
         "type": "Hex High Entropy String"
@@ -1780,7 +1601,6 @@
     "tf_files/aws/slurm/README.md": [
       {
         "hashed_secret": "fd85d792fa56981cf6a8d2a5c0857c74af86e99d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 83,
         "type": "Secret Keyword"
@@ -1789,7 +1609,6 @@
     "tf_files/azure/cloud.tf": [
       {
         "hashed_secret": "7c1a4b52b64e4106041971c345a1f3eab58fb2a4",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 424,
         "type": "Secret Keyword"
@@ -1798,7 +1617,6 @@
     "tf_files/gcp-bwg/roots/commons_setup/variables/answerfile-commons_setup-001.template.tfvars": [
       {
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 231,
         "type": "Secret Keyword"
@@ -1807,7 +1625,6 @@
     "tf_files/gcp-bwg/roots/templates/answerfile-commons_setup-001.template.tfvars": [
       {
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 231,
         "type": "Secret Keyword"
@@ -1816,7 +1633,6 @@
     "tf_files/gcp-bwg/roots/templates/answerfile-env-tenant.user.tfvars_NO_APP_SETUP": [
       {
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 262,
         "type": "Secret Keyword"
@@ -1825,21 +1641,18 @@
     "tf_files/gcp/commons/sample.tfvars": [
       {
         "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 11,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 26,
         "type": "Secret Keyword"
       },
       {
         "hashed_secret": "253c7b5e7c83a86346fc4501495b130813f08105",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 37,
         "type": "Secret Keyword"
@@ -1848,7 +1661,6 @@
     "tf_files/shared/modules/k8s_configs/creds.tpl": [
       {
         "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
-        "is_secret": false,
         "is_verified": false,
         "line_number": 8,
         "type": "Secret Keyword"
diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py
index 71575e3c5..c54f9d5aa 100644
--- a/files/scripts/healdata/heal-cedar-data-ingest.py
+++ b/files/scripts/healdata/heal-cedar-data-ingest.py
@@ -1,4 +1,5 @@
 import argparse
+import copy
 import json
 import sys
 import requests
@@ -40,6 +41,16 @@
     "study_metadata.human_subject_applicability.gender_applicability": "Not applicable"
 }
 
+# repository links
+REPOSITORY_STUDY_ID_LINK_TEMPLATE = {
+    "NIDDK Central": "https://repository.niddk.nih.gov/studies/<STUDY_ID>/",
+    "NIDA Data Share": "https://datashare.nida.nih.gov/study/<STUDY_ID>",
+    "NICHD DASH": "https://dash.nichd.nih.gov/study/<STUDY_ID>",
+    "ICPSR": "https://www.icpsr.umich.edu/web/ICPSR/studies/<STUDY_ID>",
+    "BioSystics-AP": "https://biosystics-ap.com/assays/assaystudy/<STUDY_ID>/",
+}
+
+
 def is_valid_uuid(uuid_to_test, version=4):
     """
     Check if uuid_to_test is a valid UUID.
@@ -114,6 +125,31 @@ def get_client_token(client_id: str, client_secret: str):
     return token
 
 
+def get_related_studies(serial_num, hostname):
+    related_study_result = []
+
+    if serial_num:
+        mds = requests.get(f"http://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000")
+        if mds.status_code == 200:
+            related_study_metadata = mds.json()
+
+            for (
+                related_study_metadata_key,
+                related_study_metadata_value,
+            ) in related_study_metadata.items():
+                title = (
+                    related_study_metadata_value.get(
+                        "gen3_discovery", {}
+                    )
+                    .get("study_metadata", {})
+                    .get("minimal_info", {})
+                    .get("study_name", "")
+                )
+                link = f"https://{hostname}/portal/discovery/{related_study_metadata_key}/"
+                related_study_result.append({"title": title, "link": link})
+    return related_study_result
+
+
 parser = argparse.ArgumentParser()
 
 parser.add_argument("--directory", help="CEDAR Directory ID for registering ")
@@ -214,6 +250,71 @@ def get_client_token(client_id: str, client_secret: str):
                 mds_res["gen3_discovery"]["study_metadata"].update(cedar_record)
                 mds_res["gen3_discovery"]["study_metadata"]["metadata_location"]["other_study_websites"] = cedar_record_other_study_websites
 
+                # setup citations
+                doi_citation = mds_res["gen3_discovery"]["study_metadata"].get("doi_citation", "")
+                mds_res["gen3_discovery"]["study_metadata"]["citation"]["heal_platform_citation"] = doi_citation
+
+
+                # setup repository_study_link
+                data_repositories = (
+                    mds_res
+                    .get("gen3_discovery", {})
+                    .get("study_metadata", {})
+                    .get("metadata_location", {})
+                    .get("data_repositories", [])
+                )
+                repository_citation = "Users must also include a citation to the data as specified by the local repository."
+                repository_citation_additional_text = ' The link to the study page at the local repository can be found in the "Data" tab.'
+                for repository in data_repositories:
+                    if (
+                        repository["repository_name"]
+                        and repository["repository_name"]
+                        in REPOSITORY_STUDY_ID_LINK_TEMPLATE
+                        and repository["repository_study_ID"]
+                    ):
+                        repository_study_link = REPOSITORY_STUDY_ID_LINK_TEMPLATE[
+                            repository["repository_name"]
+                        ].replace("<STUDY_ID>", repository["repository_study_ID"])
+                        repository.update({"repository_study_link": repository_study_link})
+                        if repository_citation_additional_text not in repository_citation:
+                            repository_citation += repository_citation_additional_text
+                if len(data_repositories):
+                    data_repositories[0] = {
+                        **data_repositories[0],
+                        "repository_citation": repository_citation,
+                    }
+
+                mds_res["gen3_discovery"]["study_metadata"][
+                    "metadata_location"
+                ]["data_repositories"] = copy.deepcopy(data_repositories)
+
+
+
+                # set up related studies
+                serial_num = None
+                try:
+                    serial_num = (
+                        mds_res
+                        .get("nih_reporter", {})
+                        .get("project_num_split", {})
+                        .get("serial_num", None)
+                    )
+                except Exception:
+                    print(f"Unable to get serial number for study")
+
+                if serial_num == None:
+                    print(f"Unable to get serial number for study")
+
+                related_study_result = get_related_studies(serial_num, hostname)
+                existing_related_study_result = mds_res.get("related_studies", [])
+                for related_study in related_study_result:
+                    if related_study not in existing_related_study_result:
+                        existing_related_study_result.append(copy.deepcopy(related_study))
+                mds_res["gen3_discovery"][
+                    "related_studies"
+                ] = copy.deepcopy(existing_related_study_result)
+
+
                 # merge data from cedar that is not study level metadata into a level higher
                 deleted_keys = []
                 for key, value in mds_res["gen3_discovery"]["study_metadata"].items():

From 99fc77ac5ee4a36443a9b802d903e6ffaab6f8c7 Mon Sep 17 00:00:00 2001
From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com>
Date: Thu, 7 Mar 2024 17:32:40 -0600
Subject: [PATCH 46/67] MIDRC-602 Add ecr-access job (#2480)

---
 .pre-commit-config.yaml                       |    2 +-
 .secrets.baseline                             | 3792 +++++++++++++----
 files/scripts/ecr-access-job-requirements.txt |    1 +
 files/scripts/ecr-access-job.md               |   85 +
 files/scripts/ecr-access-job.py               |  177 +
 gen3/bin/kube-setup-ecr-access-cronjob.sh     |   61 +
 kube/services/jobs/ecr-access-job.yaml        |   85 +
 7 files changed, 3339 insertions(+), 864 deletions(-)
 create mode 100644 files/scripts/ecr-access-job-requirements.txt
 create mode 100644 files/scripts/ecr-access-job.md
 create mode 100644 files/scripts/ecr-access-job.py
 create mode 100644 gen3/bin/kube-setup-ecr-access-cronjob.sh
 create mode 100644 kube/services/jobs/ecr-access-job.yaml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 2e3ce795b..82034495d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
     -   repo: git@github.com:Yelp/detect-secrets
-        rev: v0.13.1
+        rev: v1.4.0
         hooks:
         -   id: detect-secrets
             args: ['--baseline', '.secrets.baseline']
diff --git a/.secrets.baseline b/.secrets.baseline
index 200b69841..0c4eba0a8 100644
--- a/.secrets.baseline
+++ b/.secrets.baseline
@@ -1,19 +1,18 @@
 {
-  "exclude": {
-    "files": null,
-    "lines": null
-  },
-  "generated_at": "2024-03-04T21:42:56Z",
+  "version": "1.4.0",
   "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
     {
       "name": "AWSKeyDetector"
     },
     {
-      "name": "ArtifactoryDetector"
+      "name": "AzureStorageKeyDetector"
     },
     {
-      "base64_limit": 4.5,
-      "name": "Base64HighEntropyString"
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
     },
     {
       "name": "BasicAuthDetector"
@@ -22,8 +21,14 @@
       "name": "CloudantDetector"
     },
     {
-      "hex_limit": 3,
-      "name": "HexHighEntropyString"
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
     },
     {
       "name": "IbmCloudIamDetector"
@@ -35,21 +40,30 @@
       "name": "JwtTokenDetector"
     },
     {
-      "keyword_exclude": null,
-      "name": "KeywordDetector"
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
     },
     {
       "name": "MailchimpDetector"
     },
+    {
+      "name": "NpmDetector"
+    },
     {
       "name": "PrivateKeyDetector"
     },
+    {
+      "name": "SendGridDetector"
+    },
     {
       "name": "SlackDetector"
     },
     {
       "name": "SoftlayerDetector"
     },
+    {
+      "name": "SquareOAuthDetector"
+    },
     {
       "name": "StripeDetector"
     },
@@ -57,1619 +71,3671 @@
       "name": "TwilioKeyDetector"
     }
   ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".secrets.baseline"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    }
+  ],
   "results": {
     "Chef/repo/data_bags/README.md": [
       {
-        "hashed_secret": "8a9250639e092d90f164792e35073a9395bff366",
-        "is_verified": false,
-        "line_number": 45,
-        "type": "Secret Keyword"
-      },
-      {
+        "type": "Secret Keyword",
+        "filename": "Chef/repo/data_bags/README.md",
         "hashed_secret": "6367c48dd193d56ea7b0baad25b19455e529f5ee",
         "is_verified": false,
-        "line_number": 51,
-        "type": "Secret Keyword"
+        "line_number": 38
       }
     ],
-    "Docker/jenkins/Jenkins-CI-Worker/Dockerfile": [
+    "Docker/sidecar/service.key": [
       {
-        "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
+        "type": "Private Key",
+        "filename": "Docker/sidecar/service.key",
+        "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_verified": false,
-        "line_number": 124,
-        "type": "Secret Keyword"
+        "line_number": 1
       }
     ],
-    "Docker/jenkins/Jenkins-Worker/Dockerfile": [
+    "Jenkins/Stacks/Jenkins/jenkins.env.sample": [
       {
-        "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
+        "type": "Secret Keyword",
+        "filename": "Jenkins/Stacks/Jenkins/jenkins.env.sample",
+        "hashed_secret": "f41a52528dd2d592d2c05de5f388101c2948aa98",
         "is_verified": false,
-        "line_number": 139,
-        "type": "Secret Keyword"
+        "line_number": 5
       }
     ],
-    "Docker/jenkins/Jenkins/Dockerfile": [
+    "Jenkinsfile": [
       {
-        "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
+        "type": "Secret Keyword",
+        "filename": "Jenkinsfile",
+        "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf",
         "is_verified": false,
-        "line_number": 107,
-        "type": "Secret Keyword"
-      }
-    ],
-    "Docker/jenkins/Jenkins2/Dockerfile": [
+        "line_number": 144
+      },
       {
-        "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603",
+        "type": "Secret Keyword",
+        "filename": "Jenkinsfile",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 108,
-        "type": "Secret Keyword"
+        "line_number": 147
       }
     ],
-    "Docker/sidecar/service.key": [
+    "ansible/roles/slurm/README.md": [
       {
-        "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
+        "type": "Base64 High Entropy String",
+        "filename": "ansible/roles/slurm/README.md",
+        "hashed_secret": "4acfde1ff9c353ba2ef0dbe0df73bda2743cba42",
         "is_verified": false,
-        "line_number": 1,
-        "type": "Private Key"
+        "line_number": 86
       }
     ],
-    "Jenkins/Stacks/Jenkins/jenkins.env.sample": [
+    "apis_configs/fence_settings.py": [
       {
-        "hashed_secret": "eecee33686ac5861c2a7edc8b46bd0e5432bfddd",
+        "type": "Basic Auth Credentials",
+        "filename": "apis_configs/fence_settings.py",
+        "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
         "is_verified": false,
-        "line_number": 5,
-        "type": "Secret Keyword"
+        "line_number": 80
       }
     ],
-    "ansible/roles/awslogs/defaults/main.yaml": [
+    "apis_configs/peregrine_settings.py": [
       {
-        "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684",
+        "type": "Basic Auth Credentials",
+        "filename": "apis_configs/peregrine_settings.py",
+        "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
         "is_verified": false,
-        "line_number": 30,
-        "type": "Basic Auth Credentials"
+        "line_number": 46
       }
     ],
-    "ansible/roles/slurm/README.md": [
-      {
-        "hashed_secret": "4acfde1ff9c353ba2ef0dbe0df73bda2743cba42",
-        "is_verified": false,
-        "line_number": 86,
-        "type": "Base64 High Entropy String"
-      },
+    "apis_configs/sheepdog_settings.py": [
       {
-        "hashed_secret": "579649582303921502d9e6d3f8755f13fdd2b476",
+        "type": "Basic Auth Credentials",
+        "filename": "apis_configs/sheepdog_settings.py",
+        "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
         "is_verified": false,
-        "line_number": 86,
-        "type": "Secret Keyword"
+        "line_number": 46
       }
     ],
-    "apis_configs/config_helper.py": [
+    "aws-inspec/kubernetes/chef_inspec-cron.yaml": [
       {
-        "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f",
+        "type": "Secret Keyword",
+        "filename": "aws-inspec/kubernetes/chef_inspec-cron.yaml",
+        "hashed_secret": "a3ba27250861948a554629a0e21168821ddfa9f1",
         "is_verified": false,
-        "line_number": 66,
-        "type": "Basic Auth Credentials"
+        "line_number": 35
       }
     ],
-    "apis_configs/fence_credentials.json": [
+    "doc/api.md": [
       {
-        "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "type": "Hex High Entropy String",
+        "filename": "doc/api.md",
+        "hashed_secret": "625de83a7517422051911680cc803921ff99db90",
         "is_verified": false,
-        "line_number": 23,
-        "type": "Secret Keyword"
+        "line_number": 47
       }
     ],
-    "apis_configs/fence_settings.py": [
+    "doc/gen3OnK8s.md": [
       {
-        "hashed_secret": "3ef0fb8a603abdc0b6caac44a23fdc6792f77ddf",
+        "type": "Secret Keyword",
+        "filename": "doc/gen3OnK8s.md",
+        "hashed_secret": "55c100ba37d2df35ec1e5f5d6302f060387df6cc",
         "is_verified": false,
-        "line_number": 6,
-        "type": "Basic Auth Credentials"
+        "line_number": 113
       },
       {
-        "hashed_secret": "b60d121b438a380c343d5ec3c2037564b82ffef3",
+        "type": "Secret Keyword",
+        "filename": "doc/gen3OnK8s.md",
+        "hashed_secret": "262d8e9b8ac5f06e7612dfb608f7267f88679801",
         "is_verified": false,
-        "line_number": 58,
-        "type": "Secret Keyword"
+        "line_number": 120
       },
       {
-        "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
+        "type": "Secret Keyword",
+        "filename": "doc/gen3OnK8s.md",
+        "hashed_secret": "1c17e556736c4d23933f99d199e7c2c572895fd2",
+        "is_verified": false,
+        "line_number": 143
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "doc/gen3OnK8s.md",
+        "hashed_secret": "76a4acaf31b815aa2c41cc2a2176b11fa9edf00a",
+        "is_verified": false,
+        "line_number": 145
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "doc/gen3OnK8s.md",
+        "hashed_secret": "9d678cbce5a343920f754d5836f03346ee01cde5",
         "is_verified": false,
-        "line_number": 80,
-        "type": "Basic Auth Credentials"
+        "line_number": 154
       }
     ],
-    "apis_configs/indexd_settings.py": [
+    "files/scripts/psql-fips-fix.sh": [
       {
-        "hashed_secret": "0a0d18c85e096611b5685b62bc60ec534d19bacc",
+        "type": "Secret Keyword",
+        "filename": "files/scripts/psql-fips-fix.sh",
+        "hashed_secret": "2f1aa1e2a58704b452a5dd60ab1bd2b761bf296a",
         "is_verified": false,
-        "line_number": 59,
-        "type": "Basic Auth Credentials"
+        "line_number": 9
       }
     ],
-    "apis_configs/peregrine_settings.py": [
+    "gen3/bin/bucket-manifest.sh": [
       {
-        "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
+        "type": "Secret Keyword",
+        "filename": "gen3/bin/bucket-manifest.sh",
+        "hashed_secret": "2be88ca4242c76e8253ac62474851065032d6833",
         "is_verified": false,
-        "line_number": 46,
-        "type": "Basic Auth Credentials"
+        "line_number": 58
       }
     ],
-    "apis_configs/sheepdog_settings.py": [
+    "gen3/bin/bucket-replicate.sh": [
       {
-        "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3",
+        "type": "Secret Keyword",
+        "filename": "gen3/bin/bucket-replicate.sh",
+        "hashed_secret": "2be88ca4242c76e8253ac62474851065032d6833",
         "is_verified": false,
-        "line_number": 46,
-        "type": "Basic Auth Credentials"
+        "line_number": 39
       }
     ],
-    "doc/Gen3-data-upload.md": [
+    "gen3/bin/secrets.sh": [
       {
-        "hashed_secret": "b8bd20d4a2701dc3aba0efbbf325f1359392d93e",
+        "type": "Secret Keyword",
+        "filename": "gen3/bin/secrets.sh",
+        "hashed_secret": "fb6220478aaba649aac37271a1d7c6317abc03a6",
         "is_verified": false,
-        "line_number": 26,
-        "type": "Secret Keyword"
+        "line_number": 135
       }
     ],
-    "doc/api.md": [
+    "gen3/lib/aws.sh": [
       {
-        "hashed_secret": "625de83a7517422051911680cc803921ff99db90",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/aws.sh",
+        "hashed_secret": "6b44a330b450ee550c081410c6b705dfeaa105ce",
         "is_verified": false,
-        "line_number": 47,
-        "type": "Hex High Entropy String"
+        "line_number": 640
       }
     ],
-    "doc/gen3OnK8s.md": [
+    "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml": [
       {
-        "hashed_secret": "2db6d21d365f544f7ca3bcfb443ac96898a7a069",
+        "type": "Basic Auth Credentials",
+        "filename": "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml",
+        "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
         "is_verified": false,
-        "line_number": 113,
-        "type": "Secret Keyword"
-      },
+        "line_number": 33
+      }
+    ],
+    "gen3/lib/bootstrap/templates/cdis-manifest/manifests/sower/sower.json": [
       {
-        "hashed_secret": "ff9ee043d85595eb255c05dfe32ece02a53efbb2",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/bootstrap/templates/cdis-manifest/manifests/sower/sower.json",
+        "hashed_secret": "0447a636536df0264b2000403fbefd69f603ceb1",
         "is_verified": false,
-        "line_number": 143,
-        "type": "Secret Keyword"
+        "line_number": 54
       },
       {
-        "hashed_secret": "70374248fd7129088fef42b8f568443f6dce3a48",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/bootstrap/templates/cdis-manifest/manifests/sower/sower.json",
+        "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966",
         "is_verified": false,
-        "line_number": 170,
-        "type": "Secret Keyword"
+        "line_number": 60
       },
       {
-        "hashed_secret": "bcf22dfc6fb76b7366b1f1675baf2332a0e6a7ce",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/bootstrap/templates/cdis-manifest/manifests/sower/sower.json",
+        "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc",
         "is_verified": false,
-        "line_number": 189,
-        "type": "Secret Keyword"
+        "line_number": 108
       }
     ],
-    "doc/kube-setup-data-ingestion-job.md": [
+    "gen3/lib/onprem.sh": [
       {
-        "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/onprem.sh",
+        "hashed_secret": "29e52a9bac8f274fa41c51fce9c98eba0dd99cb3",
         "is_verified": false,
-        "line_number": 30,
-        "type": "Secret Keyword"
-      }
-    ],
-    "doc/logs.md": [
+        "line_number": 68
+      },
       {
-        "hashed_secret": "9addbf544119efa4a64223b649750a510f0d463f",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/onprem.sh",
+        "hashed_secret": "50f013532a9770a2c2cfdc38b7581dd01df69b70",
         "is_verified": false,
-        "line_number": 6,
-        "type": "Secret Keyword"
+        "line_number": 84
       }
     ],
-    "doc/slurm_cluster.md": [
+    "gen3/lib/testData/default/expectedFenceResult.yaml": [
       {
-        "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedFenceResult.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 184,
-        "type": "Secret Keyword"
-      }
-    ],
-    "files/dashboard/usage-reports/package-lock.json": [
+        "line_number": 68
+      },
       {
-        "hashed_secret": "e095101882f706c4de95e0f75c5bcb9666e3f448",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedFenceResult.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 10,
-        "type": "Base64 High Entropy String"
+        "line_number": 71
       },
       {
-        "hashed_secret": "5422e4f96964d5739998b25ac214520c1b113e5b",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedFenceResult.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 15,
-        "type": "Base64 High Entropy String"
-      }
-    ],
-    "gen3/bin/api.sh": [
+        "line_number": 74
+      },
       {
-        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedFenceResult.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 407,
-        "type": "Secret Keyword"
+        "line_number": 84
       },
       {
-        "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedFenceResult.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 477,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/bin/kube-dev-namespace.sh": [
+        "line_number": 87
+      },
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedFenceResult.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 135,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/bin/kube-setup-argo.sh": [
+        "line_number": 90
+      },
       {
-        "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedFenceResult.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
         "is_verified": false,
-        "line_number": 206,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/bin/kube-setup-aurora-monitoring.sh": [
+        "line_number": 93
+      },
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedFenceResult.yaml",
+        "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295",
         "is_verified": false,
-        "line_number": 59,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/bin/kube-setup-certs.sh": [
+        "line_number": 96
+      },
       {
-        "hashed_secret": "2e9ee120fd25e31048598693aca91d5473898a99",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedFenceResult.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 50,
-        "type": "Secret Keyword"
+        "line_number": 99
       }
     ],
-    "gen3/bin/kube-setup-dashboard.sh": [
+    "gen3/lib/testData/default/expectedSheepdogResult.yaml": [
       {
-        "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedSheepdogResult.yaml",
+        "hashed_secret": "ec9c944c51e87322de8d22e3ca9e2be1ad8fee0d",
         "is_verified": false,
-        "line_number": 40,
-        "type": "Secret Keyword"
+        "line_number": 60
       },
       {
-        "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedSheepdogResult.yaml",
+        "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc",
         "is_verified": false,
-        "line_number": 41,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/bin/kube-setup-data-ingestion-job.sh": [
+        "line_number": 63
+      },
       {
-        "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedSheepdogResult.yaml",
+        "hashed_secret": "e43756046ad1763d6946575fed0e05130a154bd2",
         "is_verified": false,
-        "line_number": 37,
-        "type": "Secret Keyword"
+        "line_number": 69
       },
       {
-        "hashed_secret": "8695a632956b1b0ea7b66993dcc98732da39148c",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/default/expectedSheepdogResult.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 102,
-        "type": "Secret Keyword"
+        "line_number": 72
       }
     ],
-    "gen3/bin/kube-setup-dicom-server.sh": [
+    "gen3/lib/testData/etlconvert/expected2.yaml": [
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/expected2.yaml",
+        "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474",
         "is_verified": false,
-        "line_number": 43,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/bin/kube-setup-dicom.sh": [
+        "line_number": 10
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/expected2.yaml",
+        "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55",
+        "is_verified": false,
+        "line_number": 13
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/expected2.yaml",
+        "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26",
+        "is_verified": false,
+        "line_number": 16
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/expected2.yaml",
+        "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12",
+        "is_verified": false,
+        "line_number": 18
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/expected2.yaml",
+        "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424",
+        "is_verified": false,
+        "line_number": 33
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/expected2.yaml",
+        "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba",
+        "is_verified": false,
+        "line_number": 35
+      },
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/expected2.yaml",
+        "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22",
         "is_verified": false,
-        "line_number": 78,
-        "type": "Secret Keyword"
+        "line_number": 36
       }
     ],
-    "gen3/bin/kube-setup-gen3-discovery-ai.sh": [
+    "gen3/lib/testData/etlconvert/users2.yaml": [
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/users2.yaml",
+        "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55",
+        "is_verified": false,
+        "line_number": 543
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/users2.yaml",
+        "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26",
+        "is_verified": false,
+        "line_number": 553
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/users2.yaml",
+        "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474",
+        "is_verified": false,
+        "line_number": 558
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/users2.yaml",
+        "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12",
+        "is_verified": false,
+        "line_number": 568
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/users2.yaml",
+        "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424",
+        "is_verified": false,
+        "line_number": 643
+      },
       {
-        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/users2.yaml",
+        "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba",
         "is_verified": false,
-        "line_number": 37,
-        "type": "Secret Keyword"
+        "line_number": 653
       },
       {
-        "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/lib/testData/etlconvert/users2.yaml",
+        "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22",
         "is_verified": false,
-        "line_number": 71,
-        "type": "Secret Keyword"
+        "line_number": 658
       }
     ],
-    "gen3/bin/kube-setup-jenkins.sh": [
+    "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 71
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 74
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 77
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 87
+      },
       {
-        "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 18,
-        "type": "Secret Keyword"
+        "line_number": 90
       },
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 22,
-        "type": "Secret Keyword"
+        "line_number": 93
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
+        "is_verified": false,
+        "line_number": 96
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml",
+        "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295",
+        "is_verified": false,
+        "line_number": 99
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 102
       }
     ],
-    "gen3/bin/kube-setup-jenkins2.sh": [
+    "gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml",
+        "hashed_secret": "ec9c944c51e87322de8d22e3ca9e2be1ad8fee0d",
+        "is_verified": false,
+        "line_number": 63
+      },
       {
-        "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml",
+        "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc",
         "is_verified": false,
-        "line_number": 18,
-        "type": "Secret Keyword"
+        "line_number": 66
       },
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml",
+        "hashed_secret": "e43756046ad1763d6946575fed0e05130a154bd2",
         "is_verified": false,
-        "line_number": 22,
-        "type": "Secret Keyword"
+        "line_number": 72
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 75
       }
     ],
-    "gen3/bin/kube-setup-metadata.sh": [
+    "gen3/test/secretsTest.sh": [
       {
-        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "type": "Secret Keyword",
+        "filename": "gen3/test/secretsTest.sh",
+        "hashed_secret": "c2c715092ef59cba22520f109f041efca84b8938",
         "is_verified": false,
-        "line_number": 35,
-        "type": "Secret Keyword"
+        "line_number": 25
       }
     ],
-    "gen3/bin/kube-setup-revproxy.sh": [
+    "gen3/test/terraformTest.sh": [
       {
-        "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
+        "type": "Secret Keyword",
+        "filename": "gen3/test/terraformTest.sh",
+        "hashed_secret": "6b44a330b450ee550c081410c6b705dfeaa105ce",
         "is_verified": false,
-        "line_number": 38,
-        "type": "Secret Keyword"
+        "line_number": 156
       },
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Secret Keyword",
+        "filename": "gen3/test/terraformTest.sh",
+        "hashed_secret": "d869db7fe62fb07c25a0403ecaea55031744b5fb",
         "is_verified": false,
-        "line_number": 55,
-        "type": "Secret Keyword"
+        "line_number": 163
       },
       {
-        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/test/terraformTest.sh",
+        "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009",
         "is_verified": false,
-        "line_number": 57,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/bin/kube-setup-secrets.sh": [
+        "line_number": 172
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/test/terraformTest.sh",
+        "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009",
+        "is_verified": false,
+        "line_number": 172
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "gen3/test/terraformTest.sh",
+        "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef",
+        "is_verified": false,
+        "line_number": 175
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/test/terraformTest.sh",
+        "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef",
+        "is_verified": false,
+        "line_number": 175
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "gen3/test/terraformTest.sh",
+        "hashed_secret": "212e1d3823c8c9af9e4c0c172164ee292b9a6768",
+        "is_verified": false,
+        "line_number": 311
+      },
       {
-        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "type": "Secret Keyword",
+        "filename": "gen3/test/terraformTest.sh",
+        "hashed_secret": "cb80dbb67a1a5bdf4957eea1473789f1c65357c6",
         "is_verified": false,
-        "line_number": 79,
-        "type": "Secret Keyword"
+        "line_number": 312
       },
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Secret Keyword",
+        "filename": "gen3/test/terraformTest.sh",
+        "hashed_secret": "5f35c25f4bf588b5fad46e249fcd9221f5257ce4",
         "is_verified": false,
-        "line_number": 82,
-        "type": "Secret Keyword"
+        "line_number": 313
       },
       {
-        "hashed_secret": "6f7531b95bbc99ac25a5cc82edb825f319c5dee8",
+        "type": "Secret Keyword",
+        "filename": "gen3/test/terraformTest.sh",
+        "hashed_secret": "5308421b43dde5775f1993bd25a8163070d65598",
         "is_verified": false,
-        "line_number": 95,
-        "type": "Secret Keyword"
+        "line_number": 314
       }
     ],
-    "gen3/bin/kube-setup-sftp.sh": [
+    "kube/services/access-backend/access-backend-deploy.yaml": [
       {
-        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "type": "Secret Keyword",
+        "filename": "kube/services/access-backend/access-backend-deploy.yaml",
+        "hashed_secret": "dbf88a0c3d905c669c0fd13bf8172bb34d4b1168",
         "is_verified": false,
-        "line_number": 36,
-        "type": "Secret Keyword"
-      },
+        "line_number": 60
+      }
+    ],
+    "kube/services/acronymbot/acronymbot-deploy.yaml": [
       {
-        "hashed_secret": "83d11e3aec005a3b9a2077c6800683e202a95af4",
+        "type": "Secret Keyword",
+        "filename": "kube/services/acronymbot/acronymbot-deploy.yaml",
+        "hashed_secret": "600833390a6b9891d0d8a5f6e3326abb237ac8ca",
         "is_verified": false,
-        "line_number": 51,
-        "type": "Secret Keyword"
+        "line_number": 49
       }
     ],
-    "gen3/bin/kube-setup-sheepdog.sh": [
+    "kube/services/arborist/arborist-deploy-2.yaml": [
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/arborist/arborist-deploy-2.yaml",
+        "hashed_secret": "6c57cdfdaaf3cde7a1da6aa94c7d8e46502c4bab",
         "is_verified": false,
-        "line_number": 33,
-        "type": "Secret Keyword"
+        "line_number": 59
       }
     ],
-    "gen3/bin/kube-setup-sower-jobs.sh": [
+    "kube/services/arborist/arborist-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/arborist/arborist-deploy.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 64
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/arborist/arborist-deploy.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 67
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/arborist/arborist-deploy.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 70
+      },
       {
-        "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897",
+        "type": "Secret Keyword",
+        "filename": "kube/services/arborist/arborist-deploy.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 25,
-        "type": "Secret Keyword"
+        "line_number": 77
       },
       {
-        "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a",
+        "type": "Secret Keyword",
+        "filename": "kube/services/arborist/arborist-deploy.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 26,
-        "type": "Secret Keyword"
+        "line_number": 80
       },
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/arborist/arborist-deploy.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 120,
-        "type": "Secret Keyword"
+        "line_number": 83
       },
       {
-        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "type": "Secret Keyword",
+        "filename": "kube/services/arborist/arborist-deploy.yaml",
+        "hashed_secret": "ea73fcfdaa415890d5fde24d3b2245671be32f73",
         "is_verified": false,
-        "line_number": 122,
-        "type": "Secret Keyword"
+        "line_number": 86
       }
     ],
-    "gen3/bin/kube-setup-ssjdispatcher.sh": [
+    "kube/services/argo/workflows/fence-usersync-wf.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/argo/workflows/fence-usersync-wf.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 108
+      },
       {
-        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "type": "Secret Keyword",
+        "filename": "kube/services/argo/workflows/fence-usersync-wf.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 117,
-        "type": "Secret Keyword"
+        "line_number": 111
       },
       {
-        "hashed_secret": "7992309146efaa8da936e34b0bd33242cd0e9f93",
+        "type": "Secret Keyword",
+        "filename": "kube/services/argo/workflows/fence-usersync-wf.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 184,
-        "type": "Secret Keyword"
+        "line_number": 114
       },
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/argo/workflows/fence-usersync-wf.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 197,
-        "type": "Secret Keyword"
+        "line_number": 117
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/argo/workflows/fence-usersync-wf.yaml",
+        "hashed_secret": "ea73fcfdaa415890d5fde24d3b2245671be32f73",
+        "is_verified": false,
+        "line_number": 120
       }
     ],
-    "gen3/lib/aws.sh": [
+    "kube/services/argocd/values.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/argocd/values.yaml",
+        "hashed_secret": "bfc1b86ce643b65bd540989213254b01fd6ad418",
+        "is_verified": false,
+        "line_number": 1489
+      }
+    ],
+    "kube/services/arranger/arranger-deploy.yaml": [
       {
-        "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
+        "type": "Secret Keyword",
+        "filename": "kube/services/arranger/arranger-deploy.yaml",
+        "hashed_secret": "0db22b31c9add2d3c76743c0ac6fbc99bb8b4761",
         "is_verified": false,
-        "line_number": 640,
-        "type": "Secret Keyword"
+        "line_number": 61
       },
       {
-        "hashed_secret": "5b4b6c62d3d99d202f095c38c664eded8f640ce8",
+        "type": "Secret Keyword",
+        "filename": "kube/services/arranger/arranger-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 660,
-        "type": "Secret Keyword"
+        "line_number": 64
       }
     ],
-    "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml": [
+    "kube/services/audit-service/audit-service-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/audit-service/audit-service-deploy.yaml",
+        "hashed_secret": "42cde1c58c36d8bb5804a076e55ac6ec07ef99fc",
+        "is_verified": false,
+        "line_number": 64
+      }
+    ],
+    "kube/services/aws-es-proxy/aws-es-proxy-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/aws-es-proxy/aws-es-proxy-deploy.yaml",
+        "hashed_secret": "7f834ccb442433fc12ec9532f75c3a4b6a748d4c",
+        "is_verified": false,
+        "line_number": 46
+      }
+    ],
+    "kube/services/cedar-wrapper/cedar-wrapper-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/cedar-wrapper/cedar-wrapper-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 56
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/cedar-wrapper/cedar-wrapper-deploy.yaml",
+        "hashed_secret": "5949b79e0c7082dc78d543cde662871a4f8b8913",
+        "is_verified": false,
+        "line_number": 59
+      }
+    ],
+    "kube/services/cogwheel/cogwheel-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/cogwheel/cogwheel-deploy.yaml",
+        "hashed_secret": "09b772df628fd10bca646b6a877eb661122210ab",
+        "is_verified": false,
+        "line_number": 35
+      }
+    ],
+    "kube/services/cohort-middleware/cohort-middleware-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/cohort-middleware/cohort-middleware-deploy.yaml",
+        "hashed_secret": "bf22f6c4bd03572f1ef593efc3eb1a7e0b6dcab4",
+        "is_verified": false,
+        "line_number": 62
+      }
+    ],
+    "kube/services/dashboard/dashboard-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/dashboard/dashboard-deploy.yaml",
+        "hashed_secret": "9e722d12ce045c8718ab803ed465b2fbe199f3d3",
+        "is_verified": false,
+        "line_number": 61
+      }
+    ],
+    "kube/services/datadog/values.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/datadog/values.yaml",
+        "hashed_secret": "4a8ce7ae6a8a7f2624e232b61b18c2ac9789c44b",
+        "is_verified": false,
+        "line_number": 23
+      }
+    ],
+    "kube/services/datasim/datasim-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/datasim/datasim-deploy.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 63
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/datasim/datasim-deploy.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 66
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/datasim/datasim-deploy.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 72
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/datasim/datasim-deploy.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 76
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/datasim/datasim-deploy.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
+        "is_verified": false,
+        "line_number": 79
+      }
+    ],
+    "kube/services/dicom-server/dicom-server-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/dicom-server/dicom-server-deploy.yaml",
+        "hashed_secret": "706168ac2565a93cceffe2202ac45d3d31c075fb",
+        "is_verified": false,
+        "line_number": 40
+      }
+    ],
+    "kube/services/fence/fence-canary-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-canary-deploy.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 68
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-canary-deploy.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 71
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-canary-deploy.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 74
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-canary-deploy.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 84
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-canary-deploy.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 87
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-canary-deploy.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 90
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-canary-deploy.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
+        "is_verified": false,
+        "line_number": 93
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-canary-deploy.yaml",
+        "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295",
+        "is_verified": false,
+        "line_number": 96
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-canary-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 99
+      }
+    ],
+    "kube/services/fence/fence-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-deploy.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 71
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-deploy.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 74
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-deploy.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 77
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-deploy.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 87
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-deploy.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 90
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-deploy.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 93
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-deploy.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
+        "is_verified": false,
+        "line_number": 96
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-deploy.yaml",
+        "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295",
+        "is_verified": false,
+        "line_number": 99
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fence/fence-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 102
+      }
+    ],
+    "kube/services/fenceshib/fenceshib-canary-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 62
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 65
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 68
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 78
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 81
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 84
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
+        "is_verified": false,
+        "line_number": 87
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml",
+        "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295",
+        "is_verified": false,
+        "line_number": 90
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 93
+      }
+    ],
+    "kube/services/fenceshib/fenceshib-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-deploy.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 69
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-deploy.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 72
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-deploy.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 75
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-deploy.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 85
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-deploy.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 88
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-deploy.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 91
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-deploy.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
+        "is_verified": false,
+        "line_number": 94
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-deploy.yaml",
+        "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295",
+        "is_verified": false,
+        "line_number": 97
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 100
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/fenceshib/fenceshib-deploy.yaml",
+        "hashed_secret": "6c4789c3be186fd5dcbf06723462ccdd2c86dc37",
+        "is_verified": false,
+        "line_number": 103
+      }
+    ],
+    "kube/services/frontend-framework/frontend-framework-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/frontend-framework/frontend-framework-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 54
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/frontend-framework/frontend-framework-deploy.yaml",
+        "hashed_secret": "6607b403f74e62246fc6a3c938feffc5a34a7e49",
+        "is_verified": false,
+        "line_number": 57
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/frontend-framework/frontend-framework-deploy.yaml",
+        "hashed_secret": "4b0bb3e58651fe56ee23e59aa6a3cb96dc61ddd2",
+        "is_verified": false,
+        "line_number": 60
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/frontend-framework/frontend-framework-deploy.yaml",
+        "hashed_secret": "e3c7565314f404e3883929f003c65a02a80366e9",
+        "is_verified": false,
+        "line_number": 66
+      }
+    ],
+    "kube/services/frontend-framework/frontend-framework-root-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/frontend-framework/frontend-framework-root-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 54
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/frontend-framework/frontend-framework-root-deploy.yaml",
+        "hashed_secret": "6607b403f74e62246fc6a3c938feffc5a34a7e49",
+        "is_verified": false,
+        "line_number": 57
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/frontend-framework/frontend-framework-root-deploy.yaml",
+        "hashed_secret": "4b0bb3e58651fe56ee23e59aa6a3cb96dc61ddd2",
+        "is_verified": false,
+        "line_number": 60
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/frontend-framework/frontend-framework-root-deploy.yaml",
+        "hashed_secret": "e3c7565314f404e3883929f003c65a02a80366e9",
+        "is_verified": false,
+        "line_number": 66
+      }
+    ],
+    "kube/services/gdcapi/gdcapi-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/gdcapi/gdcapi-deploy.yaml",
+        "hashed_secret": "e8c2f0bacaffbf2f9897217c6770413879945296",
+        "is_verified": false,
+        "line_number": 38
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/gdcapi/gdcapi-deploy.yaml",
+        "hashed_secret": "517cded9f3e3ab79237fde330b97a93f5a943316",
+        "is_verified": false,
+        "line_number": 41
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/gdcapi/gdcapi-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 44
+      }
+    ],
+    "kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml",
+        "hashed_secret": "38ded89f83435a558169dedb91a38f72d6cebf41",
+        "is_verified": false,
+        "line_number": 27
+      }
+    ],
+    "kube/services/google-sa-validation/google-sa-validation-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 54
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 57
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 63
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 67
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 70
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 73
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
+        "is_verified": false,
+        "line_number": 76
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml",
+        "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295",
+        "is_verified": false,
+        "line_number": 79
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 82
+      }
+    ],
+    "kube/services/guppy/guppy-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/guppy/guppy-deploy.yaml",
+        "hashed_secret": "0db22b31c9add2d3c76743c0ac6fbc99bb8b4761",
+        "is_verified": false,
+        "line_number": 65
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/guppy/guppy-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 68
+      }
+    ],
+    "kube/services/indexd/indexd-canary-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/indexd/indexd-canary-deploy.yaml",
+        "hashed_secret": "0b701c1fabb6ba47a7d47d455e3696d207014bd3",
+        "is_verified": false,
+        "line_number": 59
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/indexd/indexd-canary-deploy.yaml",
+        "hashed_secret": "aee98a99696237d70b6854ee4c2d9e42bc696039",
+        "is_verified": false,
+        "line_number": 62
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/indexd/indexd-canary-deploy.yaml",
+        "hashed_secret": "bdecca54d39013d43d3b7f05f2927eaa7df375dc",
+        "is_verified": false,
+        "line_number": 68
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/indexd/indexd-canary-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 71
+      }
+    ],
+    "kube/services/indexd/indexd-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/indexd/indexd-deploy.yaml",
+        "hashed_secret": "0b701c1fabb6ba47a7d47d455e3696d207014bd3",
+        "is_verified": false,
+        "line_number": 63
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/indexd/indexd-deploy.yaml",
+        "hashed_secret": "aee98a99696237d70b6854ee4c2d9e42bc696039",
+        "is_verified": false,
+        "line_number": 66
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/indexd/indexd-deploy.yaml",
+        "hashed_secret": "bdecca54d39013d43d3b7f05f2927eaa7df375dc",
+        "is_verified": false,
+        "line_number": 72
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/indexd/indexd-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 75
+      }
+    ],
+    "kube/services/jenkins-ci-worker/jenkins-ci-worker-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins-ci-worker/jenkins-ci-worker-deploy.yaml",
+        "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf",
+        "is_verified": false,
+        "line_number": 143
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins-ci-worker/jenkins-ci-worker-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 146
+      }
+    ],
+    "kube/services/jenkins-worker/jenkins-worker-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins-worker/jenkins-worker-deploy.yaml",
+        "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf",
+        "is_verified": false,
+        "line_number": 150
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins-worker/jenkins-worker-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 153
+      }
+    ],
+    "kube/services/jenkins/jenkins-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins/jenkins-deploy.yaml",
+        "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf",
+        "is_verified": false,
+        "line_number": 157
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins/jenkins-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 160
+      }
+    ],
+    "kube/services/jenkins2-ci-worker/jenkins2-ci-worker-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins2-ci-worker/jenkins2-ci-worker-deploy.yaml",
+        "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf",
+        "is_verified": false,
+        "line_number": 143
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins2-ci-worker/jenkins2-ci-worker-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 146
+      }
+    ],
+    "kube/services/jenkins2-worker/jenkins2-worker-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins2-worker/jenkins2-worker-deploy.yaml",
+        "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf",
+        "is_verified": false,
+        "line_number": 146
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins2-worker/jenkins2-worker-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 149
+      }
+    ],
+    "kube/services/jenkins2/jenkins2-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins2/jenkins2-deploy.yaml",
+        "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf",
+        "is_verified": false,
+        "line_number": 153
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jenkins2/jenkins2-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 156
+      }
+    ],
+    "kube/services/jobs/arborist-rm-expired-access-cronjob.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/arborist-rm-expired-access-cronjob.yaml",
+        "hashed_secret": "6c57cdfdaaf3cde7a1da6aa94c7d8e46502c4bab",
+        "is_verified": false,
+        "line_number": 37
+      }
+    ],
+    "kube/services/jobs/arborist-rm-expired-access-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/arborist-rm-expired-access-job.yaml",
+        "hashed_secret": "6c57cdfdaaf3cde7a1da6aa94c7d8e46502c4bab",
+        "is_verified": false,
+        "line_number": 37
+      }
+    ],
+    "kube/services/jobs/arboristdb-create-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/arboristdb-create-job.yaml",
+        "hashed_secret": "6c57cdfdaaf3cde7a1da6aa94c7d8e46502c4bab",
+        "is_verified": false,
+        "line_number": 33
+      }
+    ],
+    "kube/services/jobs/aws-bucket-replicate-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/aws-bucket-replicate-job.yaml",
+        "hashed_secret": "deb02468778f4041fb189654698ac948e436732d",
+        "is_verified": false,
+        "line_number": 33
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/aws-bucket-replicate-job.yaml",
+        "hashed_secret": "abe72fcb190ed9c73eb20e198c73a97605b95063",
+        "is_verified": false,
+        "line_number": 36
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/aws-bucket-replicate-job.yaml",
+        "hashed_secret": "ca3cdac59f2bfa45cb014190e4509bf6becf28fb",
+        "is_verified": false,
+        "line_number": 42
+      }
+    ],
+    "kube/services/jobs/bucket-manifest-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/bucket-manifest-job.yaml",
+        "hashed_secret": "6c36710fe8825b381388d7005f2c9b5c70175fba",
+        "is_verified": false,
+        "line_number": 33
+      }
+    ],
+    "kube/services/jobs/bucket-replicate-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/bucket-replicate-job.yaml",
+        "hashed_secret": "84954f7729144580d612cbb0517aeca8880e3483",
+        "is_verified": false,
+        "line_number": 46
+      }
+    ],
+    "kube/services/jobs/bucket-replication-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/bucket-replication-job.yaml",
+        "hashed_secret": "84954f7729144580d612cbb0517aeca8880e3483",
+        "is_verified": false,
+        "line_number": 32
+      }
+    ],
+    "kube/services/jobs/bucket-size-report-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/bucket-size-report-job.yaml",
+        "hashed_secret": "7cccf62cb63863d9d3baabed4f576eb0f7039735",
+        "is_verified": false,
+        "line_number": 34
+      }
+    ],
+    "kube/services/jobs/cedar-ingestion-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/cedar-ingestion-job.yaml",
+        "hashed_secret": "e1c426d126dcc618dcd0686fc718d509ca6ee3b8",
+        "is_verified": false,
+        "line_number": 54
+      }
+    ],
+    "kube/services/jobs/client-modify-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/client-modify-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 41
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/client-modify-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 44
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/client-modify-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 50
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/client-modify-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 54
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/client-modify-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 57
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/client-modify-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 60
+      }
+    ],
+    "kube/services/jobs/cogwheel-register-client-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/cogwheel-register-client-job.yaml",
+        "hashed_secret": "09b772df628fd10bca646b6a877eb661122210ab",
+        "is_verified": false,
+        "line_number": 40
+      }
+    ],
+    "kube/services/jobs/config-fence-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/config-fence-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 44
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/config-fence-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 54
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/config-fence-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 57
+      }
+    ],
+    "kube/services/jobs/covid19-etl-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/covid19-etl-job.yaml",
+        "hashed_secret": "a7a2b42615b2b256a7c601c77c426e5d6cafb212",
+        "is_verified": false,
+        "line_number": 34
+      }
+    ],
+    "kube/services/jobs/covid19-notebook-etl-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/covid19-notebook-etl-job.yaml",
+        "hashed_secret": "a7a2b42615b2b256a7c601c77c426e5d6cafb212",
+        "is_verified": false,
+        "line_number": 33
+      }
+    ],
+    "kube/services/jobs/data-ingestion-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/data-ingestion-job.yaml",
+        "hashed_secret": "81e4388059839f71aed21999aa51095c7e545094",
+        "is_verified": false,
+        "line_number": 34
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/data-ingestion-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 48
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/data-ingestion-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 51
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/data-ingestion-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 54
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/data-ingestion-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 60
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/data-ingestion-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 63
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/data-ingestion-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 66
+      }
+    ],
+    "kube/services/jobs/etl-cronjob.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/etl-cronjob.yaml",
+        "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966",
+        "is_verified": false,
+        "line_number": 38
+      }
+    ],
+    "kube/services/jobs/etl-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/etl-job.yaml",
+        "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966",
+        "is_verified": false,
+        "line_number": 35
+      }
+    ],
+    "kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 43
+      }
+    ],
+    "kube/services/jobs/fence-cleanup-expired-ga4gh-info-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-cleanup-expired-ga4gh-info-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 36
+      }
+    ],
+    "kube/services/jobs/fence-db-migrate-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-db-migrate-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 36
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-db-migrate-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 39
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-db-migrate-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 42
+      }
+    ],
+    "kube/services/jobs/fence-delete-expired-clients-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-delete-expired-clients-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 38
+      }
+    ],
+    "kube/services/jobs/fence-visa-update-cronjob.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-visa-update-cronjob.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 42
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-visa-update-cronjob.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 45
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-visa-update-cronjob.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 48
+      }
+    ],
+    "kube/services/jobs/fence-visa-update-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-visa-update-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 36
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-visa-update-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 39
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fence-visa-update-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 42
+      }
+    ],
+    "kube/services/jobs/fencedb-create-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/fencedb-create-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 33
+      }
+    ],
+    "kube/services/jobs/gdcdb-create-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/gdcdb-create-job.yaml",
+        "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc",
+        "is_verified": false,
+        "line_number": 33
+      }
+    ],
+    "kube/services/jobs/gen3qa-check-bucket-access-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/gen3qa-check-bucket-access-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 177
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/gen3qa-check-bucket-access-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 180
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/gen3qa-check-bucket-access-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 186
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/gen3qa-check-bucket-access-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 190
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/gen3qa-check-bucket-access-job.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
+        "is_verified": false,
+        "line_number": 193
+      }
+    ],
+    "kube/services/jobs/gentestdata-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/gentestdata-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 67
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/gentestdata-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 70
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/gentestdata-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 76
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/gentestdata-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 80
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/gentestdata-job.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
+        "is_verified": false,
+        "line_number": 83
+      }
+    ],
+    "kube/services/jobs/google-bucket-manifest-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-bucket-manifest-job.yaml",
+        "hashed_secret": "5ca8fff7767e5dd6ebed80e2c8eab66d6f3bf5eb",
+        "is_verified": false,
+        "line_number": 31
+      }
+    ],
+    "kube/services/jobs/google-bucket-replicate-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-bucket-replicate-job.yaml",
+        "hashed_secret": "b6f0ec0b08da77656ced48427841e28d7a8a81d6",
+        "is_verified": false,
+        "line_number": 35
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-bucket-replicate-job.yaml",
+        "hashed_secret": "abe72fcb190ed9c73eb20e198c73a97605b95063",
+        "is_verified": false,
+        "line_number": 38
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-bucket-replicate-job.yaml",
+        "hashed_secret": "ca3cdac59f2bfa45cb014190e4509bf6becf28fb",
+        "is_verified": false,
+        "line_number": 41
+      }
+    ],
+    "kube/services/jobs/google-create-bucket-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-create-bucket-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 78
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-create-bucket-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 81
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-create-bucket-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 84
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-create-bucket-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 91
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-create-bucket-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 94
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-create-bucket-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 97
+      }
+    ],
+    "kube/services/jobs/google-delete-expired-access-cronjob.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-access-cronjob.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 43
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-access-cronjob.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 46
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-access-cronjob.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 49
+      }
+    ],
+    "kube/services/jobs/google-delete-expired-access-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-access-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 36
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-access-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 39
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-access-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 42
+      }
+    ],
+    "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 48
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 51
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 57
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 61
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 64
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
+        "is_verified": false,
+        "line_number": 67
+      }
+    ],
+    "kube/services/jobs/google-delete-expired-service-account-job.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
+        "is_verified": false,
+        "line_number": 40
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
+        "is_verified": false,
+        "line_number": 43
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
+        "is_verified": false,
+        "line_number": 49
+      },
       {
-        "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 33,
-        "type": "Basic Auth Credentials"
+        "line_number": 53
       },
       {
-        "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 286,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/lib/bootstrap/templates/Gen3Secrets/creds.json": [
+        "line_number": 56
+      },
       {
-        "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 26,
-        "type": "Secret Keyword"
+        "line_number": 59
       }
     ],
-    "gen3/lib/bootstrap/templates/Gen3Secrets/g3auto/dbfarm/servers.json": [
+    "kube/services/jobs/google-init-proxy-groups-cronjob.yaml": [
       {
-        "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 5,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/lib/logs/utils.sh": [
+        "line_number": 48
+      },
       {
-        "hashed_secret": "76143b4ffc8aa2a53f9700ce229f904e69f1e8b5",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 3,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/lib/manifestDefaults/hatchery/hatchery.json": [
+        "line_number": 51
+      },
       {
-        "hashed_secret": "0da0e0005ca04acb407af2681d0bede6d9406039",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 78,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/lib/onprem.sh": [
+        "line_number": 54
+      },
       {
-        "hashed_secret": "29e52a9bac8f274fa41c51fce9c98eba0dd99cb3",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 68,
-        "type": "Secret Keyword"
+        "line_number": 61
       },
       {
-        "hashed_secret": "50f013532a9770a2c2cfdc38b7581dd01df69b70",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 84,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/lib/secrets/rotate-postgres.sh": [
+        "line_number": 64
+      },
       {
-        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 162,
-        "type": "Secret Keyword"
+        "line_number": 67
       },
       {
-        "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
         "is_verified": false,
-        "line_number": 250,
-        "type": "Secret Keyword"
+        "line_number": 70
       }
     ],
-    "gen3/lib/testData/etlconvert/expected2.yaml": [
+    "kube/services/jobs/google-init-proxy-groups-job.yaml": [
       {
-        "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 10,
-        "type": "Base64 High Entropy String"
+        "line_number": 40
       },
       {
-        "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 13,
-        "type": "Base64 High Entropy String"
+        "line_number": 43
       },
       {
-        "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 16,
-        "type": "Base64 High Entropy String"
+        "line_number": 46
       },
       {
-        "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 18,
-        "type": "Base64 High Entropy String"
+        "line_number": 53
       },
       {
-        "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 33,
-        "type": "Base64 High Entropy String"
+        "line_number": 56
       },
       {
-        "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 35,
-        "type": "Base64 High Entropy String"
+        "line_number": 59
       },
       {
-        "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
         "is_verified": false,
-        "line_number": 36,
-        "type": "Base64 High Entropy String"
+        "line_number": 62
       }
     ],
-    "gen3/test/secretsTest.sh": [
+    "kube/services/jobs/google-manage-account-access-cronjob.yaml": [
       {
-        "hashed_secret": "c2c715092ef59cba22520f109f041efca84b8938",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 25,
-        "type": "Secret Keyword"
-      }
-    ],
-    "gen3/test/terraformTest.sh": [
+        "line_number": 48
+      },
       {
-        "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 156,
-        "type": "Secret Keyword"
+        "line_number": 51
       },
       {
-        "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 172,
-        "type": "Base64 High Entropy String"
+        "line_number": 54
       },
       {
-        "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 61
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 175,
-        "type": "Base64 High Entropy String"
+        "line_number": 64
       },
       {
-        "hashed_secret": "329b7cd8191942bedd337107934d365c43a86e6c",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 175,
-        "type": "Secret Keyword"
+        "line_number": 67
       }
     ],
-    "kube/services/argocd/values.yaml": [
+    "kube/services/jobs/google-manage-account-access-job.yaml": [
       {
-        "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 360,
-        "type": "Private Key"
+        "line_number": 40
       },
       {
-        "hashed_secret": "edbd5e119f94badb9f99a67ac6ff4c7a5204ad61",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 379,
-        "type": "Secret Keyword"
+        "line_number": 43
       },
       {
-        "hashed_secret": "91dfd9ddb4198affc5c194cd8ce6d338fde470e2",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 412,
-        "type": "Secret Keyword"
-      }
-    ],
-    "kube/services/datadog/values.yaml": [
+        "line_number": 46
+      },
       {
-        "hashed_secret": "4a8ce7ae6a8a7f2624e232b61b18c2ac9789c44b",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
+        "is_verified": false,
+        "line_number": 53
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
+        "is_verified": false,
+        "line_number": 56
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-account-access-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 23,
-        "type": "Secret Keyword"
+        "line_number": 59
       }
     ],
-    "kube/services/fenceshib/fenceshib-configmap.yaml": [
+    "kube/services/jobs/google-manage-keys-cronjob.yaml": [
       {
-        "hashed_secret": "a985e14b9d6744a2d04f29347693b55c116e478c",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 375,
-        "type": "Base64 High Entropy String"
+        "line_number": 48
       },
       {
-        "hashed_secret": "adc747bc5eb82ef4b017f5c3759dcee5aa28c36f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 376,
-        "type": "Base64 High Entropy String"
+        "line_number": 51
       },
       {
-        "hashed_secret": "59b1702ff0eaf92c9271cbd12f587de97df7e13b",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 377,
-        "type": "Base64 High Entropy String"
+        "line_number": 54
       },
       {
-        "hashed_secret": "b4a748bbfbbca8925d932a47ab3dcb970d34caf5",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 378,
-        "type": "Base64 High Entropy String"
+        "line_number": 61
       },
       {
-        "hashed_secret": "af646701a84f7dd9f0e87753f54def881326e78a",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 379,
-        "type": "Base64 High Entropy String"
+        "line_number": 64
       },
       {
-        "hashed_secret": "20c15ad9742124dc06e1612282c49bb443ebcbd9",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 380,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 67
+      }
+    ],
+    "kube/services/jobs/google-manage-keys-job.yaml": [
       {
-        "hashed_secret": "9caded71b967a11b7a6cd0f20db91f06f3517d12",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 381,
-        "type": "Base64 High Entropy String"
+        "line_number": 40
       },
       {
-        "hashed_secret": "8f19501bc9241b71f7b6db929fb35ab12635dcd7",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 382,
-        "type": "Base64 High Entropy String"
+        "line_number": 43
       },
       {
-        "hashed_secret": "d6220f6a55df1ed11c4250f42ab07bb9da20541a",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 383,
-        "type": "Base64 High Entropy String"
+        "line_number": 46
       },
       {
-        "hashed_secret": "dadd9b96636f9529f2547d05d754dc310ceba0c3",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 384,
-        "type": "Base64 High Entropy String"
+        "line_number": 53
       },
       {
-        "hashed_secret": "3074bc66584550e20c3697a28f67a0762394943c",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 385,
-        "type": "Base64 High Entropy String"
+        "line_number": 56
       },
       {
-        "hashed_secret": "823131319b4c4b4688f44d3e832bfa9696f16b52",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-manage-keys-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 386,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 59
+      }
+    ],
+    "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml": [
       {
-        "hashed_secret": "015b780cbfb76988caf52de8ac974a6781e53110",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 387,
-        "type": "Base64 High Entropy String"
+        "line_number": 48
       },
       {
-        "hashed_secret": "5c8fac33207d74d667680ade09447ea8f43b76d7",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 388,
-        "type": "Base64 High Entropy String"
+        "line_number": 51
       },
       {
-        "hashed_secret": "c0c4bb09d8394e8f001e337bd27ccac355433d9e",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 389,
-        "type": "Base64 High Entropy String"
+        "line_number": 54
       },
       {
-        "hashed_secret": "f95631bcbbbc56e18487dcb242cfb1b3e74b16a1",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 390,
-        "type": "Base64 High Entropy String"
+        "line_number": 61
       },
       {
-        "hashed_secret": "01a692ab6232e0882a313d148981bab58ab98f53",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 391,
-        "type": "Base64 High Entropy String"
+        "line_number": 64
       },
       {
-        "hashed_secret": "658060a680d415ce6690ad2c3b622ddb33ddd50a",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 392,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 67
+      }
+    ],
+    "kube/services/jobs/google-verify-bucket-access-group-job.yaml": [
       {
-        "hashed_secret": "80915b0bd9daa5e1f95cad573892980b1b5a2294",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 393,
-        "type": "Base64 High Entropy String"
+        "line_number": 40
       },
       {
-        "hashed_secret": "cc55977b293d8cdca8a2c19dfea6874e70057c41",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 394,
-        "type": "Base64 High Entropy String"
+        "line_number": 43
       },
       {
-        "hashed_secret": "e400ed02add75dd5f3a8c212857acf12027437d1",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 395,
-        "type": "Base64 High Entropy String"
+        "line_number": 46
       },
       {
-        "hashed_secret": "2e819c8baa3b0508a32b77de258655b3f3a6f7cb",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 396,
-        "type": "Base64 High Entropy String"
+        "line_number": 53
       },
       {
-        "hashed_secret": "546ed926d58ea5492ab6adb8be94a67aa44ac433",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 397,
-        "type": "Base64 High Entropy String"
+        "line_number": 56
       },
       {
-        "hashed_secret": "f056f2deceed268e7af6dbdaf2577079c76e006a",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 398,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 59
+      }
+    ],
+    "kube/services/jobs/graph-create-job.yaml": [
       {
-        "hashed_secret": "d75efee28f4798c3a9c6f44b78a8500513ef28b2",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/graph-create-job.yaml",
+        "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc",
         "is_verified": false,
-        "line_number": 399,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 33
+      }
+    ],
+    "kube/services/jobs/indexd-authz-job.yaml": [
       {
-        "hashed_secret": "fbad0bc8f7792b03f89cd3780eb7cf79f284c525",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/indexd-authz-job.yaml",
+        "hashed_secret": "0b701c1fabb6ba47a7d47d455e3696d207014bd3",
         "is_verified": false,
-        "line_number": 419,
-        "type": "Base64 High Entropy String"
+        "line_number": 32
       },
       {
-        "hashed_secret": "3f6480956a775dacb44e2c39aa3d4722a347f7ab",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/indexd-authz-job.yaml",
+        "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc",
         "is_verified": false,
-        "line_number": 420,
-        "type": "Base64 High Entropy String"
+        "line_number": 35
       },
       {
-        "hashed_secret": "17f32ae55b14d708ca121722c2cae37189f19daf",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/indexd-authz-job.yaml",
+        "hashed_secret": "aee98a99696237d70b6854ee4c2d9e42bc696039",
         "is_verified": false,
-        "line_number": 423,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 38
+      }
+    ],
+    "kube/services/jobs/indexd-userdb-job.yaml": [
       {
-        "hashed_secret": "08a74689ca077515d406093720a7e5675fb42bb8",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/indexd-userdb-job.yaml",
+        "hashed_secret": "0b701c1fabb6ba47a7d47d455e3696d207014bd3",
         "is_verified": false,
-        "line_number": 424,
-        "type": "Base64 High Entropy String"
+        "line_number": 40
       },
       {
-        "hashed_secret": "fa577bb3b2600d2d522dcfea8f1e34896760fcf2",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/indexd-userdb-job.yaml",
+        "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc",
         "is_verified": false,
-        "line_number": 425,
-        "type": "Base64 High Entropy String"
+        "line_number": 43
       },
       {
-        "hashed_secret": "37254f15cca211a1bd5f7ceb23de2b3eb8fb33aa",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/indexd-userdb-job.yaml",
+        "hashed_secret": "aee98a99696237d70b6854ee4c2d9e42bc696039",
         "is_verified": false,
-        "line_number": 426,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 46
+      }
+    ],
+    "kube/services/jobs/metadata-aggregate-sync-job.yaml": [
       {
-        "hashed_secret": "86865593e038509467b91c2d5f36ccc09c3f422b",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/metadata-aggregate-sync-job.yaml",
+        "hashed_secret": "e14f65c8ca7f3b27a0f0f5463569954841e162c9",
         "is_verified": false,
-        "line_number": 427,
-        "type": "Base64 High Entropy String"
+        "line_number": 31
       },
       {
-        "hashed_secret": "a899a8d9e114b2a8e108f90e6a72c056db22489f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/metadata-aggregate-sync-job.yaml",
+        "hashed_secret": "c27babf45eb0ed87329e69c7d47dba611e859c5d",
         "is_verified": false,
-        "line_number": 428,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 34
+      }
+    ],
+    "kube/services/jobs/metadata-delete-expired-objects-job.yaml": [
       {
-        "hashed_secret": "756b4825f886afd83c25563ac9d45f318d695c48",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/metadata-delete-expired-objects-job.yaml",
+        "hashed_secret": "0cc8bac3fabe63722716d1e6fe04a8dded1e3ad0",
         "is_verified": false,
-        "line_number": 429,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 24
+      }
+    ],
+    "kube/services/jobs/remove-objects-from-clouds-job.yaml": [
       {
-        "hashed_secret": "89882eeb0aca97717a7e4afcf4bc08d077813c7f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/remove-objects-from-clouds-job.yaml",
+        "hashed_secret": "deb02468778f4041fb189654698ac948e436732d",
         "is_verified": false,
-        "line_number": 430,
-        "type": "Base64 High Entropy String"
+        "line_number": 34
       },
       {
-        "hashed_secret": "347140d7b7ceb4e501c3c9c2ea4f29338e2f145e",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/remove-objects-from-clouds-job.yaml",
+        "hashed_secret": "b6f0ec0b08da77656ced48427841e28d7a8a81d6",
         "is_verified": false,
-        "line_number": 431,
-        "type": "Base64 High Entropy String"
+        "line_number": 37
       },
       {
-        "hashed_secret": "61dbf70eb10d609e60c7b87faf8f755ff48abc46",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/remove-objects-from-clouds-job.yaml",
+        "hashed_secret": "ca3cdac59f2bfa45cb014190e4509bf6becf28fb",
         "is_verified": false,
-        "line_number": 432,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 43
+      }
+    ],
+    "kube/services/jobs/replicate-validation-job.yaml": [
       {
-        "hashed_secret": "24cd54c4b2f58378bba008cb2df68ac663fba7c8",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/replicate-validation-job.yaml",
+        "hashed_secret": "deb02468778f4041fb189654698ac948e436732d",
         "is_verified": false,
-        "line_number": 433,
-        "type": "Base64 High Entropy String"
+        "line_number": 34
       },
       {
-        "hashed_secret": "fa4f9626ae4b98f4b61203c5bafb6f21c9c31e5d",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/replicate-validation-job.yaml",
+        "hashed_secret": "b6f0ec0b08da77656ced48427841e28d7a8a81d6",
         "is_verified": false,
-        "line_number": 434,
-        "type": "Base64 High Entropy String"
+        "line_number": 37
       },
       {
-        "hashed_secret": "b1370003d9cc1e346c83dba33e0418c7775a0c15",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/replicate-validation-job.yaml",
+        "hashed_secret": "abe72fcb190ed9c73eb20e198c73a97605b95063",
         "is_verified": false,
-        "line_number": 435,
-        "type": "Base64 High Entropy String"
+        "line_number": 40
       },
       {
-        "hashed_secret": "c66526e195e423a7ba7d68ac661cdcd8600dcd1f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/replicate-validation-job.yaml",
+        "hashed_secret": "ca3cdac59f2bfa45cb014190e4509bf6becf28fb",
         "is_verified": false,
-        "line_number": 436,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 43
+      }
+    ],
+    "kube/services/jobs/s3sync-cronjob.yaml": [
       {
-        "hashed_secret": "d29d7044f0944eb30e02cf445f6998e3343dd811",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/s3sync-cronjob.yaml",
+        "hashed_secret": "27f6dfe15698a3bfaa183c84701cfb2bf4115415",
         "is_verified": false,
-        "line_number": 437,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 44
+      }
+    ],
+    "kube/services/jobs/usersync-job.yaml": [
       {
-        "hashed_secret": "80a869460f33722387d8d58e7d9d2e1bbd5d1fe1",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/usersync-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 438,
-        "type": "Base64 High Entropy String"
+        "line_number": 64
       },
       {
-        "hashed_secret": "4a06e2a02cbc665adccb4162dc57836895da65b8",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/usersync-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 439,
-        "type": "Base64 High Entropy String"
+        "line_number": 67
       },
       {
-        "hashed_secret": "ba2549f35835dfa101d3f660f7604dc78e3e226f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/usersync-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 440,
-        "type": "Base64 High Entropy String"
+        "line_number": 70
       },
       {
-        "hashed_secret": "f354d4ee5fdb94ad29c7b3600264467f45b80eaa",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/usersync-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 441,
-        "type": "Base64 High Entropy String"
+        "line_number": 77
       },
       {
-        "hashed_secret": "bf17b587868ba7c3db9865b114261b5b8f1df870",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/usersync-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 442,
-        "type": "Base64 High Entropy String"
+        "line_number": 80
       },
       {
-        "hashed_secret": "de1fd7a0d32cba528b4d80818c6601f2588d5383",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/usersync-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 443,
-        "type": "Base64 High Entropy String"
+        "line_number": 83
       },
       {
-        "hashed_secret": "bcad65055f6de654541db2bf27d4e27bd54d94c7",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/usersync-job.yaml",
+        "hashed_secret": "ea73fcfdaa415890d5fde24d3b2245671be32f73",
         "is_verified": false,
-        "line_number": 444,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 86
+      }
+    ],
+    "kube/services/jobs/useryaml-job.yaml": [
       {
-        "hashed_secret": "f2e16f2dd532f65f79341342fdf57a093fc408d8",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/useryaml-job.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 445,
-        "type": "Base64 High Entropy String"
+        "line_number": 40
       },
       {
-        "hashed_secret": "bb036a679a7d2df9fd2ca57068a446bf7f7dd106",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/useryaml-job.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 446,
-        "type": "Base64 High Entropy String"
+        "line_number": 43
       },
       {
-        "hashed_secret": "5aa6568b1e8185578a6e964f5c322783ad349554",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/useryaml-job.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 447,
-        "type": "Base64 High Entropy String"
+        "line_number": 46
       },
       {
-        "hashed_secret": "4d14835ff0b0bf5aad480296cb705c74ac65f413",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/useryaml-job.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 448,
-        "type": "Base64 High Entropy String"
+        "line_number": 53
       },
       {
-        "hashed_secret": "3f23f77dcf454ad73c4d61c44fd9aa584ef946c1",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/useryaml-job.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 451,
-        "type": "Base64 High Entropy String"
+        "line_number": 56
       },
       {
-        "hashed_secret": "1739fe5e5dfcf851b64f8b7b11538f1de29ce0b5",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/useryaml-job.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 452,
-        "type": "Base64 High Entropy String"
+        "line_number": 59
       },
       {
-        "hashed_secret": "8129db302110714fc735e3494bd82a65690e0963",
+        "type": "Secret Keyword",
+        "filename": "kube/services/jobs/useryaml-job.yaml",
+        "hashed_secret": "ea73fcfdaa415890d5fde24d3b2245671be32f73",
+        "is_verified": false,
+        "line_number": 65
+      }
+    ],
+    "kube/services/kayako-wrapper/kayako-wrapper-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/kayako-wrapper/kayako-wrapper-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 453,
-        "type": "Base64 High Entropy String"
+        "line_number": 56
       },
       {
-        "hashed_secret": "b48bfc62091164086a703115a0e68bdb09212591",
+        "type": "Secret Keyword",
+        "filename": "kube/services/kayako-wrapper/kayako-wrapper-deploy.yaml",
+        "hashed_secret": "fb7ea689a364feb7aafbf8d553eb77073fa7ba11",
         "is_verified": false,
-        "line_number": 454,
-        "type": "Base64 High Entropy String"
+        "line_number": 59
+      }
+    ],
+    "kube/services/kubecost-standalone/thanos-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/kubecost-standalone/thanos-deploy.yaml",
+        "hashed_secret": "064376809efc3acda5bd341aca977e149b989696",
+        "is_verified": false,
+        "line_number": 127
+      }
+    ],
+    "kube/services/kubecost-standalone/values.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/kubecost-standalone/values.yaml",
+        "hashed_secret": "ec9786daee68e3541963a51299160859fe4db663",
+        "is_verified": false,
+        "line_number": 30
+      }
+    ],
+    "kube/services/manifestservice/manifestservice-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/manifestservice/manifestservice-deploy.yaml",
+        "hashed_secret": "3da2c49c267b6c58401bbf05e379b38d20434f78",
+        "is_verified": false,
+        "line_number": 61
       },
       {
-        "hashed_secret": "a10284feaf27f84081073a3267e3dce24ca7b911",
+        "type": "Secret Keyword",
+        "filename": "kube/services/manifestservice/manifestservice-deploy.yaml",
+        "hashed_secret": "469e0c2b1a67aa94955bae023ddc727be31581a7",
         "is_verified": false,
-        "line_number": 455,
-        "type": "Base64 High Entropy String"
+        "line_number": 64
       },
       {
-        "hashed_secret": "3fd80f31de4be8dde9d2b421e832c7d4043fd49a",
+        "type": "Secret Keyword",
+        "filename": "kube/services/manifestservice/manifestservice-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 456,
-        "type": "Base64 High Entropy String"
+        "line_number": 67
       }
     ],
-    "kube/services/jobs/indexd-authz-job.yaml": [
+    "kube/services/metadata/metadata-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/metadata/metadata-deploy.yaml",
+        "hashed_secret": "e14f65c8ca7f3b27a0f0f5463569954841e162c9",
+        "is_verified": false,
+        "line_number": 61
+      },
       {
-        "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/metadata/metadata-deploy.yaml",
+        "hashed_secret": "c27babf45eb0ed87329e69c7d47dba611e859c5d",
         "is_verified": false,
-        "line_number": 87,
-        "type": "Basic Auth Credentials"
+        "line_number": 66
       }
     ],
     "kube/services/monitoring/grafana-values.yaml": [
       {
+        "type": "Secret Keyword",
+        "filename": "kube/services/monitoring/grafana-values.yaml",
         "hashed_secret": "2ae868079d293e0a185c671c7bcdac51df36e385",
         "is_verified": false,
-        "line_number": 162,
-        "type": "Secret Keyword"
+        "line_number": 162
       },
       {
-        "hashed_secret": "7a64ff8446b06d38dc271019994f13823a2cbcf4",
+        "type": "Secret Keyword",
+        "filename": "kube/services/monitoring/grafana-values.yaml",
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 166,
-        "type": "Secret Keyword"
+        "line_number": 331
       }
     ],
-    "kube/services/revproxy/helpers.js": [
+    "kube/services/monitoring/thanos-deploy.yaml": [
       {
-        "hashed_secret": "1d278d3c888d1a2fa7eed622bfc02927ce4049af",
+        "type": "Secret Keyword",
+        "filename": "kube/services/monitoring/thanos-deploy.yaml",
+        "hashed_secret": "064376809efc3acda5bd341aca977e149b989696",
         "is_verified": false,
-        "line_number": 10,
-        "type": "Base64 High Entropy String"
+        "line_number": 130
       }
     ],
-    "kube/services/revproxy/helpersTest.js": [
+    "kube/services/ohif-viewer/ohif-viewer-deploy.yaml": [
       {
-        "hashed_secret": "e029d4904cc728879d70030572bf37d4510367cb",
+        "type": "Secret Keyword",
+        "filename": "kube/services/ohif-viewer/ohif-viewer-deploy.yaml",
+        "hashed_secret": "3f87db80519a9ae7d8112f4e0d4cc81441181818",
         "is_verified": false,
-        "line_number": 22,
-        "type": "JSON Web Token"
+        "line_number": 40
       }
     ],
-    "kube/services/superset/superset-deploy.yaml": [
+    "kube/services/orthanc/orthanc-deploy.yaml": [
       {
-        "hashed_secret": "96e4aceb7cf284be363aa248a32a7cc89785a9f7",
+        "type": "Secret Keyword",
+        "filename": "kube/services/orthanc/orthanc-deploy.yaml",
+        "hashed_secret": "3f87db80519a9ae7d8112f4e0d4cc81441181818",
         "is_verified": false,
-        "line_number": 38,
-        "type": "Secret Keyword"
+        "line_number": 41
       }
     ],
-    "kube/services/superset/superset-redis.yaml": [
+    "kube/services/peregrine/peregrine-canary-deploy.yaml": [
       {
-        "hashed_secret": "4af3596275edcb7cd5cc6c3c38bc10479902a08f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/peregrine/peregrine-canary-deploy.yaml",
+        "hashed_secret": "6131c35d7eebdbc17a314bef8aac75b87323cff3",
         "is_verified": false,
-        "line_number": 165,
-        "type": "Secret Keyword"
+        "line_number": 61
       },
       {
-        "hashed_secret": "9fe1c31809da38c55b2b64bfab47b92bc5f6b7b9",
+        "type": "Secret Keyword",
+        "filename": "kube/services/peregrine/peregrine-canary-deploy.yaml",
+        "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966",
+        "is_verified": false,
+        "line_number": 64
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/peregrine/peregrine-canary-deploy.yaml",
+        "hashed_secret": "990a3202b5c94aa5e5997e7dc1a218e457f8b8ec",
+        "is_verified": false,
+        "line_number": 70
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/peregrine/peregrine-canary-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 265,
-        "type": "Secret Keyword"
+        "line_number": 73
       }
     ],
-    "kube/services/superset/values.yaml": [
+    "kube/services/peregrine/peregrine-deploy.yaml": [
       {
-        "hashed_secret": "6f803b24314c39062efe38d0c1da8c472f47eab3",
+        "type": "Secret Keyword",
+        "filename": "kube/services/peregrine/peregrine-deploy.yaml",
+        "hashed_secret": "6131c35d7eebdbc17a314bef8aac75b87323cff3",
         "is_verified": false,
-        "line_number": 54,
-        "type": "Secret Keyword"
+        "line_number": 67
       },
       {
-        "hashed_secret": "6eae3a5b062c6d0d79f070c26e6d62486b40cb46",
+        "type": "Secret Keyword",
+        "filename": "kube/services/peregrine/peregrine-deploy.yaml",
+        "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966",
         "is_verified": false,
-        "line_number": 86,
-        "type": "Secret Keyword"
+        "line_number": 70
       },
       {
-        "hashed_secret": "3eb416223e9e69e6bb8ee19793911ad1ad2027d8",
+        "type": "Secret Keyword",
+        "filename": "kube/services/peregrine/peregrine-deploy.yaml",
+        "hashed_secret": "990a3202b5c94aa5e5997e7dc1a218e457f8b8ec",
         "is_verified": false,
-        "line_number": 212,
-        "type": "Secret Keyword"
+        "line_number": 76
       },
       {
-        "hashed_secret": "ff55435345834a3fe224936776c2aa15f6ed5358",
+        "type": "Secret Keyword",
+        "filename": "kube/services/peregrine/peregrine-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 79
+      }
+    ],
+    "kube/services/pidgin/pidgin-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/pidgin/pidgin-deploy.yaml",
+        "hashed_secret": "49af232c7adfcd54a40202e06261396a757e4ddd",
         "is_verified": false,
-        "line_number": 396,
-        "type": "Secret Keyword"
+        "line_number": 59
       },
       {
-        "hashed_secret": "98a84a63e5633d17e3b27b69695f87aa7189e9dc",
+        "type": "Secret Keyword",
+        "filename": "kube/services/pidgin/pidgin-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 503,
-        "type": "Secret Keyword"
+        "line_number": 62
       }
     ],
-    "package-lock.json": [
+    "kube/services/portal/portal-deploy.yaml": [
       {
-        "hashed_secret": "0656ad0df3af4633dc369f13d5e8806973c5fd9d",
+        "type": "Secret Keyword",
+        "filename": "kube/services/portal/portal-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 1481,
-        "type": "Base64 High Entropy String"
+        "line_number": 55
       },
       {
-        "hashed_secret": "00091d875d922437c5fc9e6067a08e78c2482e87",
+        "type": "Secret Keyword",
+        "filename": "kube/services/portal/portal-deploy.yaml",
+        "hashed_secret": "5c5a8e158ad2d8544f73cd5422072d414f497faa",
         "is_verified": false,
-        "line_number": 1489,
-        "type": "Base64 High Entropy String"
+        "line_number": 58
       },
       {
-        "hashed_secret": "c4e5cc37e115bf7d86e76e3d799705bf691e4d00",
+        "type": "Secret Keyword",
+        "filename": "kube/services/portal/portal-deploy.yaml",
+        "hashed_secret": "619551216e129bbc5322678abf9c9210c0327cfb",
         "is_verified": false,
-        "line_number": 1521,
-        "type": "Base64 High Entropy String"
+        "line_number": 61
       },
       {
-        "hashed_secret": "0512e37fbedf1d16828680a038a241b4780a5c04",
+        "type": "Secret Keyword",
+        "filename": "kube/services/portal/portal-deploy.yaml",
+        "hashed_secret": "e3c7565314f404e3883929f003c65a02a80366e9",
         "is_verified": false,
-        "line_number": 1547,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 67
+      }
+    ],
+    "kube/services/portal/portal-root-deploy.yaml": [
       {
-        "hashed_secret": "01868fd50edbfe6eb91e5b01209b543adc6857af",
+        "type": "Secret Keyword",
+        "filename": "kube/services/portal/portal-root-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 1611,
-        "type": "Base64 High Entropy String"
+        "line_number": 55
       },
       {
-        "hashed_secret": "a6f48bf1e398deffc7fd31da17c3506b46c97a93",
+        "type": "Secret Keyword",
+        "filename": "kube/services/portal/portal-root-deploy.yaml",
+        "hashed_secret": "5c5a8e158ad2d8544f73cd5422072d414f497faa",
         "is_verified": false,
-        "line_number": 1640,
-        "type": "Base64 High Entropy String"
+        "line_number": 58
       },
       {
-        "hashed_secret": "85ce358dbdec0996cf3ccd2bf1c6602af68c181e",
+        "type": "Secret Keyword",
+        "filename": "kube/services/portal/portal-root-deploy.yaml",
+        "hashed_secret": "619551216e129bbc5322678abf9c9210c0327cfb",
         "is_verified": false,
-        "line_number": 1648,
-        "type": "Base64 High Entropy String"
+        "line_number": 61
       },
       {
-        "hashed_secret": "6f9bfb49cb818d2fe07592515e4c3f7a0bbd7e0e",
+        "type": "Secret Keyword",
+        "filename": "kube/services/portal/portal-root-deploy.yaml",
+        "hashed_secret": "e3c7565314f404e3883929f003c65a02a80366e9",
         "is_verified": false,
-        "line_number": 1664,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 67
+      }
+    ],
+    "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml": [
       {
-        "hashed_secret": "7098a3e6d6d2ec0a40f04fe12509c5c6f4c49c0e",
+        "type": "Secret Keyword",
+        "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml",
+        "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55",
         "is_verified": false,
-        "line_number": 1683,
-        "type": "Base64 High Entropy String"
+        "line_number": 74
       },
       {
-        "hashed_secret": "1664ad175bba1795a7ecad572bae7e0740b94f56",
+        "type": "Secret Keyword",
+        "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml",
+        "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb",
         "is_verified": false,
-        "line_number": 1733,
-        "type": "Base64 High Entropy String"
+        "line_number": 77
       },
       {
-        "hashed_secret": "1ec4ce2eb945ce2f816dcb6ebdd1e10247f439a3",
+        "type": "Secret Keyword",
+        "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml",
+        "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634",
         "is_verified": false,
-        "line_number": 1742,
-        "type": "Base64 High Entropy String"
+        "line_number": 80
       },
       {
-        "hashed_secret": "a7af5768a6d936e36f28e1030d7f894d7aaf555e",
+        "type": "Secret Keyword",
+        "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml",
+        "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd",
         "is_verified": false,
-        "line_number": 1755,
-        "type": "Base64 High Entropy String"
+        "line_number": 90
       },
       {
-        "hashed_secret": "6fbc7dd864586173160874f2a86ca7d2d552cb85",
+        "type": "Secret Keyword",
+        "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml",
+        "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d",
         "is_verified": false,
-        "line_number": 1769,
-        "type": "Base64 High Entropy String"
+        "line_number": 93
       },
       {
-        "hashed_secret": "81a961f2c89c6209328b74a8768e30fd76c3ac72",
+        "type": "Secret Keyword",
+        "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml",
+        "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb",
         "is_verified": false,
-        "line_number": 1855,
-        "type": "Base64 High Entropy String"
+        "line_number": 96
       },
       {
-        "hashed_secret": "797d4751c536c421cb82b9f62e0a804af30d78f5",
+        "type": "Secret Keyword",
+        "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml",
+        "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457",
         "is_verified": false,
-        "line_number": 1889,
-        "type": "Base64 High Entropy String"
+        "line_number": 99
       },
       {
-        "hashed_secret": "0d55babfa89f240142c0adfc7b560500a1d3ae7c",
+        "type": "Secret Keyword",
+        "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml",
+        "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295",
         "is_verified": false,
-        "line_number": 1894,
-        "type": "Base64 High Entropy String"
+        "line_number": 102
       },
       {
-        "hashed_secret": "e9fdc3025cd10bd8aa4508611e6b7b7a9d650a2c",
+        "type": "Secret Keyword",
+        "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 1921,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 105
+      }
+    ],
+    "kube/services/qa-dashboard/qa-dashboard-deployment.yaml": [
       {
-        "hashed_secret": "4cf9419259c0ce8eee84b468af3c72db8b001620",
+        "type": "Secret Keyword",
+        "filename": "kube/services/qa-dashboard/qa-dashboard-deployment.yaml",
+        "hashed_secret": "253939a955a575ac69f409e5914dd0191b704760",
         "is_verified": false,
-        "line_number": 1950,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 63
+      }
+    ],
+    "kube/services/qabot/qabot-deploy.yaml": [
       {
-        "hashed_secret": "24816e3eb4308e247bde7c1d09ffb7b79c519b71",
+        "type": "Secret Keyword",
+        "filename": "kube/services/qabot/qabot-deploy.yaml",
+        "hashed_secret": "a9fa7aa8c08b647c3fb696e6598642d4a63e25be",
         "is_verified": false,
-        "line_number": 1983,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 86
+      }
+    ],
+    "kube/services/requestor/requestor-deploy.yaml": [
       {
-        "hashed_secret": "e9adfe8a333d45f4776fe0eab31608be5d7b6a7d",
+        "type": "Secret Keyword",
+        "filename": "kube/services/requestor/requestor-deploy.yaml",
+        "hashed_secret": "15debe4170aa5b89858d939f4c0644307ae7789b",
         "is_verified": false,
-        "line_number": 2004,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 61
+      }
+    ],
+    "kube/services/revproxy/gen3.nginx.conf/indexd-service.conf": [
       {
-        "hashed_secret": "03d6fb388dd1b185129b14221f7127715822ece6",
+        "type": "Secret Keyword",
+        "filename": "kube/services/revproxy/gen3.nginx.conf/indexd-service.conf",
+        "hashed_secret": "f89523833036f85fed37ce3ebf25492189bc9397",
         "is_verified": false,
-        "line_number": 2013,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 41
+      }
+    ],
+    "kube/services/revproxy/gen3.nginx.conf/metadata-service.conf": [
       {
-        "hashed_secret": "ee161bb3f899720f95cee50a5f9ef9c9ed96278b",
+        "type": "Secret Keyword",
+        "filename": "kube/services/revproxy/gen3.nginx.conf/metadata-service.conf",
+        "hashed_secret": "18c0871af26eb9875c0f840b13211f097c133fd2",
         "is_verified": false,
-        "line_number": 2046,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 24
+      }
+    ],
+    "kube/services/revproxy/helpers.js": [
       {
-        "hashed_secret": "ebeb5b574fa1ed24a40248275e6136759e766466",
+        "type": "Base64 High Entropy String",
+        "filename": "kube/services/revproxy/helpers.js",
+        "hashed_secret": "1d278d3c888d1a2fa7eed622bfc02927ce4049af",
         "is_verified": false,
-        "line_number": 2078,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 10
+      }
+    ],
+    "kube/services/revproxy/helpersTest.js": [
       {
-        "hashed_secret": "a6a555a428522ccf439fd516ce7c7e269274363f",
+        "type": "Base64 High Entropy String",
+        "filename": "kube/services/revproxy/helpersTest.js",
+        "hashed_secret": "389c3ec21b7325359051e97ff569b078843d2d37",
         "is_verified": false,
-        "line_number": 2083,
-        "type": "Base64 High Entropy String"
+        "line_number": 19
       },
       {
-        "hashed_secret": "f7f85d9f7c87f1e576dcaf4cf50f35728f9a3265",
+        "type": "JSON Web Token",
+        "filename": "kube/services/revproxy/helpersTest.js",
+        "hashed_secret": "e029d4904cc728879d70030572bf37d4510367cb",
         "is_verified": false,
-        "line_number": 2111,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 22
+      }
+    ],
+    "kube/services/revproxy/revproxy-deploy.yaml": [
       {
-        "hashed_secret": "3f1646b60abe74297d2f37a1eee5dc771ad834fc",
+        "type": "Secret Keyword",
+        "filename": "kube/services/revproxy/revproxy-deploy.yaml",
+        "hashed_secret": "c7a87a61893a647e29289845cb51e61afb06800b",
         "is_verified": false,
-        "line_number": 2138,
-        "type": "Base64 High Entropy String"
+        "line_number": 74
       },
       {
-        "hashed_secret": "fd933c71e82d5519ae0cb0779b370d02f6935759",
+        "type": "Secret Keyword",
+        "filename": "kube/services/revproxy/revproxy-deploy.yaml",
+        "hashed_secret": "b3a4e2dea4c1fae8c58a07a84065b73b3a2d831c",
         "is_verified": false,
-        "line_number": 2143,
-        "type": "Base64 High Entropy String"
+        "line_number": 77
       },
       {
-        "hashed_secret": "7090aa59cb52ad1f1810b08c4ac1ddf5c8fce523",
+        "type": "Secret Keyword",
+        "filename": "kube/services/revproxy/revproxy-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
         "is_verified": false,
-        "line_number": 2150,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 80
+      }
+    ],
+    "kube/services/sftp/sftp-deploy.yaml": [
       {
-        "hashed_secret": "756444bea4ea3d67844d8ddf58ad32356e9c2430",
+        "type": "Secret Keyword",
+        "filename": "kube/services/sftp/sftp-deploy.yaml",
+        "hashed_secret": "9fdebf62e477d59d25730744c8b3089c67c3db85",
         "is_verified": false,
-        "line_number": 2188,
-        "type": "Base64 High Entropy String"
-      },
+        "line_number": 39
+      }
+    ],
+    "kube/services/sheepdog/sheepdog-canary-deploy.yaml": [
       {
-        "hashed_secret": "f74135fdd6b8dafdfb01ebbc61c5e5c24ee27cf8",
+        "type": "Secret Keyword",
+        "filename": "kube/services/sheepdog/sheepdog-canary-deploy.yaml",
+        "hashed_secret": "ec9c944c51e87322de8d22e3ca9e2be1ad8fee0d",
         "is_verified": false,
-        "line_number": 2291,
-        "type": "Base64 High Entropy String"
+        "line_number": 58
       },
       {
-        "hashed_secret": "56fbae787f4aed7d0632e95840d71bd378d3a36f",
+        "type": "Secret Keyword",
+        "filename": "kube/services/sheepdog/sheepdog-canary-deploy.yaml",
+        "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc",
         "is_verified": false,
-        "line_number": 2303,
-        "type": "Base64 High Entropy String"
+        "line_number": 61
       },
       {
-        "hashed_secret": "81cb6be182eb79444202c4563080aee75296a672",
+        "type": "Secret Keyword",
+        "filename": "kube/services/sheepdog/sheepdog-canary-deploy.yaml",
+        "hashed_secret": "e43756046ad1763d6946575fed0e05130a154bd2",
         "is_verified": false,
-        "line_number": 2308,
-        "type": "Base64 High Entropy String"
+        "line_number": 67
       },
       {
-        "hashed_secret": "f0f3f7bce32184893046ac5f8cc80da56c3ca539",
+        "type": "Secret Keyword",
+        "filename": "kube/services/sheepdog/sheepdog-canary-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 70
+      }
+    ],
+    "kube/services/sheepdog/sheepdog-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/sheepdog/sheepdog-deploy.yaml",
+        "hashed_secret": "ec9c944c51e87322de8d22e3ca9e2be1ad8fee0d",
         "is_verified": false,
-        "line_number": 2317,
-        "type": "Base64 High Entropy String"
+        "line_number": 63
       },
       {
-        "hashed_secret": "097893233346336f4003acfb6eb173ee59e648f0",
+        "type": "Secret Keyword",
+        "filename": "kube/services/sheepdog/sheepdog-deploy.yaml",
+        "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc",
         "is_verified": false,
-        "line_number": 2327,
-        "type": "Base64 High Entropy String"
+        "line_number": 66
       },
       {
-        "hashed_secret": "bb14c3b4ef4a9f2e86ffdd44b88d9b6729419671",
+        "type": "Secret Keyword",
+        "filename": "kube/services/sheepdog/sheepdog-deploy.yaml",
+        "hashed_secret": "e43756046ad1763d6946575fed0e05130a154bd2",
         "is_verified": false,
-        "line_number": 2332,
-        "type": "Base64 High Entropy String"
+        "line_number": 72
       },
       {
-        "hashed_secret": "71344a35cff67ef081920095d1406601fb5e9b97",
+        "type": "Secret Keyword",
+        "filename": "kube/services/sheepdog/sheepdog-deploy.yaml",
+        "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d",
+        "is_verified": false,
+        "line_number": 75
+      }
+    ],
+    "kube/services/shiny/shiny-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/shiny/shiny-deploy.yaml",
+        "hashed_secret": "327a1bbc6dc0ce857472ee9162a3415133862d50",
+        "is_verified": false,
+        "line_number": 43
+      }
+    ],
+    "kube/services/ssjdispatcher/ssjdispatcher-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/ssjdispatcher/ssjdispatcher-deploy.yaml",
+        "hashed_secret": "7f932449df74fc78573fea502df8a484aef3f69d",
+        "is_verified": false,
+        "line_number": 61
+      }
+    ],
+    "kube/services/superset/superset-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/superset/superset-deploy.yaml",
+        "hashed_secret": "3e9d1737117ff62b23e37aedc72b522b0134997a",
         "is_verified": false,
-        "line_number": 2340,
-        "type": "Base64 High Entropy String"
+        "line_number": 235
       },
       {
-        "hashed_secret": "eb3db6990fd43477a35dfeffc90b3f1ffa83c7bd",
+        "type": "Secret Keyword",
+        "filename": "kube/services/superset/superset-deploy.yaml",
+        "hashed_secret": "6ac08eaa58d425783ff8b5a38fe16ee66c0bce15",
+        "is_verified": false,
+        "line_number": 311
+      }
+    ],
+    "kube/services/superset/superset-redis.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/superset/superset-redis.yaml",
+        "hashed_secret": "9fe1c31809da38c55b2b64bfab47b92bc5f6b7b9",
+        "is_verified": false,
+        "line_number": 265
+      }
+    ],
+    "kube/services/superset/values.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/superset/values.yaml",
+        "hashed_secret": "9a09d4081ddc128a80384712ce6df3578e6bc58e",
         "is_verified": false,
-        "line_number": 2349,
-        "type": "Base64 High Entropy String"
+        "line_number": 173
       },
       {
-        "hashed_secret": "266288bdc14807b538d1e48a5891e361fa9b4a14",
+        "type": "Secret Keyword",
+        "filename": "kube/services/superset/values.yaml",
+        "hashed_secret": "118c413f3fc929a1624f4c3e1da1e3d24377a693",
         "is_verified": false,
-        "line_number": 2357,
-        "type": "Base64 High Entropy String"
+        "line_number": 299
       },
       {
-        "hashed_secret": "800477261175fd21f23e7321923e1fba6ae55471",
+        "type": "Secret Keyword",
+        "filename": "kube/services/superset/values.yaml",
+        "hashed_secret": "d2a8d1ddfa75398366cff06545380c73481ec17d",
         "is_verified": false,
-        "line_number": 2369,
-        "type": "Base64 High Entropy String"
+        "line_number": 445
       },
       {
-        "hashed_secret": "3f0c251b9c2c21454445a98fde6915ceacde2136",
+        "type": "Secret Keyword",
+        "filename": "kube/services/superset/values.yaml",
+        "hashed_secret": "98a84a63e5633d17e3b27b69695f87aa7189e9dc",
+        "is_verified": false,
+        "line_number": 459
+      }
+    ],
+    "kube/services/thor/thor-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/thor/thor-deploy.yaml",
+        "hashed_secret": "1f3f96a3887209d0dda357e5516231ee9c5cd9a7",
+        "is_verified": false,
+        "line_number": 100
+      }
+    ],
+    "kube/services/tube/tube-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/tube/tube-deploy.yaml",
+        "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966",
         "is_verified": false,
-        "line_number": 2387,
-        "type": "Base64 High Entropy String"
+        "line_number": 58
       }
     ],
-    "tf_files/aws/cognito/README.md": [
+    "kube/services/ws-storage/ws-storage-deploy.yaml": [
       {
-        "hashed_secret": "f6920f370a30262b7dd70e97293c73ec89739b70",
+        "type": "Secret Keyword",
+        "filename": "kube/services/ws-storage/ws-storage-deploy.yaml",
+        "hashed_secret": "ec2d9395e11f353370a4abac21a1565641b35ce9",
         "is_verified": false,
-        "line_number": 106,
-        "type": "Secret Keyword"
+        "line_number": 66
+      }
+    ],
+    "kube/services/wts/wts-deploy.yaml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kube/services/wts/wts-deploy.yaml",
+        "hashed_secret": "5de687ae886f19c3cb68d4980e3f2e77cca3db9e",
+        "is_verified": false,
+        "line_number": 65
+      }
+    ],
+    "packer/buildAll.sh": [
+      {
+        "type": "Secret Keyword",
+        "filename": "packer/buildAll.sh",
+        "hashed_secret": "6e1d66a1596528c308e601c10aa0b92d53606ab9",
+        "is_verified": false,
+        "line_number": 15
+      }
+    ],
+    "packer/variables.example.json": [
+      {
+        "type": "Secret Keyword",
+        "filename": "packer/variables.example.json",
+        "hashed_secret": "a3a0648a036bebf78ba1a1eb498a66081059da10",
+        "is_verified": false,
+        "line_number": 5
       }
     ],
     "tf_files/aws/commons/README.md": [
       {
-        "hashed_secret": "d02e53411e8cb4cd709778f173f7bc9a3455f8ed",
+        "type": "Secret Keyword",
+        "filename": "tf_files/aws/commons/README.md",
+        "hashed_secret": "5f02a3fb14ab1ce5c18c362b04b8ffc603ea5951",
         "is_verified": false,
-        "line_number": 60,
-        "type": "Secret Keyword"
+        "line_number": 60
       },
       {
-        "hashed_secret": "9dc0da3613af850c5a018b0a88a5626fb8888e4e",
+        "type": "Secret Keyword",
+        "filename": "tf_files/aws/commons/README.md",
+        "hashed_secret": "49cfceed8aa8df159e53aa5c5951cad48a3f1216",
         "is_verified": false,
-        "line_number": 78,
-        "type": "Secret Keyword"
+        "line_number": 67
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tf_files/aws/commons/README.md",
+        "hashed_secret": "18ad13589ca5fb3c432d7d9f0fe49f8ed6e2c478",
+        "is_verified": false,
+        "line_number": 70
       }
     ],
     "tf_files/aws/eks/sample.tfvars": [
       {
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/eks/sample.tfvars",
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
         "is_verified": false,
-        "line_number": 107,
-        "type": "Hex High Entropy String"
+        "line_number": 107
       }
     ],
     "tf_files/aws/eks/variables.tf": [
       {
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/eks/variables.tf",
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
         "is_verified": false,
-        "line_number": 133,
-        "type": "Hex High Entropy String"
+        "line_number": 133
       }
     ],
     "tf_files/aws/modules/common-logging/README.md": [
       {
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/common-logging/README.md",
         "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017",
         "is_verified": false,
-        "line_number": 57,
-        "type": "Base64 High Entropy String"
-      },
-      {
-        "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2",
-        "is_verified": false,
-        "line_number": 59,
-        "type": "Hex High Entropy String"
+        "line_number": 57
       }
     ],
     "tf_files/aws/modules/common-logging/lambda_function.py": [
       {
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/common-logging/lambda_function.py",
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
         "is_verified": false,
-        "line_number": 18,
-        "type": "Hex High Entropy String"
+        "line_number": 18
       },
       {
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/common-logging/lambda_function.py",
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
         "is_verified": false,
-        "line_number": 18,
-        "type": "Base64 High Entropy String"
+        "line_number": 18
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/common-logging/lambda_function.py",
+        "hashed_secret": "a4752db26b4774d3429878f36ceb7b61805ffd94",
+        "is_verified": false,
+        "line_number": 18
       },
       {
-        "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/common-logging/lambda_function.py",
+        "hashed_secret": "b979d8d0c0e8413c20a5597f789e31f0a2b2ff3a",
         "is_verified": false,
-        "line_number": 18,
-        "type": "Hex High Entropy String"
+        "line_number": 18
       },
       {
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/common-logging/lambda_function.py",
         "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff",
         "is_verified": false,
-        "line_number": 30,
-        "type": "Hex High Entropy String"
+        "line_number": 30
       }
     ],
     "tf_files/aws/modules/common-logging/testLambda.py": [
       {
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/common-logging/testLambda.py",
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
         "is_verified": false,
-        "line_number": 5,
-        "type": "Hex High Entropy String"
+        "line_number": 5
       },
       {
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/common-logging/testLambda.py",
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
         "is_verified": false,
-        "line_number": 5,
-        "type": "Base64 High Entropy String"
+        "line_number": 5
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/common-logging/testLambda.py",
+        "hashed_secret": "a4752db26b4774d3429878f36ceb7b61805ffd94",
+        "is_verified": false,
+        "line_number": 5
       },
       {
-        "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/common-logging/testLambda.py",
+        "hashed_secret": "b979d8d0c0e8413c20a5597f789e31f0a2b2ff3a",
         "is_verified": false,
-        "line_number": 5,
-        "type": "Hex High Entropy String"
+        "line_number": 5
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/common-logging/testLambda.py",
+        "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff",
+        "is_verified": false,
+        "line_number": 10
       }
     ],
     "tf_files/aws/modules/eks/variables.tf": [
       {
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/eks/variables.tf",
         "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884",
         "is_verified": false,
-        "line_number": 113,
-        "type": "Hex High Entropy String"
+        "line_number": 113
       }
     ],
     "tf_files/aws/modules/management-logs/README.md": [
       {
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/README.md",
         "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017",
         "is_verified": false,
-        "line_number": 54,
-        "type": "Base64 High Entropy String"
-      },
-      {
-        "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2",
-        "is_verified": false,
-        "line_number": 56,
-        "type": "Hex High Entropy String"
+        "line_number": 54
       }
     ],
     "tf_files/aws/modules/management-logs/lambda_function.py": [
       {
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/lambda_function.py",
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
         "is_verified": false,
-        "line_number": 18,
-        "type": "Hex High Entropy String"
+        "line_number": 18
       },
       {
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/lambda_function.py",
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
         "is_verified": false,
-        "line_number": 18,
-        "type": "Base64 High Entropy String"
+        "line_number": 18
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/lambda_function.py",
+        "hashed_secret": "a4752db26b4774d3429878f36ceb7b61805ffd94",
+        "is_verified": false,
+        "line_number": 18
       },
       {
-        "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/lambda_function.py",
+        "hashed_secret": "b979d8d0c0e8413c20a5597f789e31f0a2b2ff3a",
         "is_verified": false,
-        "line_number": 18,
-        "type": "Hex High Entropy String"
+        "line_number": 18
       },
       {
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/lambda_function.py",
         "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff",
         "is_verified": false,
-        "line_number": 30,
-        "type": "Hex High Entropy String"
+        "line_number": 30
       }
     ],
     "tf_files/aws/modules/management-logs/testLambda.py": [
       {
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/testLambda.py",
         "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de",
         "is_verified": false,
-        "line_number": 5,
-        "type": "Hex High Entropy String"
+        "line_number": 5
       },
       {
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/testLambda.py",
         "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef",
         "is_verified": false,
-        "line_number": 5,
-        "type": "Base64 High Entropy String"
+        "line_number": 5
       },
       {
-        "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38",
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/testLambda.py",
+        "hashed_secret": "a4752db26b4774d3429878f36ceb7b61805ffd94",
         "is_verified": false,
-        "line_number": 5,
-        "type": "Hex High Entropy String"
+        "line_number": 5
       },
       {
-        "hashed_secret": "3cf8eb4e9254e1d6cc523da01f8b798b9a83101a",
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/testLambda.py",
+        "hashed_secret": "b979d8d0c0e8413c20a5597f789e31f0a2b2ff3a",
         "is_verified": false,
-        "line_number": 6,
-        "type": "Base64 High Entropy String"
+        "line_number": 5
       },
       {
-        "hashed_secret": "51118900cd675df1b44f254057398f3e52902a5d",
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/testLambda.py",
+        "hashed_secret": "3cf8eb4e9254e1d6cc523da01f8b798b9a83101a",
         "is_verified": false,
-        "line_number": 6,
-        "type": "Hex High Entropy String"
+        "line_number": 6
       },
       {
+        "type": "Hex High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/testLambda.py",
         "hashed_secret": "60a6dfc8d43cd2f5c6292899fc2f94f2d4fc32c4",
         "is_verified": false,
-        "line_number": 6,
-        "type": "Hex High Entropy String"
+        "line_number": 6
+      },
+      {
+        "type": "Base64 High Entropy String",
+        "filename": "tf_files/aws/modules/management-logs/testLambda.py",
+        "hashed_secret": "d484ccb4ced21e0149078377f14b913bf5c613d0",
+        "is_verified": false,
+        "line_number": 6
       }
     ],
     "tf_files/aws/slurm/README.md": [
       {
-        "hashed_secret": "fd85d792fa56981cf6a8d2a5c0857c74af86e99d",
+        "type": "Secret Keyword",
+        "filename": "tf_files/aws/slurm/README.md",
+        "hashed_secret": "c16686250cd583de64e02a47a8b194cd5578b2a1",
         "is_verified": false,
-        "line_number": 83,
-        "type": "Secret Keyword"
+        "line_number": 83
       }
     ],
     "tf_files/azure/cloud.tf": [
       {
-        "hashed_secret": "7c1a4b52b64e4106041971c345a1f3eab58fb2a4",
+        "type": "Secret Keyword",
+        "filename": "tf_files/azure/cloud.tf",
+        "hashed_secret": "38d930120a56321ceaa147b2bc1f19db53a0b993",
         "is_verified": false,
-        "line_number": 424,
-        "type": "Secret Keyword"
+        "line_number": 361
       }
     ],
     "tf_files/gcp-bwg/roots/commons_setup/variables/answerfile-commons_setup-001.template.tfvars": [
       {
+        "type": "Secret Keyword",
+        "filename": "tf_files/gcp-bwg/roots/commons_setup/variables/answerfile-commons_setup-001.template.tfvars",
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
         "is_verified": false,
-        "line_number": 231,
-        "type": "Secret Keyword"
+        "line_number": 231
       }
     ],
     "tf_files/gcp-bwg/roots/templates/answerfile-commons_setup-001.template.tfvars": [
       {
+        "type": "Secret Keyword",
+        "filename": "tf_files/gcp-bwg/roots/templates/answerfile-commons_setup-001.template.tfvars",
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
         "is_verified": false,
-        "line_number": 231,
-        "type": "Secret Keyword"
+        "line_number": 231
       }
     ],
     "tf_files/gcp-bwg/roots/templates/answerfile-env-tenant.user.tfvars_NO_APP_SETUP": [
       {
+        "type": "Secret Keyword",
+        "filename": "tf_files/gcp-bwg/roots/templates/answerfile-env-tenant.user.tfvars_NO_APP_SETUP",
         "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227",
         "is_verified": false,
-        "line_number": 262,
-        "type": "Secret Keyword"
+        "line_number": 262
       }
     ],
-    "tf_files/gcp/commons/sample.tfvars": [
+    "tf_files/gcp/commons/root.tf": [
       {
-        "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd",
+        "type": "Secret Keyword",
+        "filename": "tf_files/gcp/commons/root.tf",
+        "hashed_secret": "013b6be0bd7ef38a9ee3472cec65c208a19421e6",
         "is_verified": false,
-        "line_number": 11,
-        "type": "Secret Keyword"
-      },
+        "line_number": 65
+      }
+    ],
+    "tf_files/gcp/commons/sample.tfvars": [
       {
-        "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d",
+        "type": "Secret Keyword",
+        "filename": "tf_files/gcp/commons/sample.tfvars",
+        "hashed_secret": "6b44a330b450ee550c081410c6b705dfeaa105ce",
         "is_verified": false,
-        "line_number": 26,
-        "type": "Secret Keyword"
+        "line_number": 26
       },
       {
-        "hashed_secret": "253c7b5e7c83a86346fc4501495b130813f08105",
-        "is_verified": false,
-        "line_number": 37,
-        "type": "Secret Keyword"
-      }
-    ],
-    "tf_files/shared/modules/k8s_configs/creds.tpl": [
-      {
-        "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c",
+        "type": "Secret Keyword",
+        "filename": "tf_files/gcp/commons/sample.tfvars",
+        "hashed_secret": "791191ef9eafc75f5dd28e37df837b4991556876",
         "is_verified": false,
-        "line_number": 8,
-        "type": "Secret Keyword"
+        "line_number": 31
       }
     ]
   },
-  "version": "0.13.1",
-  "word_list": {
-    "file": null,
-    "hash": null
-  }
+  "generated_at": "2024-03-07T21:26:14Z"
 }
diff --git a/files/scripts/ecr-access-job-requirements.txt b/files/scripts/ecr-access-job-requirements.txt
new file mode 100644
index 000000000..bb6d4b847
--- /dev/null
+++ b/files/scripts/ecr-access-job-requirements.txt
@@ -0,0 +1 @@
+boto3<2
diff --git a/files/scripts/ecr-access-job.md b/files/scripts/ecr-access-job.md
new file mode 100644
index 000000000..9659b186b
--- /dev/null
+++ b/files/scripts/ecr-access-job.md
@@ -0,0 +1,85 @@
+# ecr-access-job
+
+### How to run
+
+Configure `global.ecr-access-job-role-arn` to the ARN of the `EcrRepoPolicyUpdateRole` role (described below) in the `manifest.json` file.
+
+Run `gen3 kube-setup-ecr-access-cronjob` to set up the ECR access cronjob.
+
+### What does it do?
+
+The job runs the `ecr-access-job.py` script.
+
+This script updates the configuration of ECR repositories so that users can access the repositories that were created for them.
+
+It queries a DynamoDB table which has the following (simplified) structure:
+| user_id            | workspace_type       | account_id |
+| ------------------ | -------------------- | ---------- |
+| user1@username.com | Direct Pay           | 123456     |
+| user2@username.com | Direct Pay           | 789012     |
+| user1@username.com | Other workspace type | <null>     |
+
+and then allows each AWS account to acccess the appropriate ECR repositories. The users' ECR repositories are based on their username as stored in the table. For example, `user1@username.com`'s ECR repository is assumed to be `nextflow-approved/user1-40username-2ecom`.
+
+### Access needed
+
+- "EcrRepoPolicyUpdateRole" role in the account (Acct1) that contains the ECR repositories:
+
+**Note:** `kube-setup-ecr-access-cronjob.sh` assumes this role already exists.
+
+Permissions:
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "UpdateEcrRepoPolicy",
+            "Effect": "Allow",
+            "Action": "ecr:SetRepositoryPolicy",
+            "Resource": "arn:aws:ecr:us-east-1:<Acct1 ID>:repository/nextflow-approved/*"
+        }
+    ]
+}
+```
+
+Trust policy (allows Acct2):
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+        "Sid": "AllowAssumingRole",
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "arn:aws:iam::<Acct2 ID>:root"
+            },
+            "Action": "sts:AssumeRole"
+        }
+    ]
+}
+```
+
+- Policy in the account (Acct2) that contains the DynamoDB table (created automatically by `kube-setup-ecr-access-job.sh`):
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Sid": "ReadDynamoDB",
+            "Effect": "Allow",
+            "Action": [
+                "dynamodb:Scan"
+            ],
+            "Resource": "arn:aws:dynamodb:<region>:<Acct2 ID>:table/<table name>"
+        },
+        {
+            "Sid": "AssumeEcrRole",
+            "Effect": "Allow",
+            "Action": [
+                "sts:AssumeRole"
+            ],
+            "Resource": "arn:aws:iam::<Acct1 ID>:role/<role name>"
+        }
+    ]
+}
+```
diff --git a/files/scripts/ecr-access-job.py b/files/scripts/ecr-access-job.py
new file mode 100644
index 000000000..828d94c96
--- /dev/null
+++ b/files/scripts/ecr-access-job.py
@@ -0,0 +1,177 @@
+"""
+See documentation at https://github.com/uc-cdis/cloud-automation/blob/master/files/scripts/ecr-access-job.md
+"""
+
+from decimal import Decimal
+import json
+import os
+from typing import List
+import uuid
+
+import boto3
+from boto3.dynamodb.conditions import Attr
+
+
+REGION = "us-east-1"
+
+# for local testing. in production, use a service account instead of a key.
+MAIN_ACCOUNT_CREDS = {"key_id": os.environ.get("KEY_ID"), "key_secret": os.environ.get("KEY_SECRET")}
+
+
+def escapism(string: str) -> str:
+    """
+    This is a direct translation of Hatchery's `escapism` golang function to python.
+    We need to escape the username in the same way it's escaped by Hatchery's `escapism` function because
+    special chars cannot be used in an ECR repo name, and so that the ECR repo generated here matches the
+    name expected by Hatchery.
+    """
+    safeBytes = "abcdefghijklmnopqrstuvwxyz0123456789"
+    escaped = ""
+    for v in string:
+        if v not in safeBytes:
+            hexCode = "{0:02x}".format(ord(v))
+            escaped += "-" + hexCode
+        else:
+            escaped += v
+    return escaped
+
+
+def get_configs() -> (str, str):
+    table_name = os.environ.get("PAY_MODELS_DYNAMODB_TABLE")
+    if not table_name:
+        raise Exception("Missing 'PAY_MODELS_DYNAMODB_TABLE' environment variable")
+
+    ecr_role_arn = os.environ.get("ECR_ACCESS_JOB_ARN")
+    if not ecr_role_arn:
+        raise Exception("Missing 'ECR_ACCESS_JOB_ARN' environment variable")
+
+    return table_name, ecr_role_arn
+
+
+def query_usernames_and_account_ids(table_name: str) -> List[dict]:
+    """
+    Returns:
+        List[dict]: [ { "user_id": "user1@username.com", "account_id": "123456" } ]
+    """
+    if MAIN_ACCOUNT_CREDS["key_id"]:
+        session = boto3.Session(
+            aws_access_key_id=MAIN_ACCOUNT_CREDS["key_id"],
+            aws_secret_access_key=MAIN_ACCOUNT_CREDS["key_secret"],
+        )
+    else:
+        session = boto3.Session()
+    dynamodb = session.resource("dynamodb", region_name=REGION)
+    table = dynamodb.Table(table_name)
+
+    # get usernames and AWS account IDs from DynamoDB
+    queried_keys = ["user_id", "account_id"]
+    filter_expr = Attr("workspace_type").eq("Direct Pay")
+    proj = ", ".join("#" + key for key in queried_keys)
+    expr = {"#" + key: key for key in queried_keys}
+    response = table.scan(
+        FilterExpression=filter_expr,
+        ProjectionExpression=proj,
+        ExpressionAttributeNames=expr,
+    )
+    assert response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 200, response
+    items = response["Items"]
+    # if the response is paginated, get the rest of the items
+    while response["Count"] > 0:
+        if "LastEvaluatedKey" not in response:
+            break
+        response = table.scan(
+            FilterExpression=filter_expr,
+            ProjectionExpression=proj,
+            ExpressionAttributeNames=expr,
+            ExclusiveStartKey=response["LastEvaluatedKey"],
+        )
+        assert (
+            response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 200
+        ), response
+        items.extend(response["Items"])
+
+    return items
+
+
+def update_access_in_ecr(repo_to_account_ids: List[dict], ecr_role_arn: str) -> None:
+    # get access to ECR in the account that contains the ECR repos
+    if MAIN_ACCOUNT_CREDS["key_id"]:
+        sts = boto3.client(
+            "sts",
+            aws_access_key_id=MAIN_ACCOUNT_CREDS["key_id"],
+            aws_secret_access_key=MAIN_ACCOUNT_CREDS["key_secret"],
+        )
+    else:
+        sts = boto3.client("sts")
+    assumed_role = sts.assume_role(
+        RoleArn=ecr_role_arn,
+        DurationSeconds=900,  # minimum time for aws assume role as per boto docs
+        RoleSessionName=f"ecr-access-assume-role-{str(uuid.uuid4())[:8]}",
+    )
+    assert "Credentials" in assumed_role, "Unable to assume role"
+    ecr = boto3.client(
+        "ecr",
+        aws_access_key_id=assumed_role["Credentials"]["AccessKeyId"],
+        aws_secret_access_key=assumed_role["Credentials"]["SecretAccessKey"],
+        aws_session_token=assumed_role["Credentials"]["SessionToken"],
+    )
+
+    # for each ECR repo, whitelist the account IDs so users can access the repo
+    for repo, account_ids in repo_to_account_ids.items():
+        print(f"Allowing AWS accounts {account_ids} to use ECR repository '{repo}'")
+        policy = {
+            "Version": "2008-10-17",
+            "Statement": [
+                {
+                    "Sid": "AllowCrossAccountPull",
+                    "Effect": "Allow",
+                    "Principal": {
+                        "AWS": [
+                            f"arn:aws:iam::{account_id}:root"
+                            for account_id in account_ids
+                        ]
+                    },
+                    "Action": [
+                        "ecr:BatchCheckLayerAvailability",
+                        "ecr:BatchGetImage",
+                        "ecr:GetAuthorizationToken",
+                        "ecr:GetDownloadUrlForLayer",
+                    ],
+                }
+            ],
+        }
+        # Note that this is overwriting the repo policy, not appending to it. This means we can't have 2 dynamodb
+        # tables pointing at the same set of ECR repos: the repos would only allow the accounts in the table for
+        # which the script was run most recently. eg QA and Staging can't use the same ECR repos.
+        # Appending is not possible since this code will eventually rely on Arborist for authorization information
+        # and we'll need to overwrite in order to remove expired access.
+        try:
+            ecr.set_repository_policy(
+                repositoryName=repo,
+                policyText=json.dumps(policy),
+            )
+        except Exception as e:
+            print(f"  Unable to update '{repo}'; skipping it: {e}")
+
+
+def main() -> None:
+    table_name, ecr_role_arn = get_configs()
+    items = query_usernames_and_account_ids(table_name)
+
+    # construct mapping: { ECR repo url: [ AWS account IDs with access ] }
+    ecr_repo_prefix = "nextflow-approved"
+    repo_to_account_ids = {
+        f"{ecr_repo_prefix}/{escapism(e['user_id'])}": [e["account_id"]]
+        for e in items
+        if "account_id" in e
+    }
+    print(
+        "Mapping of ECR repository to allowed AWS accounts:\n",
+        json.dumps(repo_to_account_ids, indent=2),
+    )
+
+    update_access_in_ecr(repo_to_account_ids, ecr_role_arn)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/gen3/bin/kube-setup-ecr-access-cronjob.sh b/gen3/bin/kube-setup-ecr-access-cronjob.sh
new file mode 100644
index 000000000..d23afc862
--- /dev/null
+++ b/gen3/bin/kube-setup-ecr-access-cronjob.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+source "${GEN3_HOME}/gen3/lib/utils.sh"
+gen3_load "gen3/lib/kube-setup-init"
+
+setup_ecr_access_job() {
+    if g3kubectl get configmap manifest-global > /dev/null; then
+      ecrRoleArn=$(g3kubectl get configmap manifest-global -o jsonpath={.data.ecr-access-job-role-arn})
+    fi
+    if [ -z "$ecrRoleArn" ]; then
+      gen3_log_err "Missing 'global.ecr-access-job-role-arn' configuration in manifest.json"
+      return 1
+    fi
+
+    local saName="ecr-access-job-sa"
+    if ! g3kubectl get sa "$saName" > /dev/null 2>&1; then
+      tempFile="ecr-access-job-policy.json"
+      cat - > $tempFile <<EOM
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "ReadDynamoDB",
+      "Effect": "Allow",
+      "Action": [
+        "dynamodb:Scan"
+      ],
+      "Resource": "arn:aws:dynamodb:*:*:table/*"
+    },
+    {
+      "Sid": "AssumeEcrRole",
+      "Effect": "Allow",
+      "Action": [
+        "sts:AssumeRole"
+      ],
+      "Resource": "$ecrRoleArn"
+    }
+  ]
+}
+EOM
+      local role_name
+      if ! role_name="$(gen3 iam-serviceaccount -c "${saName}" -p $tempFile)" || [[ -z "$role_name" ]]; then
+        gen3_log_err "Failed to create iam service account"
+        rm $tempFile
+        return 1
+      fi
+      rm $tempFile
+      gen3_log_info "created service account '${saName}' and role '${role_name}'"
+    else
+      gen3_log_info "service account '${saName}' already exist"
+    fi
+}
+
+setup_ecr_access_job
+
+# Run every hour, at the top of the hour
+gen3 job cron ecr-access "0 * * * *"
+
+cat <<EOM
+The ecr-access cronjob has been configured.
+EOM
diff --git a/kube/services/jobs/ecr-access-job.yaml b/kube/services/jobs/ecr-access-job.yaml
new file mode 100644
index 000000000..11979a123
--- /dev/null
+++ b/kube/services/jobs/ecr-access-job.yaml
@@ -0,0 +1,85 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ecr-access
+spec:
+  template:
+    metadata:
+      labels:
+        app: gen3job
+    spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: karpenter.sh/capacity-type
+                    operator: In
+                    values:
+                      - on-demand
+            - weight: 99
+              preference:
+                matchExpressions:
+                  - key: eks.amazonaws.com/capacityType
+                    operator: In
+                    values:
+                      - ONDEMAND
+      restartPolicy: Never
+      serviceAccountName: ecr-access-job-sa
+      securityContext:
+        fsGroup: 1000
+      containers:
+        - name: awshelper
+          GEN3_AWSHELPER_IMAGE|-image: quay.io/cdis/awshelper:master-|
+          imagePullPolicy: Always
+          resources:
+            limits:
+              cpu: 0.5
+              memory: 1Gi
+          env:
+            - name: SLACK_WEBHOOK
+              valueFrom:
+                configMapKeyRef:
+                  name: global
+                  key: slack_webhook
+                  optional: true
+            - name: HOSTNAME
+              valueFrom:
+                  configMapKeyRef:
+                    name: global
+                    key: hostname
+            - name: PAY_MODELS_DYNAMODB_TABLE
+              valueFrom:
+                configMapKeyRef:
+                  name: manifest-hatchery
+                  key: pay-models-dynamodb-table
+                  optional: true
+            - name: ECR_ACCESS_JOB_ARN
+              valueFrom:
+                configMapKeyRef:
+                  name: manifest-global
+                  key: ecr-access-job-role-arn
+                  optional: true
+          command: ["/bin/bash"]
+          args:
+            - "-c"
+            - |
+              set -e
+
+              cd cloud-automation/files/scripts/
+              echo Installing requirements...
+              pip3 install -r ecr-access-job-requirements.txt
+              python3 ecr-access-job.py
+              exitcode=$?
+
+              if [[ "${SLACK_WEBHOOK}" != 'None' ]]; then
+                if [[ $exitcode == 1 ]]; then
+                  curl -X POST --data-urlencode "payload={\"text\": \"JOBFAIL: ECR access job on ${HOSTNAME}\"}" "${SLACK_WEBHOOK}"
+                else
+                  curl -X POST --data-urlencode "payload={\"text\": \"SUCCESS: ECR access job on ${HOSTNAME}\"}" "${SLACK_WEBHOOK}"
+                fi
+              fi
+
+              echo "Exit code: $exitcode"
+              exit "$exitcode"

From b077ca609ef2ebc0bb0e0ec21bd1c2f34fa14fa1 Mon Sep 17 00:00:00 2001
From: Jian <52763034+tianj7@users.noreply.github.com>
Date: Fri, 8 Mar 2024 16:28:46 -0600
Subject: [PATCH 47/67] Add new middleware url (#2497)

* add new middleware url

* feat(argo-wrapper): newline

---------

Co-authored-by: Andrew Prokhorenkov <andrew.prokhorenkov@gmail.com>
---
 kube/services/argo-wrapper/config.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/kube/services/argo-wrapper/config.ini b/kube/services/argo-wrapper/config.ini
index 40ac392fd..0693ee2e2 100644
--- a/kube/services/argo-wrapper/config.ini
+++ b/kube/services/argo-wrapper/config.ini
@@ -3,3 +3,4 @@ ARGO_ACCESS_METHOD = access
 ARGO_HOST = $ARGO_HOST
 ARGO_NAMESPACE = $ARGO_NAMESPACE
 COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = http://cohort-middleware-service/cohortdefinition-stats/by-source-id/{}/by-team-project?team-project={}
+COHORT_MIDDLEWARE_URL = http://cohort-middleware-service

From f84581c1fd8899d7cbb09d48464bb6a9a6ceaa47 Mon Sep 17 00:00:00 2001
From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com>
Date: Mon, 11 Mar 2024 09:35:52 -0600
Subject: [PATCH 48/67] Update gen3-helpers to fix failing tests (#2443)

* updating the awsrole script to better handle variables and flag

* adding "migrate to vpc cni" script to cloud-auto

* removing script- wrong branch

* updating kube-setup-argo script for IRSA

* changing the script to get the 3 private ip to account for fargate instances and fix the "ec2 test filter" test

* changing -S to -e to verify if docker is the container runtime due to the way that Jenkins pods are setup

* testing mount change

* testing another mount change

* reverting change

* mounting containerd instead

* modifying jenkins shell commands

* reverting ecr change

* reverting jenkisfile change and then changing permissions on containerd.sock instead of docker.sock

* adding more jobs to the "excludeJob" list

* commenting out jupyter metric test as we currently don't use prometheus in most environments

* commented out the wrong line in the wrong script

* removing terraform test as we are now using Atlantis and Terragrunt to manage our infastructure and will no longer use the "workon" command

* fixing snapshot script to grab init containers and all the main containers seperately

* updating to use "pip3" instead of /usr/bin/pip3"

* changing the evicted pod cleanup to produce no output so the healthcheck will pass in Jenkins.

also, adding in the proper path for pip in the pytest stage of the pipeline

* trying another method to get rid of the json error when running the gen3 healthcheck test

* Update healthcheck.sh

moving clear_evicted_pods function call to run inside the healthcheck function
---
 Jenkinsfile                 |  12 +-
 gen3/bin/awsrole.sh         |  48 ++--
 gen3/bin/healthcheck.sh     |   6 +-
 gen3/bin/kube-setup-argo.sh |   2 +-
 gen3/lib/logs/snapshot.sh   |   2 +-
 gen3/test/ec2Test.sh        |   2 +-
 gen3/test/ecrTest.sh        |   4 +-
 gen3/test/jobTest.sh        |   2 +-
 gen3/test/jupyterTest.sh    |   2 +-
 gen3/test/terraformTest.sh  | 461 ------------------------------------
 10 files changed, 39 insertions(+), 502 deletions(-)
 delete mode 100644 gen3/test/terraformTest.sh

diff --git a/Jenkinsfile b/Jenkinsfile
index 9c70a2e37..908c2d01a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -134,8 +134,8 @@ spec:
       readOnly: true
       mountPath: "/usr/local/share/ca-certificates/cdis/cdis-ca.crt"
       subPath: "ca.pem"
-    - name: dockersock
-      mountPath: "/var/run/docker.sock"
+    - name: containerdsock
+      mountPath: "/var/run/containerd/containerd.sock"
   serviceAccount: jenkins-service
   serviceAccountName: jenkins-service
   volumes:
@@ -145,9 +145,9 @@ spec:
   - name: ca-volume
     secret:
       secretName: "service-ca"
-  - name: dockersock
+  - name: containerdsock
     hostPath:
-      path: /var/run/docker.sock
+      path: /var/run/containerd/containerd.sock
 '''
         defaultContainer 'shell'
         }
@@ -293,8 +293,8 @@ spec:
                 script {
                     try {
                         if(!skipUnitTests) {
-                            sh '/usr/bin/pip3 install boto3 --upgrade --user'
-                            sh '/usr/bin/pip3 install kubernetes --upgrade --user'
+                            sh '/usr/local/bin/pip3 install boto3 --upgrade --user'
+                            sh '/usr/local/bin/pip3 install kubernetes --upgrade --user'
                             sh 'python3 -m pytest cloud-automation/apis_configs/'
                             sh 'python3 -m pytest cloud-automation/gen3/lib/dcf/'
                             sh 'cd cloud-automation/tf_files/aws/modules/common-logging && python3 -m pytest testLambda.py'
diff --git a/gen3/bin/awsrole.sh b/gen3/bin/awsrole.sh
index 144b7a4fe..dd19ea7a4 100644
--- a/gen3/bin/awsrole.sh
+++ b/gen3/bin/awsrole.sh
@@ -25,16 +25,16 @@ gen3_awsrole_help() {
 function gen3_awsrole_ar_policy() {
   local serviceAccount="$1"
   shift || return 1
-  if [[ ! -z $1 ]]; then
-    local namespace=$1
+  if [[ -z $1 ]] || [[ $1 == -* ]]; then
+    namespace=$(gen3 db namespace)
   else
-    local namespace=$(gen3 db namespace)
+    namespace=$1
+    shift
   fi
   local issuer_url
   local account_id
   local vpc_name
-  shift || return 1
-  local flag=$1
+  local flag=$flag
 
   vpc_name="$(gen3 api environment)" || return 1
   issuer_url="$(aws eks describe-cluster \
@@ -46,7 +46,7 @@ function gen3_awsrole_ar_policy() {
 
   local provider_arn="arn:aws:iam::${account_id}:oidc-provider/${issuer_url}"
 
-  if [[ "$flag" == "all_namespaces" ]]; then
+  if [[ "$flag" == "-all_namespaces" ]]; then
     # Use a trust policy that allows role to be used by multiple namespaces.
     cat - <<EOF
 {
@@ -110,7 +110,6 @@ EOF
   fi
 }
 
-
 #
 # Annotate the given service account with the given IAM role
 #
@@ -162,6 +161,7 @@ _get_entity_type() {
 #
 # @param rolename
 # @param saName for assume-role policy document
+# @param flag (optional) - specify a flag to use a different trust policy
 #
 _tfplan_role() {
   local rolename="$1"
@@ -169,15 +169,11 @@ _tfplan_role() {
   local saName="$1"
   shift || return 1
   local namespace="$1"
-  shift || return 1
-  local flag=""
-  # Check if the "all_namespaces" flag is provided
-  if [[ "$1" == "-f" || "$1" == "--flag" ]]; then
-    flag="$2"
-    shift 2
-  fi
   local arDoc
+  local flag=$flag
+
   arDoc="$(gen3_awsrole_ar_policy "$saName" "$namespace" "$flag")" || return 1
+
   gen3 workon default "${rolename}_role"
   gen3 cd
   cat << EOF > config.tfvars
@@ -230,10 +226,14 @@ gen3_awsrole_create() {
     gen3_log_err "use: gen3 awsrole create roleName saName"
     return 1
   fi
-  if [[ ! -z $1 ]]; then
-    local namespace=$1
+  if [[ -z $1 ]] || [[ $1 == -* ]]; then
+    namespace=$(gen3 db namespace)
   else
-    local namespace=$(gen3 db namespace)
+    namespace=$1
+    shift
+  fi
+  if [[ ! -z $1 ]]; then
+    flag=$1
   fi
   # do simple validation of name
   local regexp="^[a-z][a-z0-9\-]*$"
@@ -247,13 +247,7 @@ EOF
     gen3_log_err $errMsg
     return 1
   fi
-  shift || return 1
-  local flag=""
-  # Check if the "all_namespaces" flag is provided
-  if [[ "$1" == "-f" || "$1" == "--flag" ]]; then
-    flag="$2"
-    shift 2
-  fi
+
 
   # check if the name is already used by another entity
   local entity_type
@@ -271,9 +265,11 @@ EOF
   fi
 
   TF_IN_AUTOMATION="true"
-  if ! _tfplan_role $rolename $saName $namespace -f $flag; then
+
+  if ! _tfplan_role $rolename $saName $namespace $flag; then
     return 1
   fi
+
   if ! _tfapply_role $rolename; then
     return 1
   fi
@@ -422,4 +418,4 @@ gen3_awsrole() {
 # Let testsuite source file
 if [[ -z "$GEN3_SOURCE_ONLY" ]]; then
   gen3_awsrole "$@"
-fi
+fi
\ No newline at end of file
diff --git a/gen3/bin/healthcheck.sh b/gen3/bin/healthcheck.sh
index 149cb1aaa..b658ff033 100644
--- a/gen3/bin/healthcheck.sh
+++ b/gen3/bin/healthcheck.sh
@@ -137,6 +137,10 @@ gen3_healthcheck() {
     internetAccessExplicitProxy=false
   fi
 
+  gen3_log_info "Clearing Evicted pods"
+  sleep 5
+  clear_evicted_pods
+
   local healthJson=$(cat - <<EOM
   {
     "pendingTimeoutPods": $pendingTimeoutPods,
@@ -210,5 +214,3 @@ clear_evicted_pods() {
 }
 
 gen3_healthcheck "$@"
-
-clear_evicted_pods
diff --git a/gen3/bin/kube-setup-argo.sh b/gen3/bin/kube-setup-argo.sh
index 20676145b..4c6c55eee 100644
--- a/gen3/bin/kube-setup-argo.sh
+++ b/gen3/bin/kube-setup-argo.sh
@@ -156,7 +156,7 @@ EOF
       g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} --overwrite -n $argo_namespace
       g3kubectl annotate serviceaccount argo eks.amazonaws.com/role-arn=${roleArn} --overwrite -n $nameSpace
   else
-        gen3 awsrole create $roleName argo $nameSpace -f all_namespaces
+        gen3 awsrole create $roleName argo $nameSpace -all_namespaces
         roleArn=$(aws iam get-role --role-name "${roleName}" --query 'Role.Arn' --output text)
         g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} -n $argo_namespace
   fi
diff --git a/gen3/lib/logs/snapshot.sh b/gen3/lib/logs/snapshot.sh
index d3d3b2c6c..ae769a285 100644
--- a/gen3/lib/logs/snapshot.sh
+++ b/gen3/lib/logs/snapshot.sh
@@ -39,7 +39,7 @@ gen3_logs_snapshot_all() {
   # For each pod for which we can list the containers, get the pod name and get its list of containers
   # (container names + initContainers names). Diplay them as lines of "<pod name>  <container name>".
   g3kubectl get pods -o json | \
-    jq -r '.items | map(select(.status.phase != "Pending" and .status.phase != "Unknown")) | map( {pod: .metadata.name, containers: [(.spec.containers | select(.!=null) | map(.name)), (.spec.initContainers | select(.!=null) | map(.name)) | add ] } ) | map( .pod as $pod | .containers | map( { pod: $pod, cont: .})[]) | map(select(.cont != "pause" and .cont != "jupyterhub"))[] | .pod + "  " + .cont' | \
+    jq -r '.items | map(select(.status.phase != "Pending" and .status.phase != "Unknown")) | .[] | .metadata.name as $pod | (.spec.containers + .spec.initContainers) | map(select(.name != "pause" and .name != "jupyterhub")) | .[] | {pod: $pod, cont: .name} | "\(.pod)  \(.cont)"' | \
     while read -r line; do
       gen3_logs_snapshot_container $line
     done
diff --git a/gen3/test/ec2Test.sh b/gen3/test/ec2Test.sh
index 21310a24c..4981c925c 100644
--- a/gen3/test/ec2Test.sh
+++ b/gen3/test/ec2Test.sh
@@ -1,6 +1,6 @@
 
 
-if ! EC2_TEST_IP="$(g3kubectl get nodes -o json | jq -r -e '.items[0].status.addresses[] | select(.type == "InternalIP") | .address')" || [[ -z "$EC2_TEST_IP" ]]; then
+if ! EC2_TEST_IP="$(g3kubectl get nodes -o json | jq -r -e '.items[3].status.addresses[] | select(.type == "InternalIP") | .address')" || [[ -z "$EC2_TEST_IP" ]]; then
   gen3_log_err "ec2Test failed to acquire IP address of a k8s node to test against"
 fi
 
diff --git a/gen3/test/ecrTest.sh b/gen3/test/ecrTest.sh
index 91edf798b..57847abe5 100644
--- a/gen3/test/ecrTest.sh
+++ b/gen3/test/ecrTest.sh
@@ -10,8 +10,8 @@ test_ecr_login() {
 
 test_ecr_setup() {
   if [[ -n "$JENKINS_HOME" ]]; then
-    # give ourselves read/write permissions on /var/run/docker.sock
-    sudo chmod a+rw /var/run/docker.sock; because $? "ecr_setup modified docker.sock"
+    # give ourselves permissions on /run/containerd/containerd.sock
+    sudo chown root:sudo /run/containerd/containerd.sock; because $? "ecr_setup modified containerd.sock"
   fi
 }
 
diff --git a/gen3/test/jobTest.sh b/gen3/test/jobTest.sh
index 84a4d046b..bb37b4f72 100644
--- a/gen3/test/jobTest.sh
+++ b/gen3/test/jobTest.sh
@@ -6,7 +6,7 @@
 excludeJob() {
   local jobKey="$1"
   local excludeList=(
-    /aws-bucket- /bucket- /covid19- /data-ingestion- /google- /nb-etl- /remove-objects-from- /replicate- /s3sync- /fence-cleanup
+    /aws-bucket- /bucket- /covid19- /data-ingestion- /google- /nb-etl- /remove-objects-from- /replicate- /s3sync- /fence-cleanup /etl- /indexd- /metadata-
   )
   for exclude in "${excludeList[@]}"; do
     if [[ "$it" =~ $exclude ]]; then return 0; fi
diff --git a/gen3/test/jupyterTest.sh b/gen3/test/jupyterTest.sh
index f0e327d71..db6a62618 100644
--- a/gen3/test/jupyterTest.sh
+++ b/gen3/test/jupyterTest.sh
@@ -30,7 +30,7 @@ test_jupyter_metrics() {
 }
 
 shunit_runtest "test_jupyter_idle" "jupyter"
-shunit_runtest "test_jupyter_metrics" "jupyter"
+# shunit_runtest "test_jupyter_metrics" "jupyter"
 shunit_runtest "test_jupyter_prepuller" "local,jupyter"
 shunit_runtest "test_jupyter_namespace" "local,jupyter"
 shunit_runtest "test_jupyter_setup" "jupyter"
diff --git a/gen3/test/terraformTest.sh b/gen3/test/terraformTest.sh
deleted file mode 100644
index 17bcc03c2..000000000
--- a/gen3/test/terraformTest.sh
+++ /dev/null
@@ -1,461 +0,0 @@
-GEN3_TEST_PROFILE="${GEN3_TEST_PROFILE:-cdistest}"
-GEN3_TEST_WORKSPACE="gen3test"
-GEN3_TEST_ACCOUNT=707767160287
-
-#
-# TODO - generalize these tests to setup their own test VPC,
-# rather than relying on qaplanetv1 or devplanetv1 being there
-#
-
-#
-# Little macos/linux stat wrapper
-#
-file_mode() {
-  if [[ $(uname -s) == 'Linux' ]]; then
-    stat -c %a "$1"
-  else
-    stat -f %p "$1"
-  fi
-}
-
-test_workspace() {  
-  gen3 workon $GEN3_TEST_PROFILE $GEN3_TEST_WORKSPACE; because $? "Calling gen3 workon multiple times should be harmless"
-  [[ $GEN3_PROFILE = $GEN3_TEST_PROFILE ]]; because $? "gen3 workon sets the GEN3_PROFILE env variable: $GEN3_PROFILE"
-  [[ $GEN3_WORKSPACE = $GEN3_TEST_WORKSPACE ]]; because $? "gen3 workon sets the GEN3_WORKSPACE env variable: $GEN3_WORKSPACE"
-  [[ $GEN3_FLAVOR = "AWS" || \
-    ($GEN3_FLAVOR == "GCP" && $GEN3_PROFILE =~ ^gcp-) || \
-    ($GEN3_FLAVOR == "ONPREM" && $GEN3_PROFILE =~ ^onprem-) ]]; because $? "GEN3_FLAVOR is gcp for gcp-* profiles, else AWS"
-  [[ $GEN3_FLAVOR != "AWS" || $GEN3_S3_BUCKET = "cdis-state-ac${GEN3_TEST_ACCOUNT}-gen3" || $GEN3_S3_BUCKET = "cdis-terraform-state.account-${GEN3_TEST_ACCOUNT}.gen3" ]]; because $? "gen3 workon sets the GEN3_S3_BUCKET env variable: $GEN3_S3_BUCKET"
-  [[ (! -z $GEN3_WORKDIR) && -d $GEN3_WORKDIR ]]; because $? "gen3 workon sets the GEN3_WORKDIR env variable, and initializes the folder: $GEN3_WORKDIR"
-  [[ $(file_mode $GEN3_WORKDIR) =~ 700$ ]]; because $? "gen3 workon sets the GEN3_WORKDIR to mode 0700, because secrets are in there"
-  gen3 cd && [[ $(pwd) = "$GEN3_WORKDIR" ]]; because $? "gen3 cd should take us to the workspace by default: $(pwd) =? $GEN3_WORKDIR"
-  for fileName in README.md config.tfvars backend.tfvars; do
-    [[ -f $fileName ]]; because $? "gen3 workon ensures we have a $fileName - local copy || s3 copy || generated from template"
-  done
-  [[ ! -z "$MD5" ]]; because $? "commons.sh sets MD5 to $MD5"
-
-  if [[ $GEN3_TEST_WORKSPACE =~ __custom$ ]]; then
-    [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_WORKDIR" ]]; because $? "a __custom workspace loads from the workspace folder"
-  elif [[ "$GEN3_TEST_PROFILE" =~ ^gcp- ]]; then
-    [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/gcp/commons" ]]; because $? "a gcp- profile currently only support a commons workspace"
-  elif [[ "$GEN3_TEST_PROFILE" =~ ^onprem- ]]; then
-    for fileName in README.md creds.json 00configmap.yaml kube-setup.sh; do
-      filePath="onprem_scripts/$fileName"
-      [[ -f $filePath ]]; because $? "gen3 workon ensures we have a $filePath generated from template"
-    done
-  else  # aws profile
-    [[ "$GEN3_TFSCRIPT_FOLDER" =~ ^"$GEN3_HOME/tf_files/aws/" ]]; because $? "an aws workspace references the aws/ folder: $GEN3_TFSCRIPT_FOLDER"
-  fi
-}
-
-workspace_cleanup() {
-  # try to avoid accidentally erasing the user's data ...
-  cd /tmp && [[ -n "$GEN3_WORKDIR" && "$GEN3_WORKDIR" =~ /gen3/ && -f "$GEN3_WORKDIR/config.tfvars" ]] && /bin/rm -rf "$GEN3_WORKDIR";
-    because $? "was able to cleanup $GEN3_WORKDIR"
-}
-
-test_uservpc_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_user"
-  test_workspace
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/user_vpc" ]]; because $? "a _user workspace should use the ./aws/user_vpc resources: $GEN3_TFSCRIPT_FOLDER"
-  workspace_cleanup
-}
-
-test_usergeneric_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_usergeneric"
-  test_workspace
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/user_generic" ]]; because $? "a _usergeneric workspace should use the ./aws/user_generic resources: $GEN3_TFSCRIPT_FOLDER"
-  cat << EOF > config.tfvars
-username="frickjack"
-EOF
-  gen3 tfplan; because $? "_usergeneric tfplan should work";
-  workspace_cleanup
-}
-
-test_snapshot_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_snapshot"
-  test_workspace
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/rds_snapshot" ]]; because $? "a _snapshot workspace should use the ./aws/rds_snapshot resources: $GEN3_TFSCRIPT_FOLDER"
-  workspace_cleanup
-}
-
-test_databucket_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_databucket"
-  test_workspace
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/data_bucket" ]]; because $? "a _databucket workspace should use the ./aws/data_bucket resources: $GEN3_TFSCRIPT_FOLDER"
-  cat - > config.tfvars <<EOM
-bucket_name="gen3test-databucket.gen3"
-environment="qaplanetv1"
-EOM
-  gen3 tfplan; because $? "_databucket tfplan should work"
-  workspace_cleanup
-}
-
-test_arp_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_role_policy_attachment"
-  test_workspace
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/role_policy_attachment" ]]; because $? "a _role_policy_attachment workspace should use the ./aws/role_policy_attachment resources: $GEN3_TFSCRIPT_FOLDER"
-  workspace_cleanup
-}
-
-test_cognito_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}__cognito"
-  test_workspace
-  cat - > config.tfvars <<EOM
-vpc_name                 = "qaplanetv1"
-cognito_provider_name    = "bogus.federation.tld"
-cognito_domain_name      = "qaplanetv1"
-cognito_callback_urls    = ["https://qa.planx-pla.net/","https://qa.planx-pla.net/login/cognito/login/","https://qa.planx-pla.net/user/login/cognito/login/"]
-cognito_provider_details = {"MetadataURL"="https://bogus.federation.tld/federationmetadata/2007-06/federationmetadata.xml"}
-
-tags                     = {
-  "Organization" = "PlanX"
-  "Environment"  = "qaplanetv1"
-}
-EOM
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/cognito" ]]; because $? "a __cognito workspace should use the ./aws/congnito resources: $GEN3_TFSCRIPT_FOLDER"
-  gen3 tfplan; because $? "tfplan cognito should run ok"
-  workspace_cleanup
-}
-
-
-test_commons_workspace() {
-  test_workspace
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/commons" ]]; because $? "a generic workspace should use the ./aws/commons resources: $GEN3_TFSCRIPT_FOLDER"
-  # terraform plan fails if it can't lookup the cert for the commons in the account
-  cat - > config.tfvars <<EOM
-# VPC name is also used in DB name, so only alphanumeric characters
-vpc_name="gen3test"
-#
-vpc_cidr_block="172.24.64.0/20"
-
-dictionary_url="https://s3.amazonaws.com/dictionary-artifacts/YOUR/DICTIONARY/schema.json"
-portal_app="dev"
-
-aws_cert_name="arn:aws:acm:REGION:ACCOUNT-NUMBER:certificate/CERT-ID"
-
-fence_db_size    = 10
-sheepdog_db_size = 10
-indexd_db_size   = 10
-
-fence_db_instance    = "db.t2.micro"
-sheepdog_db_instance = "db.t2.micro"
-indexd_db_instance   = "db.t2.micro"
-
-# This indexd guid prefix should come from Trevar/ZAC
-#indexd_prefix=ENTER_UNIQUE_GUID_PREFIX
-
-hostname="YOUR.API.HOSTNAME"
-#
-# Bucket in bionimbus account hosts user.yaml
-# config for all commons:
-#   s3://cdis-gen3-users/CONFIG_FOLDER/user.yaml
-#
-config_folder="PUT-SOMETHING-HERE"
-
-google_client_secret="YOUR.GOOGLE.SECRET"
-google_client_id="YOUR.GOOGLE.CLIENT"
-
-# Following variables can be randomly generated passwords
-
-hmac_encryption_key="whatever="
-
-gdcapi_secret_key="whatever"
-
-# don't use ( ) " ' { } < > @ in password
-db_password_fence="whatever"
-
-db_password_gdcapi="whatever"
-db_password_sheepdog="whatever"
-db_password_peregrine="whatever"
-
-db_password_indexd="g6pmYkcoR7qECjGoErzVb5gkX3kum0yo"
-
-# password for write access to indexd
-gdcapi_indexd_password="oYva39mIPV5uXskv7jWnKuVZBUFBQcxd"
-
-fence_snapshot=""
-gdcapi_snapshot=""
-indexd_snapshot=""
-# mailgun for sending alert e-mails
-mailgun_api_key=""
-mailgun_api_url=""
-mailgun_smtp_host=""
-
-kube_ssh_key=""
-EOM
-  [[ "$(pwd)" =~ "/$GEN3_WORKSPACE"$ ]]; because $? "commons workspace should have base $GEN3_WORKSPACE - $(pwd)"
-  gen3 tfplan; because $? "tfplan should run even with some invalid config variables"
-  [[ -f "$GEN3_WORKDIR/plan.terraform" ]]; because $? "'gen3 tfplan' generates a plan.terraform file used by 'gen3 tfapply'"
-  workspace_cleanup
-}
-
-test_custom_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}__custom"
-  test_workspace
-
-  local sourceFolder="../../../../../cloud-automation/tf_files/aws/modules/s3-bucket"
-  if [[ ! -d "$sourceFolder" ]]; then
-    # Jenkins has a different relative path setup
-    sourceFolder="../../../../cloud-automation/tf_files/aws/modules/s3-bucket"
-  fi
-  cat - > bucket.tf <<EOM
-provider "aws" {}
-
-module "s3_bucket" {
-  bucket_name       = "frickjack-crazy-test"
-  environment       = "qaplanetv1"
-  source            = "$sourceFolder"
-  cloud_trail_count = "0"
-}
-EOM
-  gen3 workon . .
-  gen3 tfplan; because $? "tfplan __custom should run ok"
-  workspace_cleanup
-}
-
-test_rds_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}__rds"
-  test_workspace
-  cat - > config.tfvars <<EOM
-rds_instance_allocated_storage            = 20
-rds_instance_engine                       = "postgres"
-rds_instance_engine_version               = "10.14"
-rds_instance_username                     = "jenkins"
-rds_instance_db_subnet_group_name         = "qaplanetv1_private_group"
-rds_instance_identifier                   = "jenkins"
-rds_instance_port                         = 5432
-rds_instance_create_monitoring_role       = true
-EOM
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/rds" ]]; because $? "a __rds workspace should use the ./aws/rds resources: $GEN3_TFSCRIPT_FOLDER"
-  gen3 tfplan; because $? "tfplan rds should run ok"
-  workspace_cleanup
-}
-
-test_role_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_role"
-  test_workspace
-  cat - > config.tfvars <<EOM
-rolename="jenkins_testsuite"
-description="Role created with gen3 awsrole"
-path="/gen3_service/"
-arpolicy=<<EDOC
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Principal": {"Service": "ec2.amazonaws.com"},
-      "Action": "sts:AssumeRole"
-    },
-    {
-      "Sid": "",
-      "Effect": "Allow",
-      "Principal": {
-        "Federated": "arn:aws:iam::707767160287:oidc-provider/oidc.eks.us-east-1.amazonaws.com/id/7BCF89168EA2F7291A266833B57566CC"
-      },
-      "Action": "sts:AssumeRoleWithWebIdentity",
-      "Condition": {
-        "StringEquals": {
-          "oidc.eks.us-east-1.amazonaws.com/id/7BCF89168EA2F7291A266833B57566CC:aud": "sts.amazonaws.com",
-          "oidc.eks.us-east-1.amazonaws.com/id/7BCF89168EA2F7291A266833B57566CC:sub": "system:serviceaccount:reuben:sa_awsrole_testsuite"
-        }
-      }
-    }
-  ]
-}
-EDOC
-EOM
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/role" ]]; because $? "a _role workspace should use the ./aws/role resources: $GEN3_TFSCRIPT_FOLDER"
-  gen3 tfplan; because $? "tfplan role should run ok"
-  workspace_cleanup
-}
-
-test_eks_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_eks"
-  test_workspace
-  cat - > config.tfvars <<EOM
-vpc_name                     = "devplanetv1"
-instance_type                = "t3.2xlarge"
-jupyter_instance_type        = "t3.xlarge"
-ec2_keyname                  = "devplanetv1_automation_dev"
-users_policy                 = "devplanetv1"
-worker_drive_size            = 50
-eks_version                  = "1.15"
-#deploy_jupyter_pool         = "yes"
-#kernel                       = "4.19.30"
-workers_subnet_size          = 23
-#jupyter_bootstrap_script     = "bootstrap-2.1.0.sh"
-#bootstrap_script             = "bootstrap-2.0.0.sh"
-jupyter_worker_drive_size    = 80
-#proxy_name                   = "_squid_auto_setup_autoscaling_grp_member"
-jupyter_asg_desired_capacity = 3
-jupyter_asg_max_size         = 10
-jupyter_asg_min_size         = 3
-iam-serviceaccount           = true
-domain_test                  = "www.google.com"
-ha_squid                     = true
-jupyter_asg_desired_capacity = 0
-jupyter_asg_min_size = 0
-EOM
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/eks" ]]; because $? "a _eks workspace should use the ./aws/eks resources: $GEN3_TFSCRIPT_FOLDER"
-  gen3 tfplan; because $? "tfplan eks should run ok"
-  workspace_cleanup
-}
-
-test_encrypted-rds_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}__encrypted-rds"
-  test_workspace
-  cat - > config.tfvars <<EOM
-vpc_name = "devplanetv1"
-db_password_fence = "fence"
-db_password_sheepdog = "sheepdog"
-db_password_peregrine = "peregrine"
-db_password_indexd = "indexd"
-EOM
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/encrypted-rds" ]]; because $? "a __encrypted-rds workspace should use the ./aws/encrypted-rds resources: $GEN3_TFSCRIPT_FOLDER"
-  gen3 tfplan; because $? "tfplan encrypted-rds should run ok"
-  workspace_cleanup
-}
-
-test_dbq_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}__data-bucket-queue"
-  test_workspace
-  cat - > config.tfvars <<EOM
-bucket_name="qaplanetv1-data-bucket"
-EOM
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/data-bucket-queue" ]]; because $? "a __data-bucket-queue workspace should use the ./aws/data-bucket-queue resources: $GEN3_TFSCRIPT_FOLDER"
-  gen3 tfplan; because $? "tfplan data-bucket-queue should run ok"
-  workspace_cleanup
-}
-
-test_sftp_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}__sftp"
-  test_workspace
-  cat - > config.tfvars <<EOM
-ssh_key = "test-key"
-s3_bucket_name = "test-bucket"
-EOM
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/sftp" ]]; because $? "a __sftp workspace should use the ./aws/sftp resources: $GEN3_TFSCRIPT_FOLDER"
-  gen3 tfplan; because $? "tfplan sftp should run ok"
-  workspace_cleanup
-}
-
-test_gcp_workspace() {
-  GEN3_TEST_PROFILE="gcp-dcf-integration"
-  test_workspace
-  workspace_cleanup
-}
-
-test_onprem_workspace() {
-  GEN3_TEST_PROFILE="onprem-${GEN3_TEST_PROFILE}"
-  test_workspace
-  workspace_cleanup
-}
-
-test_utilityvm_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_utilityvm"
-  test_workspace
-  test_workspace
-  cat - > config.tfvars <<EOM
-bootstrap_path = "cloud-automation/flavors/adminvm/"
-bootstrap_script = "ubuntu-18-init.sh"
-vm_name = "jenkinstest"
-vm_hostname = "jenkinstest"
-vpc_id = "vpc-00d2fc7e8fd84fce8"
-vpc_subnet_id = "subnet-07929c80bc1a6619e"
-# secgroup egress whitelist
-vpc_cidr_list = ["172.26.128.0/20", "52.0.0.0/8", "54.0.0.0/8"]
-aws_account_id = "707767160287"
-extra_vars = []
-instance_type = "t3.micro"
-ssh_key_name = "reubenplanetv1_automation_dev"
-EOM
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/utility_vm" ]]; because $? "a __sftp workspace should use the ./aws/sftp resources: $GEN3_TFSCRIPT_FOLDER"
-  gen3 tfplan; because $? "tfplan utility_vm should run ok"
-  workspace_cleanup
-}
-
-test_utilityadmin_workspace() {
-  GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}__utility_admin"
-  test_workspace
-  test_workspace
-  cat - > config.tfvars <<EOM
-bootstrap_path = "cloud-automation/flavors/adminvm/"
-bootstrap_script = "ubuntu-18-init.sh"
-vm_name = "admintest"
-vm_hostname = "admintest"
-vpc_id = "vpc-00d2fc7e8fd84fce8"
-vpc_subnet_id = "subnet-07929c80bc1a6619e"
-# secgroup egress whitelist
-vpc_cidr_list = ["172.26.128.0/20", "52.0.0.0/8", "54.0.0.0/8"]
-aws_account_id = "707767160287"
-extra_vars = []
-instance_type = "t3.micro"
-ssh_key_name = "reubenplanetv1_automation_dev"
-EOM
-  [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/utility_admin" ]]; because $? "a __sftp workspace should use the ./aws/sftp resources: $GEN3_TFSCRIPT_FOLDER"
-  gen3 tfplan; because $? "tfplan utility_admin should run ok"
-  workspace_cleanup
-}
-
-test_trash() {
-  gen3 workon $GEN3_TEST_PROFILE $GEN3_TEST_WORKSPACE; because $? "Calling gen3 workon multiple times should be harmless"
-  [[ -d $GEN3_WORKDIR ]]; because $? "gen3 workon should create $GEN3_WORKDIR"
-  gen3 trash --apply; because $? "gen3 trash should mv a workspace to the trash"
-  [[ ! -d $GEN3_WORKDIR ]]; because $? "the workdir should be gone after trash - $GEN3_WORKDIR"
-  gen3 workon $GEN3_TEST_PROFILE $GEN3_TEST_WORKSPACE; because $? "Calling gen3 workon after trash should recreate a workspace"
-  [[ -d $GEN3_WORKDIR ]]; because $? "gen3 workon should create $GEN3_WORKDIR"
-}
-
-test_ls() {
-  gen3 ls | grep -e "${GEN3_TEST_PROFILE} \s*${GEN3_TEST_WORKSPACE}"; because $? "gen3 ls should include test workspace in result: $GEN3_TEST_PROFILE $GEN3_TEST_WORKSPACE"
-}
-
-
-test_refresh() {
-  gen3 workon $GEN3_TEST_PROFILE $GEN3_TEST_WORKSPACE
-  gen3 --dryrun refresh; because $? "--dryrun refresh should be harmless in test workspace"
-  gen3 refresh; because $? "refresh should be harmless in test workspace"
-  gen3 cd && [[ $(pwd) = "$GEN3_WORKDIR" ]]; because $? "gen3 cd should put us into the workspace"
-  for fileName in README.md config.tfvars backend.tfvars; do
-    hashStr=$($MD5 $fileName | awk '{ print $1 }')
-    backupPath="backups/${fileName}.${hashStr}"
-    [[ -f $backupPath ]]; because $? "gen3 refresh should back $fileName to $backupPath"
-  done
-}
-
-
-test_tfoutput() {
-  # Test runs in a subshell, so we won't stay in the devplanetv1 workspace
-  gen3 workon "${GEN3_TEST_PROFILE}" devplanetv1; because $? "devplanetv1 has some state to run tfoutput against"
-  gen3 tfoutput; because $? "tfoutput should run successfully against devplanetv1"
-  vpcName=$(gen3 tfoutput vpc_name)
-  [[ $vpcName = $GEN3_WORKSPACE ]]; because $? "tfoutput vpc_name works: $vpcName =? $GEN3_WORKSPACE"
-}
-
-shunit_runtest "test_workspace" "terraform"
-shunit_runtest "test_arp_workspace" "terraform"
-shunit_runtest "test_cognito_workspace" "terraform"
-shunit_runtest "test_commons_workspace" "terraform"
-shunit_runtest "test_custom_workspace" "terraform"
-shunit_runtest "test_databucket_workspace" "terraform"
-shunit_runtest "test_eks_workspace" "terraform"
-shunit_runtest "test_encrypted-rds_workspace" "terraform"
-shunit_runtest "test_rds_workspace" "terraform"
-shunit_runtest "test_role_workspace" "terraform"
-shunit_runtest "test_snapshot_workspace" "terraform"
-shunit_runtest "test_usergeneric_workspace" "terraform"
-shunit_runtest "test_uservpc_workspace" "terraform"
-shunit_runtest "test_utilityvm_workspace" "terraform"
-shunit_runtest "test_utilityadmin_workspace" "terraform"
-if [[ -z "$JENKINS_HOME" ]]; then
-  # jenkins does not have Google configurations yet
-  shunit_runtest "test_gcp_workspace" "terraform"
-fi
-shunit_runtest "test_onprem_workspace" "terraform"
-shunit_runtest "test_sftp_workspace" "terraform"
-shunit_runtest "test_trash" "terraform"
-shunit_runtest "test_refresh" "terraform"
-shunit_runtest "test_tfoutput" "terraform"
-shunit_runtest "test_ls" "terraform"

From 6e8e372469956f601d10dbccf8b6d9219223053f Mon Sep 17 00:00:00 2001
From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com>
Date: Tue, 12 Mar 2024 10:05:49 -0500
Subject: [PATCH 49/67] Split build-push workflows (#2499)

---
 .github/workflows/build_awshelper.yaml  | 21 +++++++++++++
 .github/workflows/build_python3.10.yaml | 23 ++++++++++++++
 .github/workflows/build_python3.9.yaml  | 23 ++++++++++++++
 .github/workflows/image_build_push.yaml | 42 -------------------------
 4 files changed, 67 insertions(+), 42 deletions(-)
 create mode 100644 .github/workflows/build_awshelper.yaml
 create mode 100644 .github/workflows/build_python3.10.yaml
 create mode 100644 .github/workflows/build_python3.9.yaml
 delete mode 100644 .github/workflows/image_build_push.yaml

diff --git a/.github/workflows/build_awshelper.yaml b/.github/workflows/build_awshelper.yaml
new file mode 100644
index 000000000..f42a13610
--- /dev/null
+++ b/.github/workflows/build_awshelper.yaml
@@ -0,0 +1,21 @@
+name: Build awshelper image
+
+on:
+  push:
+    paths:
+    - .github/workflows/build_awshelper.yaml
+    - Docker/awshelper/**
+
+jobs:
+  awshelper:
+    name: Build and push
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./Docker/awshelper/Dockerfile"
+      OVERRIDE_REPO_NAME: "awshelper"
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
diff --git a/.github/workflows/build_python3.10.yaml b/.github/workflows/build_python3.10.yaml
new file mode 100644
index 000000000..993da1468
--- /dev/null
+++ b/.github/workflows/build_python3.10.yaml
@@ -0,0 +1,23 @@
+name: Build Python 3.10 image
+
+on:
+  push:
+    paths:
+    - .github/workflows/build_python3.10.yaml
+    - Docker/python-nginx/python3.10-buster/**
+
+jobs:
+  python_3-10:
+    name: Build and push
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.10-buster/Dockerfile"
+      DOCKERFILE_BUILD_CONTEXT: "./Docker/python-nginx/python3.10-buster"
+      OVERRIDE_REPO_NAME: "python"
+      OVERRIDE_TAG_NAME: "python3.10-buster-$(echo ${GITHUB_REF#refs/*/} | tr / _)"
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
diff --git a/.github/workflows/build_python3.9.yaml b/.github/workflows/build_python3.9.yaml
new file mode 100644
index 000000000..5bc8bc462
--- /dev/null
+++ b/.github/workflows/build_python3.9.yaml
@@ -0,0 +1,23 @@
+name: Build Python 3.9 image
+
+on:
+  push:
+    paths:
+    - .github/workflows/build_python3.9.yaml
+    - Docker/python-nginx/python3.9-buster/**
+
+jobs:
+  python_3-9:
+    name: Build and push
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.9-buster/Dockerfile"
+      DOCKERFILE_BUILD_CONTEXT: "./Docker/python-nginx/python3.9-buster"
+      OVERRIDE_REPO_NAME: "python"
+      OVERRIDE_TAG_NAME: "python3.9-buster-$(echo ${GITHUB_REF#refs/*/} | tr / _)"
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
diff --git a/.github/workflows/image_build_push.yaml b/.github/workflows/image_build_push.yaml
deleted file mode 100644
index d5bfea351..000000000
--- a/.github/workflows/image_build_push.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-name: Build Python Base Images
-
-on: push
-
-jobs:
-  python_3-9:
-    name: Python 3.9
-    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
-    with:
-      DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.9-buster/Dockerfile"
-      DOCKERFILE_BUILD_CONTEXT: "./Docker/python-nginx/python3.9-buster"
-      OVERRIDE_REPO_NAME: "python"
-      OVERRIDE_TAG_NAME: "python3.9-buster-$(echo ${GITHUB_REF#refs/*/} | tr / _)"
-    secrets:
-      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
-      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
-      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
-      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
-  python_3-10:
-    name: Python 3.10
-    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
-    with:
-      DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.10-buster/Dockerfile"
-      DOCKERFILE_BUILD_CONTEXT: "./Docker/python-nginx/python3.10-buster"
-      OVERRIDE_REPO_NAME: "python"
-      OVERRIDE_TAG_NAME: "python3.10-buster-$(echo ${GITHUB_REF#refs/*/} | tr / _)"
-    secrets:
-      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
-      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
-      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
-      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
-  awshelper:
-    name: AwsHelper
-    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
-    with:
-      DOCKERFILE_LOCATION: "./Docker/awshelper/Dockerfile"
-      OVERRIDE_REPO_NAME: "awshelper"
-    secrets:
-      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
-      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
-      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
-      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}

From b2195b1ae6f9d802d346629d126e9d3ba408eccd Mon Sep 17 00:00:00 2001
From: Pauline <4224001+paulineribeyre@users.noreply.github.com>
Date: Tue, 12 Mar 2024 10:07:34 -0500
Subject: [PATCH 50/67] fix wf naming

---
 .github/workflows/build_awshelper.yaml  | 2 +-
 .github/workflows/build_python3.10.yaml | 2 +-
 .github/workflows/build_python3.9.yaml  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build_awshelper.yaml b/.github/workflows/build_awshelper.yaml
index f42a13610..3d2da5393 100644
--- a/.github/workflows/build_awshelper.yaml
+++ b/.github/workflows/build_awshelper.yaml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   awshelper:
-    name: Build and push
+    name: awshelper
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/awshelper/Dockerfile"
diff --git a/.github/workflows/build_python3.10.yaml b/.github/workflows/build_python3.10.yaml
index 993da1468..80d2d7623 100644
--- a/.github/workflows/build_python3.10.yaml
+++ b/.github/workflows/build_python3.10.yaml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   python_3-10:
-    name: Build and push
+    name: Python 3.10
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.10-buster/Dockerfile"
diff --git a/.github/workflows/build_python3.9.yaml b/.github/workflows/build_python3.9.yaml
index 5bc8bc462..540e0d4ec 100644
--- a/.github/workflows/build_python3.9.yaml
+++ b/.github/workflows/build_python3.9.yaml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   python_3-9:
-    name: Build and push
+    name: Python 3.9
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
     with:
       DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.9-buster/Dockerfile"

From b6031e029db84ab0190d2a263c16b418b113482d Mon Sep 17 00:00:00 2001
From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com>
Date: Tue, 12 Mar 2024 12:03:41 -0500
Subject: [PATCH 51/67] Build awshelper workflow: always build (#2501)

---
 .github/workflows/build_awshelper.yaml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/build_awshelper.yaml b/.github/workflows/build_awshelper.yaml
index 3d2da5393..36b5745db 100644
--- a/.github/workflows/build_awshelper.yaml
+++ b/.github/workflows/build_awshelper.yaml
@@ -1,10 +1,8 @@
 name: Build awshelper image
 
-on:
-  push:
-    paths:
-    - .github/workflows/build_awshelper.yaml
-    - Docker/awshelper/**
+# Always build this image because it contains all the cloud-automation files.
+# Some jobs depend on arbitrary files and we need to test them with updated awshelper images.
+on: push
 
 jobs:
   awshelper:

From 24492c2d6868ce49a474617544b575a38697d0af Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Tue, 12 Mar 2024 16:23:59 -0400
Subject: [PATCH 52/67] Adding a gen3 db namespace to the temp files so they
 don't overlap (#2502)

---
 gen3/bin/kube-setup-revproxy.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gen3/bin/kube-setup-revproxy.sh b/gen3/bin/kube-setup-revproxy.sh
index 5db9850a1..fd30b478b 100644
--- a/gen3/bin/kube-setup-revproxy.sh
+++ b/gen3/bin/kube-setup-revproxy.sh
@@ -114,8 +114,8 @@ done
 
 if g3k_manifest_lookup .argo.argo_server_service_url 2> /dev/null; then
   argo_server_service_url=$(g3k_manifest_lookup .argo.argo_server_service_url)
-  g3k_kv_filter "${scriptDir}/gen3.nginx.conf/argo-server.conf" SERVICE_URL "${argo_server_service_url}" > /tmp/argo-server-with-url.conf
-  filePath="/tmp/argo-server-with-url.conf"
+  g3k_kv_filter "${scriptDir}/gen3.nginx.conf/argo-server.conf" SERVICE_URL "${argo_server_service_url}" > /tmp/argo-server-with-url$(gen3 db namespace).conf
+  filePath="/tmp/argo-server-with-url$(gen3 db namespace).conf"
   if [[ -f "$filePath" ]]; then
     confFileList+=("--from-file" "$filePath")
   fi 

From e979669cd92cf1ecad69f2bc2837a8fed35e2926 Mon Sep 17 00:00:00 2001
From: Mingfei Shao <2475897+mfshao@users.noreply.github.com>
Date: Wed, 13 Mar 2024 12:08:30 -0500
Subject: [PATCH 53/67] HP-1310 feat: updated related studies logic (#2498)

* feat: updated related studies logic

* update
---
 .../healdata/heal-cedar-data-ingest.py        | 174 +++++++++++-------
 1 file changed, 107 insertions(+), 67 deletions(-)

diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py
index c54f9d5aa..e0c4b3c46 100644
--- a/files/scripts/healdata/heal-cedar-data-ingest.py
+++ b/files/scripts/healdata/heal-cedar-data-ingest.py
@@ -13,7 +13,7 @@
     "study_metadata.study_type.study_subject_type": "Subject Type",
     "study_metadata.human_subject_applicability.gender_applicability": "Gender",
     "study_metadata.human_subject_applicability.age_applicability": "Age",
-    "research_program": "Research Program"
+    "research_program": "Research Program",
 }
 
 # Defines how to handle special cases for values in filters
@@ -33,7 +33,7 @@
     "Gender Queer": "Genderqueer/gender nonconforming/neither exclusively male nor female",
     "Intersex": "Genderqueer/gender nonconforming/neither exclusively male nor female",
     "Intersexed": "Genderqueer/gender nonconforming/neither exclusively male nor female",
-    "Buisness Development": "Business Development"
+    "Buisness Development": "Business Development",
 }
 
 # Defines field that we don't want to include in the filters
@@ -54,24 +54,25 @@
 def is_valid_uuid(uuid_to_test, version=4):
     """
     Check if uuid_to_test is a valid UUID.
-    
+
      Parameters
     ----------
     uuid_to_test : str
     version : {1, 2, 3, 4}
-    
+
      Returns
     -------
     `True` if uuid_to_test is a valid UUID, otherwise `False`.
-    
+
     """
-    
+
     try:
         uuid_obj = UUID(uuid_to_test, version=version)
     except ValueError:
         return False
     return str(uuid_obj) == uuid_to_test
 
+
 def update_filter_metadata(metadata_to_update):
     filter_metadata = []
     for metadata_field_key, filter_field_key in FILTER_FIELD_MAPPINGS.items():
@@ -83,20 +84,21 @@ def update_filter_metadata(metadata_to_update):
                 print(filter_field_values)
                 raise TypeError("Neither a string nor a list")
             for filter_field_value in filter_field_values:
-                if (metadata_field_key, filter_field_value) in OMITTED_VALUES_MAPPING.items():
+                if (
+                    metadata_field_key,
+                    filter_field_value,
+                ) in OMITTED_VALUES_MAPPING.items():
                     continue
                 if filter_field_value in SPECIAL_VALUE_MAPPINGS:
                     filter_field_value = SPECIAL_VALUE_MAPPINGS[filter_field_value]
-                filter_metadata.append({"key": filter_field_key, "value": filter_field_value})
+                filter_metadata.append(
+                    {"key": filter_field_key, "value": filter_field_value}
+                )
     filter_metadata = pydash.uniq(filter_metadata)
     metadata_to_update["advSearchFilters"] = filter_metadata
     # Retain these from existing tags
     save_tags = ["Data Repository"]
-    tags = [
-        tag
-        for tag in metadata_to_update["tags"]
-        if tag["category"] in save_tags
-    ]
+    tags = [tag for tag in metadata_to_update["tags"] if tag["category"] in save_tags]
     # Add any new tags from advSearchFilters
     for f in metadata_to_update["advSearchFilters"]:
         if f["key"] == "Gender":
@@ -111,25 +113,30 @@ def update_filter_metadata(metadata_to_update):
 def get_client_token(client_id: str, client_secret: str):
     try:
         token_url = f"http://revproxy-service/user/oauth2/token"
-        headers = {'Content-Type': 'application/x-www-form-urlencoded'}
-        params = {'grant_type': 'client_credentials'}
-        data = 'scope=openid user data'
+        headers = {"Content-Type": "application/x-www-form-urlencoded"}
+        params = {"grant_type": "client_credentials"}
+        data = "scope=openid user data"
 
         token_result = requests.post(
-            token_url, params=params, headers=headers, data=data,
+            token_url,
+            params=params,
+            headers=headers,
+            data=data,
             auth=(client_id, client_secret),
         )
-        token =  token_result.json()["access_token"]
+        token = token_result.json()["access_token"]
     except:
         raise Exception("Could not get token")
     return token
 
 
-def get_related_studies(serial_num, hostname):
+def get_related_studies(serial_num, guid, hostname):
     related_study_result = []
 
     if serial_num:
-        mds = requests.get(f"http://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000")
+        mds = requests.get(
+            f"http://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000"
+        )
         if mds.status_code == 200:
             related_study_metadata = mds.json()
 
@@ -137,15 +144,22 @@ def get_related_studies(serial_num, hostname):
                 related_study_metadata_key,
                 related_study_metadata_value,
             ) in related_study_metadata.items():
+                if related_study_metadata_key == guid or (
+                    related_study_metadata_value["_guid_type"] != "discovery_metadata"
+                    and related_study_metadata_value["_guid_type"]
+                    != "unregistered_discovery_metadata"
+                ):
+                    # do nothing for self, or for archived studies
+                    continue
                 title = (
-                    related_study_metadata_value.get(
-                        "gen3_discovery", {}
-                    )
+                    related_study_metadata_value.get("gen3_discovery", {})
                     .get("study_metadata", {})
                     .get("minimal_info", {})
                     .get("study_name", "")
                 )
-                link = f"https://{hostname}/portal/discovery/{related_study_metadata_key}/"
+                link = (
+                    f"https://{hostname}/portal/discovery/{related_study_metadata_key}/"
+                )
                 related_study_result.append({"title": title, "link": link})
     return related_study_result
 
@@ -180,7 +194,7 @@ def get_related_studies(serial_num, hostname):
 
 print("Getting CEDAR client access token")
 access_token = get_client_token(client_id, client_secret)
-token_header = {"Authorization": 'bearer ' + access_token}
+token_header = {"Authorization": "bearer " + access_token}
 
 limit = 10
 offset = 0
@@ -192,16 +206,21 @@ def get_related_studies(serial_num, hostname):
     print("Directory ID is not in UUID format!")
     sys.exit(1)
 
-while((limit + offset <= total)):
+while limit + offset <= total:
     # Get the metadata from cedar to register
     print("Querying CEDAR...")
-    cedar = requests.get(f"http://revproxy-service/cedar/get-instance-by-directory/{dir_id}?limit={limit}&offset={offset}", headers=token_header)
+    cedar = requests.get(
+        f"http://revproxy-service/cedar/get-instance-by-directory/{dir_id}?limit={limit}&offset={offset}",
+        headers=token_header,
+    )
 
     # If we get metadata back now register with MDS
     if cedar.status_code == 200:
         metadata_return = cedar.json()
         if "metadata" not in metadata_return:
-            print("Got 200 from CEDAR wrapper but no metadata in body, something is not right!")
+            print(
+                "Got 200 from CEDAR wrapper but no metadata in body, something is not right!"
+            )
             sys.exit(1)
 
         total = metadata_return["metadata"]["totalCount"]
@@ -209,13 +228,17 @@ def get_related_studies(serial_num, hostname):
         print(f"Successfully got {returned_records} record(s) from CEDAR directory")
         for cedar_record in metadata_return["metadata"]["records"]:
             # get the appl id from cedar for querying in our MDS
-            cedar_appl_id = pydash.get(cedar_record, "metadata_location.nih_application_id")
+            cedar_appl_id = pydash.get(
+                cedar_record, "metadata_location.nih_application_id"
+            )
             if cedar_appl_id is None:
                 print("This record doesn't have appl_id, skipping...")
                 continue
 
             # Get the metadata record for the nih_application_id
-            mds = requests.get(f"http://revproxy-service/mds/metadata?gen3_discovery.study_metadata.metadata_location.nih_application_id={cedar_appl_id}&data=true")
+            mds = requests.get(
+                f"http://revproxy-service/mds/metadata?gen3_discovery.study_metadata.metadata_location.nih_application_id={cedar_appl_id}&data=true"
+            )
             if mds.status_code == 200:
                 mds_res = mds.json()
 
@@ -234,9 +257,13 @@ def get_related_studies(serial_num, hostname):
                 if mds_res["_guid_type"] == "discovery_metadata":
                     print("Metadata is already registered. Updating MDS record")
                 elif mds_res["_guid_type"] == "unregistered_discovery_metadata":
-                    print("Metadata has not been registered. Registering it in MDS record")
+                    print(
+                        "Metadata has not been registered. Registering it in MDS record"
+                    )
                 else:
-                    print(f"This metadata data record has a special GUID type \"{mds_res['_guid_type']}\" and will be skipped")
+                    print(
+                        f"This metadata data record has a special GUID type \"{mds_res['_guid_type']}\" and will be skipped"
+                    )
                     continue
 
                 if "clinicaltrials_gov" in cedar_record:
@@ -244,21 +271,27 @@ def get_related_studies(serial_num, hostname):
                     del cedar_record["clinicaltrials_gov"]
 
                 # some special handing for this field, because its parent will be deleted before we merging the CEDAR and MDS SLMD to avoid duplicated values
-                cedar_record_other_study_websites = cedar_record.get("metadata_location", {}).get("other_study_websites", [])
+                cedar_record_other_study_websites = cedar_record.get(
+                    "metadata_location", {}
+                ).get("other_study_websites", [])
                 del cedar_record["metadata_location"]
 
                 mds_res["gen3_discovery"]["study_metadata"].update(cedar_record)
-                mds_res["gen3_discovery"]["study_metadata"]["metadata_location"]["other_study_websites"] = cedar_record_other_study_websites
+                mds_res["gen3_discovery"]["study_metadata"]["metadata_location"][
+                    "other_study_websites"
+                ] = cedar_record_other_study_websites
 
                 # setup citations
-                doi_citation = mds_res["gen3_discovery"]["study_metadata"].get("doi_citation", "")
-                mds_res["gen3_discovery"]["study_metadata"]["citation"]["heal_platform_citation"] = doi_citation
-
+                doi_citation = mds_res["gen3_discovery"]["study_metadata"].get(
+                    "doi_citation", ""
+                )
+                mds_res["gen3_discovery"]["study_metadata"]["citation"][
+                    "heal_platform_citation"
+                ] = doi_citation
 
                 # setup repository_study_link
                 data_repositories = (
-                    mds_res
-                    .get("gen3_discovery", {})
+                    mds_res.get("gen3_discovery", {})
                     .get("study_metadata", {})
                     .get("metadata_location", {})
                     .get("data_repositories", [])
@@ -275,8 +308,13 @@ def get_related_studies(serial_num, hostname):
                         repository_study_link = REPOSITORY_STUDY_ID_LINK_TEMPLATE[
                             repository["repository_name"]
                         ].replace("<STUDY_ID>", repository["repository_study_ID"])
-                        repository.update({"repository_study_link": repository_study_link})
-                        if repository_citation_additional_text not in repository_citation:
+                        repository.update(
+                            {"repository_study_link": repository_study_link}
+                        )
+                        if (
+                            repository_citation_additional_text
+                            not in repository_citation
+                        ):
                             repository_citation += repository_citation_additional_text
                 if len(data_repositories):
                     data_repositories[0] = {
@@ -284,36 +322,28 @@ def get_related_studies(serial_num, hostname):
                         "repository_citation": repository_citation,
                     }
 
-                mds_res["gen3_discovery"]["study_metadata"][
-                    "metadata_location"
-                ]["data_repositories"] = copy.deepcopy(data_repositories)
-
-
+                mds_res["gen3_discovery"]["study_metadata"]["metadata_location"][
+                    "data_repositories"
+                ] = copy.deepcopy(data_repositories)
 
                 # set up related studies
                 serial_num = None
                 try:
                     serial_num = (
-                        mds_res
-                        .get("nih_reporter", {})
+                        mds_res.get("nih_reporter", {})
                         .get("project_num_split", {})
                         .get("serial_num", None)
                     )
                 except Exception:
-                    print(f"Unable to get serial number for study")
-
-                if serial_num == None:
-                    print(f"Unable to get serial number for study")
+                    print("Unable to get serial number for study")
 
-                related_study_result = get_related_studies(serial_num, hostname)
-                existing_related_study_result = mds_res.get("related_studies", [])
-                for related_study in related_study_result:
-                    if related_study not in existing_related_study_result:
-                        existing_related_study_result.append(copy.deepcopy(related_study))
-                mds_res["gen3_discovery"][
-                    "related_studies"
-                ] = copy.deepcopy(existing_related_study_result)
+                if serial_num is None:
+                    print("Unable to get serial number for study")
 
+                related_study_result = get_related_studies(
+                    serial_num, mds_record_guid, hostname
+                )
+                mds_res["gen3_discovery"]["related_studies"] = copy.deepcopy(related_study_result)
 
                 # merge data from cedar that is not study level metadata into a level higher
                 deleted_keys = []
@@ -324,29 +354,39 @@ def get_related_studies(serial_num, hostname):
                 for key in deleted_keys:
                     del mds_res["gen3_discovery"]["study_metadata"][key]
 
-                mds_discovery_data_body = update_filter_metadata(mds_res["gen3_discovery"])
+                mds_discovery_data_body = update_filter_metadata(
+                    mds_res["gen3_discovery"]
+                )
 
                 mds_cedar_register_data_body["gen3_discovery"] = mds_discovery_data_body
                 if mds_clinical_trials:
-                    mds_cedar_register_data_body["clinicaltrials_gov"] = {**mds_cedar_register_data_body.get("clinicaltrials_gov", {}), **mds_clinical_trials}
+                    mds_cedar_register_data_body["clinicaltrials_gov"] = {
+                        **mds_cedar_register_data_body.get("clinicaltrials_gov", {}),
+                        **mds_clinical_trials,
+                    }
 
                 mds_cedar_register_data_body["_guid_type"] = "discovery_metadata"
 
                 print(f"Metadata {mds_record_guid} is now being registered.")
-                mds_put = requests.put(f"http://revproxy-service/mds/metadata/{mds_record_guid}",
+                mds_put = requests.put(
+                    f"http://revproxy-service/mds/metadata/{mds_record_guid}",
                     headers=token_header,
-                    json = mds_cedar_register_data_body
+                    json=mds_cedar_register_data_body,
                 )
                 if mds_put.status_code == 200:
                     print(f"Successfully registered: {mds_record_guid}")
                 else:
-                    print(f"Failed to register: {mds_record_guid}. Might not be MDS admin")
+                    print(
+                        f"Failed to register: {mds_record_guid}. Might not be MDS admin"
+                    )
                     print(f"Status from MDS: {mds_put.status_code}")
             else:
                 print(f"Failed to get information from MDS: {mds.status_code}")
-    
+
     else:
-        print(f"Failed to get information from CEDAR wrapper service: {cedar.status_code}")
+        print(
+            f"Failed to get information from CEDAR wrapper service: {cedar.status_code}"
+        )
 
     if offset + limit == total:
         break

From ec6510ff37b03662497ac5e651b36d70f4101e68 Mon Sep 17 00:00:00 2001
From: Jian <52763034+tianj7@users.noreply.github.com>
Date: Wed, 13 Mar 2024 16:30:06 -0500
Subject: [PATCH 54/67] add alt text to maintenance page images (#2500)

---
 files/dashboard/maintenance-page/index.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/files/dashboard/maintenance-page/index.html b/files/dashboard/maintenance-page/index.html
index a3e34479b..fac49e64e 100644
--- a/files/dashboard/maintenance-page/index.html
+++ b/files/dashboard/maintenance-page/index.html
@@ -16,7 +16,7 @@
         <div class='nav-bar'>
             <div class='nav-bar__header'>
                 <div class='nav-bar__logo'>
-                    <img src='img/header-logo.png' class='nav-bar__logo-img' />
+                    <img src='img/header-logo.png' class='nav-bar__logo-img' alt='Gen3 Data Commons'/>
                 </div>
             </div>
         </div>
@@ -27,12 +27,12 @@ <h1>This site is under maintenance...</h1>
                     Please check back later.
                 </p>
             </div>
-            <img src='img/error.svg' class='error-img' />
+            <img src='img/error.svg' class='error-img' alt='A shiba dog looking into the distance'/>
         </div>
         <div class='footer'>
             <div class="footer__logo-area">
-                <img class='footer__img' src='img/gen3.png' />
-                <img class='footer__img' src='img/createdby.png' />
+                <img class='footer__img' src='img/gen3.png' alt='Gen3 Data Commons'/>
+                <img class='footer__img' src='img/createdby.png' alt='Center for Translational Data Science at the University of Chicago'/>
             </div>
         </div>
     </div>

From 5b75af3a39cc8ebcfeed9c75a4fe55f0db004ae1 Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Mon, 18 Mar 2024 10:41:44 -0600
Subject: [PATCH 55/67] Update ingress.yaml (#2506)

---
 kube/services/ingress/ingress.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kube/services/ingress/ingress.yaml b/kube/services/ingress/ingress.yaml
index 3f1f31259..1db08e8ef 100644
--- a/kube/services/ingress/ingress.yaml
+++ b/kube/services/ingress/ingress.yaml
@@ -11,7 +11,7 @@ metadata:
     alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
     alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=600
     alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}'
-    alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04
+    alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-FIPS-2023-04
 spec:
   ingressClassName: alb
   rules:

From bff3a57818d24f416a3a518bebfe956e281bad80 Mon Sep 17 00:00:00 2001
From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com>
Date: Tue, 19 Mar 2024 09:53:47 -0500
Subject: [PATCH 56/67] MIDRC-543 Let Hatchery assume role (#2504)

---
 gen3/bin/kube-setup-hatchery.sh        | 31 ++++++++++++++++++++++++--
 kube/services/jobs/ecr-access-job.yaml |  2 --
 2 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/gen3/bin/kube-setup-hatchery.sh b/gen3/bin/kube-setup-hatchery.sh
index 5454d1e24..bdcff8ed0 100644
--- a/gen3/bin/kube-setup-hatchery.sh
+++ b/gen3/bin/kube-setup-hatchery.sh
@@ -76,15 +76,38 @@ else
     exists_or_create_gen3_license_table "$TARGET_TABLE"
 fi
 
+# if `nextflow-global.imagebuilder-reader-role-arn` is set in hatchery config, allow hatchery
+# to assume the configured role
+imagebuilderRoleArn=$(g3kubectl get configmap manifest-hatchery -o jsonpath={.data.nextflow-global} | jq -r '."imagebuilder-reader-role-arn"')
+assumeImageBuilderRolePolicyBlock=""
+if [ -z "$imagebuilderRoleArn" ]; then
+    gen3_log_info "No 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration, not granting AssumeRole"
+else
+    gen3_log_info "Found 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration, granting AssumeRole"
+    assumeImageBuilderRolePolicyBlock=$( cat <<EOM
+        {
+            "Sid": "AssumeImageBuilderReaderRole",
+            "Effect": "Allow",
+            "Action": [
+                "sts:AssumeRole"
+            ],
+            "Resource": "$imagebuilderRoleArn"
+        },
+EOM
+)
+fi
+
 policy=$( cat <<EOM
 {
     "Version": "2012-10-17",
     "Statement": [
         {
+            "Sid": "AssumeCsocAdminRole",
             "Effect": "Allow",
             "Action": "sts:AssumeRole",
             "Resource": "arn:aws:iam::*:role/csoc_adminvm*"
         },
+$assumeImageBuilderRolePolicyBlock
         {
             "Effect": "Allow",
             "Action": "ec2:*",
@@ -152,8 +175,13 @@ policy=$( cat <<EOM
 }
 EOM
 )
+
 saName=$(echo "hatchery-service-account" | head -c63)
-echo $saName
+echo Service account name: $saName
+echo Policy document: $policy
+
+# if the policy has changed and must be updated, run:
+# `kubectl delete sa hatchery-service-account && gen3 kube-setup-hatchery`
 if ! g3kubectl get sa "$saName" -o json | jq -e '.metadata.annotations | ."eks.amazonaws.com/role-arn"' > /dev/null 2>&1; then
     roleName="$(gen3 api safe-name hatchery-sa)"
     gen3 awsrole create $roleName $saName
@@ -176,7 +204,6 @@ if ! g3kubectl get sa "$saName" -o json | jq -e '.metadata.annotations | ."eks.a
         # create the new version
         gen3_aws_run aws iam create-policy-version --policy-arn "$policyArn" --policy-document "$policy" --set-as-default
     fi
-
     gen3_log_info "Attaching policy '${policyName}' to role '${roleName}'"
     gen3 awsrole attach-policy ${policyArn} --role-name ${roleName} --force-aws-cli || exit 1
     gen3 awsrole attach-policy "arn:aws:iam::aws:policy/AWSResourceAccessManagerFullAccess" --role-name ${roleName} --force-aws-cli || exit 1
diff --git a/kube/services/jobs/ecr-access-job.yaml b/kube/services/jobs/ecr-access-job.yaml
index 11979a123..89bb49d6d 100644
--- a/kube/services/jobs/ecr-access-job.yaml
+++ b/kube/services/jobs/ecr-access-job.yaml
@@ -65,8 +65,6 @@ spec:
           args:
             - "-c"
             - |
-              set -e
-
               cd cloud-automation/files/scripts/
               echo Installing requirements...
               pip3 install -r ecr-access-job-requirements.txt

From 6d67d747679ed73edf55fc3484297732f59e4000 Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Wed, 20 Mar 2024 10:10:41 -0400
Subject: [PATCH 57/67] Feat/scaling va workflows (#2507)

* Raising total parallelism to 13, to enable stress tests in va-testing. This should be merged with a value of 10, to allow 5 WFs in each env

* Bumping the parallelism for Argo up to 5
---
 kube/services/argo/values.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/kube/services/argo/values.yaml b/kube/services/argo/values.yaml
index 2b46ced0f..23dda4a5a 100644
--- a/kube/services/argo/values.yaml
+++ b/kube/services/argo/values.yaml
@@ -1,6 +1,6 @@
 controller:
-  parallelism: 8
-  namespaceParallelism: 3
+  parallelism: 10
+  namespaceParallelism: 5
   metricsConfig:
     # -- Enables prometheus metrics server
     enabled: true

From 31e6e49015d84782e1c5687175dc11e36bfae4f8 Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Wed, 20 Mar 2024 15:41:29 -0400
Subject: [PATCH 58/67] Creating a new type of DB dump that grabs stuff for
 va-testing (#2508)

* Creating a new type of DB dump that grabs stuff for va-testing

* Missed changes to dbdump script

* Changing job name
---
 gen3/bin/dbbackup.sh                          | 13 ++-
 .../jobs/psql-db-dump-va-testing-job.yaml     | 80 +++++++++++++++++++
 2 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 kube/services/jobs/psql-db-dump-va-testing-job.yaml

diff --git a/gen3/bin/dbbackup.sh b/gen3/bin/dbbackup.sh
index 29f267221..eb9611a90 100644
--- a/gen3/bin/dbbackup.sh
+++ b/gen3/bin/dbbackup.sh
@@ -173,6 +173,10 @@ db_restore() {
     gen3 job run psql-db-prep-restore
 }
 
+va_testing_db_dump() {
+  gen3 job run psql-db-dump-va-testing
+}
+
 
 # main function to determine whether dump or restore
 main() {
@@ -191,8 +195,15 @@ main() {
             create_s3_bucket
             db_restore
             ;;
+        va-dump)
+            gen3_log_info "Running a va-testing DB dump..."
+            create_policy
+            create_service_account_and_role
+            create_s3_bucket
+            va_testing_db_dump
+            ;;
         *)
-            echo "Invalid command. Usage: gen3 dbbackup [dump|restore]"
+            echo "Invalid command. Usage: gen3 dbbackup [dump|restore|va-dump]"
             return 1
             ;;
     esac
diff --git a/kube/services/jobs/psql-db-dump-va-testing-job.yaml b/kube/services/jobs/psql-db-dump-va-testing-job.yaml
new file mode 100644
index 000000000..8a8037e16
--- /dev/null
+++ b/kube/services/jobs/psql-db-dump-va-testing-job.yaml
@@ -0,0 +1,80 @@
+---
+# NOTE: This job was created specifically to dump all the databases in va-testing, in preparation for a move to second cluster
+# If you aren't doing that, this probably is not the job you're looking for
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: psql-db-dump-va-testing
+spec:
+  template:
+    metadata:
+      labels:
+        app: gen3job
+    spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: karpenter.sh/capacity-type
+                    operator: In
+                    values:
+                      - on-demand
+            - weight: 99
+              preference:
+                matchExpressions:
+                  - key: eks.amazonaws.com/capacityType
+                    operator: In
+                    values:
+                      - ONDEMAND
+      serviceAccountName: dbbackup-sa
+      containers:
+        - name: pgdump
+          image: quay.io/cdis/awshelper:master
+          imagePullPolicy: Always
+          env:
+            - name: gen3Env
+              valueFrom:
+                configMapKeyRef:
+                  name: global
+                  key: environment
+            - name: JENKINS_HOME
+              value: "devterm"
+            - name: GEN3_HOME
+              value: /home/ubuntu/cloud-automation
+          command: ["/bin/bash"]
+          args:
+            - "-c"
+            - |
+              source "${GEN3_HOME}/gen3/lib/utils.sh"
+              gen3_load "gen3/gen3setup"
+              account_id=$(aws sts get-caller-identity --query "Account" --output text)
+              default_bucket_name="gen3-db-backups-${account_id}"
+              default_databases=("fence" "indexd" "sheepdog" "peregrine" "arborist" "argo" "atlas" "metadata" "ohdsi" "omop-data" "wts")
+              s3_dir="va-testing-$(date +"%Y-%m-%d-%H-%M-%S")"
+              databases=("${default_databases[@]}")
+              bucket_name=$default_bucket_name
+
+              for database in "${databases[@]}"; do
+                gen3_log_info "Starting database backup for ${database}"
+                gen3 db backup "${database}" > "${database}.sql"
+
+                if [ $? -eq 0 ] && [ -f "${database}.sql" ]; then
+                  gen3_log_info "Uploading backup file ${database}.sql to s3://${bucket_name}/${s3_dir}/${database}.sql"
+                  aws s3 cp "${database}.sql" "s3://${bucket_name}/${s3_dir}/${database}.sql"
+
+                  if [ $? -eq 0 ]; then
+                    gen3_log_info "Successfully uploaded ${database}.sql to S3"
+                  else
+                    gen3_log_err "Failed to upload ${database}.sql to S3"
+                  fi
+                  gen3_log_info "Deleting temporary backup file ${database}.sql"
+                  rm -f "${database}.sql"
+                else
+                  gen3_log_err "Backup operation failed for ${database}"
+                  rm -f "${database}.sql"
+                fi
+              done
+              sleep 600
+      restartPolicy: Never

From 6de65e70a7065789f6250ad05e94f816bf8eeeaf Mon Sep 17 00:00:00 2001
From: Michael Lukowski <lukowski9@gmail.com>
Date: Wed, 20 Mar 2024 15:32:38 -0500
Subject: [PATCH 59/67] add whitelist for qdr staging (#2509)

---
 files/squid_whitelist/web_whitelist | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist
index c191b2e8c..afacba9e4 100644
--- a/files/squid_whitelist/web_whitelist
+++ b/files/squid_whitelist/web_whitelist
@@ -165,3 +165,5 @@ www.rabbitmq.com
 www.uniprot.org
 vpodc.org
 yahoo.com
+idp.stage.qdr.org
+stage.qdr.org
\ No newline at end of file

From cde8a9666a53fe9c2345f4562f202a701e9a172a Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Mon, 25 Mar 2024 11:26:58 -0400
Subject: [PATCH 60/67] Cronjobs aren't beta (#2511)

---
 .../services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml b/kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml
index 74d7fc9a4..93eaf7652 100644
--- a/kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml
+++ b/kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml
@@ -1,5 +1,5 @@
 ---
-apiVersion: batch/v1beta1
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: fence-cleanup-expired-ga4gh-info

From 79f305a7741a221a6a6e0236c08e0f610cc589f0 Mon Sep 17 00:00:00 2001
From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com>
Date: Thu, 28 Mar 2024 15:13:38 -0500
Subject: [PATCH 61/67] MIDRC-672 Fix ECR access job role name conflict (#2515)

---
 gen3/bin/iam-serviceaccount.sh            | 25 +++++++++++++++--------
 gen3/bin/kube-setup-ecr-access-cronjob.sh |  4 ++--
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/gen3/bin/iam-serviceaccount.sh b/gen3/bin/iam-serviceaccount.sh
index 0c5a8bba3..1ea055f66 100644
--- a/gen3/bin/iam-serviceaccount.sh
+++ b/gen3/bin/iam-serviceaccount.sh
@@ -115,7 +115,7 @@ EOF
 # @return the resulting json from awscli
 ##
 function create_role(){
-  local role_name="${vpc_name}-${SERVICE_ACCOUNT_NAME}-role"
+  local role_name="${1}"
   if [[ ${#role_name} -gt 63 ]]; then
     role_name=$(echo "$role_name" | head -c63)
     gen3_log_warning "Role name has been truncated, due to amazon role name 64 character limit. New role name is $role_name"
@@ -123,8 +123,8 @@ function create_role(){
   local assume_role_policy_path="$(create_assume_role_policy)"
 
   gen3_log_info "Entering create_role"
-  gen3_log_info "  ${role_name}"
-  gen3_log_info "  ${assume_role_policy_path}"
+  gen3_log_info "  Role: ${role_name}"
+  gen3_log_info "  Policy path: ${assume_role_policy_path}"
 
   local role_json
   role_json=$(aws iam create-role \
@@ -156,8 +156,8 @@ function add_policy_to_role(){
   local role_name="${2}"
 
   gen3_log_info "Entering add_policy_to_role"
-  gen3_log_info "  ${policy}"
-  gen3_log_info "  ${role_name}"
+  gen3_log_info "  Policy: ${policy}"
+  gen3_log_info "  Role: ${role_name}"
 
   local result
   if [[ ${policy} =~ arn:aws:iam::aws:policy/[a-zA-Z0-9]+ ]]
@@ -198,8 +198,8 @@ function create_role_with_policy() {
   local role_name="${2}"
 
   gen3_log_info "Entering create_role_with_policy"
-  gen3_log_info "  ${policy}"
-  gen3_log_info "  ${role_name}"
+  gen3_log_info "  Policy: ${policy}"
+  gen3_log_info "  Role: ${role_name}"
 
   local created_role_json
   created_role_json="$(create_role ${role_name})" || return $?
@@ -357,7 +357,10 @@ function main() {
 
   local policy_validation
   local policy_source
-  local role_name="${vpc_name}-${SERVICE_ACCOUNT_NAME}-role"
+  local role_name=$ROLE_NAME
+  if [ -z "${role_name}" ]; then
+    role_name="${vpc_name}-${SERVICE_ACCOUNT_NAME}-role"
+  fi
 
   if [ -z ${NAMESPACE_SCRIPT} ];
   then
@@ -481,6 +484,12 @@ while getopts "$OPTSPEC" optchar; do
           ACTION="c"
           SERVICE_ACCOUNT_NAME=${OPTARG#*=}
           ;;
+        role-name)
+          ROLE_NAME="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+          ;;
+        role-name=*)
+          ROLE_NAME=${OPTARG#*=}
+          ;;
         list)
           ACTION="l"
           SERVICE_ACCOUNT_NAME="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
diff --git a/gen3/bin/kube-setup-ecr-access-cronjob.sh b/gen3/bin/kube-setup-ecr-access-cronjob.sh
index d23afc862..5c645ad35 100644
--- a/gen3/bin/kube-setup-ecr-access-cronjob.sh
+++ b/gen3/bin/kube-setup-ecr-access-cronjob.sh
@@ -38,8 +38,8 @@ setup_ecr_access_job() {
   ]
 }
 EOM
-      local role_name
-      if ! role_name="$(gen3 iam-serviceaccount -c "${saName}" -p $tempFile)" || [[ -z "$role_name" ]]; then
+      local safe_role_name=$(gen3 api safe-name ${saName}-role | head -c63)
+      if ! role_name="$(gen3 iam-serviceaccount -c "${saName}" -p $tempFile --role-name $safe_role_name)" || [[ -z "$role_name" ]]; then
         gen3_log_err "Failed to create iam service account"
         rm $tempFile
         return 1

From 797fdf3fcd4f2ce8d66582a6e7891bfbba5bffe1 Mon Sep 17 00:00:00 2001
From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
Date: Fri, 29 Mar 2024 11:59:34 -0400
Subject: [PATCH 62/67] Adding awslabs.github.io to the squid whitelist (#2516)

---
 files/squid_whitelist/web_whitelist | 1 +
 1 file changed, 1 insertion(+)

diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist
index afacba9e4..6896314ab 100644
--- a/files/squid_whitelist/web_whitelist
+++ b/files/squid_whitelist/web_whitelist
@@ -7,6 +7,7 @@ achecker.ca
 apache.github.io
 api.epigraphdb.org
 api.monqcle.com
+awslabs.github.io
 biodata-integration-tests.net
 marketing.biorender.com
 clinicaltrials.gov

From 07813b6fff305398b90c6dcdc810e253fb41b086 Mon Sep 17 00:00:00 2001
From: "J. Q" <55899496+jawadqur@users.noreply.github.com>
Date: Thu, 4 Apr 2024 10:37:08 -0500
Subject: [PATCH 63/67] Fail if fence-create client fails in kube-setup-ohdsi
 (#2514)

Co-authored-by: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com>
---
 gen3/bin/kube-setup-ohdsi.sh | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/gen3/bin/kube-setup-ohdsi.sh b/gen3/bin/kube-setup-ohdsi.sh
index 14b35a714..3d8165547 100644
--- a/gen3/bin/kube-setup-ohdsi.sh
+++ b/gen3/bin/kube-setup-ohdsi.sh
@@ -14,13 +14,8 @@ new_client() {
   local secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client atlas --urls https://${atlas_hostname}/WebAPI/user/oauth/callback?client_name=OidcClient --username atlas --allowed-scopes openid profile email user | tail -1)
   # secrets looks like ('CLIENT_ID', 'CLIENT_SECRET')
   if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
-      # try delete client
-      g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client atlas > /dev/null 2>&1
-      secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client atlas --urls https://${atlas_hostname}/WebAPI/user/oauth/callback?client_name=OidcClient --username atlas --allowed-scopes openid profile email user | tail -1)
-      if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then
-          gen3_log_err "kube-setup-ohdsi" "Failed generating oidc client for atlas: $secrets"
-          return 1
-      fi
+    gen3_log_err "kube-setup-ohdsi" "Failed generating oidc client for atlas: $secrets"
+    return 1
   fi
   local FENCE_CLIENT_ID="${BASH_REMATCH[2]}"
   local FENCE_CLIENT_SECRET="${BASH_REMATCH[3]}"

From 775d224e4ffc301e04a78c7878b499caf05d1f0f Mon Sep 17 00:00:00 2001
From: "J. Q" <55899496+jawadqur@users.noreply.github.com>
Date: Thu, 4 Apr 2024 13:05:45 -0500
Subject: [PATCH 64/67] Update ecr.sh (#2518)

---
 gen3/bin/ecr.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gen3/bin/ecr.sh b/gen3/bin/ecr.sh
index 930202a87..f3f13b993 100644
--- a/gen3/bin/ecr.sh
+++ b/gen3/bin/ecr.sh
@@ -32,6 +32,7 @@ accountList=(
 205252583234
 885078588865
 922467707295
+533267425233
 )
 
 principalStr=""

From 9c2f09eefbcdc1cc29286481cb0bafac170a4bf1 Mon Sep 17 00:00:00 2001
From: "J. Q" <55899496+jawadqur@users.noreply.github.com>
Date: Thu, 4 Apr 2024 13:16:43 -0500
Subject: [PATCH 65/67] Update ecr.sh (#2519)

---
 gen3/bin/ecr.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gen3/bin/ecr.sh b/gen3/bin/ecr.sh
index f3f13b993..36af791ef 100644
--- a/gen3/bin/ecr.sh
+++ b/gen3/bin/ecr.sh
@@ -33,6 +33,7 @@ accountList=(
 885078588865
 922467707295
 533267425233
+048463324059
 )
 
 principalStr=""

From 147ea5e0086a0753536a0f2f027471544b638bea Mon Sep 17 00:00:00 2001
From: emalinowski <emalinowski@uchicago.edu>
Date: Mon, 15 Apr 2024 06:14:13 -0600
Subject: [PATCH 66/67] fix(sqs-helper): Updated sqs helper script to create
 more than one workspace GPE-998 (#2295)

* fix(sqs-helper): Updated sqs helper script to create more than one workspace

* fix(sqs-helper): Updated scripts calling sqs helper to give simpler sqs names

* fix(sqs-helper): Updated sqs variables to be consistent

---------

Co-authored-by: Edward Malinowski <edwardmalinowski@Eds-MacBook-Pro.attlocal.net>
---
 gen3/bin/kube-setup-audit-service.sh |  2 +-
 gen3/bin/kube-setup-fence.sh         |  2 +-
 gen3/bin/kube-setup-karpenter.sh     |  6 +++---
 gen3/bin/sqs.sh                      | 15 ++++++++-------
 4 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/gen3/bin/kube-setup-audit-service.sh b/gen3/bin/kube-setup-audit-service.sh
index b7565194c..92c70f352 100644
--- a/gen3/bin/kube-setup-audit-service.sh
+++ b/gen3/bin/kube-setup-audit-service.sh
@@ -65,7 +65,7 @@ EOM
 }
 
 setup_audit_sqs() {
-  local sqsName="$(gen3 api safe-name audit-sqs)"
+  local sqsName="audit-sqs"
   sqsInfo="$(gen3 sqs create-queue-if-not-exist $sqsName)" || exit 1
   sqsUrl="$(jq -e -r '.["url"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-url' from output: $sqsInfo"; exit 1; }
   sqsArn="$(jq -e -r '.["arn"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-arn' from output: $sqsInfo"; exit 1; }
diff --git a/gen3/bin/kube-setup-fence.sh b/gen3/bin/kube-setup-fence.sh
index 03edabbf4..cc0516c93 100644
--- a/gen3/bin/kube-setup-fence.sh
+++ b/gen3/bin/kube-setup-fence.sh
@@ -9,7 +9,7 @@ source "${GEN3_HOME}/gen3/lib/utils.sh"
 gen3_load "gen3/lib/kube-setup-init"
 
 setup_audit_sqs() {
-  local sqsName="$(gen3 api safe-name audit-sqs)"
+  local sqsName="audit-sqs"
   sqsInfo="$(gen3 sqs create-queue-if-not-exist $sqsName)" || exit 1
   sqsUrl="$(jq -e -r '.["url"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-url' from output: $sqsInfo"; exit 1; }
   sqsArn="$(jq -e -r '.["arn"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-arn' from output: $sqsInfo"; exit 1; }
diff --git a/gen3/bin/kube-setup-karpenter.sh b/gen3/bin/kube-setup-karpenter.sh
index 2737ed6ee..949c1ccd1 100644
--- a/gen3/bin/kube-setup-karpenter.sh
+++ b/gen3/bin/kube-setup-karpenter.sh
@@ -31,7 +31,7 @@ gen3_deploy_karpenter() {
       else
         karpenter=${karpenter:-v0.22.0}
       fi    
-      local queue_name="karpenter-sqs-${vpc_name}"
+      local queue_name="$(gen3 api safe-name karpenter-sqs)"
       echo '{
           "Statement": [
               {
@@ -202,9 +202,9 @@ gen3_update_karpenter_configs() {
 }
 
 gen3_create_karpenter_sqs_eventbridge() {
-  local queue_name="karpenter-sqs-${vpc_name}"
+  local queue_name="$(gen3 api safe-name karpenter-sqs)"
   local eventbridge_rule_name="karpenter-eventbridge-${vpc_name}"
-  #gen3 sqs create-queue-if-not-exist $queue_name >> "$XDG_RUNTIME_DIR/sqs-${vpc_name}.json"
+  gen3 sqs create-queue-if-not-exist karpenter-sqs >> "$XDG_RUNTIME_DIR/sqs-${vpc_name}.json"
   local queue_url=$(cat "$XDG_RUNTIME_DIR/sqs-${vpc_name}.json" | jq -r '.url')
   local queue_arn=$(cat "$XDG_RUNTIME_DIR/sqs-${vpc_name}.json" | jq -r '.arn')
   # Create eventbridge rules
diff --git a/gen3/bin/sqs.sh b/gen3/bin/sqs.sh
index dccb1ff7b..7448437a0 100644
--- a/gen3/bin/sqs.sh
+++ b/gen3/bin/sqs.sh
@@ -50,15 +50,15 @@ EOM
 # @sqsName
 #
 gen3_sqs_create_queue() {
-  local sqsName=$1
-  if ! shift || [[ -z "$sqsName" ]]; then
-    gen3_log_err "Must provide 'sqsName' to 'gen3_sqs_create_queue'"
+  local serviceName=$1
+  if ! shift || [[ -z "$serviceName" ]]; then
+    gen3_log_err "Must provide 'serviceName' to 'gen3_sqs_create_queue'"
     return 1
   fi
+  local sqsName="$(gen3 api safe-name $serviceName)"
   gen3_log_info "Creating SQS '$sqsName'"
-  local prefix="$(gen3 api safe-name sqs-create)"
   ( # subshell - do not pollute parent environment
-    gen3 workon default ${prefix}__sqs 1>&2
+    gen3 workon default ${sqsName}__sqs 1>&2
     gen3 cd 1>&2
     cat << EOF > config.tfvars
 sqs_name="$sqsName"
@@ -76,7 +76,8 @@ EOF
 # @sqsName
 #
 gen3_sqs_create_queue_if_not_exist() {
-  local sqsName=$1
+  local serviceName=$1
+  local sqsName="$(gen3 api safe-name $serviceName)"
   if ! shift || [[ -z "$sqsName" ]]; then
     gen3_log_err "Must provide 'sqsName' to 'gen3_sqs_create_queue'"
     return 1
@@ -90,7 +91,7 @@ gen3_sqs_create_queue_if_not_exist() {
     gen3_log_info "The '$sqsName' SQS already exists"
   else
     # create the queue
-    sqsInfo="$(gen3_sqs_create_queue $sqsName)" || exit 1
+    sqsInfo="$(gen3_sqs_create_queue $serviceName)" || exit 1
     sqsUrl="$(jq -e -r '.["sqs-url"].value' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-url' from output: $sqsInfo"; exit 1; }
     sqsArn="$(jq -e -r '.["sqs-arn"].value' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-arn' from output: $sqsInfo"; exit 1; }
   fi

From d55a3862609339149ac9373d708aac6546267618 Mon Sep 17 00:00:00 2001
From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com>
Date: Mon, 15 Apr 2024 16:34:51 -0600
Subject: [PATCH 67/67] Update web_wildcard_whitelist (#2523)

---
 files/squid_whitelist/web_wildcard_whitelist | 1 +
 1 file changed, 1 insertion(+)

diff --git a/files/squid_whitelist/web_wildcard_whitelist b/files/squid_whitelist/web_wildcard_whitelist
index b71ee76c2..1374c5d67 100644
--- a/files/squid_whitelist/web_wildcard_whitelist
+++ b/files/squid_whitelist/web_wildcard_whitelist
@@ -11,6 +11,7 @@
 .bioconductor.org
 .bionimbus.org
 .bitbucket.org
+.blob.core.windows.net
 .bloodpac.org
 .braincommons.org
 .bsc.es