Adding a worker k8s pod dedicated to the webcrawler. (#5247)

.github/workflows/deploy-connectors.yml

@@ -52,6 +52,7 @@ jobs:
           ./k8s/deploy-image.sh gcr.io/$GCLOUD_PROJECT_ID/connectors-image:${{ steps.short_sha.outputs.short_sha }} connectors-deployment
           ./k8s/deploy-image.sh gcr.io/$GCLOUD_PROJECT_ID/connectors-image:${{ steps.short_sha.outputs.short_sha }} connectors-worker-deployment
           ./k8s/deploy-image.sh gcr.io/$GCLOUD_PROJECT_ID/connectors-image:${{ steps.short_sha.outputs.short_sha }} connectors-worker-notion-deployment
+          ./k8s/deploy-image.sh gcr.io/$GCLOUD_PROJECT_ID/connectors-image:${{ steps.short_sha.outputs.short_sha }} connectors-worker-webcrawler-deployment
 
       - name: Wait for rollout to complete
         run: |
@@ -61,3 +62,5 @@ jobs:
           kubectl rollout status deployment/connectors-worker-deployment --timeout=10m
           echo "Waiting for rollout to complete (notion worker)"
           kubectl rollout status deployment/connectors-worker-notion-deployment --timeout=10m
+          echo "Waiting for rollout to complete (webcrawler worker)"
+          kubectl rollout status deployment/connectors-worker-webcrawler-deployment --timeout=10m

k8s/apply_infra.sh

@@ -95,6 +95,7 @@ apply_deployment front-edge-deployment
 apply_deployment connectors-deployment
 apply_deployment connectors-worker-deployment
 apply_deployment connectors-worker-notion-deployment
+apply_deployment connectors-worker-webcrawler-deployment
 apply_deployment docs-deployment
 apply_deployment metabase-deployment
 apply_deployment alerting-temporal-deployment

k8s/deployments/connectors-worker-deployment.yaml

@@ -29,7 +29,6 @@ spec:
               "google_drive",
               "intercom",
               "slack",
-              "webcrawler",
             ]
           imagePullPolicy: Always
           envFrom:

k8s/deployments/connectors-worker-webcrawler-deployment.yaml

@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: connectors-worker-webcrawler-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: connectors-worker
+      worker: webcrawler
+  template:
+    metadata:
+      labels:
+        app: connectors-worker
+        name: connectors-worker-pod
+        worker: webcrawler
+        admission.datadoghq.com/enabled: "true"
+      annotations:
+        ad.datadoghq.com/web.logs: '[{"source": "connectors-worker","service": "connectors-worker","tags": ["env:prod"]}]'
+    spec:
+      containers:
+        - name: web
+          image: gcr.io/or1g1n-186209/connectors-image:latest
+          command: ["npm", "run", "start:worker"]
+          args: ["--", "--workers", "webcrawler"]
+          imagePullPolicy: Always
+          envFrom:
+            - configMapRef:
+                name: connectors-worker-specific-config
+            - secretRef:
+                name: connectors-secrets
+          env:
+            - name: DD_AGENT_HOST
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+
+          volumeMounts:
+            - name: cert-volume
+              mountPath: /etc/certs
+            - name: private-key-volume
+              mountPath: /etc/private-keys
+
+          resources:
+            requests:
+              cpu: 3000m
+              memory: 8Gi
+              ephemeral-storage: 4Gi
+
+            limits:
+              cpu: 3000m
+              memory: 8Gi
+              ephemeral-storage: 4Gi
+
+      volumes:
+        - name: cert-volume
+          secret:
+            secretName: temporal-cert
+
+        - name: private-key-volume
+          secret:
+            secretName: github-app-private-key