diff --git a/.github/workflows/deploy-connectors.yml b/.github/workflows/deploy-connectors.yml index b0be7e5b4ba7..5bfaa221bbe9 100644 --- a/.github/workflows/deploy-connectors.yml +++ b/.github/workflows/deploy-connectors.yml @@ -52,6 +52,7 @@ jobs: ./k8s/deploy-image.sh gcr.io/$GCLOUD_PROJECT_ID/connectors-image:${{ steps.short_sha.outputs.short_sha }} connectors-deployment ./k8s/deploy-image.sh gcr.io/$GCLOUD_PROJECT_ID/connectors-image:${{ steps.short_sha.outputs.short_sha }} connectors-worker-deployment ./k8s/deploy-image.sh gcr.io/$GCLOUD_PROJECT_ID/connectors-image:${{ steps.short_sha.outputs.short_sha }} connectors-worker-notion-deployment + ./k8s/deploy-image.sh gcr.io/$GCLOUD_PROJECT_ID/connectors-image:${{ steps.short_sha.outputs.short_sha }} connectors-worker-webcrawler-deployment - name: Wait for rollout to complete run: | @@ -61,3 +62,5 @@ jobs: kubectl rollout status deployment/connectors-worker-deployment --timeout=10m echo "Waiting for rollout to complete (notion worker)" kubectl rollout status deployment/connectors-worker-notion-deployment --timeout=10m + echo "Waiting for rollout to complete (webcrawler worker)" + kubectl rollout status deployment/connectors-worker-webcrawler-deployment --timeout=10m diff --git a/k8s/apply_infra.sh b/k8s/apply_infra.sh index 1859f24faa4d..2d2c46b930fe 100755 --- a/k8s/apply_infra.sh +++ b/k8s/apply_infra.sh @@ -95,6 +95,7 @@ apply_deployment front-edge-deployment apply_deployment connectors-deployment apply_deployment connectors-worker-deployment apply_deployment connectors-worker-notion-deployment +apply_deployment connectors-worker-webcrawler-deployment apply_deployment docs-deployment apply_deployment metabase-deployment apply_deployment alerting-temporal-deployment diff --git a/k8s/deployments/connectors-worker-deployment.yaml b/k8s/deployments/connectors-worker-deployment.yaml index 1ebf209c95e3..d95a45c8e6de 100644 --- a/k8s/deployments/connectors-worker-deployment.yaml +++ b/k8s/deployments/connectors-worker-deployment.yaml @@ -29,7 +29,6 @@ spec: "google_drive", "intercom", "slack", - "webcrawler", ] imagePullPolicy: Always envFrom: diff --git a/k8s/deployments/connectors-worker-webcrawler-deployment.yaml b/k8s/deployments/connectors-worker-webcrawler-deployment.yaml new file mode 100644 index 000000000000..7cb8bc15614a --- /dev/null +++ b/k8s/deployments/connectors-worker-webcrawler-deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: connectors-worker-webcrawler-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: connectors-worker + worker: webcrawler + template: + metadata: + labels: + app: connectors-worker + name: connectors-worker-pod + worker: webcrawler + admission.datadoghq.com/enabled: "true" + annotations: + ad.datadoghq.com/web.logs: '[{"source": "connectors-worker","service": "connectors-worker","tags": ["env:prod"]}]' + spec: + containers: + - name: web + image: gcr.io/or1g1n-186209/connectors-image:latest + command: ["npm", "run", "start:worker"] + args: ["--", "--workers", "webcrawler"] + imagePullPolicy: Always + envFrom: + - configMapRef: + name: connectors-worker-specific-config + - secretRef: + name: connectors-secrets + env: + - name: DD_AGENT_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + + volumeMounts: + - name: cert-volume + mountPath: /etc/certs + - name: private-key-volume + mountPath: /etc/private-keys + + resources: + requests: + cpu: 3000m + memory: 8Gi + ephemeral-storage: 4Gi + + limits: + cpu: 3000m + memory: 8Gi + ephemeral-storage: 4Gi + + volumes: + - name: cert-volume + secret: + secretName: temporal-cert + + - name: private-key-volume + secret: + secretName: github-app-private-key