feat: new viz service (#6469)

* feat: viz service skeleton, infra and certificate

* add lint WF

* r

---------

Co-authored-by: Henry Fontanier <[email protected]>

.github/workflows/build-and-lint-viz.yml

@@ -0,0 +1,23 @@
+name: Lint & Build (viz)
+
+on:
+  push:
+    paths:
+      - types/**
+      - viz/**
+      - .github/workflows/build-and-lint-viz.yml
+
+jobs:
+  check-eslint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20.13.0
+          cache: "npm"
+          cache-dependency-path: ./viz/package-lock.json
+      - working-directory: types
+        run: npm install && npm run build
+      - working-directory: viz
+        run: npm install && npm run build && npm run lint

.github/workflows/deploy-viz.yml

@@ -0,0 +1,56 @@
+name: Deploy Viz
+
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: deploy_viz
+  cancel-in-progress: false
+
+env:
+  GCLOUD_PROJECT_ID: ${{ secrets.GCLOUD_PROJECT_ID }}
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+
+    if: github.ref == 'refs/heads/main'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Get short sha
+        id: short_sha
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: "Authenticate with Google Cloud"
+        uses: "google-github-actions/auth@v1"
+        with:
+          credentials_json: "${{ secrets.GCLOUD_SA_KEY }}"
+
+      - name: "Set up Cloud SDK"
+        uses: "google-github-actions/setup-gcloud@v1"
+
+      - name: Install gke-gcloud-auth-plugin
+        run: |
+          gcloud components install gke-gcloud-auth-plugin
+
+      - name: Setup kubectl
+        run: |
+          gcloud container clusters get-credentials dust-kube --region us-central1
+
+      - name: Build the image on Cloud Build
+        run: |
+          chmod +x ./k8s/cloud-build.sh
+          ./k8s/cloud-build.sh viz ./viz/Dockerfile ./
+
+      - name: Deploy the image on Kubernetes
+        run: |
+          chmod +x ./k8s/deploy-image.sh
+          ./k8s/deploy-image.sh gcr.io/$GCLOUD_PROJECT_ID/viz-image:${{ steps.short_sha.outputs.short_sha }} viz-deployment
+
+      - name: Wait for rollout to complete
+        run: |
+          echo "Waiting for rollout to complete"
+          kubectl rollout status deployment/viz-deployment --timeout=10m

k8s/apply_infra.sh

@@ -75,6 +75,7 @@ kubectl apply -f "$(dirname "$0")/configmaps/core-configmap.yaml"
 kubectl apply -f "$(dirname "$0")/configmaps/core-sqlite-worker-configmap.yaml"
 kubectl apply -f "$(dirname "$0")/configmaps/oauth-configmap.yaml"
 kubectl apply -f "$(dirname "$0")/configmaps/prodbox-configmap.yaml"
+kubectl apply -f "$(dirname "$0")/configmaps/viz-configmap.yaml"
 
 echo "-----------------------------------"
 echo "Applying backend configs"
@@ -86,6 +87,7 @@ kubectl apply -f "$(dirname "$0")/backend-configs/connectors-backend-config.yaml
 kubectl apply -f "$(dirname "$0")/backend-configs/metabase-backend-config.yaml"
 kubectl apply -f "$(dirname "$0")/backend-configs/core-backend-config.yaml"
 kubectl apply -f "$(dirname "$0")/backend-configs/oauth-backend-config.yaml"
+kubectl apply -f "$(dirname "$0")/backend-configs/viz-backend-config.yaml"
 
 echo "-----------------------------------"
 echo "Applying managed certificates"
@@ -95,6 +97,7 @@ kubectl apply -f "$(dirname "$0")/managed-certs/front-managed-cert.yaml"
 kubectl apply -f "$(dirname "$0")/managed-certs/front-edge-managed-cert.yaml"
 kubectl apply -f "$(dirname "$0")/managed-certs/connectors-managed-cert.yaml"
 kubectl apply -f "$(dirname "$0")/managed-certs/metabase-managed-cert.yaml"
+kubectl apply -f "$(dirname "$0")/managed-certs/viz-managed-cert.yaml"
 
 
 echo "-----------------------------------"
@@ -122,6 +125,7 @@ apply_deployment core-deployment
 apply_deployment core-sqlite-worker-deployment
 apply_deployment oauth-deployment
 apply_deployment prodbox-deployment
+apply_deployment viz-deployment
 
 echo "-----------------------------------"
 echo "Applying HPAs"
@@ -142,6 +146,7 @@ kubectl apply -f "$(dirname "$0")/services/metabase-service.yaml"
 kubectl apply -f "$(dirname "$0")/services/core-service.yaml"
 kubectl apply -f "$(dirname "$0")/services/core-sqlite-worker-headless-service.yaml"
 kubectl apply -f "$(dirname "$0")/services/oauth-service.yaml"
+kubectl apply -f "$(dirname "$0")/services/viz-service.yaml"
 
 
 echo "-----------------------------------"

k8s/backend-configs/viz-backend-config.yaml

@@ -0,0 +1,9 @@
+apiVersion: cloud.google.com/v1
+kind: BackendConfig
+metadata:
+  name: viz-backendconfig
+spec:
+  sessionAffinity:
+    affinityType: "GENERATED_COOKIE"
+    affinityCookieTtlSec: 600
+  timeoutSec: 30

k8s/configmaps/viz-configmap.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: viz-config
+data:
+  DD_ENV: "prod"
+  DD_SERVICE: "viz"
+  NODE_OPTIONS: "-r dd-trace/init"
+  DD_LOGS_INJECTION: "true"
+  DD_RUNTIME_METRICS_ENABLED: "true"
+  NODE_ENV: "production"

k8s/deployments/viz-deployment.yaml

@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: viz-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: viz
+  template:
+    metadata:
+      labels:
+        app: viz
+        name: viz-pod
+        admission.datadoghq.com/enabled: "true"
+      annotations:
+        ad.datadoghq.com/web.logs: '[{"source": "viz","service": "viz","tags": ["env:prod"]}]'
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+        - name: web
+          image: gcr.io/or1g1n-186209/viz-image:latest
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 3007
+          readinessProbe:
+            httpGet:
+              path: /api/healthz
+              port: 3007
+            initialDelaySeconds: 5
+            periodSeconds: 5
+
+          envFrom:
+            - configMapRef:
+                name: viz-config
+
+          env:
+            - name: DD_AGENT_HOST
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+
+          resources:
+            requests:
+              cpu: 200m
+              memory: 200Mi
+
+            limits:
+              cpu: 200m
+              memory: 200Mi

k8s/ingress.yaml

@@ -53,3 +53,14 @@ spec:
                 name: metabase-service
                 port:
                   number: 80
+
+    - host: viz.dust.tt
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: viz-service
+                port:
+                  number: 80

k8s/managed-certs/viz-managed-cert.yaml

@@ -0,0 +1,7 @@
+apiVersion: networking.gke.io/v1
+kind: ManagedCertificate
+metadata:
+  name: viz-managed-cert
+spec:
+  domains:
+    - viz.dust.tt

k8s/services/viz-service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: viz-service
+  annotations:
+    cloud.google.com/backend-config: '{"default": "viz-backendconfig"}'
+spec:
+  selector:
+    app: viz
+    name: viz-pod
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 3007
+  type: ClusterIP

viz/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

viz/.gitignore

@@ -0,0 +1,36 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+.yarn/install-state.gz
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

viz/Dockerfile

@@ -0,0 +1,24 @@
+FROM node:20.13.0 as viz
+
+RUN apt-get update && apt-get install -y vim redis-tools postgresql-client htop
+
+WORKDIR /types
+COPY /types/package*.json ./
+COPY /types/ .
+RUN npm ci
+RUN npm run build
+
+WORKDIR /app
+
+COPY /viz/package*.json ./
+RUN npm ci
+
+COPY /viz .
+
+ARG COMMIT_HASH
+ENV NEXT_PUBLIC_COMMIT_HASH=${COMMIT_HASH}
+
+
+RUN npm run build
+
+CMD ["npm", "--silent", "run", "start"]

viz/README.md

@@ -0,0 +1 @@
+# [Dust](https://dust.tt)

viz/app/api/healthz/route.ts

@@ -0,0 +1,5 @@
+import { NextResponse } from "next/server";
+
+export async function GET() {
+  return NextResponse.json("Hello, World!", { status: 200 });
+}

viz/app/globals.css

@@ -0,0 +1,33 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+:root {
+  --foreground-rgb: 0, 0, 0;
+  --background-start-rgb: 214, 219, 220;
+  --background-end-rgb: 255, 255, 255;
+}
+
+@media (prefers-color-scheme: dark) {
+  :root {
+    --foreground-rgb: 255, 255, 255;
+    --background-start-rgb: 0, 0, 0;
+    --background-end-rgb: 0, 0, 0;
+  }
+}
+
+body {
+  color: rgb(var(--foreground-rgb));
+  background: linear-gradient(
+      to bottom,
+      transparent,
+      rgb(var(--background-end-rgb))
+    )
+    rgb(var(--background-start-rgb));
+}
+
+@layer utilities {
+  .text-balance {
+    text-wrap: balance;
+  }
+}

viz/app/layout.tsx

@@ -0,0 +1,21 @@
+import type { Metadata } from "next";
+import { Inter } from "next/font/google";
+import "./globals.css";
+
+const inter = Inter({ subsets: ["latin"] });
+
+export const metadata: Metadata = {
+  title: "Dust Viz",
+};
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode;
+}>) {
+  return (
+    <html lang="en">
+      <body className={inter.className}>{children}</body>
+    </html>
+  );
+}

viz/app/page.tsx

@@ -0,0 +1,11 @@
+export default function Home() {
+  return (
+    <main className="flex min-h-screen flex-col items-center justify-between p-24">
+      <div className="z-10 w-full max-w-5xl items-center justify-between font-mono text-sm lg:flex">
+        <p className="fixed left-0 top-0 flex w-full justify-center border-b border-gray-300 bg-gradient-to-b from-zinc-200 pb-6 pt-8 backdrop-blur-2xl dark:border-neutral-800 dark:bg-zinc-800/30 dark:from-inherit lg:static lg:w-auto  lg:rounded-xl lg:border lg:bg-gray-200 lg:p-4 lg:dark:bg-zinc-800/30">
+          Hello World!
+        </p>
+      </div>
+    </main>
+  );
+}

viz/next.config.mjs

@@ -0,0 +1,4 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {};
+
+export default nextConfig;