Aric types v4 (#2605)

* Adding @dust-tt/types empty package

Remove readme file

Clean up

adjust name + tsconfig moduleResolution

Moving KeyType for POC

Add dts-cli

Fix front end github workflow

Adjust Docker build information for front

Adjust Dockerfile location

Testing new settings

npm install types

npm install

Fix types copy

Start command

Clean up

Fix format

Remove demo page

Cleaner build method

Deploy front edge adjusted

Fix _DOCKERFILE_PATH

Displaying initial pwd

print args

Fix deploy

Testing front and connectors deployment on edges

Add dts-cli to front/package-lock.json dependencies

Fix types/package-lock.json

Revert to connectors/package*.json from main

Harmonize Dockerfile and package-*.json

Build and lint types in build-and-lint-connectors

Adjust github workflow deploy-connectors-edge

Fix deploy-connectors-edge

Testing docker build for alerting/temporal without the deploy steps

Add build only Github workflow for alerting/temporal

Trigger workflow from branch

Adjust docker file path for alerting/temporal

Adjust all workflows

Fix connectors deployment

Add PostContentFragment io-ts

Format types

npm format

front> npm run format

Clean up after self review

types> add npm run lint to the CI build and lint workflow

npm run format

Downgrading typescript version for types/

fix front/package conflict

* Fix front workflow

---------

Co-authored-by: ARIC LASRY <[email protected]>

.github/workflows/build-and-lint-connectors.yml

@@ -16,5 +16,7 @@ jobs:
           node-version: 18.15.0
           cache: "npm"
           cache-dependency-path: ./connectors/package-lock.json
+      - working-directory: types
+        run: npm install &&  npm run build          
       - working-directory: connectors
         run: npm install && npm run build && npm run lint && npm run format:check

.github/workflows/build-and-lint-front.yml

@@ -16,5 +16,7 @@ jobs:
           node-version: 18.15.0
           cache: "npm"
           cache-dependency-path: ./front/package-lock.json
+      - working-directory: types
+        run: npm install && npm run build
       - working-directory: front
         run: npm install && npm run tsc && npm run lint && npm run format:check

.github/workflows/build-and-lint-types.yml

@@ -0,0 +1,21 @@
+name: Build & Lint (types)
+
+on:
+  push:
+    paths:
+      - types/**
+      - .github/workflows/build-and-lint-types.yml
+
+jobs:
+  check-eslint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18.15.0
+          cache: "npm"
+          cache-dependency-path: ./types/package-lock.json
+      - working-directory: types
+        run: npm install && npm run lint && npm run build
+

.github/workflows/deploy-alerting-temporal.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Build the image on Cloud Build
         run: |
           chmod +x ./k8s/cloud-build.sh
-          ./k8s/cloud-build.sh alerting-temporal alerting/temporal
+          ./k8s/cloud-build.sh alerting-temporal ./Dockerfile ./alerting/temporal/
 
       - name: Deploy the image on Kubernetes
         run: |

.github/workflows/deploy-blog.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Build the image on Cloud Build
         run: |
           chmod +x ./k8s/cloud-build.sh
-          ./k8s/cloud-build.sh blog
+          ./k8s/cloud-build.sh blog ./Dockerfile ./blog/
 
       - name: Deploy the image on Kubernetes
         run: |

.github/workflows/deploy-connectors-edge.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Build the image on Cloud Build
         run: |
           chmod +x ./k8s/cloud-build.sh
-          ./k8s/cloud-build.sh connectors
+          ./k8s/cloud-build.sh connectors ./connectors/Dockerfile ./
 
       - name: Deploy the image on Kubernetes
         run: |

.github/workflows/deploy-connectors.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Build the image on Cloud Build
         run: |
           chmod +x ./k8s/cloud-build.sh
-          ./k8s/cloud-build.sh connectors
+          ./k8s/cloud-build.sh connectors ./connectors/Dockerfile ./
 
       - name: Deploy the image on Kubernetes
         run: |

.github/workflows/deploy-core.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Build the image on Cloud Build
         run: |
           chmod +x ./k8s/cloud-build.sh
-          ./k8s/cloud-build.sh core
+          ./k8s/cloud-build.sh core ./Dockerfile ./core/
 
       - name: Deploy the image on Kubernetes
         run: |

.github/workflows/deploy-docs.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Build the image on Cloud Build
         run: |
           chmod +x ./k8s/cloud-build.sh
-          ./k8s/cloud-build.sh docs
+          ./k8s/cloud-build.sh docs ./Dockerfile ./docs/
 
       - name: Deploy the image on Kubernetes
         run: |

.github/workflows/deploy-front-edge.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Build the image on Cloud Build
         run: |
           chmod +x ./k8s/cloud-build.sh
-          ./k8s/cloud-build.sh front
+          ./k8s/cloud-build.sh front ./front/Dockerfile ./
 
       - name: Deploy the image on Kubernetes
         run: |

.github/workflows/deploy-front.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Build the image on Cloud Build
         run: |
           chmod +x ./k8s/cloud-build.sh
-          ./k8s/cloud-build.sh front
+          ./k8s/cloud-build.sh front ./front/Dockerfile ./
 
       - name: Deploy the image on Kubernetes
         run: |

connectors/Dockerfile

@@ -3,7 +3,7 @@
 FROM node:18.15.0 as build
 
 WORKDIR /tmp/
-COPY ./admin/docker_build/install_poppler_tools.sh ./
+COPY ./connectors/admin/docker_build/install_poppler_tools.sh ./
 RUN chmod +x ./install_poppler_tools.sh
 RUN ./install_poppler_tools.sh
 # end installing poppler tools
@@ -14,13 +14,16 @@ ENV LD_LIBRARY_PATH=/usr/local/lib
 COPY --from=build /tmp/poppler-23.07.0/build/utils/pdftotext /usr/bin/pdftotext
 COPY --from=build /tmp/poppler-23.07.0/build/libpoppler.so.130 /usr/lib/libpoppler.so.130
 
-WORKDIR /app
-
-COPY package*.json ./
+WORKDIR /types
+COPY /types/package*.json ./
+COPY /types/ .
 RUN npm ci
 
-COPY . .
+WORKDIR /app
 
+COPY ./connectors/package*.json ./
+RUN npm ci
+COPY ./connectors/ .
 RUN npm run build
 
 

connectors/package.json

@@ -13,6 +13,7 @@
     "initdb": "env $(cat .env.local) npx tsx src/admin/db.ts"
   },
   "dependencies": {
+    "@dust-tt/types": "file:../types",
     "@nangohq/node": "^0.21.4",
     "@notionhq/client": "^2.2.4",
     "@slack/web-api": "^6.8.1",

connectors/src/lib/dust_api.ts

@@ -1,3 +1,4 @@
+import { PostContentFragmentRequestBodySchema } from "@dust-tt/types";
 import { createParser } from "eventsource-parser";
 import * as t from "io-ts";
 
@@ -84,22 +85,6 @@ const PostMessagesRequestBodySchemaIoTs = t.type({
   }),
 });
 
-export const PostContentFragmentRequestBodySchemaIoTs = t.type({
-  title: t.string,
-  content: t.string,
-  url: t.union([t.string, t.null]),
-  contentType: t.literal("slack_thread_content"),
-  context: t.union([
-    t.type({
-      username: t.string,
-      fullName: t.union([t.string, t.null]),
-      email: t.union([t.string, t.null]),
-      profilePictureUrl: t.union([t.string, t.null]),
-    }),
-    t.null,
-  ]),
-});
-
 const PostConversationsRequestBodySchemaIoTs = t.type({
   title: t.union([t.string, t.null]),
   visibility: t.union([
@@ -108,10 +93,7 @@ const PostConversationsRequestBodySchemaIoTs = t.type({
     t.literal("deleted"),
   ]),
   message: t.union([PostMessagesRequestBodySchemaIoTs, t.undefined]),
-  contentFragment: t.union([
-    PostContentFragmentRequestBodySchemaIoTs,
-    t.undefined,
-  ]),
+  contentFragment: t.union([PostContentFragmentRequestBodySchema, t.undefined]),
 });
 
 type PostConversationsRequestBodySchema = t.TypeOf<
@@ -123,7 +105,7 @@ export type PostMessagesRequestBodySchema = t.TypeOf<
 >;
 
 export type PostContentFragmentRequestBody = t.TypeOf<
-  typeof PostContentFragmentRequestBodySchemaIoTs
+  typeof PostContentFragmentRequestBodySchema
 >;
 
 // Event sent when the user message is created.

front/Dockerfile

@@ -1,14 +1,20 @@
 FROM node:18.15.0 as front
 
+WORKDIR /types
+COPY /types/package*.json ./
+COPY /types/ .
+RUN npm ci
+
 WORKDIR /app
 
-COPY package*.json ./
+COPY /front/package*.json ./
 RUN npm ci
 
-COPY . .
+COPY /front .
+
 # fake database URIs are needed because Sequelize will throw if the `url` parameter
 # is undefined, and `next build` imports the `models.ts` file while "Collecting page data"
 RUN FRONT_DATABASE_URI="sqlite:foo.sqlite" XP1_DATABASE_URI="sqlite:bar.sqlite" npm run build
 
 
-CMD ["npm", "--silent", "run", "start"]
+CMD ["npm", "--silent", "run", "start"]

front/package.json

@@ -14,6 +14,7 @@
   },
   "dependencies": {
     "@dust-tt/sparkle": "^0.2.47",
+    "@dust-tt/types": "file:../types",
     "@emoji-mart/data": "^1.1.2",
     "@emoji-mart/react": "^1.1.1",
     "@headlessui/react": "^1.7.7",

front/pages/api/v1/w/[wId]/assistant/conversations/[cId]/content_fragments.ts

@@ -1,3 +1,4 @@
+import { PostContentFragmentRequestBodySchema } from "@dust-tt/types";
 import { isLeft } from "fp-ts/lib/Either";
 import * as t from "io-ts";
 import * as reporter from "io-ts-reporters";
@@ -16,25 +17,6 @@ export type PostContentFragmentsResponseBody = {
   contentFragment: ContentFragmentType;
 };
 
-export const PostContentFragmentRequestBodySchema = t.type({
-  title: t.string,
-  content: t.string,
-  url: t.union([t.string, t.null]),
-  contentType: t.union([
-    t.literal("slack_thread_content"),
-    t.literal("file_attachment"),
-  ]),
-  context: t.union([
-    t.type({
-      profilePictureUrl: t.union([t.string, t.null]),
-      fullName: t.union([t.string, t.null]),
-      email: t.union([t.string, t.null]),
-      username: t.union([t.string, t.null]),
-    }),
-    t.null,
-  ]),
-});
-
 export type PostContentFragmentRequestBody = t.TypeOf<
   typeof PostContentFragmentRequestBodySchema
 >;

front/pages/api/v1/w/[wId]/assistant/conversations/index.ts

@@ -1,3 +1,4 @@
+import { PostContentFragmentRequestBodySchema } from "@dust-tt/types";
 import { isLeft } from "fp-ts/lib/Either";
 import * as t from "io-ts";
 import * as reporter from "io-ts-reporters";
@@ -19,8 +20,6 @@ import {
   UserMessageType,
 } from "@app/types/assistant/conversation";
 
-import { PostContentFragmentRequestBodySchema } from "./[cId]/content_fragments";
-
 const PostConversationsRequestBodySchema = t.type({
   title: t.union([t.string, t.null]),
   visibility: t.union([

front/pages/api/w/[wId]/keys/[secret]/disable.ts

@@ -1,9 +1,9 @@
+import { KeyType } from "@dust-tt/types";
 import { NextApiRequest, NextApiResponse } from "next";
 
 import { Authenticator, getSession } from "@app/lib/auth";
 import { Key } from "@app/lib/models";
 import { withLogging } from "@app/logger/withlogging";
-import { KeyType } from "@app/types/key";
 
 export type PostKeysResponseBody = {
   key: KeyType;

front/pages/api/w/[wId]/keys/index.ts

@@ -1,10 +1,10 @@
+import { KeyType } from "@dust-tt/types";
 import { NextApiRequest, NextApiResponse } from "next";
 
 import { Authenticator, getSession } from "@app/lib/auth";
 import { Key } from "@app/lib/models";
 import { new_id } from "@app/lib/utils";
 import { withLogging } from "@app/logger/withlogging";
-import { KeyType } from "@app/types/key";
 
 export type GetKeysResponseBody = {
   keys: KeyType[];

front/pages/w/[wId]/a/index.tsx

@@ -1,4 +1,5 @@
 import { Button, CommandLineIcon, Page, PlusIcon } from "@dust-tt/sparkle";
+import { KeyType } from "@dust-tt/types";
 import { EyeIcon, EyeSlashIcon } from "@heroicons/react/24/outline";
 import { GetServerSideProps, InferGetServerSidePropsType } from "next";
 import Link from "next/link";
@@ -25,7 +26,6 @@ import { modelProviders, serviceProviders } from "@app/lib/providers";
 import { useKeys, useProviders } from "@app/lib/swr";
 import { classNames } from "@app/lib/utils";
 import { AppType } from "@app/types/app";
-import { KeyType } from "@app/types/key";
 import { SubscriptionType } from "@app/types/plan";
 import { UserType, WorkspaceType } from "@app/types/user";