- Help you set up an AI project with ease. Here are two example use cases:
+ Help you set up an AI project with ease. Here are two example use
+ cases:
diff --git a/src/app/sign-in/[[...sign-in]]/page.tsx b/src/app/sign-in/[[...sign-in]]/page.tsx
index 4121caad..fa8b0d2f 100644
--- a/src/app/sign-in/[[...sign-in]]/page.tsx
+++ b/src/app/sign-in/[[...sign-in]]/page.tsx
@@ -1,5 +1,3 @@
-import { SignIn } from "@clerk/nextjs";
-
export default function Page() {
- return
;
-}
\ No newline at end of file
+ return ;
+}
diff --git a/src/components/Examples.tsx b/src/components/Examples.tsx
index 3d830187..6f951156 100644
--- a/src/components/Examples.tsx
+++ b/src/components/Examples.tsx
@@ -1,20 +1,13 @@
"use client";
import { useState } from "react";
import QAModal from "./QAModal";
-import TextToImgModal from "./TextToImgModal";
import Image from "next/image";
const examples = [
- {
- name: "Text to image",
- title: "Powered by stable-diffusion on Replicate",
- imageUrl:
- "https://tjzk.replicate.delivery/models_models_featured_image/710f5e9f-9561-4e4f-9d1e-614205f62597/stable-diffusion.webp",
- },
{
name: "Q&A on documents",
title: "Ask questions about an a16z blog",
imageUrl:
- "https://a16z.com/wp-content/uploads/2021/02/Default-Website-FeatureImage.jpg"
+ "https://a16z.com/wp-content/uploads/2021/02/Default-Website-FeatureImage.jpg",
},
];
@@ -24,10 +17,6 @@ export default function Examples() {
return (
-
(
- i === 0 ? setTextToImageModalOpen(true) : setQAModalOpen(true)
- }
+ onClick={() => setQAModalOpen(true)}
className="col-span-1 flex flex-col rounded-lg bg-slate-800 text-center shadow relative ring-1 ring-white/10 cursor-pointer hover:ring-sky-300/70 transition"
>
diff --git a/src/components/Navbar.tsx b/src/components/Navbar.tsx
index 0ac3b042..06a59dd7 100644
--- a/src/components/Navbar.tsx
+++ b/src/components/Navbar.tsx
@@ -1,5 +1,3 @@
-import { UserButton } from "@clerk/nextjs";
-import { auth } from "@clerk/nextjs";
import Image from "next/image";
import Link from "next/link";
const navigation = [
@@ -15,7 +13,6 @@ function classNames(...classes: string[]) {
}
export default function Navbar() {
- const { userId } = auth();
return (
@@ -69,18 +66,6 @@ export default function Navbar() {
-
- {userId ? (
-
- ) : (
-
- Sign In
-
- )}
-
diff --git a/src/components/QAModal.tsx b/src/components/QAModal.tsx
index f3d9cd8b..70ff889c 100644
--- a/src/components/QAModal.tsx
+++ b/src/components/QAModal.tsx
@@ -11,7 +11,7 @@ export default function QAModal({
}) {
const { completion, input, isLoading, handleInputChange, handleSubmit } =
useCompletion({
- api: "/api/qa-pinecone",
+ api: "/api/qa-pg-vector",
});
return (
diff --git a/src/middleware.ts b/src/middleware.ts
deleted file mode 100644
index b9fa7737..00000000
--- a/src/middleware.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { authMiddleware } from "@clerk/nextjs";
-
-// This requires user to sign in to see any page or call any API route
-export default authMiddleware();
-
-// Use the following code instead to expose /api or / as public routes
-// export default authMiddleware({
-// publicRoutes: ["/", "/api(.*)"],
-// });
-
-export const config = {
- matcher: ["/((?!.*\\..*|_next).*)", "/", "/(api|trpc)(.*)"],
-};
diff --git a/src/scripts/indexBlogPGVectorWithEmbed.mjs b/src/scripts/indexBlogPGVectorWithEmbed.mjs
new file mode 100644
index 00000000..db185f12
--- /dev/null
+++ b/src/scripts/indexBlogPGVectorWithEmbed.mjs
@@ -0,0 +1,81 @@
+// Call embeding API and insert to supabase
+// Ref: https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase
+
+import dotenv from "dotenv";
+import { Document } from "langchain/document";
+import { createClient } from "@supabase/supabase-js";
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
+import { pipeline } from "@xenova/transformers";
+
+import fs from "fs";
+import path from "path";
+
+dotenv.config({ path: `.env.local` });
+
+const fileNames = fs.readdirSync("blogs");
+const splitter = RecursiveCharacterTextSplitter.fromLanguage("markdown", {
+ chunkSize: 1000,
+ chunkOverlap: 50,
+});
+
+let langchainDocs = await Promise.all(
+ fileNames.map(async (fileName) => {
+ const filePath = path.join("blogs", fileName);
+ const fileContent = fs.readFileSync(filePath, "utf8");
+ const splitDocs = await splitter.splitText(fileContent);
+ return splitDocs.map((doc) => {
+ return new Document({
+ metadata: { fileName },
+ pageContent: doc,
+ });
+ });
+ })
+);
+
+const auth = {
+ detectSessionInUrl: false,
+ persistSession: false,
+ autoRefreshToken: false,
+};
+
+const client = createClient(
+ process.env.SUPABASE_URL,
+ process.env.SUPABASE_PRIVATE_KEY,
+ { auth }
+);
+
+const embeddingPromises = langchainDocs[0].map((doc) => {
+ return generateEmbedding(doc.pageContent);
+});
+
+const returnedEmbeddings = await Promise.all(embeddingPromises);
+
+let insertData = [];
+langchainDocs[0].map((doc, index) => {
+ insertData.push({
+ content: doc.pageContent,
+ embedding: returnedEmbeddings[index],
+ metadata: doc.metadata,
+ });
+});
+console.log("insertData", insertData);
+
+const { error } = await client.from("documents").insert(insertData);
+console.debug(error);
+
+export async function generateEmbedding(content) {
+ const generateEmbedding = await pipeline(
+ "feature-extraction",
+ "Xenova/all-MiniLM-L6-v2"
+ );
+
+ // Generate a vector using Transformers.js
+ const output = await generateEmbedding(content, {
+ pooling: "mean",
+ normalize: true,
+ });
+
+ // Extract the embedding output
+ const embedding = Array.from(output.data);
+ return embedding;
+}
diff --git a/src/util.ts b/src/util.ts
new file mode 100644
index 00000000..25d61981
--- /dev/null
+++ b/src/util.ts
@@ -0,0 +1,53 @@
+import { SupabaseClient } from "@supabase/supabase-js";
+import { pipeline } from "@xenova/transformers";
+
+const embedding_endpoint = process.env.SUPABASE_EMBEDDING_ENDPOINT!;
+
+export async function vectorSearch(
+ client: SupabaseClient,
+ contentToSearch: string
+) {
+ const embedding = await generateEmbedding(contentToSearch);
+ const result = await client.rpc("match_documents", {
+ query_embedding: embedding,
+ match_count: 3,
+ });
+
+ if (result.error) {
+ console.error("ERROR: ", result.error);
+ }
+ return result.data || "";
+}
+
+export async function generateEmbeddingSupabase(content: string) {
+ const headers = new Headers();
+ headers.append("Authorization", "Bearer " + process.env.SUPABASE_PRIVATE_KEY);
+ headers.append("Content-Type", "application/json");
+ const body = JSON.stringify({
+ input: content,
+ });
+ const response = await fetch(embedding_endpoint, {
+ method: "POST",
+ headers,
+ body,
+ });
+ const json = await response.json();
+ return json.embedding;
+}
+
+export async function generateEmbedding(content: string) {
+ const generateEmbedding = await pipeline(
+ "feature-extraction",
+ "Xenova/all-MiniLM-L6-v2"
+ );
+
+ // Generate a vector using Transformers.js
+ const output = await generateEmbedding(content, {
+ pooling: "mean",
+ normalize: true,
+ });
+
+ // Extract the embedding output
+ const embedding = Array.from(output.data);
+ return embedding;
+}
diff --git a/supabase/.gitignore b/supabase/.gitignore
new file mode 100644
index 00000000..773c7c3e
--- /dev/null
+++ b/supabase/.gitignore
@@ -0,0 +1,3 @@
+# Supabase
+.branches
+.temp
diff --git a/supabase/config.toml b/supabase/config.toml
new file mode 100644
index 00000000..728bba02
--- /dev/null
+++ b/supabase/config.toml
@@ -0,0 +1,82 @@
+# A string used to distinguish different Supabase projects on the same host. Defaults to the working
+# directory name when running `supabase init`.
+project_id = "local-ai-getting-started"
+
+[api]
+# Port to use for the API URL.
+port = 54321
+# Schemas to expose in your API. Tables, views and stored procedures in this schema will get API
+# endpoints. public and storage are always included.
+schemas = ["public", "storage", "graphql_public"]
+# Extra schemas to add to the search_path of every request. public is always included.
+extra_search_path = ["public", "extensions"]
+# The maximum number of rows returns from a view, table, or stored procedure. Limits payload size
+# for accidental or malicious requests.
+max_rows = 1000
+
+[db]
+# Port to use for the local database URL.
+port = 54322
+# The database major version to use. This has to be the same as your remote database's. Run `SHOW
+# server_version;` on the remote database to check.
+major_version = 15
+
+[studio]
+# Port to use for Supabase Studio.
+port = 54323
+
+# Email testing server. Emails sent with the local dev setup are not actually sent - rather, they
+# are monitored, and you can view the emails that would have been sent from the web interface.
+[inbucket]
+# Port to use for the email testing server web interface.
+port = 54324
+smtp_port = 54325
+pop3_port = 54326
+
+[storage]
+# The maximum file size allowed (e.g. "5MB", "500KB").
+file_size_limit = "50MiB"
+
+[auth]
+# The base URL of your website. Used as an allow-list for redirects and for constructing URLs used
+# in emails.
+site_url = "http://localhost:3000"
+# A list of *exact* URLs that auth providers are permitted to redirect to post authentication.
+additional_redirect_urls = ["https://localhost:3000"]
+# How long tokens are valid for, in seconds. Defaults to 3600 (1 hour), maximum 604,800 seconds (one
+# week).
+jwt_expiry = 3600
+# Allow/disallow new user signups to your project.
+enable_signup = true
+
+[auth.email]
+# Allow/disallow new user signups via email to your project.
+enable_signup = true
+# If enabled, a user will be required to confirm any email change on both the old, and new email
+# addresses. If disabled, only the new email is required to confirm.
+double_confirm_changes = true
+# If enabled, users need to confirm their email address before signing in.
+enable_confirmations = false
+
+# Use an external OAuth provider. The full list of providers are: `apple`, `azure`, `bitbucket`,
+# `discord`, `facebook`, `github`, `gitlab`, `google`, `keycloak`, `linkedin`, `notion`, `twitch`,
+# `twitter`, `slack`, `spotify`, `workos`, `zoom`.
+[auth.external.apple]
+enabled = false
+client_id = ""
+secret = ""
+# Overrides the default auth redirectUrl.
+redirect_uri = ""
+# Overrides the default auth provider URL. Used to support self-hosted gitlab, single-tenant Azure,
+# or any other third-party OIDC providers.
+url = ""
+
+[analytics]
+enabled = false
+port = 54327
+vector_port = 54328
+# Setup BigQuery project to enable log viewer on local development stack.
+# See: https://supabase.com/docs/guides/getting-started/local-development#enabling-local-logging
+gcp_project_id = ""
+gcp_project_number = ""
+gcp_jwt_path = "supabase/gcloud.json"
diff --git a/supabase/functions/embed/index.ts b/supabase/functions/embed/index.ts
new file mode 100644
index 00000000..776d1b9e
--- /dev/null
+++ b/supabase/functions/embed/index.ts
@@ -0,0 +1,40 @@
+// Follow this setup guide to integrate the Deno language server with your editor:
+// https://deno.land/manual/getting_started/setup_your_environment
+// This enables autocomplete, go to definition, etc.
+
+import { serve } from "https://deno.land/std@0.168.0/http/server.ts";
+import {
+ env,
+ pipeline,
+} from "https://cdn.jsdelivr.net/npm/@xenova/transformers@2.5.0";
+
+// Configuration for Deno runtime
+env.useBrowserCache = false;
+env.allowLocalModels = false;
+
+const pipe = await pipeline("feature-extraction", "Supabase/gte-small");
+
+serve(async (req) => {
+ // Extract input string from JSON body
+ const { input } = await req.json();
+
+ // Generate the embedding from the user input
+ const output = await pipe(input, {
+ pooling: "mean",
+ normalize: true,
+ });
+
+ // Extract the embedding output
+ const embedding = Array.from(output.data);
+
+ // Return the embedding
+ return new Response(JSON.stringify({ embedding }), {
+ headers: { "Content-Type": "application/json" },
+ });
+});
+
+// To invoke:
+// curl -i --location --request POST 'http://localhost:54321/functions/v1/' \
+// --header 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6ImFub24iLCJleHAiOjE5ODM4MTI5OTZ9.CRXP1A7WOeoJeXxjNni43kdQwgnWNReilDMblYTn_I0' \
+// --header 'Content-Type: application/json' \
+// --data '{"name":"Functions"}'
diff --git a/supabase/migrations/20231028002341_create_vector_db.sql b/supabase/migrations/20231028002341_create_vector_db.sql
new file mode 100644
index 00000000..07bd5c59
--- /dev/null
+++ b/supabase/migrations/20231028002341_create_vector_db.sql
@@ -0,0 +1,40 @@
+-- Reference: https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase#create-a-table-and-search-function-in-your-database
+-- Visit Supabase blogpost for more: https://supabase.com/blog/openai-embeddings-postgres-vector
+-- Enable the pgvector extension to work with embedding vectors
+create extension vector;
+
+-- Create a table to store your documents
+create table documents (
+ id bigserial primary key,
+ content text, -- corresponds to Document.pageContent
+ metadata jsonb, -- corresponds to Document.metadata
+ embedding vector(384) -- 384 works for OpenAI embeddings, change if needed
+);
+
+-- Create a function to search for documents
+create function match_documents (
+ query_embedding vector(384),
+ match_count int DEFAULT null,
+ filter jsonb DEFAULT '{}'
+) returns table (
+ id bigint,
+ content text,
+ metadata jsonb,
+ similarity float
+)
+language plpgsql
+as $$
+#variable_conflict use_column
+begin
+ return query
+ select
+ id,
+ content,
+ metadata,
+ 1 - (documents.embedding <=> query_embedding) as similarity
+ from documents
+ where metadata @> filter
+ order by documents.embedding <=> query_embedding
+ limit match_count;
+end;
+$$;
\ No newline at end of file
diff --git a/supabase/seed.sql b/supabase/seed.sql
new file mode 100644
index 00000000..e69de29b