Fix self host .env examples and documentation

.env.example

@@ -0,0 +1,13 @@
+# In tracecat/.env
+TRACECAT__SIGNING_SECRET=${ openssl rand -hex 32 }
+TRACECAT__SERVICE_KEY=${ openssl rand -hex 32 }
+TRACECAT__DB_ENCRYPTION_KEY=your-fernet-key
+SUPABASE_JWT_SECRET=super-secret-jwt-token-with-at-least-32-characters-long
+SUPABASE_JWT_ALGORITHM="HS256"
+SUPABASE_PSQL_URL=postgresql://postgres:[email protected]:54322/postgres
+OPENAI_API_KEY=your-openai-api-key
+RESEND_API_KEY=your-resend-api-key
+TRACECAT__SELF_HOSTED_DB_BACKEND=postgres
+TRACECAT__RUNNER_URL= #ngrok url
+TRACECAT__DB_URI=postgresql://postgres:[email protected]:54322/postgres
+LOG_LEVEL=INFO

docker-compose.yaml

@@ -62,4 +62,4 @@ services:
       - nginx-network
 
 networks:
-  nginx-network:
+  nginx-network:

docs/installation.mdx

@@ -16,7 +16,7 @@ Install Tracecat using Docker Compose.
 - [Docker compose](https://docs.docker.com/compose/install/)
 - [Supabase CLI](https://supabase.com/docs/guides/cli/getting-started#installing-the-supabase-cli)
 - [pnpm](https://pnpm.io/installation#using-corepack)
-- [Resend API Key](https://resend.com/docs/introduction) 
+- [Resend API Key](https://resend.com/docs/introduction)
 - [OpenAI API Key](https://platform.openai.com/docs/overview)
 - [ngrok](https://ngrok.com/)
 
@@ -64,12 +64,13 @@ SUPABASE_JWT_SECRET=your-supabase-jwt-secret
 SUPABASE_JWT_ALGORITHM="HS256"
 SUPABASE_PSQL_URL=your-supabase-psql-url
 # If running locally this will be
-# SUPABASE_PSQL_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres
+# SUPABASE_PSQL_URL=postgresql://postgres:postgres@host.docker.internal:54322/postgres
 # NOTE: The port here is the container's port (default '5432') to allow for services in the local docker network to reach the database.
 OPENAI_API_KEY=your-openai-api-key
 RESEND_API_KEY=your-resend-api-key
-TRACECAT__SELF_HOSTED_DB_BACKEND="postgres" # Or "sqlite", but we don't recommend this for now
+TRACECAT__DB_URI=postgresql://postgres:[email protected]:54322/postgres
 ```
+
 _Note: to generate a `TRACECAT__DB_ENCRYPTION_KEY` you'll need to create a Fernet key: https://cryptography.io/en/latest/fernet/._
 
 ### Expose the runner
@@ -82,68 +83,75 @@ We recommend you grab a free static url from their page.
 TRACECAT__RUNNER_URL=https://your-runner-url
 ```
 
-
-
 ## Create `.env.local` file in `tracecat/frontend`
+
 Our frontend stack is React + Typescript + NextJS. Add the following:
 
 ```bash
 # In tracecat/frontend/.env.local
 NODE_ENV="development"
 NEXT_PUBLIC_APP_ENV="dev"
 NEXT_PUBLIC_SELF_HOSTED=true
-NEXT_PUBLIC_APP_URL="http://localhost:8000"
+NEXT_PUBLIC_APP_URL="http://localhost:3000"
+NEXT_PUBLIC_API_URL="http://localhost:8000"
 NEXT_PUBLIC_SUPABASE_URL="http://localhost:54321"
 NEXT_PUBLIC_SUPABASE_ANON_KEY= ... # From output of supabase start
 ```
 
 ## Start Tracecat
+
 ### Backend
+
 At this point, the Supabase stack should already be running. Run the following to bring up Tracecat `nginx`, `api`, and `runner` services:
+
 ```bash
-docker compose -f docker-compose.self-host.yaml up
+docker compose -f up
 ```
 
 Allow incoming webhooks for the runner:
+
 ```bash
 # Tracecat runner listens on port 8001
 ngrok http --domain=your-ngrok-url-here.ngrok-free.app 8001
 ```
 
 ### Frontend
+
 Inside `tracecat/frontend`, install dependencies with
+
 ```bash
 pnpm install
 ```
+
 Then run the frontend in development mode:
+
 ```bash
 pnpm dev
 ```
+
 Alternatively if you'd like to build it:
+
 ```bash
 pnpm run build
 pnpm run start
 ```
 
 _Note: We're working on adding the frontend to the docker compose stack!_
 
-
-
-
 ## Getting started
+
 You can access Tracecat's frontend at [http://localhost:3000](http://localhost:3000)
 and the OpenAPI docs at [http://localhost:8000/docs](http://localhost:8000/docs).
 
-Once the frontend and backend services are running, you should see a email + password login page at [http://localhost:3000](http://localhost:3000). 
+Once the frontend and backend services are running, you should see a email + password login page at [http://localhost:3000](http://localhost:3000).
 
 ### Create a user and log in
-To create a user account, visit the Supabase dashboard at [http://localhost:54323](http://localhost:54323) and click on `Authentication` on the left side panel. You should see a dropdown `Add User` at the top right of the page, where you can create a user. 
-After doing so, you should be able to log into through the frontend.
 
+To create a user account, visit the Supabase dashboard at [http://localhost:54323](http://localhost:54323) and click on `Authentication` on the left side panel. You should see a dropdown `Add User` at the top right of the page, where you can create a user.
+After doing so, you should be able to log into through the frontend.
 
 ## Shut down
+
 ```bash
 docker compose -f docker-compose.self-host.yaml down --remove-orphans && supabase stop
 ```
-
-

frontend/.env.example

@@ -1,13 +1,16 @@
-NEXT_PUBLIC_APP_ENV=dev
-NEXT_PUBLIC_APP_URL=
-NEXT_PUBLIC_SUPABASE_URL=
+# In tracecat/frontend/.env.local
+NODE_ENV="development"
+NEXT_PUBLIC_APP_ENV="dev"
+NEXT_PUBLIC_SELF_HOSTED=true
+NEXT_PUBLIC_APP_URL="http://localhost:3000"
+NEXT_PUBLIC_API_URL="http://localhost:8000"
+NEXT_PUBLIC_SUPABASE_URL="http://localhost:54321"
 NEXT_PUBLIC_SUPABASE_ANON_KEY=
 
 # Tracecat self-hosted only, please ignore if Cloud:
 NEXT_PUBLIC_SELF_HOSTED=true
-
 # Tracecat Cloud only, please ignore if self-hosted:
 NEXT_PUBLIC_POSTHOG_KEY=
 NEXT_PUBLIC_POSTHOG_INGEST_HOST="https://www.your-domain.com/ingest"
 NEXT_PUBLIC_POSTHOG_HOST="https://app.posthog.com"
-NEXT_PUBLIC_DISABLE_SESSION_RECORDING=true
+NEXT_PUBLIC_DISABLE_SESSION_RECORDING=true

frontend/src/env.mjs

@@ -4,8 +4,10 @@ import { z } from "zod"
 export const env = createEnv({
   client: {
     NEXT_PUBLIC_APP_URL: z.string().min(1),
+    NEXT_PUBLIC_API_URL: z.string().min(1),
   },
   runtimeEnv: {
     NEXT_PUBLIC_APP_URL: process.env.NEXT_PUBLIC_APP_URL,
+    NEXT_PUBLIC_API_URL: process.env.NEXT_PUBLIC_API_URL,
   },
 })

frontend/src/lib/api.ts

@@ -3,7 +3,7 @@ import { type Session } from "@supabase/supabase-js"
 import axios from "axios"
 
 const client = axios.create({
-  baseURL: process.env.NEXT_PUBLIC_APP_URL,
+  baseURL: process.env.NEXT_PUBLIC_API_URL,
 })
 export type Client = typeof client
 
@@ -19,7 +19,7 @@ export const getAuthenticatedClient = (session: Session | null) => {
 }
 
 export async function streamingResponse(endpoint: string, init?: RequestInit) {
-  return fetch(`${process.env.NEXT_PUBLIC_APP_URL}${endpoint}`, init)
+  return fetch(`${process.env.NEXT_PUBLIC_API_URL}${endpoint}`, init)
 }
 
 export async function* streamGenerator(
@@ -33,7 +33,7 @@ export async function* streamGenerator(
   }
   const { headers, ...rest } = init ?? {}
   const response = await fetch(
-    `${process.env.NEXT_PUBLIC_APP_URL}${endpoint}`,
+    `${process.env.NEXT_PUBLIC_API_URL}${endpoint}`,
     {
       ...rest,
       headers: {

frontend/src/lib/auth.ts

@@ -92,7 +92,7 @@ export async function signInWithEmailMagicLink(formData: FormData) {
 }
 
 export async function newUserFlow(session: Session) {
-  const response = await fetch(`${process.env.NEXT_PUBLIC_APP_URL}/users`, {
+  const response = await fetch(`${process.env.NEXT_PUBLIC_API_URL}/users`, {
     method: "PUT",
     headers: {
       Authorization: `Bearer ${session.access_token}`,

supabase/config.toml

@@ -143,7 +143,7 @@ redirect_uri = "http://localhost:54321/auth/v1/callback"
 url = ""
 
 [analytics]
-enabled = false
+enabled = true
 port = 54327
 vector_port = 54328
 # Configure one of the supported backends: `postgres`, `bigquery`.

tracecat/config.py

@@ -7,7 +7,4 @@
 TRACECAT__API_URL = os.environ.get("TRACECAT__API_URL", "http://api:8000")
 TRACECAT__RUNNER_URL = os.environ.get("TRACECAT__RUNNER_URL", "http://runner:8000")
 
-TRACECAT__SERVICE_ROLES_WHITELIST = ["tracecat-runner", "tracecat-api"]
-TRACECAT__SELF_HOSTED_DB_BACKEND = os.environ.get(
-    "TRACECAT__SELF_HOSTED_DB_BACKEND", "postgres"
-)
+TRACECAT__SERVICE_ROLES_WHITELIST = ["tracecat-runner", "tracecat-api"]

tracecat/db.py

@@ -17,7 +17,6 @@
 from tracecat.config import (
     TRACECAT__APP_ENV,
     TRACECAT__RUNNER_URL,
-    TRACECAT__SELF_HOSTED_DB_BACKEND,
 )
 from tracecat.labels.mitre import get_mitre_tactics_techniques
 
@@ -33,14 +32,7 @@
     "Sinkholed",
 ]
 STORAGE_PATH.mkdir(parents=True, exist_ok=True)
-if TRACECAT__APP_ENV == "prod":
-    TRACECAT__DB_URI = os.environ["TRACECAT__DB_URI"]
-else:
-    # Attempt to use supabase first
-    if TRACECAT__SELF_HOSTED_DB_BACKEND == "postgres":
-        TRACECAT__DB_URI = os.environ["TRACECAT__DB_URI"]
-    else:
-        TRACECAT__DB_URI = f"sqlite:////{STORAGE_PATH}/database.db"
+TRACECAT__DB_URI = os.environ["TRACECAT__DB_URI"]
 
 
 class User(SQLModel, table=True):
@@ -233,14 +225,11 @@ def create_db_engine() -> Engine:
             "connect_args": {"sslmode": "require"},
         }
     else:
-        if TRACECAT__SELF_HOSTED_DB_BACKEND == "postgres":
-            engine_kwargs = {
-                "pool_timeout": 30,
-                "pool_recycle": 3600,
-                "connect_args": {"sslmode": "disable"},
-            }
-        else:
-            engine_kwargs = {"connect_args": {"check_same_thread": False}}
+        engine_kwargs = {
+            "pool_timeout": 30,
+            "pool_recycle": 3600,
+            "connect_args": {"sslmode": "disable"},
+        }
     engine = create_engine(TRACECAT__DB_URI, **engine_kwargs)
     return engine