From 2e73a13898134d5c947ccb00dfdd68b1cdb25cb3 Mon Sep 17 00:00:00 2001 From: Daryl Lim Date: Mon, 25 Mar 2024 01:02:37 -0700 Subject: [PATCH 1/4] build(engine): Add resend api key --- aws/stack.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aws/stack.py b/aws/stack.py index f8d652046..2c41d6e44 100644 --- a/aws/stack.py +++ b/aws/stack.py @@ -244,6 +244,9 @@ def __init__(self, scope: Construct, id: str, **kwargs) -> None: "OPENAI_API_KEY": ecs.Secret.from_secrets_manager( tracecat_secret, field="openai-api-key" ), + "RESEND_API_KEY": ecs.Secret.from_secrets_manager( + tracecat_secret, field="resend-api-key" + ), } if TRACECAT__APP_ENV == "prod": shared_env = { From 97b90d63cf7c3c692d443ddfbc69e791d818093a Mon Sep 17 00:00:00 2001 From: Daryl Lim Date: Mon, 25 Mar 2024 03:53:35 -0700 Subject: [PATCH 2/4] feat(engine): Supabase init + use localhost --- supabase/.gitignore | 4 ++ supabase/config.toml | 163 +++++++++++++++++++++++++++++++++++++++++++ supabase/seed.sql | 0 3 files changed, 167 insertions(+) create mode 100644 supabase/.gitignore create mode 100644 supabase/config.toml create mode 100644 supabase/seed.sql diff --git a/supabase/.gitignore b/supabase/.gitignore new file mode 100644 index 000000000..a3ad88055 --- /dev/null +++ b/supabase/.gitignore @@ -0,0 +1,4 @@ +# Supabase +.branches +.temp +.env diff --git a/supabase/config.toml b/supabase/config.toml new file mode 100644 index 000000000..38d353929 --- /dev/null +++ b/supabase/config.toml @@ -0,0 +1,163 @@ +# A string used to distinguish different Supabase projects on the same host. Defaults to the +# working directory name when running `supabase init`. +project_id = "tracecat" + +[api] +enabled = true +# Port to use for the API URL. +port = 54321 +# Schemas to expose in your API. Tables, views and stored procedures in this schema will get API +# endpoints. public and storage are always included. +schemas = ["public", "storage", "graphql_public"] +# Extra schemas to add to the search_path of every request. public is always included. +extra_search_path = ["public", "extensions"] +# The maximum number of rows returns from a view, table, or stored procedure. Limits payload size +# for accidental or malicious requests. +max_rows = 1000 + +[db] +# Port to use for the local database URL. +port = 54322 +# Port used by db diff command to initialize the shadow database. +shadow_port = 54320 +# The database major version to use. This has to be the same as your remote database's. Run `SHOW +# server_version;` on the remote database to check. +major_version = 15 + +[db.pooler] +enabled = false +# Port to use for the local connection pooler. +port = 54329 +# Specifies when a server connection can be reused by other clients. +# Configure one of the supported pooler modes: `transaction`, `session`. +pool_mode = "transaction" +# How many server connections to allow per user/database pair. +default_pool_size = 20 +# Maximum number of client connections allowed. +max_client_conn = 100 + +[realtime] +enabled = false +# Bind realtime via either IPv4 or IPv6. (default: IPv6) +# ip_version = "IPv6" +# The maximum length in bytes of HTTP request headers. (default: 4096) +# max_header_length = 4096 + +[studio] +enabled = true +# Port to use for Supabase Studio. +port = 54323 +# External URL of the API server that frontend connects to. +api_url = "http://localhost" +# OpenAI API Key to use for Supabase AI in the Supabase Studio. +openai_api_key = "env(OPENAI_API_KEY)" + +# Email testing server. Emails sent with the local dev setup are not actually sent - rather, they +# are monitored, and you can view the emails that would have been sent from the web interface. +[inbucket] +enabled = false +# Port to use for the email testing server web interface. +port = 54324 +# Uncomment to expose additional ports for testing user applications that send emails. +# smtp_port = 54325 +# pop3_port = 54326 + + +[storage] +enabled = true +# The maximum file size allowed (e.g. "5MB", "500KB"). +file_size_limit = "50MiB" + +[auth] +enabled = true +# The base URL of your website. Used as an allow-list for redirects and for constructing URLs used +# in emails. +site_url = "http://localhost:3000" +additional_redirect_urls = [ + "https://localhost:3000/auth/callback", # Used for OAuth2/magic link redirect +] # A list of *exact* URLs that auth providers are permitted to redirect to post authentication. +# How long tokens are valid for, in seconds. Defaults to 3600 (1 hour), maximum 604,800 (1 week). +jwt_expiry = 3600 +# If disabled, the refresh token will never expire. +enable_refresh_token_rotation = true +# Allows refresh tokens to be reused after expiry, up to the specified interval in seconds. +# Requires enable_refresh_token_rotation = true. +refresh_token_reuse_interval = 10 +# Allow/disallow new user signups to your project. +enable_signup = true +# Allow/disallow testing manual linking of accounts +enable_manual_linking = false + +[auth.email] +# Allow/disallow new user signups via email to your project. +enable_signup = true +# If enabled, a user will be required to confirm any email change on both the old, and new email +# addresses. If disabled, only the new email is required to confirm. +double_confirm_changes = true +# If enabled, users need to confirm their email address before signing in. +enable_confirmations = false + +# Uncomment to customize email template +# [auth.email.template.invite] +# subject = "You have been invited" +# content_path = "./supabase/templates/invite.html" + +[auth.sms] +# Allow/disallow new user signups via SMS to your project. +enable_signup = false +# If enabled, users need to confirm their phone number before signing in. +enable_confirmations = false +# Template for sending OTP to users +template = "Your code is {{ .Code }} ." + +# Use pre-defined map of phone number to OTP for testing. +[auth.sms.test_otp] +# 4152127777 = "123456" + +# This hook runs before a token is issued and allows you to add additional claims based on the authentication method used. +[auth.hook.custom_access_token] +# enabled = true +# uri = "pg-functions:////" + + +# Configure one of the supported SMS providers: `twilio`, `twilio_verify`, `messagebird`, `textlocal`, `vonage`. +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +# DO NOT commit your Twilio auth token to git. Use environment variable substitution instead: +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +# Use an external OAuth provider. The full list of providers are: `apple`, `azure`, `bitbucket`, +# `discord`, `facebook`, `github`, `gitlab`, `google`, `keycloak`, `linkedin_oidc`, `notion`, `twitch`, +# `twitter`, `slack`, `spotify`, `workos`, `zoom`. +[auth.external.google] +enabled = false +client_id = "env(GOOGLE_OAUTH_CLIENT_ID)" +# DO NOT commit your OAuth provider secret to git. Use environment variable substitution instead: +secret = "env(GOOGLE_OAUTH_CLIENT_SECRET)" +# Overrides the default auth redirectUrl. +redirect_uri = "http://localhost:54321/auth/v1/callback" +# Overrides the default auth provider URL. Used to support self-hosted gitlab, single-tenant Azure, +# or any other third-party OIDC providers. +url = "" + +[analytics] +enabled = false +port = 54327 +vector_port = 54328 +# Configure one of the supported backends: `postgres`, `bigquery`. +backend = "postgres" + +# Experimental features may be deprecated any time +[experimental] +# Configures Postgres storage engine to use OrioleDB (S3) +orioledb_version = "" +# Configures S3 bucket URL, eg. .s3-.amazonaws.com +s3_host = "env(S3_HOST)" +# Configures S3 bucket region, eg. us-east-1 +s3_region = "env(S3_REGION)" +# Configures AWS_ACCESS_KEY_ID for S3 bucket +s3_access_key = "env(S3_ACCESS_KEY)" +# Configures AWS_SECRET_ACCESS_KEY for S3 bucket +s3_secret_key = "env(S3_SECRET_KEY)" diff --git a/supabase/seed.sql b/supabase/seed.sql new file mode 100644 index 000000000..e69de29bb From f9c5624a5320dc0f1d423db6ec22c94fd95b1c74 Mon Sep 17 00:00:00 2001 From: Daryl Lim Date: Mon, 25 Mar 2024 06:07:28 -0700 Subject: [PATCH 3/4] feat(ui): Add login for self host --- frontend/.env.example | 3 ++ frontend/src/app/auth/callback/route.ts | 25 ++------------- frontend/src/components/auth/forms.tsx | 9 +++--- frontend/src/components/auth/login.tsx | 42 ++++++++++++++----------- frontend/src/lib/auth.ts | 42 ++++++++++++++++++++++--- 5 files changed, 72 insertions(+), 49 deletions(-) diff --git a/frontend/.env.example b/frontend/.env.example index d8701f9ca..860eb4851 100644 --- a/frontend/.env.example +++ b/frontend/.env.example @@ -3,6 +3,9 @@ NEXT_PUBLIC_APP_URL= NEXT_PUBLIC_SUPABASE_URL= NEXT_PUBLIC_SUPABASE_ANON_KEY= +# Tracecat self-hosted only, please ignore if Cloud: +NEXT_PUBLIC_SELF_HOSTED=true + # Tracecat Cloud only, please ignore if self-hosted: NEXT_PUBLIC_POSTHOG_KEY= NEXT_PUBLIC_POSTHOG_INGEST_HOST="https://www.your-domain.com/ingest" diff --git a/frontend/src/app/auth/callback/route.ts b/frontend/src/app/auth/callback/route.ts index cbe4afc54..1b1c3f426 100644 --- a/frontend/src/app/auth/callback/route.ts +++ b/frontend/src/app/auth/callback/route.ts @@ -2,7 +2,7 @@ import { redirect } from "next/navigation" import { NextResponse } from "next/server" import { createClient } from "@/utils/supabase/server" -import { createWorkflow } from "@/lib/flow" +import { newUserFlow } from "@/lib/auth" export async function GET(request: Request) { // The `/auth/callback` route is required for the server-side auth flow implemented @@ -27,28 +27,7 @@ export async function GET(request: Request) { console.error("Failed to get session") return redirect("/?level=error&message=Could not authenticate user") } - const response = await fetch(`${process.env.NEXT_PUBLIC_APP_URL}/users`, { - method: "PUT", - headers: { - Authorization: `Bearer ${session.access_token}`, - }, - }) - - // If the user already exists, we'll get a 409 conflict - if (!response.ok && response.status !== 409) { - console.error("Failed to create user") - return redirect("/?level=error&message=Could not authenticate user") - } - - if (response.status !== 409) { - console.log("New user created") - await createWorkflow( - session, - "My first workflow", - "Welcome to Tracecat. This is your first workflow!" - ) - console.log("Created first workflow for new user") - } + await newUserFlow(session) // URL to redirect to after sign up process completes return NextResponse.redirect(`${origin}/workflows`) } diff --git a/frontend/src/components/auth/forms.tsx b/frontend/src/components/auth/forms.tsx index b1e830461..f4d84a401 100644 --- a/frontend/src/components/auth/forms.tsx +++ b/frontend/src/components/auth/forms.tsx @@ -14,7 +14,7 @@ import { SubmitButton } from "./submit-button" export function SignInForm({ searchParams, }: { - searchParams: { message: string } + searchParams: { level?: AlertLevel; message?: string } }) { const [isLoading, setIsLoading] = useState(false) async function onSubmit(event: React.SyntheticEvent) { @@ -63,9 +63,10 @@ export function SignInForm({ Sign In {searchParams?.message && ( -

- {searchParams.message} -

+ )} diff --git a/frontend/src/components/auth/login.tsx b/frontend/src/components/auth/login.tsx index 7b7eef28f..800aeec28 100644 --- a/frontend/src/components/auth/login.tsx +++ b/frontend/src/components/auth/login.tsx @@ -11,7 +11,7 @@ import { CardHeader, CardTitle, } from "@/components/ui/card" -import { PasswordlessSignInForm } from "@/components/auth/forms" +import { PasswordlessSignInForm, SignInForm } from "@/components/auth/forms" import { GithubOAuthButton, GoogleOAuthButton, @@ -61,23 +61,29 @@ export default async function Login({ Enter your email below to create your account or sign in - -
- - -
-
-
- -
-
- - Or continue with - -
-
- - + {Boolean(process.env.NEXT_PUBLIC_SELF_HOSTED) ? ( + + ) : ( + <> + +
+ + +
+
+
+ +
+
+ + Or continue with + +
+
+ + + + )} ) diff --git a/frontend/src/lib/auth.ts b/frontend/src/lib/auth.ts index b28593980..90a75fc65 100644 --- a/frontend/src/lib/auth.ts +++ b/frontend/src/lib/auth.ts @@ -3,6 +3,9 @@ import { headers } from "next/headers" import { redirect } from "next/navigation" import { createClient } from "@/utils/supabase/server" +import { Session } from "@supabase/supabase-js" + +import { createWorkflow } from "@/lib/flow" type ThirdPartyAuthProvider = "google" | "github" @@ -11,15 +14,21 @@ export async function signInFlow(formData: FormData) { const password = formData.get("password") as string const supabase = createClient() - const { error } = await supabase.auth.signInWithPassword({ + const { + data: { session }, + error, + } = await supabase.auth.signInWithPassword({ email, password, }) - if (error) { + if (error || !session) { + console.error("error", error, "session", session) return redirect("/?level=error&message=Could not authenticate user") } + await newUserFlow(session) + return redirect("/workflows") } @@ -41,7 +50,7 @@ export async function signUpFlow(formData: FormData) { return redirect("/?level=error&message=Could not authenticate user") } - return redirect("/?message=Check email to continue sign in process") + return redirect("/?message=Check your email to continue") } export async function thirdPartyAuthFlow(provider: ThirdPartyAuthProvider) { @@ -79,5 +88,30 @@ export async function signInWithEmailMagicLink(formData: FormData) { if (error) { return redirect("/?level=error&message=Could not authenticate user") } - return redirect("/?message=Check email to continue sign in process") + return redirect("/?message=Check your email to continue") +} + +export async function newUserFlow(session: Session) { + const response = await fetch(`${process.env.NEXT_PUBLIC_APP_URL}/users`, { + method: "PUT", + headers: { + Authorization: `Bearer ${session.access_token}`, + }, + }) + + // If the user already exists, we'll get a 409 conflict + if (!response.ok && response.status !== 409) { + console.error("Failed to create user") + return redirect("/?level=error&message=Could not authenticate user") + } + + if (response.status !== 409) { + console.log("New user created") + await createWorkflow( + session, + "My first workflow", + "Welcome to Tracecat. This is your first workflow!" + ) + console.log("Created first workflow for new user") + } } From 01bd0a32067247ad909edc35088e1989ceea75c3 Mon Sep 17 00:00:00 2001 From: Daryl Lim Date: Mon, 25 Mar 2024 06:08:31 -0700 Subject: [PATCH 4/4] build: Add support for self host --- docker-compose.self-host.yaml | 53 +++++++++++++++++++++++++++++++++++ docker-compose.yaml | 5 ++++ frontend/.dockerignore | 3 ++ frontend/Dockerfile | 21 ++++++++++++++ frontend/package.json | 2 +- tracecat/config.py | 3 ++ tracecat/db.py | 21 ++++++++++++-- 7 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 docker-compose.self-host.yaml create mode 100644 frontend/.dockerignore create mode 100644 frontend/Dockerfile diff --git a/docker-compose.self-host.yaml b/docker-compose.self-host.yaml new file mode 100644 index 000000000..3bea67ef2 --- /dev/null +++ b/docker-compose.self-host.yaml @@ -0,0 +1,53 @@ +version: "3.8" +services: + api: + extends: + file: docker-compose.yaml + service: api + environment: + TRACECAT__SELF_HOSTED_DB_BACKEND: ${TRACECAT__SELF_HOSTED_DB_BACKEND} + networks: + - nginx-network + - supabase_network_tracecat + + runner: + extends: + file: docker-compose.yaml + service: runner + environment: + TRACECAT__SELF_HOSTED_DB_BACKEND: ${TRACECAT__SELF_HOSTED_DB_BACKEND} + networks: + - nginx-network + - supabase_network_tracecat + + nginx: + extends: + file: docker-compose.yaml + service: nginx + networks: + - nginx-network + - supabase_network_tracecat + + # frontend: + # build: + # context: ./frontend + # container_name: frontend + # restart: unless-stopped + # depends_on: + # - api + # environment: + # NEXT_PUBLIC_APP_ENV: "development" + # NEXT_PUBLIC_APP_URL: ${NEXT_PUBLIC_APP_URL} + # NEXT_PUBLIC_SUPABASE_URL: ${NEXT_PUBLIC_SUPABASE_URL} + # NEXT_PUBLIC_SUPABASE_ANON_KEY: ${NEXT_PUBLIC_SUPABASE_ANON_KEY} + # NEXT_PUBLIC_SELF_HOSTED: ${NEXT_PUBLIC_SELF_HOSTED} + # ports: + # - "3000:3000" + # networks: + # - nginx-network + # - supabase_network_tracecat + +networks: + nginx-network: + supabase_network_tracecat: + external: true diff --git a/docker-compose.yaml b/docker-compose.yaml index f6b66f2a3..8283aeba1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -12,10 +12,12 @@ services: TRACECAT__SIGNING_SECRET: ${TRACECAT__SIGNING_SECRET} TRACECAT__SERVICE_KEY: ${TRACECAT__SERVICE_KEY} TRACECAT__DB_ENCRYPTION_KEY: ${TRACECAT__DB_ENCRYPTION_KEY} + TRACECAT__RUNNER_URL: ${TRACECAT__RUNNER_URL} SUPABASE_JWT_SECRET: ${SUPABASE_JWT_SECRET} SUPABASE_JWT_ALGORITHM: ${SUPABASE_JWT_ALGORITHM} SUPABASE_PSQL_URL: ${SUPABASE_PSQL_URL} OPENAI_API_KEY: ${OPENAI_API_KEY} + LOG_LEVEL: ${LOG_LEVEL} container_name: api restart: unless-stopped depends_on: @@ -35,7 +37,10 @@ services: TRACECAT__SIGNING_SECRET: ${TRACECAT__SIGNING_SECRET} TRACECAT__SERVICE_KEY: ${TRACECAT__SERVICE_KEY} TRACECAT__DB_ENCRYPTION_KEY: ${TRACECAT__DB_ENCRYPTION_KEY} + TRACECAT__RUNNER_URL: ${TRACECAT__RUNNER_URL} OPENAI_API_KEY: ${OPENAI_API_KEY} + SUPABASE_PSQL_URL: ${SUPABASE_PSQL_URL} + LOG_LEVEL: ${LOG_LEVEL} container_name: runner restart: unless-stopped depends_on: diff --git a/frontend/.dockerignore b/frontend/.dockerignore new file mode 100644 index 000000000..493057bbb --- /dev/null +++ b/frontend/.dockerignore @@ -0,0 +1,3 @@ +node_modules +build +.next diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 000000000..e90f4125b --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,21 @@ +FROM node:21-slim AS base +ENV PNPM_HOME="/pnpm" +ENV PATH="$PNPM_HOME:$PATH" +RUN corepack enable +COPY . /app +WORKDIR /app + +FROM base AS prod-deps +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile + +FROM base AS builder +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile +RUN pnpm run build + +FROM base +COPY --from=prod-deps /app/node_modules /app/node_modules +COPY --from=builder /app/next.config.mjs ./ +COPY --from=builder /app/public ./public +COPY --from=builder /app/.next ./.next +EXPOSE 3000 +CMD [ "pnpm", "start" ] diff --git a/frontend/package.json b/frontend/package.json index c58c49107..0e2e6b1f7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -11,7 +11,7 @@ "format": "pnpm prettier --check --ignore-path .gitignore .", "lint:fix": "next lint --fix", "lint": "next lint", - "prepare": "cd .. && husky frontend/.husky", + "prepare": "cd .. && husky frontend/.husky || true", "preview": "next build && next start", "start": "next start", "typecheck": "tsc --noEmit" diff --git a/tracecat/config.py b/tracecat/config.py index 69bd64093..88f9bef0a 100644 --- a/tracecat/config.py +++ b/tracecat/config.py @@ -8,3 +8,6 @@ TRACECAT__RUNNER_URL = os.environ.get("TRACECAT__RUNNER_URL", "http://runner:8000") TRACECAT__SERVICE_ROLES_WHITELIST = ["tracecat-runner", "tracecat-api"] +TRACECAT__SELF_HOSTED_DB_BACKEND = os.environ.get( + "TRACECAT__SELF_HOSTED_DB_BACKEND", "postgres" +) diff --git a/tracecat/db.py b/tracecat/db.py index 81e003be5..4e0ac93b8 100644 --- a/tracecat/db.py +++ b/tracecat/db.py @@ -14,7 +14,11 @@ from tracecat import auth from tracecat.auth import decrypt_key, encrypt_key -from tracecat.config import TRACECAT__APP_ENV, TRACECAT__RUNNER_URL +from tracecat.config import ( + TRACECAT__APP_ENV, + TRACECAT__RUNNER_URL, + TRACECAT__SELF_HOSTED_DB_BACKEND, +) from tracecat.labels.mitre import get_mitre_tactics_techniques STORAGE_PATH = Path( @@ -32,7 +36,11 @@ if TRACECAT__APP_ENV == "prod": TRACECAT__DB_URI = os.environ["TRACECAT__DB_URI"] else: - TRACECAT__DB_URI = f"sqlite:////{STORAGE_PATH}/database.db" + # Attempt to use supabase first + if TRACECAT__SELF_HOSTED_DB_BACKEND == "postgres": + TRACECAT__DB_URI = os.environ["SUPABASE_PSQL_URL"] + else: + TRACECAT__DB_URI = f"sqlite:////{STORAGE_PATH}/database.db" class User(SQLModel, table=True): @@ -215,7 +223,14 @@ def create_db_engine() -> Engine: "connect_args": {"sslmode": "require"}, } else: - engine_kwargs = {"connect_args": {"check_same_thread": False}} + if TRACECAT__SELF_HOSTED_DB_BACKEND == "postgres": + engine_kwargs = { + "pool_timeout": 30, + "pool_recycle": 3600, + "connect_args": {"sslmode": "disable"}, + } + else: + engine_kwargs = {"connect_args": {"check_same_thread": False}} engine = create_engine(TRACECAT__DB_URI, **engine_kwargs) return engine