From 9f9d5a9b717da9157942d6940c81bb927cf0c14d Mon Sep 17 00:00:00 2001 From: eden wang <64514273+eyw520@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:25:31 -0500 Subject: [PATCH 01/47] fix: remove `Request Body` field from `llms-full.txt` generation. (#2043) --- CONTRIBUTING.md | 14 -------------- .../bundle/src/server/getMarkdownForPath.ts | 7 ------- 2 files changed, 21 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dc7c185353..2a21943ea8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,20 +54,6 @@ To run the integration tests: `pnpm test:ete`. Many of our tests rely on [Vite](https://vitejs.dev/) snapshots. To rewrite snapshots, use `-u`: `pnpm test -u` and `pnpm test:ete -u`. -### CLI - -To build the CLI, run either: - -- `pnpm dist:cli:dev`. This compiles and bundles a CLI that communicates with our dev cloud environment. The CLI is outputted to `packages/cli/cli/dist/dev/cli.cjs`. - -- `pnpm dist:cli:prod`. This compiles and bundles a CLI that communicates with our production cloud environment. The CLI is outputted to `packages/cli/cli/dist/prod/cli.cjs`. - -To run the locally-generated CLI, run: - -``` -FERN_NO_VERSION_REDIRECTION=true node -``` - ### Docs UI To build and run the NextJS docs UI, run: diff --git a/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts b/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts index 495281f30d..1071ea9552 100644 --- a/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts +++ b/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts @@ -162,13 +162,6 @@ export function endpointDefinitionToMarkdown( `- ${pascalCaseHeaderKey(param.key)}${getShorthand(param.valueShape, types, param.description)}` ) .join("\n"), - endpoint.requests?.[0] != null ? "## Request Body" : undefined, - typeof endpoint.requests?.[0]?.description === "string" - ? endpoint.requests?.[0]?.description - : undefined, - endpoint.requests?.[0] != null - ? `\`\`\`json\n${JSON.stringify(endpoint.requests[0].body)}\n\`\`\`` - : undefined, endpoint.responseHeaders?.length ? "## Response Headers" : undefined, endpoint.responseHeaders ?.map( From 4e37e5131eadb9b351588d700e9615ea9feae826 Mon Sep 17 00:00:00 2001 From: Andrew Jiang Date: Tue, 21 Jan 2025 18:46:18 -0500 Subject: [PATCH 02/47] fix: safari css behavior for accordion and backdrop blur (#2045) --- packages/fern-docs/components/src/accordion.scss | 5 +++++ .../fern-docs/search-ui/src/components/desktop/desktop.scss | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/fern-docs/components/src/accordion.scss b/packages/fern-docs/components/src/accordion.scss index d2ef736bde..277a6995a3 100644 --- a/packages/fern-docs/components/src/accordion.scss +++ b/packages/fern-docs/components/src/accordion.scss @@ -15,6 +15,11 @@ @apply hover:bg-tag-default flex cursor-pointer items-center gap-3 rounded-[inherit] p-4 transition-colors hover:transition-none data-[state=open]:rounded-b-none; @apply list-none; + // Hide the default arrow on WebKit browsers + &::-webkit-details-marker { + @apply hidden; + } + .fern-accordion-trigger-arrow { @apply t-muted ease-shift size-4 shrink-0 transition-transform duration-[400ms]; } diff --git a/packages/fern-docs/search-ui/src/components/desktop/desktop.scss b/packages/fern-docs/search-ui/src/components/desktop/desktop.scss index db91b4692d..c37e399873 100644 --- a/packages/fern-docs/search-ui/src/components/desktop/desktop.scss +++ b/packages/fern-docs/search-ui/src/components/desktop/desktop.scss @@ -2,7 +2,6 @@ overflow: hidden; border: 1px solid var(--grayscale-a6); background-color: var(--grayscale-a1); - backdrop-filter: blur(40px); border-radius: 0.75rem; transition: transform 100ms ease; position: relative; From cc20642b52c029ad2accf4ff2cd169269dc8b77b Mon Sep 17 00:00:00 2001 From: Deep Singhvi Date: Wed, 22 Jan 2025 13:01:27 -0500 Subject: [PATCH 03/47] chore(docs): start instrumenting braintrust (#2049) --- packages/fern-docs/bundle/package.json | 1 + .../app/api/fern-docs/search/v2/chat/route.ts | 111 ++-- pnpm-lock.yaml | 486 +++++++++++++++++- turbo.json | 1 + 4 files changed, 534 insertions(+), 65 deletions(-) diff --git a/packages/fern-docs/bundle/package.json b/packages/fern-docs/bundle/package.json index c4769212b5..fabf290a9a 100644 --- a/packages/fern-docs/bundle/package.json +++ b/packages/fern-docs/bundle/package.json @@ -57,6 +57,7 @@ "@workos-inc/node": "^7.31.0", "ai": "^4.0.18", "algoliasearch": "^5.13.0", + "braintrust": "^0.0.182", "cssnano": "^6.0.3", "es-toolkit": "^1.27.0", "esbuild": "0.20.2", diff --git a/packages/fern-docs/bundle/src/app/api/fern-docs/search/v2/chat/route.ts b/packages/fern-docs/bundle/src/app/api/fern-docs/search/v2/chat/route.ts index 5908f03894..daae0ba3f9 100644 --- a/packages/fern-docs/bundle/src/app/api/fern-docs/search/v2/chat/route.ts +++ b/packages/fern-docs/bundle/src/app/api/fern-docs/search/v2/chat/route.ts @@ -13,6 +13,7 @@ import { } from "@fern-docs/search-server/turbopuffer"; import { COOKIE_FERN_TOKEN, withoutStaging } from "@fern-docs/utils"; import { embed, EmbeddingModel, streamText, tool } from "ai"; +import { initLogger, traced, wrapAISDKModel } from "braintrust"; import { cookies } from "next/headers"; import { NextRequest, NextResponse } from "next/server"; import { z } from "zod"; @@ -21,12 +22,18 @@ export const maxDuration = 60; export const revalidate = 0; export async function POST(req: NextRequest) { + const _logger = initLogger({ + projectName: "Braintrust Evaluation", + apiKey: process.env.BRAINTRUST_API_KEY, + }); const bedrock = createAmazonBedrock({ region: "us-west-2", accessKeyId: process.env.AWS_ACCESS_KEY_ID, secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, }); - const languageModel = bedrock("anthropic.claude-3-5-sonnet-20241022-v2:0"); + const languageModel = wrapAISDKModel( + bedrock("anthropic.claude-3-5-sonnet-20241022-v2:0") + ); const openai = createOpenAI({ apiKey: openaiApiKey() }); const embeddingModel = openai.embedding("text-embedding-3-small"); @@ -78,65 +85,53 @@ export async function POST(req: NextRequest) { date: new Date().toDateString(), documents, }); - - const result = streamText({ - model: languageModel, - system, - messages, - maxSteps: 10, - maxRetries: 3, - tools: { - search: tool({ - description: - "Search the knowledge base for the user's query. Semantic search is enabled.", - parameters: z.object({ - query: z.string(), + const result = traced(() => + streamText({ + model: languageModel, + system, + messages, + maxSteps: 10, + maxRetries: 3, + tools: { + search: tool({ + description: + "Search the knowledge base for the user's query. Semantic search is enabled.", + parameters: z.object({ + query: z.string(), + }), + async execute({ query }) { + const response = await runQueryTurbopuffer(query, { + embeddingModel, + namespace, + authed: user != null, + roles: user?.roles ?? [], + }); + return response.map((hit) => { + const { domain, pathname, hash } = hit.attributes; + const url = `https://${domain}${pathname}${hash ?? ""}`; + return { url, ...hit.attributes }; + }); + }, }), - async execute({ query }) { - const response = await runQueryTurbopuffer(query, { - embeddingModel, - namespace, - authed: user != null, - roles: user?.roles ?? [], - }); - return response.map((hit) => { - const { domain, pathname, hash } = hit.attributes; - const url = `https://${domain}${pathname}${hash ?? ""}`; - return { url, ...hit.attributes }; - }); - }, - }), - }, - experimental_telemetry: { - isEnabled: true, - recordInputs: true, - recordOutputs: true, - functionId: "ask_ai_chat", - metadata: { - domain, - languageModel: languageModel.modelId, - embeddingModel: embeddingModel.modelId, - db: "turbopuffer", - namespace, }, - }, - onFinish: async (e) => { - const end = Date.now(); - await track("ask_ai", { - languageModel: languageModel.modelId, - embeddingModel: embeddingModel.modelId, - durationMs: end - start, - domain, - namespace, - numToolCalls: e.toolCalls.length, - finishReason: e.finishReason, - ...e.usage, - }); - e.warnings?.forEach((warning) => { - console.warn(warning); - }); - }, - }); + onFinish: async (e) => { + const end = Date.now(); + await track("ask_ai", { + languageModel: languageModel.modelId, + embeddingModel: embeddingModel.modelId, + durationMs: end - start, + domain, + namespace, + numToolCalls: e.toolCalls.length, + finishReason: e.finishReason, + ...e.usage, + }); + e.warnings?.forEach((warning) => { + console.warn(warning); + }); + }, + }) + ); const response = result.toDataStreamResponse(); response.headers.set("Access-Control-Allow-Origin", "*"); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fd7b6be066..0e42a993f8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -273,7 +273,7 @@ importers: version: 3.0.3 tsup: specifier: ^8.3.5 - version: 8.3.5(@swc/core@1.5.7)(jiti@1.21.7)(postcss@8.4.31)(tsx@4.19.2)(typescript@5.7.2)(yaml@2.4.2) + version: 8.3.5(@swc/core@1.5.7)(jiti@1.21.7)(postcss@8.4.31)(tsx@4.19.2)(typescript@5.7.2)(yaml@2.7.0) typescript: specifier: 5.7.2 version: 5.7.2 @@ -789,6 +789,9 @@ importers: algoliasearch: specifier: ^5.13.0 version: 5.13.0 + braintrust: + specifier: ^0.0.182 + version: 0.0.182(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.682.0))(react@18.3.1)(sswr@2.1.0(svelte@5.19.2))(svelte@5.19.2)(vue@3.5.13(typescript@5.7.2))(zod@3.23.8) cssnano: specifier: ^6.0.3 version: 6.1.2(postcss@8.4.31) @@ -2486,7 +2489,7 @@ importers: version: 3.4.2 tsup: specifier: ^8.3.5 - version: 8.3.5(@swc/core@1.5.7)(jiti@1.21.7)(postcss@8.4.31)(tsx@4.19.2)(typescript@5.7.2)(yaml@2.4.2) + version: 8.3.5(@swc/core@1.5.7)(jiti@1.21.7)(postcss@8.4.31)(tsx@4.19.2)(typescript@5.7.2)(yaml@2.7.0) typescript: specifier: 5.7.2 version: 5.7.2 @@ -3061,6 +3064,15 @@ packages: peerDependencies: zod: ^3.0.0 + '@ai-sdk/provider-utils@1.0.22': + resolution: {integrity: sha512-YHK2rpj++wnLVc9vPGzGFP3Pjeld2MwhKinetA0zKXOoHAT/Jit5O8kZsxcSlJPu9wvcGT1UGZEjZrtO7PfFOQ==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + peerDependenciesMeta: + zod: + optional: true + '@ai-sdk/provider-utils@2.0.4': resolution: {integrity: sha512-GMhcQCZbwM6RoZCri0MWeEWXRt/T+uCxsmHEsTwNvEH3GDjNzchfX25C8ftry2MeEOOn6KfqCLSKomcgK6RoOg==} engines: {node: '>=18'} @@ -3079,6 +3091,10 @@ packages: zod: optional: true + '@ai-sdk/provider@0.0.26': + resolution: {integrity: sha512-dQkfBDs2lTYpKM8389oopPdQgIU007GQyCbuPPrV+K6MtSII3HBfE0stUIMXUb44L+LK1t6GXPP7wjSzjO6uKg==} + engines: {node: '>=18'} + '@ai-sdk/provider@1.0.2': resolution: {integrity: sha512-YYtP6xWQyaAf5LiWLJ+ycGTOeBLWrED7LUrvc+SQIWhGaneylqbaGsyQL7VouQUeQ4JZ1qKYZuhmi3W56HADPA==} engines: {node: '>=18'} @@ -3087,6 +3103,18 @@ packages: resolution: {integrity: sha512-lJi5zwDosvvZER3e/pB8lj1MN3o3S7zJliQq56BRr4e9V3fcRyFtwP0JRxaRS5vHYX3OJ154VezVoQNrk0eaKw==} engines: {node: '>=18'} + '@ai-sdk/react@0.0.70': + resolution: {integrity: sha512-GnwbtjW4/4z7MleLiW+TOZC2M29eCg1tOUpuEiYFMmFNZK8mkrqM0PFZMo6UsYeUYMWqEOOcPOU9OQVJMJh7IQ==} + engines: {node: '>=18'} + peerDependencies: + react: 18.3.1 + zod: ^3.0.0 + peerDependenciesMeta: + react: + optional: true + zod: + optional: true + '@ai-sdk/react@1.0.6': resolution: {integrity: sha512-8Hkserq0Ge6AEi7N4hlv2FkfglAGbkoAXEZ8YSp255c3PbnZz6+/5fppw+aROmZMOfNwallSRuy1i/iPa2rBpQ==} engines: {node: '>=18'} @@ -3099,6 +3127,33 @@ packages: zod: optional: true + '@ai-sdk/solid@0.0.54': + resolution: {integrity: sha512-96KWTVK+opdFeRubqrgaJXoNiDP89gNxFRWUp0PJOotZW816AbhUf4EnDjBjXTLjXL1n0h8tGSE9sZsRkj9wQQ==} + engines: {node: '>=18'} + peerDependencies: + solid-js: ^1.7.7 + peerDependenciesMeta: + solid-js: + optional: true + + '@ai-sdk/svelte@0.0.57': + resolution: {integrity: sha512-SyF9ItIR9ALP9yDNAD+2/5Vl1IT6kchgyDH8xkmhysfJI6WrvJbtO1wdQ0nylvPLcsPoYu+cAlz1krU4lFHcYw==} + engines: {node: '>=18'} + peerDependencies: + svelte: ^3.0.0 || ^4.0.0 || ^5.0.0 + peerDependenciesMeta: + svelte: + optional: true + + '@ai-sdk/ui-utils@0.0.50': + resolution: {integrity: sha512-Z5QYJVW+5XpSaJ4jYCCAVG7zIAuKOOdikhgpksneNmKvx61ACFaf98pmOd+xnjahl0pIlc/QIe6O4yVaJ1sEaw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + peerDependenciesMeta: + zod: + optional: true + '@ai-sdk/ui-utils@1.0.5': resolution: {integrity: sha512-DGJSbDf+vJyWmFNexSPUsS1AAy7gtsmFmoSyNbNbJjwl9hRIf2dknfA1V0ahx6pg3NNklNYFm53L8Nphjovfvg==} engines: {node: '>=18'} @@ -3108,6 +3163,15 @@ packages: zod: optional: true + '@ai-sdk/vue@0.0.59': + resolution: {integrity: sha512-+ofYlnqdc8c4F6tM0IKF0+7NagZRAiqBJpGDJ+6EYhDW8FHLUP/JFBgu32SjxSxC6IKFZxEnl68ZoP/Z38EMlw==} + engines: {node: '>=18'} + peerDependencies: + vue: ^3.3.4 + peerDependenciesMeta: + vue: + optional: true + '@algolia/cache-browser-local-storage@4.24.0': resolution: {integrity: sha512-t63W9BnoXVrGy9iYHBgObNXqYXM3tYXCjDSHeNwnsc324r4o5UiVKUiAB4THQ5z9U5hTj6qUvwg/Ez43ZD85ww==} @@ -3272,6 +3336,11 @@ packages: react: 18.3.1 react-dom: 18.3.1 + '@asteasolutions/zod-to-openapi@6.4.0': + resolution: {integrity: sha512-8cxfF7AHHx2PqnN4Cd8/O8CBu/nVYJP9DpnfVLW3BFb66VJDnqI/CczZnkqMc3SNh6J9GiX7JbJ5T4BSP4HZ2Q==} + peerDependencies: + zod: ^3.20.2 + '@aws-cdk/asset-awscli-v1@2.2.202': resolution: {integrity: sha512-JqlF0D4+EVugnG5dAsNZMqhu3HW7ehOXm5SDMxMbXNDMdsF0pxtQKNHRl52z1U9igsHmaFpUgSGjbhAJ+0JONg==} @@ -4343,6 +4412,9 @@ packages: '@braintree/sanitize-url@7.1.0': resolution: {integrity: sha512-o+UlMLt49RvtCASlOMW0AkHnabN9wR9rwCCherxO0yG4Npy34GkvrAqdXQvrhNs+jh+gkK8gB8Lf05qL/O7KWg==} + '@braintrust/core@0.0.76': + resolution: {integrity: sha512-kCefHWOqxVtLhNS9DPW9itG99M1ivfRfkPAF8RIJMa1Wi13e6Yq7iNvOK//gdZr+ekj24VlyupaPn1B47NA+ng==} + '@cfworker/json-schema@4.0.3': resolution: {integrity: sha512-ZykIcDTVv5UNmKWSTLAs3VukO6NDJkkSKxrgUTDPBkAlORVT3H9n5DbRjRl8xIotklscHdbLIa0b9+y3mQq73g==} @@ -7269,7 +7341,7 @@ packages: resolution: {integrity: sha512-GWrNeElMYHO8FVETjW205u2s9IXFs46fmVKY8T1dHgksCm3JV8w4k14gM2eaZbOUOH/tGcOuz5YbqJl8iKkA8w==} engines: {node: '>=18.0.0'} peerDependencies: - next: npm:@fern-api/next@14.2.9-fork.2 + next: ^13.5.0 || ^14.0.0 || ^15.0.0 react: 18.3.1 react-dom: 18.3.1 storybook: ^8.4.4 @@ -8005,6 +8077,15 @@ packages: '@opentelemetry/api': optional: true + '@vercel/functions@1.5.2': + resolution: {integrity: sha512-9XuynFoM/J1X+LSahgjhuAZCbZ96vm9mpXapCkSS1MX890U7zLh7n2RW/2KLNuxsXt8u8h2dOCw+Njtg+7pXgQ==} + engines: {node: '>= 16'} + peerDependencies: + '@aws-sdk/credential-provider-web-identity': '*' + peerDependenciesMeta: + '@aws-sdk/credential-provider-web-identity': + optional: true + '@vercel/kv@2.0.0': resolution: {integrity: sha512-zdVrhbzZBYo5d1Hfn4bKtqCeKf0FuzW8rSHauzQVMUgv1+1JOwof2mWcBuI+YMJy8s0G0oqAUfQ7HgUDzb8EbA==} engines: {node: '>=14.6'} @@ -8083,18 +8164,47 @@ packages: '@vue/compiler-core@3.4.27': resolution: {integrity: sha512-E+RyqY24KnyDXsCuQrI+mlcdW3ALND6U7Gqa/+bVwbcpcR3BRRIckFoz7Qyd4TTlnugtwuI7YgjbvsLmxb+yvg==} + '@vue/compiler-core@3.5.13': + resolution: {integrity: sha512-oOdAkwqUfW1WqpwSYJce06wvt6HljgY3fGeM9NcVA1HaYOij3mZG9Rkysn0OHuyUAGMbEbARIpsG+LPVlBJ5/Q==} + '@vue/compiler-dom@3.4.27': resolution: {integrity: sha512-kUTvochG/oVgE1w5ViSr3KUBh9X7CWirebA3bezTbB5ZKBQZwR2Mwj9uoSKRMFcz4gSMzzLXBPD6KpCLb9nvWw==} + '@vue/compiler-dom@3.5.13': + resolution: {integrity: sha512-ZOJ46sMOKUjO3e94wPdCzQ6P1Lx/vhp2RSvfaab88Ajexs0AHeV0uasYhi99WPaogmBlRHNRuly8xV75cNTMDA==} + '@vue/compiler-sfc@3.4.27': resolution: {integrity: sha512-nDwntUEADssW8e0rrmE0+OrONwmRlegDA1pD6QhVeXxjIytV03yDqTey9SBDiALsvAd5U4ZrEKbMyVXhX6mCGA==} + '@vue/compiler-sfc@3.5.13': + resolution: {integrity: sha512-6VdaljMpD82w6c2749Zhf5T9u5uLBWKnVue6XWxprDobftnletJ8+oel7sexFfM3qIxNmVE7LSFGTpv6obNyaQ==} + '@vue/compiler-ssr@3.4.27': resolution: {integrity: sha512-CVRzSJIltzMG5FcidsW0jKNQnNRYC8bT21VegyMMtHmhW3UOI7knmUehzswXLrExDLE6lQCZdrhD4ogI7c+vuw==} + '@vue/compiler-ssr@3.5.13': + resolution: {integrity: sha512-wMH6vrYHxQl/IybKJagqbquvxpWCuVYpoUJfCqFZwa/JY1GdATAQ+TgVtgrwwMZ0D07QhA99rs/EAAWfvG6KpA==} + + '@vue/reactivity@3.5.13': + resolution: {integrity: sha512-NaCwtw8o48B9I6L1zl2p41OHo/2Z4wqYGGIK1Khu5T7yxrn+ATOixn/Udn2m+6kZKB/J7cuT9DbWWhRxqixACg==} + + '@vue/runtime-core@3.5.13': + resolution: {integrity: sha512-Fj4YRQ3Az0WTZw1sFe+QDb0aXCerigEpw418pw1HBUKFtnQHWzwojaukAs2X/c9DQz4MQ4bsXTGlcpGxU/RCIw==} + + '@vue/runtime-dom@3.5.13': + resolution: {integrity: sha512-dLaj94s93NYLqjLiyFzVs9X6dWhTdAlEAciC3Moq7gzAc13VJUdCnjjRurNM6uTLFATRHexHCTu/Xp3eW6yoog==} + + '@vue/server-renderer@3.5.13': + resolution: {integrity: sha512-wAi4IRJV/2SAW3htkTlB+dHeRmpTiVIK1OGLWV1yeStVSebSQQOwGwIq0D3ZIoBj2C2qpgz5+vX9iEBkTdk5YA==} + peerDependencies: + vue: 3.5.13 + '@vue/shared@3.4.27': resolution: {integrity: sha512-DL3NmY2OFlqmYYrzp39yi3LDkKxa5vZVwxWdQ3rG0ekuWscHraeIbnI8t+aZK7qhYqEqWKTUdijadunb9pnrgA==} + '@vue/shared@3.5.13': + resolution: {integrity: sha512-/hnE/qP5ZoGpol0a5mDi45bOd7t3tjYJBjsgCsivow7D48cJeV5l05RD82lPqi7gRiphZM37rnhW1l6ZoCNNnQ==} + '@webassemblyjs/ast@1.12.1': resolution: {integrity: sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==} @@ -8587,6 +8697,11 @@ packages: peerDependencies: acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + acorn-typescript@1.4.13: + resolution: {integrity: sha512-xsc9Xv0xlVfwp2o7sQ+GCQ1PgbkdcpWdTzrwXxO3xDMTAywVS3oXVOcOHuRjAPkS4P9b+yc/qNF15460v+jp4Q==} + peerDependencies: + acorn: '>=8.9.0' + acorn-walk@8.3.2: resolution: {integrity: sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==} engines: {node: '>=0.4.0'} @@ -8621,6 +8736,27 @@ packages: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} engines: {node: '>=8'} + ai@3.4.33: + resolution: {integrity: sha512-plBlrVZKwPoRTmM8+D1sJac9Bq8eaa2jiZlHLZIWekKWI1yMWYZvCCEezY9ASPwRhULYDJB2VhKOBUUeg3S5JQ==} + engines: {node: '>=18'} + peerDependencies: + openai: ^4.42.0 + react: 18.3.1 + sswr: ^2.1.0 + svelte: ^3.0.0 || ^4.0.0 || ^5.0.0 + zod: ^3.0.0 + peerDependenciesMeta: + openai: + optional: true + react: + optional: true + sswr: + optional: true + svelte: + optional: true + zod: + optional: true + ai@4.0.18: resolution: {integrity: sha512-BTWzalLNE1LQphEka5xzJXDs5v4xXy1Uzr7dAVk+C/CnO3WNpuMBgrCymwUv0VrWaWc8xMQuh+OqsT7P7JyekQ==} engines: {node: '>=18'} @@ -9061,6 +9197,12 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} + braintrust@0.0.182: + resolution: {integrity: sha512-97shS1R8+821l+Ny6ZPt5h0ppG8k4sNEvWx3J388svObPV5n6Vz/zcMzplk7HCGKwz/QyNJL+8NJyUB/v00IAQ==} + hasBin: true + peerDependencies: + zod: ^3.0.0 + brorand@1.1.0: resolution: {integrity: sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==} @@ -9347,6 +9489,10 @@ packages: resolution: {integrity: sha512-p+hyTPxSZWG1c3Qy1DLBoGZhpeA3Y6AMlKrtbGpMMSKpezbSLel8gW4e5You4FNlHb3wS/M1JU594OAWe/Totg==} engines: {node: '>=10.0'} + cli-progress@3.12.0: + resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==} + engines: {node: '>=4'} + cli-spinners@2.9.2: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} engines: {node: '>=6'} @@ -10579,6 +10725,9 @@ packages: jiti: optional: true + esm-env@1.2.2: + resolution: {integrity: sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==} + esniff@1.1.3: resolution: {integrity: sha512-SLBLpfE7xWgF/HbzhVuAwqnJDRqSCNZqcqaIMVm+f+PbTp1kFRWu6BuT83SATb4Tp+ovr+S+u7vDH7/UErAOkw==} engines: {node: '>=0.10'} @@ -10605,6 +10754,9 @@ packages: resolution: {integrity: sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==} engines: {node: '>=0.10'} + esrap@1.4.3: + resolution: {integrity: sha512-Xddc1RsoFJ4z9nR7W7BFaEPIp4UXoeQ0+077UdWLxbafMQFyU79sQJMk7kxNgRwQ9/aVgaKacCHC2pUACGwmYw==} + esrecurse@4.3.0: resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} engines: {node: '>=4.0'} @@ -10675,6 +10827,10 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} + eventsource-parser@1.1.2: + resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==} + engines: {node: '>=14.18'} + eventsource-parser@3.0.0: resolution: {integrity: sha512-T1C0XCUimhxVQzW4zFipdx0SficT651NnkR0ZSH3yQwh+mFMdLfgjABVi4YtMTtaL4s168593DaoaRLMqryavA==} engines: {node: '>=18.0.0'} @@ -11940,6 +12096,9 @@ packages: is-reference@3.0.2: resolution: {integrity: sha512-v3rht/LgVcsdZa3O2Nqs+NMowLOxeOm7Ay9+/ARQ2F+qEoANRcqrjAZKGN0v8ymUetZGgkp26LTnGT7H0Qo9Pg==} + is-reference@3.0.3: + resolution: {integrity: sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==} + is-regex@1.1.4: resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==} engines: {node: '>= 0.4'} @@ -12575,6 +12734,9 @@ packages: localforage@1.10.0: resolution: {integrity: sha512-14/H1aX7hzBBmmh7sGPd+AOMkkIrHM3Z1PAyGgZigA1H1p5O5ANnMyWzvpAETtG68/dC4pC0ncy3+PPGzXZHPg==} + locate-character@3.0.0: + resolution: {integrity: sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==} + locate-path@5.0.0: resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} engines: {node: '>=8'} @@ -13522,6 +13684,9 @@ packages: openapi-types@12.1.3: resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} + openapi3-ts@4.4.0: + resolution: {integrity: sha512-9asTNB9IkKEzWMcHmVZE7Ts3kC9G7AFHfs8i7caD8HbI76gEjdkId4z/AkP83xdZsH7PLAnnbl47qZkXuxpArw==} + opener@1.5.2: resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} hasBin: true @@ -15230,6 +15395,10 @@ packages: resolution: {integrity: sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==} engines: {node: '>=12'} + slugify@1.6.6: + resolution: {integrity: sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==} + engines: {node: '>=8.0.0'} + sonner@1.5.0: resolution: {integrity: sha512-FBjhG/gnnbN6FY0jaNnqZOMmB73R+5IiyYAw8yBj7L54ER7HB3fOSE5OFiQiE2iXWxeXKvg6fIP4LtVppHEdJA==} peerDependencies: @@ -15293,6 +15462,11 @@ packages: sprintf-kit@2.0.1: resolution: {integrity: sha512-2PNlcs3j5JflQKcg4wpdqpZ+AjhQJ2OZEo34NXDtlB0tIPG84xaaXhpA8XFacFiwjKA4m49UOYG83y3hbMn/gQ==} + sswr@2.1.0: + resolution: {integrity: sha512-Cqc355SYlTAaUt8iDPaC/4DPPXK925PePLMxyBKuWd5kKc5mwsG3nT9+Mq2tyguL5s7b4Jg+IRMpTRsNTAfpSQ==} + peerDependencies: + svelte: ^4.0.0 || ^5.0.0-next.0 + stable-hash@0.0.4: resolution: {integrity: sha512-LjdcbuBeLcdETCrPn9i8AYAZ1eCtu4ECAWtP7UleOiZ9LzVxRzzUZEoZ8zB24nhkQnDWyET0I+3sWokSDS3E7g==} @@ -15651,6 +15825,10 @@ packages: peerDependencies: react: 18.3.1 + svelte@5.19.2: + resolution: {integrity: sha512-Ww1uLgdX5MdQrAO5zfU1dWUh6zqiPR6uIbwqm8a+4eQ+tNEYHRPgypvKKfHh9lmTkmJ30PWZ2O5qX8aS+PblRQ==} + engines: {node: '>=18'} + svg-tags@1.0.0: resolution: {integrity: sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA==} @@ -15664,6 +15842,14 @@ packages: peerDependencies: react: 18.3.1 + swrev@4.0.0: + resolution: {integrity: sha512-LqVcOHSB4cPGgitD1riJ1Hh4vdmITOp+BkmfmXRh4hSF/t7EnS4iD+SOTmq7w5pPm/SiPeto4ADbKS6dHUDWFA==} + + swrv@1.1.0: + resolution: {integrity: sha512-pjllRDr2s0iTwiE5Isvip51dZGR7GjLH1gCSVyE8bQnbAx6xackXsFdojau+1O5u98yHF5V73HQGOFxKUXO9gQ==} + peerDependencies: + vue: '>=3.2.26 < 4' + symbol-observable@4.0.0: resolution: {integrity: sha512-b19dMThMV4HVFynSAM1++gBHAbk2Tc/osgLIBZMKsyqh34jb2e8Os7T6ZW/Bt3pJFdBTd2JwAnAAEQV7rSNvcQ==} engines: {node: '>=0.10'} @@ -16583,6 +16769,14 @@ packages: vscode-uri@3.0.8: resolution: {integrity: sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw==} + vue@3.5.13: + resolution: {integrity: sha512-wmeiSMxkZCSc+PM2w2VRsOYAZC8GdipNFRTsLSfodVqI9mbejKeXEGr8SckuLnrQPGe3oJN5c3K0vpoU9q/wCQ==} + peerDependencies: + typescript: 5.7.2 + peerDependenciesMeta: + typescript: + optional: true + w3c-xmlserializer@5.0.0: resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==} engines: {node: '>=18'} @@ -16865,6 +17059,11 @@ packages: engines: {node: '>= 14'} hasBin: true + yaml@2.7.0: + resolution: {integrity: sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==} + engines: {node: '>= 14'} + hasBin: true + yargs-parser@20.2.9: resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} engines: {node: '>=10'} @@ -16909,6 +17108,9 @@ packages: zen-observable@0.8.15: resolution: {integrity: sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ==} + zimmerframe@1.1.2: + resolution: {integrity: sha512-rAbqEGa8ovJy4pyBxZM70hg4pE6gDgaQ0Sl9M3enG3I0d6H4XSAM3GeNGLKnsBpuijUow064sf7ww1nutC5/3w==} + zip-stream@4.1.1: resolution: {integrity: sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==} engines: {node: '>= 10'} @@ -17011,6 +17213,15 @@ snapshots: '@ai-sdk/provider-utils': 2.0.4(zod@3.23.8) zod: 3.23.8 + '@ai-sdk/provider-utils@1.0.22(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 0.0.26 + eventsource-parser: 1.1.2 + nanoid: 3.3.8 + secure-json-parse: 2.7.0 + optionalDependencies: + zod: 3.23.8 + '@ai-sdk/provider-utils@2.0.4(zod@3.23.8)': dependencies: '@ai-sdk/provider': 1.0.2 @@ -17029,6 +17240,10 @@ snapshots: optionalDependencies: zod: 3.23.8 + '@ai-sdk/provider@0.0.26': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/provider@1.0.2': dependencies: json-schema: 0.4.0 @@ -17037,6 +17252,16 @@ snapshots: dependencies: json-schema: 0.4.0 + '@ai-sdk/react@0.0.70(react@18.3.1)(zod@3.23.8)': + dependencies: + '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8) + '@ai-sdk/ui-utils': 0.0.50(zod@3.23.8) + swr: 2.2.5(react@18.3.1) + throttleit: 2.1.0 + optionalDependencies: + react: 18.3.1 + zod: 3.23.8 + '@ai-sdk/react@1.0.6(react@18.3.1)(zod@3.23.8)': dependencies: '@ai-sdk/provider-utils': 2.0.4(zod@3.23.8) @@ -17047,6 +17272,33 @@ snapshots: react: 18.3.1 zod: 3.23.8 + '@ai-sdk/solid@0.0.54(zod@3.23.8)': + dependencies: + '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8) + '@ai-sdk/ui-utils': 0.0.50(zod@3.23.8) + transitivePeerDependencies: + - zod + + '@ai-sdk/svelte@0.0.57(svelte@5.19.2)(zod@3.23.8)': + dependencies: + '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8) + '@ai-sdk/ui-utils': 0.0.50(zod@3.23.8) + sswr: 2.1.0(svelte@5.19.2) + optionalDependencies: + svelte: 5.19.2 + transitivePeerDependencies: + - zod + + '@ai-sdk/ui-utils@0.0.50(zod@3.23.8)': + dependencies: + '@ai-sdk/provider': 0.0.26 + '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8) + json-schema: 0.4.0 + secure-json-parse: 2.7.0 + zod-to-json-schema: 3.24.1(zod@3.23.8) + optionalDependencies: + zod: 3.23.8 + '@ai-sdk/ui-utils@1.0.5(zod@3.23.8)': dependencies: '@ai-sdk/provider': 1.0.2 @@ -17055,6 +17307,16 @@ snapshots: optionalDependencies: zod: 3.23.8 + '@ai-sdk/vue@0.0.59(vue@3.5.13(typescript@5.7.2))(zod@3.23.8)': + dependencies: + '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8) + '@ai-sdk/ui-utils': 0.0.50(zod@3.23.8) + swrv: 1.1.0(vue@3.5.13(typescript@5.7.2)) + optionalDependencies: + vue: 3.5.13(typescript@5.7.2) + transitivePeerDependencies: + - zod + '@algolia/cache-browser-local-storage@4.24.0': dependencies: '@algolia/cache-common': 4.24.0 @@ -17376,6 +17638,11 @@ snapshots: transitivePeerDependencies: - '@internationalized/date' + '@asteasolutions/zod-to-openapi@6.4.0(zod@3.24.1)': + dependencies: + openapi3-ts: 4.4.0 + zod: 3.24.1 + '@aws-cdk/asset-awscli-v1@2.2.202': {} '@aws-cdk/asset-kubectl-v20@2.1.2': {} @@ -18259,6 +18526,16 @@ snapshots: '@smithy/types': 3.6.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.682.0)': + dependencies: + '@aws-sdk/client-sts': 3.682.0 + '@aws-sdk/core': 3.723.0 + '@aws-sdk/types': 3.723.0 + '@smithy/property-provider': 4.0.0 + '@smithy/types': 4.0.0 + tslib: 2.8.1 + optional: true + '@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.723.0)': dependencies: '@aws-sdk/client-sts': 3.723.0 @@ -19546,6 +19823,12 @@ snapshots: '@braintree/sanitize-url@7.1.0': {} + '@braintrust/core@0.0.76': + dependencies: + '@asteasolutions/zod-to-openapi': 6.4.0(zod@3.24.1) + uuid: 9.0.1 + zod: 3.24.1 + '@cfworker/json-schema@4.0.3': {} '@chevrotain/cst-dts-gen@11.0.3': @@ -24297,6 +24580,10 @@ snapshots: transitivePeerDependencies: - typescript + '@vercel/functions@1.5.2(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.682.0))': + optionalDependencies: + '@aws-sdk/credential-provider-web-identity': 3.723.0(@aws-sdk/client-sts@3.682.0) + '@vercel/kv@2.0.0': dependencies: '@upstash/redis': 1.34.0 @@ -24421,11 +24708,24 @@ snapshots: estree-walker: 2.0.2 source-map-js: 1.2.1 + '@vue/compiler-core@3.5.13': + dependencies: + '@babel/parser': 7.26.2 + '@vue/shared': 3.5.13 + entities: 4.5.0 + estree-walker: 2.0.2 + source-map-js: 1.2.1 + '@vue/compiler-dom@3.4.27': dependencies: '@vue/compiler-core': 3.4.27 '@vue/shared': 3.4.27 + '@vue/compiler-dom@3.5.13': + dependencies: + '@vue/compiler-core': 3.5.13 + '@vue/shared': 3.5.13 + '@vue/compiler-sfc@3.4.27': dependencies: '@babel/parser': 7.24.5 @@ -24438,13 +24738,54 @@ snapshots: postcss: 8.4.31 source-map-js: 1.2.0 + '@vue/compiler-sfc@3.5.13': + dependencies: + '@babel/parser': 7.26.2 + '@vue/compiler-core': 3.5.13 + '@vue/compiler-dom': 3.5.13 + '@vue/compiler-ssr': 3.5.13 + '@vue/shared': 3.5.13 + estree-walker: 2.0.2 + magic-string: 0.30.12 + postcss: 8.4.31 + source-map-js: 1.2.1 + '@vue/compiler-ssr@3.4.27': dependencies: '@vue/compiler-dom': 3.4.27 '@vue/shared': 3.4.27 + '@vue/compiler-ssr@3.5.13': + dependencies: + '@vue/compiler-dom': 3.5.13 + '@vue/shared': 3.5.13 + + '@vue/reactivity@3.5.13': + dependencies: + '@vue/shared': 3.5.13 + + '@vue/runtime-core@3.5.13': + dependencies: + '@vue/reactivity': 3.5.13 + '@vue/shared': 3.5.13 + + '@vue/runtime-dom@3.5.13': + dependencies: + '@vue/reactivity': 3.5.13 + '@vue/runtime-core': 3.5.13 + '@vue/shared': 3.5.13 + csstype: 3.1.3 + + '@vue/server-renderer@3.5.13(vue@3.5.13(typescript@5.7.2))': + dependencies: + '@vue/compiler-ssr': 3.5.13 + '@vue/shared': 3.5.13 + vue: 3.5.13(typescript@5.7.2) + '@vue/shared@3.4.27': {} + '@vue/shared@3.5.13': {} + '@webassemblyjs/ast@1.12.1': dependencies: '@webassemblyjs/helper-numbers': 1.11.6 @@ -25511,6 +25852,10 @@ snapshots: dependencies: acorn: 8.14.0 + acorn-typescript@1.4.13(acorn@8.14.0): + dependencies: + acorn: 8.14.0 + acorn-walk@8.3.2: {} acorn@8.11.3: {} @@ -25541,6 +25886,30 @@ snapshots: clean-stack: 2.2.0 indent-string: 4.0.0 + ai@3.4.33(react@18.3.1)(sswr@2.1.0(svelte@5.19.2))(svelte@5.19.2)(vue@3.5.13(typescript@5.7.2))(zod@3.23.8): + dependencies: + '@ai-sdk/provider': 0.0.26 + '@ai-sdk/provider-utils': 1.0.22(zod@3.23.8) + '@ai-sdk/react': 0.0.70(react@18.3.1)(zod@3.23.8) + '@ai-sdk/solid': 0.0.54(zod@3.23.8) + '@ai-sdk/svelte': 0.0.57(svelte@5.19.2)(zod@3.23.8) + '@ai-sdk/ui-utils': 0.0.50(zod@3.23.8) + '@ai-sdk/vue': 0.0.59(vue@3.5.13(typescript@5.7.2))(zod@3.23.8) + '@opentelemetry/api': 1.9.0 + eventsource-parser: 1.1.2 + json-schema: 0.4.0 + jsondiffpatch: 0.6.0 + secure-json-parse: 2.7.0 + zod-to-json-schema: 3.24.1(zod@3.23.8) + optionalDependencies: + react: 18.3.1 + sswr: 2.1.0(svelte@5.19.2) + svelte: 5.19.2 + zod: 3.23.8 + transitivePeerDependencies: + - solid-js + - vue + ai@4.0.18(react@18.3.1)(zod@3.23.8): dependencies: '@ai-sdk/provider': 1.0.2 @@ -26120,6 +26489,39 @@ snapshots: dependencies: fill-range: 7.1.1 + braintrust@0.0.182(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.682.0))(react@18.3.1)(sswr@2.1.0(svelte@5.19.2))(svelte@5.19.2)(vue@3.5.13(typescript@5.7.2))(zod@3.23.8): + dependencies: + '@ai-sdk/provider': 1.0.4 + '@braintrust/core': 0.0.76 + '@next/env': 14.2.9 + '@vercel/functions': 1.5.2(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.682.0)) + ai: 3.4.33(react@18.3.1)(sswr@2.1.0(svelte@5.19.2))(svelte@5.19.2)(vue@3.5.13(typescript@5.7.2))(zod@3.23.8) + argparse: 2.0.1 + chalk: 4.1.2 + cli-progress: 3.12.0 + dotenv: 16.4.5 + esbuild: 0.20.2 + eventsource-parser: 1.1.2 + graceful-fs: 4.2.11 + minimatch: 9.0.4 + mustache: 4.2.0 + pluralize: 8.0.0 + simple-git: 3.24.0 + slugify: 1.6.6 + source-map: 0.7.4 + uuid: 9.0.1 + zod: 3.23.8 + zod-to-json-schema: 3.24.1(zod@3.23.8) + transitivePeerDependencies: + - '@aws-sdk/credential-provider-web-identity' + - openai + - react + - solid-js + - sswr + - supports-color + - svelte + - vue + brorand@1.1.0: {} browser-assert@1.2.1: {} @@ -26437,6 +26839,10 @@ snapshots: timers-ext: 0.1.7 type: 2.7.2 + cli-progress@3.12.0: + dependencies: + string-width: 4.2.3 + cli-spinners@2.9.2: {} cli-sprintf-format@1.1.1: @@ -28012,6 +28418,8 @@ snapshots: transitivePeerDependencies: - supports-color + esm-env@1.2.2: {} + esniff@1.1.3: dependencies: d: 1.0.2 @@ -28038,6 +28446,10 @@ snapshots: dependencies: estraverse: 5.3.0 + esrap@1.4.3: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.0 + esrecurse@4.3.0: dependencies: estraverse: 5.3.0 @@ -28111,6 +28523,8 @@ snapshots: events@3.3.0: {} + eventsource-parser@1.1.2: {} + eventsource-parser@3.0.0: {} evp_bytestokey@1.0.3: @@ -29643,6 +30057,10 @@ snapshots: dependencies: '@types/estree': 1.0.6 + is-reference@3.0.3: + dependencies: + '@types/estree': 1.0.6 + is-regex@1.1.4: dependencies: call-bind: 1.0.7 @@ -30518,6 +30936,8 @@ snapshots: dependencies: lie: 3.1.1 + locate-character@3.0.0: {} + locate-path@5.0.0: dependencies: p-locate: 4.1.0 @@ -31947,6 +32367,10 @@ snapshots: openapi-types@12.1.3: {} + openapi3-ts@4.4.0: + dependencies: + yaml: 2.7.0 + opener@1.5.2: {} optimism@0.18.0: @@ -32336,14 +32760,14 @@ snapshots: postcss: 8.4.31 ts-node: 10.9.2(@swc/core@1.5.7)(@types/node@18.19.33)(typescript@5.7.2) - postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.4.31)(tsx@4.19.2)(yaml@2.4.2): + postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.4.31)(tsx@4.19.2)(yaml@2.7.0): dependencies: lilconfig: 3.1.1 optionalDependencies: jiti: 1.21.7 postcss: 8.4.31 tsx: 4.19.2 - yaml: 2.4.2 + yaml: 2.7.0 postcss-loader@8.1.1(postcss@8.4.31)(typescript@5.7.2)(webpack@5.94.0(@swc/core@1.5.7)(esbuild@0.20.2)): dependencies: @@ -33906,6 +34330,8 @@ snapshots: ansi-styles: 6.2.1 is-fullwidth-code-point: 4.0.0 + slugify@1.6.6: {} + sonner@1.5.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: react: 18.3.1 @@ -33968,6 +34394,11 @@ snapshots: dependencies: es5-ext: 0.10.64 + sswr@2.1.0(svelte@5.19.2): + dependencies: + svelte: 5.19.2 + swrev: 4.0.0 + stable-hash@0.0.4: {} stack-trace@0.0.10: {} @@ -34384,6 +34815,23 @@ snapshots: dependencies: react: 18.3.1 + svelte@5.19.2: + dependencies: + '@ampproject/remapping': 2.3.0 + '@jridgewell/sourcemap-codec': 1.5.0 + '@types/estree': 1.0.6 + acorn: 8.14.0 + acorn-typescript: 1.4.13(acorn@8.14.0) + aria-query: 5.3.2 + axobject-query: 4.1.0 + clsx: 2.1.1 + esm-env: 1.2.2 + esrap: 1.4.3 + is-reference: 3.0.3 + locate-character: 3.0.0 + magic-string: 0.30.12 + zimmerframe: 1.1.2 + svg-tags@1.0.0: {} svgo@3.3.2: @@ -34402,6 +34850,12 @@ snapshots: react: 18.3.1 use-sync-external-store: 1.2.2(react@18.3.1) + swrev@4.0.0: {} + + swrv@1.1.0(vue@3.5.13(typescript@5.7.2)): + dependencies: + vue: 3.5.13(typescript@5.7.2) + symbol-observable@4.0.0: {} symbol-tree@3.2.4: {} @@ -34791,7 +35245,7 @@ snapshots: tslib@2.8.1: {} - tsup@8.3.5(@swc/core@1.5.7)(jiti@1.21.7)(postcss@8.4.31)(tsx@4.19.2)(typescript@5.7.2)(yaml@2.4.2): + tsup@8.3.5(@swc/core@1.5.7)(jiti@1.21.7)(postcss@8.4.31)(tsx@4.19.2)(typescript@5.7.2)(yaml@2.7.0): dependencies: bundle-require: 5.0.0(esbuild@0.20.2) cac: 6.7.14 @@ -34801,7 +35255,7 @@ snapshots: esbuild: 0.20.2 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.4.31)(tsx@4.19.2)(yaml@2.4.2) + postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.4.31)(tsx@4.19.2)(yaml@2.7.0) resolve-from: 5.0.0 rollup: 4.24.3 source-map: 0.8.0-beta.0 @@ -35482,6 +35936,16 @@ snapshots: vscode-uri@3.0.8: {} + vue@3.5.13(typescript@5.7.2): + dependencies: + '@vue/compiler-dom': 3.5.13 + '@vue/compiler-sfc': 3.5.13 + '@vue/runtime-dom': 3.5.13 + '@vue/server-renderer': 3.5.13(vue@3.5.13(typescript@5.7.2)) + '@vue/shared': 3.5.13 + optionalDependencies: + typescript: 5.7.2 + w3c-xmlserializer@5.0.0: dependencies: xml-name-validator: 5.0.0 @@ -35904,6 +36368,8 @@ snapshots: yaml@2.4.2: {} + yaml@2.7.0: {} + yargs-parser@20.2.9: {} yargs-parser@21.1.1: {} @@ -35953,6 +36419,8 @@ snapshots: zen-observable@0.8.15: {} + zimmerframe@1.1.2: {} + zip-stream@4.1.1: dependencies: archiver-utils: 3.0.4 @@ -35969,6 +36437,10 @@ snapshots: dependencies: zod: 3.23.8 + zod-to-json-schema@3.24.1(zod@3.23.8): + dependencies: + zod: 3.23.8 + zod-to-json-schema@3.24.1(zod@3.24.1): dependencies: zod: 3.24.1 diff --git a/turbo.json b/turbo.json index 076cb7b733..0c31ac678e 100644 --- a/turbo.json +++ b/turbo.json @@ -53,6 +53,7 @@ "docs:build": { "dependsOn": ["^build", "^compile", "^docs:build"], "env": [ + "BRAINTRUST_API_KEY", "ALGOLIA_API_KEY", "ALGOLIA_APP_ID", "ALGOLIA_SEARCH_API_KEY", From 1db1d7aff63fb9432b4751df4bb679a3d151d3e2 Mon Sep 17 00:00:00 2001 From: Andrew Jiang Date: Wed, 22 Jan 2025 14:28:55 -0500 Subject: [PATCH 04/47] fix: add another test for mdx sanitizer (#2050) --- packages/fern-docs/mdx/package.json | 20 +- .../src/mdast-utils/mdast-from-markdown.ts | 5 +- packages/fern-docs/mdx/src/parse.ts | 5 +- .../__test__/remark-sanitize-acorn.test.ts | 5 + .../mdx/src/plugins/remark-sanitize-acorn.ts | 2 + .../__test__/sanitize-mdx-expression.test.ts | 204 ++++++++++------ .../src/sanitize/sanitize-mdx-expression.ts | 98 ++++++-- .../ui/src/mdx/bundlers/mdx-bundler.ts | 2 +- .../ui/src/mdx/bundlers/next-mdx-remote.ts | 2 +- pnpm-lock.yaml | 222 ++++++++++-------- 10 files changed, 350 insertions(+), 215 deletions(-) diff --git a/packages/fern-docs/mdx/package.json b/packages/fern-docs/mdx/package.json index 48b6fbfd77..57b768c725 100644 --- a/packages/fern-docs/mdx/package.json +++ b/packages/fern-docs/mdx/package.json @@ -39,33 +39,33 @@ "dependencies": { "@fern-api/fdr-sdk": "workspace:*", "@fern-api/ui-core-utils": "workspace:*", - "@types/estree": "^1.0.5", + "@types/estree": "^1.0.6", "@types/hast": "^3.0.4", - "@types/mdast": "^4.0.3", + "@types/mdast": "^4.0.4", "@types/unist": "^3.0.3", "collapse-white-space": "^2.1.0", "es-toolkit": "^1.27.0", - "estree-util-value-to-estree": "^3.1.2", + "estree-util-value-to-estree": "^3.2.1", "estree-walker": "^3.0.3", "github-slugger": "^2.0.0", "gray-matter": "^4.0.3", "hast-util-heading-rank": "^3.0.0", - "hast-util-to-estree": "^3.1.0", - "hast-util-to-string": "^3.0.0", + "hast-util-to-estree": "^3.1.1", + "hast-util-to-string": "^3.0.1", "hastscript": "^9.0.0", - "mdast-util-from-markdown": "^2.0.0", + "mdast-util-from-markdown": "^2.0.2", "mdast-util-gfm": "^3.0.0", "mdast-util-math": "^3.0.0", "mdast-util-mdx": "^3.0.0", - "mdast-util-mdx-jsx": "^3.1.0", + "mdast-util-mdx-jsx": "^3.2.0", "mdast-util-mdxjs-esm": "^2.0.1", - "mdast-util-to-hast": "^13.1.0", - "mdast-util-to-markdown": "^2.1.0", + "mdast-util-to-hast": "^13.2.0", + "mdast-util-to-markdown": "^2.1.2", "micromark-extension-gfm": "^3.0.0", "micromark-extension-math": "^3.1.0", "micromark-extension-mdxjs": "^3.0.0", "rehype-slug": "^6.0.0", - "style-to-object": "^1.0.5", + "style-to-object": "^1.0.8", "ts-essentials": "^10.0.1", "unist-util-visit": "^5.0.0", "vfile-message": "^4.0.2" diff --git a/packages/fern-docs/mdx/src/mdast-utils/mdast-from-markdown.ts b/packages/fern-docs/mdx/src/mdast-utils/mdast-from-markdown.ts index 40c237082a..b335c755fd 100644 --- a/packages/fern-docs/mdx/src/mdast-utils/mdast-from-markdown.ts +++ b/packages/fern-docs/mdx/src/mdast-utils/mdast-from-markdown.ts @@ -13,7 +13,10 @@ export function mdastFromMarkdown( format: "mdx" | "md" = "mdx" ): MdastRoot { if (format === "md") { - return fromMarkdown(content); + return fromMarkdown(content, { + extensions: [math(), gfm()], + mdastExtensions: [mathFromMarkdown(), gfmFromMarkdown()], + }); } else if (format === "mdx") { return fromMarkdown(content, { extensions: [mdxjs(), math(), gfm()], diff --git a/packages/fern-docs/mdx/src/parse.ts b/packages/fern-docs/mdx/src/parse.ts index 0fb0c1074e..94a4309b7c 100644 --- a/packages/fern-docs/mdx/src/parse.ts +++ b/packages/fern-docs/mdx/src/parse.ts @@ -41,7 +41,10 @@ export function toTree( jsxElements: string[]; esmElements: string[]; } { - content = sanitize ? sanitizeMdxExpression(sanitizeBreaks(content)) : content; + content = + sanitize && format === "mdx" + ? sanitizeMdxExpression(sanitizeBreaks(content))[0] + : content; const mdast = mdastFromMarkdown(content, format); diff --git a/packages/fern-docs/mdx/src/plugins/__test__/remark-sanitize-acorn.test.ts b/packages/fern-docs/mdx/src/plugins/__test__/remark-sanitize-acorn.test.ts index 8e7e65c82a..3cc5b04bb3 100644 --- a/packages/fern-docs/mdx/src/plugins/__test__/remark-sanitize-acorn.test.ts +++ b/packages/fern-docs/mdx/src/plugins/__test__/remark-sanitize-acorn.test.ts @@ -324,4 +324,9 @@ describe("remarkSanitizeAcorn", () => { const result = sanitizeAcorns("export const foo = await something();"); expect(result).toBe("\\{export const foo = await something();}\n"); }); + + it("should not escape frontmatter but escape other identifiers", () => { + const result = sanitizeAcorns("{frontmatter.foo} and {something}"); + expect(result).toBe("{frontmatter.foo} and \\{something}\n"); + }); }); diff --git a/packages/fern-docs/mdx/src/plugins/remark-sanitize-acorn.ts b/packages/fern-docs/mdx/src/plugins/remark-sanitize-acorn.ts index 810b67db14..1fae4d983b 100644 --- a/packages/fern-docs/mdx/src/plugins/remark-sanitize-acorn.ts +++ b/packages/fern-docs/mdx/src/plugins/remark-sanitize-acorn.ts @@ -14,6 +14,7 @@ interface Options { * Not allowed: Promise, crypto, etc. * * Note: `props` will be used by @mdx-js/esbuild + * and `frontmatter` is injected by @mdx-js/mdx */ const ALLOWED_GLOBAL_IDENTIFIERS: string[] = [ "Math", @@ -24,6 +25,7 @@ const ALLOWED_GLOBAL_IDENTIFIERS: string[] = [ "Map", "URL", "props", + "frontmatter", ]; /** diff --git a/packages/fern-docs/mdx/src/sanitize/__test__/sanitize-mdx-expression.test.ts b/packages/fern-docs/mdx/src/sanitize/__test__/sanitize-mdx-expression.test.ts index 213b23e06c..650df7938b 100644 --- a/packages/fern-docs/mdx/src/sanitize/__test__/sanitize-mdx-expression.test.ts +++ b/packages/fern-docs/mdx/src/sanitize/__test__/sanitize-mdx-expression.test.ts @@ -2,83 +2,112 @@ import { sanitizeMdxExpression } from "../sanitize-mdx-expression"; describe("sanitizeMdxExpression", () => { it("should escape base cases", () => { - expect(sanitizeMdxExpression("{")).toBe("\\{"); - expect(sanitizeMdxExpression("<")).toBe("\\<"); - expect(sanitizeMdxExpression("{{")).toBe("\\{\\{"); - expect(sanitizeMdxExpression("<<")).toBe("\\<\\<"); + expect(sanitizeMdxExpression("{")).toStrictEqual(["\\{", true]); + expect(sanitizeMdxExpression("<")).toStrictEqual(["\\<", true]); + expect(sanitizeMdxExpression("{{")).toStrictEqual(["\\{\\{", true]); + expect(sanitizeMdxExpression("<<")).toStrictEqual(["\\<\\<", true]); - expect(sanitizeMdxExpression("{a}{")).toBe("{a}\\{"); - expect(sanitizeMdxExpression("<")).toBe("\\<"); - expect(sanitizeMdxExpression("{{a}")).toBe("\\{{a}"); - expect(sanitizeMdxExpression("{a{}")).toBe("\\{a{}"); - expect(sanitizeMdxExpression("<")).toStrictEqual([ + "\\<", + true, + ]); + expect(sanitizeMdxExpression("{{a}")).toStrictEqual(["\\{{a}", true]); + expect(sanitizeMdxExpression("{a{}")).toStrictEqual(["\\{a{}", true]); + expect(sanitizeMdxExpression("")).toBe("\\<.>"); - expect(sanitizeMdxExpression("")).toBe("\\"); - expect(sanitizeMdxExpression("")).toBe("\\"); - expect(sanitizeMdxExpression("")).toBe("\\"); - expect(sanitizeMdxExpression("")).toBe("\\"); - expect(sanitizeMdxExpression("")).toBe("\\"); - expect(sanitizeMdxExpression("")).toBe("\\"); - expect(sanitizeMdxExpression("")).toBe("\\"); - expect(sanitizeMdxExpression("")).toBe("\\"); - expect(sanitizeMdxExpression("> ")).toBe("> \\"); + expect(sanitizeMdxExpression("")).toStrictEqual(["\\<.>", true]); + expect(sanitizeMdxExpression("")).toStrictEqual(["\\", true]); + expect(sanitizeMdxExpression("")).toStrictEqual(["\\", true]); + expect(sanitizeMdxExpression("")).toStrictEqual(["\\", true]); + expect(sanitizeMdxExpression("")).toStrictEqual(["\\", true]); + expect(sanitizeMdxExpression("")).toStrictEqual(["\\", true]); + expect(sanitizeMdxExpression("")).toStrictEqual(["\\", true]); + expect(sanitizeMdxExpression("")).toStrictEqual(["\\", true]); + expect(sanitizeMdxExpression("")).toStrictEqual(["\\", true]); + expect(sanitizeMdxExpression("> ")).toStrictEqual([ + "> \\", + true, + ]); - expect(sanitizeMdxExpression("a { b")).toBe("a \\{ b"); - expect(sanitizeMdxExpression("> {a\nb}")).toBe("> \\{a\nb}"); - expect(sanitizeMdxExpression("")).toBe( - "\\" - ); - expect(sanitizeMdxExpression("")).toBe("\\"); - expect(sanitizeMdxExpression("a { b { c } d")).toBe("a \\{ b { c } d"); - expect(sanitizeMdxExpression('a {"b" "c"} d')).toBe('a \\{"b" "c"} d'); - expect(sanitizeMdxExpression('a {var b = "c"} d')).toBe( - 'a \\{var b = "c"} d' - ); + expect(sanitizeMdxExpression("a { b")).toStrictEqual(["a \\{ b", true]); + expect(sanitizeMdxExpression("> {a\nb}")).toStrictEqual([ + "> \\{a\nb}", + true, + ]); + expect(sanitizeMdxExpression("")).toStrictEqual([ + "\\", + true, + ]); + expect(sanitizeMdxExpression("")).toStrictEqual([ + "\\", + true, + ]); + expect(sanitizeMdxExpression("a { b { c } d")).toStrictEqual([ + "a \\{ b { c } d", + true, + ]); + expect(sanitizeMdxExpression('a {"b" "c"} d')).toStrictEqual([ + 'a \\{"b" "c"} d', + true, + ]); + expect(sanitizeMdxExpression('a {var b = "c"} d')).toStrictEqual([ + 'a \\{var b = "c"} d', + true, + ]); }); it("should escape only the part of the line that contains the error", () => { - expect(sanitizeMdxExpression("a { b { c } d e")).toBe("a \\{ b { c } d e"); - expect(sanitizeMdxExpression("")).toBe( - "\\" - ); - expect(sanitizeMdxExpression("")).toBe( - "\\" - ); - expect(sanitizeMdxExpression("")).toBe( - "" - ); - expect(sanitizeMdxExpression("")).toBe( - "" - ); - expect(sanitizeMdxExpression("{b")).toBe( - "\\{b" - ); - expect(sanitizeMdxExpression("{b}")).toBe( - "{b}" - ); - expect(sanitizeMdxExpression("{b + a}")).toBe( - "{b + a}" - ); + expect(sanitizeMdxExpression("a { b { c } d e")).toStrictEqual([ + "a \\{ b { c } d e", + true, + ]); + expect(sanitizeMdxExpression("")).toStrictEqual([ + "\\", + true, + ]); + expect(sanitizeMdxExpression("")).toStrictEqual([ + "<Something \\{...props} d>", + true, + ]); + expect(sanitizeMdxExpression("")).toStrictEqual([ + "", + true, + ]); + expect(sanitizeMdxExpression("")).toStrictEqual([ + "", + true, + ]); + expect(sanitizeMdxExpression("{b")).toStrictEqual([ + "\\{b", + true, + ]); + expect(sanitizeMdxExpression("{b}")).toStrictEqual([ + "{b}", + true, + ]); + expect( + sanitizeMdxExpression("{b + a}") + ).toStrictEqual(["{b + a}", true]); - expect(sanitizeMdxExpression("This is a test. a < b, but b > c. {c}")).toBe( - "This is a test. a < b, but b > c. {c}" - ); + expect( + sanitizeMdxExpression("This is a test. a < b, but b > c. {c}") + ).toStrictEqual(["This is a test. a < b, but b > c. {c}", true]); expect( sanitizeMdxExpression("This is a test. a <= b, but b > c. {c} d") - ).toBe("This is a test. a \\<= b, but b > c. {c} d"); + ).toStrictEqual(["This is a test. a \\<= b, but b > c. {c} d", true]); }); it("should handle complex cases", () => { @@ -86,27 +115,46 @@ describe("sanitizeMdxExpression", () => { sanitizeMdxExpression( "previous billing period) Ex. January {M1:{VM:VM0}}, February" ) - ).toMatchInlineSnapshot( - '"previous billing period) Ex. January \\{M1:\\{VM:VM0}}, February"' - ); + ).toStrictEqual([ + "previous billing period) Ex. January \\{M1:\\{VM:VM0}}, February", + true, + ]); expect( sanitizeMdxExpression( "from the previous month Ex. January15 {M1:{VM:VM0,on, 4}} February15" ) - ).toMatchInlineSnapshot( - '"from the previous month Ex. January15 \\{M1:\\{VM:VM0,on, 4}} February15"' - ); + ).toStrictEqual([ + "from the previous month Ex. January15 \\{M1:\\{VM:VM0,on, 4}} February15", + true, + ]); expect( sanitizeMdxExpression( "{ M1:2, M1:4 } => {M1:6} 2] Minimum - min of all the values for the" ) - ).toMatchInlineSnapshot( - '"\\{ M1:2, M1:4 } => \\{M1:6} 2] Minimum - min of all the values for the"' - ); + ).toStrictEqual([ + "\\{ M1:2, M1:4 } => \\{M1:6} 2] Minimum - min of all the values for the", + true, + ]); }); it("should avoid escaping math expressions", () => { - expect(sanitizeMdxExpression("$$x^2$$")).toBe("$$x^2$$"); - expect(sanitizeMdxExpression("$${x^2}$$")).toBe("$${x^2}$$"); + expect(sanitizeMdxExpression("$$x^2$$")).toStrictEqual(["$$x^2$$", true]); + expect(sanitizeMdxExpression("$${x^2}$$")).toStrictEqual([ + "$${x^2}$$", + true, + ]); + }); + + it("should handle end-tag-mismatch", () => { + expect(sanitizeMdxExpression("")).toStrictEqual(["<a>", true]); + + expect( + sanitizeMdxExpression( + "This is the JSON that can be generated in the Google Cloud Console at https://console.cloud.google.com/iam-admin/serviceaccounts/details//keys." + ) + ).toStrictEqual([ + "This is the JSON that can be generated in the Google Cloud Console at https://console.cloud.google.com/iam-admin/serviceaccounts/details/<service-account-id>/keys.", + true, + ]); }); }); diff --git a/packages/fern-docs/mdx/src/sanitize/sanitize-mdx-expression.ts b/packages/fern-docs/mdx/src/sanitize/sanitize-mdx-expression.ts index bdac09a4d9..cd5e1bb955 100644 --- a/packages/fern-docs/mdx/src/sanitize/sanitize-mdx-expression.ts +++ b/packages/fern-docs/mdx/src/sanitize/sanitize-mdx-expression.ts @@ -18,6 +18,7 @@ const RULE_IDS = { NON_SPREAD: "non-spread", SPREAD_EXTRA: "spread-extra", ACORN: "acorn", + END_TAG_MISMATCH: "end-tag-mismatch", } as const; /** @@ -25,15 +26,24 @@ const RULE_IDS = { * This function attempts to sanitize the markdown by escaping the curly braces, but listening for VFileMessage errors. * * This is not really efficient because it can loop a lot and depends on try-catching errors, but it performs a bit of "magic" to improve quality of life. + * + * @param content - the markdown content to sanitize + * @returns the sanitized markdown content and a boolean indicating is sanitized (false if it failed to sanitize) */ -export function sanitizeMdxExpression(content: string): string { +export function sanitizeMdxExpression( + content: string +): [string, handled: boolean] { // these are errors encountered but sanitized const errors: ErrorContext[] = []; let loops = 0; while (loops++ < 100) { - if (loops === 100) { - console.error("Infinite Loop Detected: sanitizing acorn failed"); + if (loops >= 100) { + console.error( + "Infinite Loop Detected: sanitizing acorn failed:", + content + ); + return [content, false]; } try { @@ -49,6 +59,15 @@ export function sanitizeMdxExpression(content: string): string { }; errors.push(errorContext); + if (e.ruleId === RULE_IDS.END_TAG_MISMATCH) { + const [newContent, handled] = handleEndTagMismatch(content, e); + errorContext.handled = handled; + if (handled) { + content = newContent; + continue; + } + } + if ( e.ruleId === RULE_IDS.UNEXPECTED_EOF || e.ruleId === RULE_IDS.UNEXPECTED_CHARACTER || @@ -107,17 +126,15 @@ export function sanitizeMdxExpression(content: string): string { } } - if (errors.length > 0) { - console.debug( - "MDX sanitization errors:", - errors.map( - (e) => - `handled=${e.handled}, rule=${e.error.ruleId}, line=\`${e.affectedLine}\`` - ) - ); - } + errors.forEach((e) => { + if (!e.handled) { + console.error( + `Unhandled MDX sanitization error: ${String(e.error.message)}: ${e.affectedLine}` + ); + } + }); - return content; + return [content, errors.length === 0 || errors.every((e) => e.handled)]; } function escapeAllUnescapedOpeningBrackets(content: string): string { @@ -127,7 +144,7 @@ function escapeAllUnescapedOpeningBrackets(content: string): string { function handleUnexpectedEOF( content: string, _e: VFileMessage -): [string, boolean] { +): [string, handled: boolean] { const stack: [number, "{" | "<"][] = []; const needsEscaping: number[] = []; @@ -144,8 +161,11 @@ function handleUnexpectedEOF( continue; } - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const [start, openingChar] = stack.pop()!; + const popped = stack.pop(); + if (!popped) { + continue; + } + const [start, openingChar] = popped; if ( (char === ">" && openingChar === "<") || (char === "}" && openingChar === "{") @@ -174,7 +194,7 @@ function handleUnexpectedEOF( function handleSpreadExtra( content: string, e: VFileMessage -): [string, boolean] { +): [string, handled: boolean] { let handled = false; const start = e.place == null ? -1 : getStart(content.split("\n"), e.place); @@ -204,7 +224,7 @@ function handleSpreadExtra( function escapeCurrentLine( content: string, e: VFileMessage -): [string, boolean] { +): [string, handled: boolean] { const line = e.line ?? (isPoint(e.place) ? e.place?.line : undefined); if (line == null) { @@ -227,7 +247,7 @@ function escapeCurrentLine( function handleUnexpectedCharacter( content: string, e: VFileMessage -): [string, boolean] { +): [string, handled: boolean] { const start = e.place == null ? -1 : getStart(content.split("\n"), e.place); const lastAlligatorBracket = content.slice(0, start).lastIndexOf("<"); @@ -251,3 +271,43 @@ function handleUnexpectedCharacter( true, ]; } + +function handleEndTagMismatch( + content: string, + e: VFileMessage +): [string, handled: boolean] { + // reason example: Expected a closing tag for `` (1:139-1:159) before the end of `paragraph` + // parse (1:139-1:159) to get the start and end of the tag + const tagLocationStr = e.reason?.match(/\((\d+:\d+-\d+:\d+)\)/)?.[1]; + if (tagLocationStr == null) { + return [content, false]; + } + const [tagStartLine, tagStartColumn, tagEndLine, tagEndColumn] = + tagLocationStr.split(/[:-]/g).map((s) => parseInt(s)); + + if (!tagStartLine || !tagStartColumn || !tagEndLine || !tagEndColumn) { + return [content, false]; + } + + const lines = content.split("\n"); + + const start = getStart(lines, { + line: tagStartLine, + column: tagStartColumn, + }); + + const end = getStart(lines, { + line: tagEndLine, + column: tagEndColumn, + }); + + return [ + content.slice(0, start) + + content + .slice(start, end) + .replaceAll(//g, ">") + + content.slice(end), + true, + ]; +} diff --git a/packages/fern-docs/ui/src/mdx/bundlers/mdx-bundler.ts b/packages/fern-docs/ui/src/mdx/bundlers/mdx-bundler.ts index 95f1e936b7..38d9f69903 100644 --- a/packages/fern-docs/ui/src/mdx/bundlers/mdx-bundler.ts +++ b/packages/fern-docs/ui/src/mdx/bundlers/mdx-bundler.ts @@ -48,7 +48,7 @@ export async function serializeMdx( } content = sanitizeBreaks(content); - content = sanitizeMdxExpression(content); + content = sanitizeMdxExpression(content)[0]; if (process.platform === "win32") { process.env.ESBUILD_BINARY_PATH = path.join( diff --git a/packages/fern-docs/ui/src/mdx/bundlers/next-mdx-remote.ts b/packages/fern-docs/ui/src/mdx/bundlers/next-mdx-remote.ts index 0209c45e1d..6894b2cd83 100644 --- a/packages/fern-docs/ui/src/mdx/bundlers/next-mdx-remote.ts +++ b/packages/fern-docs/ui/src/mdx/bundlers/next-mdx-remote.ts @@ -111,7 +111,7 @@ export async function serializeMdx( // } content = sanitizeBreaks(content); - content = sanitizeMdxExpression(content); + content = sanitizeMdxExpression(content)[0]; try { const result = await serialize< diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0e42a993f8..bb39279723 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1468,14 +1468,14 @@ importers: specifier: workspace:* version: link:../../commons/core-utils '@types/estree': - specifier: ^1.0.5 + specifier: ^1.0.6 version: 1.0.6 '@types/hast': specifier: ^3.0.4 version: 3.0.4 '@types/mdast': - specifier: ^4.0.3 - version: 4.0.3 + specifier: ^4.0.4 + version: 4.0.4 '@types/unist': specifier: ^3.0.3 version: 3.0.3 @@ -1486,8 +1486,8 @@ importers: specifier: ^1.27.0 version: 1.30.0 estree-util-value-to-estree: - specifier: ^3.1.2 - version: 3.1.2 + specifier: ^3.2.1 + version: 3.2.1 estree-walker: specifier: ^3.0.3 version: 3.0.3 @@ -1501,17 +1501,17 @@ importers: specifier: ^3.0.0 version: 3.0.0 hast-util-to-estree: - specifier: ^3.1.0 - version: 3.1.0 + specifier: ^3.1.1 + version: 3.1.1 hast-util-to-string: - specifier: ^3.0.0 - version: 3.0.0 + specifier: ^3.0.1 + version: 3.0.1 hastscript: specifier: ^9.0.0 version: 9.0.0 mdast-util-from-markdown: - specifier: ^2.0.0 - version: 2.0.0 + specifier: ^2.0.2 + version: 2.0.2 mdast-util-gfm: specifier: ^3.0.0 version: 3.0.0 @@ -1522,17 +1522,17 @@ importers: specifier: ^3.0.0 version: 3.0.0 mdast-util-mdx-jsx: - specifier: ^3.1.0 - version: 3.1.2 + specifier: ^3.2.0 + version: 3.2.0 mdast-util-mdxjs-esm: specifier: ^2.0.1 version: 2.0.1 mdast-util-to-hast: - specifier: ^13.1.0 - version: 13.1.0 + specifier: ^13.2.0 + version: 13.2.0 mdast-util-to-markdown: - specifier: ^2.1.0 - version: 2.1.0 + specifier: ^2.1.2 + version: 2.1.2 micromark-extension-gfm: specifier: ^3.0.0 version: 3.0.0 @@ -1546,8 +1546,8 @@ importers: specifier: ^6.0.0 version: 6.0.0 style-to-object: - specifier: ^1.0.5 - version: 1.0.6 + specifier: ^1.0.8 + version: 1.0.8 ts-essentials: specifier: ^10.0.1 version: 10.0.1(typescript@5.7.2) @@ -7769,6 +7769,9 @@ packages: '@types/mdast@4.0.3': resolution: {integrity: sha512-LsjtqsyF+d2/yFOYaN22dHZI1Cpwkrj+g06G8+qtUKlhovPW89YhqSnfKtMbkgmEtYpH2gydRNULd6y8mciAFg==} + '@types/mdast@4.0.4': + resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} + '@types/mdx@2.0.13': resolution: {integrity: sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw==} @@ -10781,8 +10784,8 @@ packages: estree-util-to-js@2.0.0: resolution: {integrity: sha512-WDF+xj5rRWmD5tj6bIqRi6CkLIXbbNQUcxQHzGysQzvHmdYG2G7p/Tf0J0gpxGgkeMZNTIjT/AoSvC9Xehcgdg==} - estree-util-value-to-estree@3.1.2: - resolution: {integrity: sha512-S0gW2+XZkmsx00tU2uJ4L9hUT7IFabbml9pHh2WQqFmAbxit++YGZne0sKJbNwkj9Wvg9E4uqWl4nCIFQMmfag==} + estree-util-value-to-estree@3.2.1: + resolution: {integrity: sha512-Vt2UOjyPbNQQgT5eJh+K5aATti0OjCIAGc9SgMdOFYbohuifsWclR74l0iZTJwePMgWYdX1hlVS+dedH9XV8kw==} estree-util-visit@2.0.0: resolution: {integrity: sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==} @@ -11572,8 +11575,8 @@ packages: hast-util-raw@7.2.3: resolution: {integrity: sha512-RujVQfVsOrxzPOPSzZFiwofMArbQke6DJjnFfceiEbFh7S05CbPt0cYN+A5YeD3pso0JQk6O1aHBnx9+Pm2uqg==} - hast-util-to-estree@3.1.0: - resolution: {integrity: sha512-lfX5g6hqVh9kjS/B9E2gSkvHH4SZNiQFiqWS0x9fENzEl+8W12RqdRxX6d/Cwxi30tPQs3bIO+aolQJNp1bIyw==} + hast-util-to-estree@3.1.1: + resolution: {integrity: sha512-IWtwwmPskfSmma9RpzCappDUitC8t5jhAynHhc1m2+5trOgsrp7txscUSavc5Ic8PATyAjfrCK1wgtxh2cICVQ==} hast-util-to-html@9.0.3: resolution: {integrity: sha512-M17uBDzMJ9RPCqLMO92gNNUDuBSq10a25SDBI08iCCxmorf4Yy6sYHK57n9WAbRAAaU+DuR4W6GN9K4DFZesYg==} @@ -11587,6 +11590,9 @@ packages: hast-util-to-string@3.0.0: resolution: {integrity: sha512-OGkAxX1Ua3cbcW6EJ5pT/tslVb90uViVkcJ4ZZIMW/R33DX/AkcJcRrPebPwJkHYwlDHXz4aIwvAAaAdtrACFA==} + hast-util-to-string@3.0.1: + resolution: {integrity: sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A==} + hast-util-to-text@4.0.2: resolution: {integrity: sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==} @@ -11838,8 +11844,8 @@ packages: inline-style-parser@0.1.1: resolution: {integrity: sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==} - inline-style-parser@0.2.3: - resolution: {integrity: sha512-qlD8YNDqyTKTyuITrDOffsl6Tdhv+UC4hcdAVuQsK4IMQ99nSgd1MIA/Q+jQYoh9r3hVUXhYh7urSRmXPkW04g==} + inline-style-parser@0.2.4: + resolution: {integrity: sha512-0aO8FkhNZlj/ZIbNi7Lxxr12obT7cL1moPfE4tg1LkX7LlLfC6DeX4l2ZEud1ukP9jNQyNnfzQVqwbwmAATY4Q==} inquirer@8.2.6: resolution: {integrity: sha512-M1WuAmb7pn9zdFRtQYk26ZBoY043Sse0wVDdk4Bppr+JOXyQYybdtvK+l9wUibhtjdjvtoiNy8tk+EgsYIUqKg==} @@ -12968,8 +12974,8 @@ packages: mdast-util-from-markdown@1.3.1: resolution: {integrity: sha512-4xTO/M8c82qBcnQc1tgpNtubGUW/Y1tBQ1B0i5CtSoelOLKFYlElIr3bvgREYYO5iRqbMY1YuqZng0GVOI8Qww==} - mdast-util-from-markdown@2.0.0: - resolution: {integrity: sha512-n7MTOr/z+8NAX/wmhhDji8O3bRvPTV/U0oTCaZJkjhPSKTPhS3xufVhKGF8s1pJ7Ox4QgoIU7KHseh09S+9rTA==} + mdast-util-from-markdown@2.0.2: + resolution: {integrity: sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA==} mdast-util-frontmatter@2.0.1: resolution: {integrity: sha512-LRqI9+wdgC25P0URIJY9vwocIzCcksduHQ9OF2joxQoyTNVduwLAFUzjoopuRJbJAReaKrNQKAZKL3uCMugWJA==} @@ -12998,8 +13004,8 @@ packages: mdast-util-mdx-expression@2.0.0: resolution: {integrity: sha512-fGCu8eWdKUKNu5mohVGkhBXCXGnOTLuFqOvGMvdikr+J1w7lDJgxThOKpwRWzzbyXAU2hhSwsmssOY4yTokluw==} - mdast-util-mdx-jsx@3.1.2: - resolution: {integrity: sha512-eKMQDeywY2wlHc97k5eD8VC+9ASMjN8ItEZQNGwJ6E0XWKiW/Z0V5/H8pvoXUf+y+Mj0VIgeRRbujBmFn4FTyA==} + mdast-util-mdx-jsx@3.2.0: + resolution: {integrity: sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q==} mdast-util-mdx@3.0.0: resolution: {integrity: sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w==} @@ -13013,11 +13019,11 @@ packages: mdast-util-to-hast@12.3.0: resolution: {integrity: sha512-pits93r8PhnIoU4Vy9bjW39M2jJ6/tdHyja9rrot9uujkN7UTU9SDnE6WNJz/IGyQk3XHX6yNNtrBH6cQzm8Hw==} - mdast-util-to-hast@13.1.0: - resolution: {integrity: sha512-/e2l/6+OdGp/FB+ctrJ9Avz71AN/GRH3oi/3KAx/kMnoUsD6q0woXlDT8lLEeViVKE7oZxE7RXzvO3T8kF2/sA==} + mdast-util-to-hast@13.2.0: + resolution: {integrity: sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA==} - mdast-util-to-markdown@2.1.0: - resolution: {integrity: sha512-SR2VnIEdVNCJbP6y7kVTJgPLifdr8WEU440fQec7qHoHOUz/oJ2jmNRqdDQ3rbiStOXb2mCDGTuwsK5OPUgYlQ==} + mdast-util-to-markdown@2.1.2: + resolution: {integrity: sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==} mdast-util-to-string@3.2.0: resolution: {integrity: sha512-V4Zn/ncyN1QNSqSBxTrMOLpjr+IKdHl2v3KVLoWmDPscP4r9GcCi71gjgvUV1SFSKh92AjAG4peFuBl2/YgCJg==} @@ -15683,8 +15689,8 @@ packages: style-to-object@0.4.4: resolution: {integrity: sha512-HYNoHZa2GorYNyqiCaBgsxvcJIn7OHq6inEga+E6Ke3m5JkoqpQbnFssk4jwe+K7AhGa2fcha4wSOf1Kn01dMg==} - style-to-object@1.0.6: - resolution: {integrity: sha512-khxq+Qm3xEyZfKd/y9L3oIWQimxuc4STrQKtQn8aSDRHb8mFgpukgX1hdzfrMEW6JCjyJ8p89x+IUMVnCBI1PA==} + style-to-object@1.0.8: + resolution: {integrity: sha512-xT47I/Eo0rwJmaXC4oilDGDWLohVhR6o/xAQcPQN8q6QBuZVL8qMYL85kLmST5cPjAorwvqIA4qXTRQoYHaL6g==} styled-jsx@5.1.1: resolution: {integrity: sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==} @@ -20968,7 +20974,7 @@ snapshots: estree-util-is-identifier-name: 3.0.0 estree-util-to-js: 2.0.0 estree-walker: 3.0.3 - hast-util-to-estree: 3.1.0 + hast-util-to-estree: 3.1.1 hast-util-to-jsx-runtime: 2.3.0 markdown-extensions: 2.0.0 periscopic: 3.1.0 @@ -24226,6 +24232,10 @@ snapshots: dependencies: '@types/unist': 3.0.3 + '@types/mdast@4.0.4': + dependencies: + '@types/unist': 3.0.3 + '@types/mdx@2.0.13': {} '@types/mime@1.3.5': {} @@ -28477,7 +28487,7 @@ snapshots: astring: 1.8.6 source-map: 0.7.4 - estree-util-value-to-estree@3.1.2: + estree-util-value-to-estree@3.2.1: dependencies: '@types/estree': 1.0.6 @@ -29432,7 +29442,7 @@ snapshots: web-namespaces: 2.0.1 zwitch: 2.0.4 - hast-util-to-estree@3.1.0: + hast-util-to-estree@3.1.1: dependencies: '@types/estree': 1.0.6 '@types/estree-jsx': 1.0.5 @@ -29443,11 +29453,11 @@ snapshots: estree-util-is-identifier-name: 3.0.0 hast-util-whitespace: 3.0.0 mdast-util-mdx-expression: 2.0.0 - mdast-util-mdx-jsx: 3.1.2 + mdast-util-mdx-jsx: 3.2.0 mdast-util-mdxjs-esm: 2.0.1 property-information: 6.5.0 space-separated-tokens: 2.0.2 - style-to-object: 0.4.4 + style-to-object: 1.0.8 unist-util-position: 5.0.0 zwitch: 2.0.4 transitivePeerDependencies: @@ -29461,7 +29471,7 @@ snapshots: comma-separated-tokens: 2.0.3 hast-util-whitespace: 3.0.0 html-void-elements: 3.0.0 - mdast-util-to-hast: 13.1.0 + mdast-util-to-hast: 13.2.0 property-information: 6.5.0 space-separated-tokens: 2.0.2 stringify-entities: 4.0.4 @@ -29477,11 +29487,11 @@ snapshots: estree-util-is-identifier-name: 3.0.0 hast-util-whitespace: 3.0.0 mdast-util-mdx-expression: 2.0.0 - mdast-util-mdx-jsx: 3.1.2 + mdast-util-mdx-jsx: 3.2.0 mdast-util-mdxjs-esm: 2.0.1 property-information: 6.5.0 space-separated-tokens: 2.0.2 - style-to-object: 1.0.6 + style-to-object: 1.0.8 unist-util-position: 5.0.0 vfile-message: 4.0.2 transitivePeerDependencies: @@ -29500,6 +29510,10 @@ snapshots: dependencies: '@types/hast': 3.0.4 + hast-util-to-string@3.0.1: + dependencies: + '@types/hast': 3.0.4 + hast-util-to-text@4.0.2: dependencies: '@types/hast': 3.0.4 @@ -29796,7 +29810,7 @@ snapshots: inline-style-parser@0.1.1: {} - inline-style-parser@0.2.3: {} + inline-style-parser@0.2.4: {} inquirer@8.2.6: dependencies: @@ -31155,7 +31169,7 @@ snapshots: mdast-util-find-and-replace@3.0.1: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 escape-string-regexp: 5.0.0 unist-util-is: 6.0.0 unist-util-visit-parents: 6.0.1 @@ -31177,9 +31191,9 @@ snapshots: transitivePeerDependencies: - supports-color - mdast-util-from-markdown@2.0.0: + mdast-util-from-markdown@2.0.2: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 '@types/unist': 3.0.3 decode-named-character-reference: 1.0.2 devlop: 1.1.0 @@ -31196,18 +31210,18 @@ snapshots: mdast-util-frontmatter@2.0.1: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 devlop: 1.1.0 escape-string-regexp: 5.0.0 - mdast-util-from-markdown: 2.0.0 - mdast-util-to-markdown: 2.1.0 + mdast-util-from-markdown: 2.0.2 + mdast-util-to-markdown: 2.1.2 micromark-extension-frontmatter: 2.0.0 transitivePeerDependencies: - supports-color mdast-util-gfm-autolink-literal@2.0.0: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 ccount: 2.0.1 devlop: 1.1.0 mdast-util-find-and-replace: 3.0.1 @@ -31215,61 +31229,61 @@ snapshots: mdast-util-gfm-footnote@2.0.0: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.0 - mdast-util-to-markdown: 2.1.0 + mdast-util-from-markdown: 2.0.2 + mdast-util-to-markdown: 2.1.2 micromark-util-normalize-identifier: 2.0.0 transitivePeerDependencies: - supports-color mdast-util-gfm-strikethrough@2.0.0: dependencies: - '@types/mdast': 4.0.3 - mdast-util-from-markdown: 2.0.0 - mdast-util-to-markdown: 2.1.0 + '@types/mdast': 4.0.4 + mdast-util-from-markdown: 2.0.2 + mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color mdast-util-gfm-table@2.0.0: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 devlop: 1.1.0 markdown-table: 3.0.3 - mdast-util-from-markdown: 2.0.0 - mdast-util-to-markdown: 2.1.0 + mdast-util-from-markdown: 2.0.2 + mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color mdast-util-gfm-task-list-item@2.0.0: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.0 - mdast-util-to-markdown: 2.1.0 + mdast-util-from-markdown: 2.0.2 + mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color mdast-util-gfm@3.0.0: dependencies: - mdast-util-from-markdown: 2.0.0 + mdast-util-from-markdown: 2.0.2 mdast-util-gfm-autolink-literal: 2.0.0 mdast-util-gfm-footnote: 2.0.0 mdast-util-gfm-strikethrough: 2.0.0 mdast-util-gfm-table: 2.0.0 mdast-util-gfm-task-list-item: 2.0.0 - mdast-util-to-markdown: 2.1.0 + mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color mdast-util-math@3.0.0: dependencies: '@types/hast': 3.0.4 - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 devlop: 1.1.0 longest-streak: 3.1.0 - mdast-util-from-markdown: 2.0.0 - mdast-util-to-markdown: 2.1.0 + mdast-util-from-markdown: 2.0.2 + mdast-util-to-markdown: 2.1.2 unist-util-remove-position: 5.0.0 transitivePeerDependencies: - supports-color @@ -31278,26 +31292,25 @@ snapshots: dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.0 - mdast-util-to-markdown: 2.1.0 + mdast-util-from-markdown: 2.0.2 + mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdx-jsx@3.1.2: + mdast-util-mdx-jsx@3.2.0: dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 '@types/unist': 3.0.3 ccount: 2.0.1 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.0 - mdast-util-to-markdown: 2.1.0 + mdast-util-from-markdown: 2.0.2 + mdast-util-to-markdown: 2.1.2 parse-entities: 4.0.1 stringify-entities: 4.0.4 - unist-util-remove-position: 5.0.0 unist-util-stringify-position: 4.0.0 vfile-message: 4.0.2 transitivePeerDependencies: @@ -31305,11 +31318,11 @@ snapshots: mdast-util-mdx@3.0.0: dependencies: - mdast-util-from-markdown: 2.0.0 + mdast-util-from-markdown: 2.0.2 mdast-util-mdx-expression: 2.0.0 - mdast-util-mdx-jsx: 3.1.2 + mdast-util-mdx-jsx: 3.2.0 mdast-util-mdxjs-esm: 2.0.1 - mdast-util-to-markdown: 2.1.0 + mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color @@ -31317,16 +31330,16 @@ snapshots: dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.0 - mdast-util-to-markdown: 2.1.0 + mdast-util-from-markdown: 2.0.2 + mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color mdast-util-phrasing@4.1.0: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 unist-util-is: 6.0.0 mdast-util-to-hast@12.3.0: @@ -31340,10 +31353,10 @@ snapshots: unist-util-position: 4.0.4 unist-util-visit: 4.1.2 - mdast-util-to-hast@13.1.0: + mdast-util-to-hast@13.2.0: dependencies: '@types/hast': 3.0.4 - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 '@ungap/structured-clone': 1.2.0 devlop: 1.1.0 micromark-util-sanitize-uri: 2.0.0 @@ -31352,13 +31365,14 @@ snapshots: unist-util-visit: 5.0.0 vfile: 6.0.1 - mdast-util-to-markdown@2.1.0: + mdast-util-to-markdown@2.1.2: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 '@types/unist': 3.0.3 longest-streak: 3.1.0 mdast-util-phrasing: 4.1.0 mdast-util-to-string: 4.0.0 + micromark-util-classify-character: 2.0.0 micromark-util-decode-string: 2.0.0 unist-util-visit: 5.0.0 zwitch: 2.0.4 @@ -31369,7 +31383,7 @@ snapshots: mdast-util-to-string@4.0.0: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 mdn-data@2.0.28: {} @@ -33336,7 +33350,7 @@ snapshots: devlop: 1.1.0 hast-util-to-jsx-runtime: 2.3.0 html-url-attributes: 3.0.0 - mdast-util-to-hast: 13.1.0 + mdast-util-to-hast: 13.2.0 react: 18.3.1 remark-parse: 11.0.0 remark-rehype: 11.1.0 @@ -33640,14 +33654,14 @@ snapshots: '@types/hast': 3.0.4 github-slugger: 2.0.0 hast-util-heading-rank: 3.0.0 - hast-util-to-string: 3.0.0 + hast-util-to-string: 3.0.1 unist-util-visit: 5.0.0 relateurl@0.2.7: {} remark-frontmatter@5.0.0: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 mdast-util-frontmatter: 2.0.1 micromark-extension-frontmatter: 2.0.0 unified: 11.0.4 @@ -33656,13 +33670,13 @@ snapshots: remark-gemoji@8.0.0: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 gemoji: 8.1.0 mdast-util-find-and-replace: 3.0.1 remark-gfm@4.0.0: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 mdast-util-gfm: 3.0.0 micromark-extension-gfm: 3.0.0 remark-parse: 11.0.0 @@ -33673,7 +33687,7 @@ snapshots: remark-math@6.0.0: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 mdast-util-math: 3.0.0 micromark-extension-math: 3.1.0 unified: 11.0.4 @@ -33682,9 +33696,9 @@ snapshots: remark-mdx-frontmatter@4.0.0: dependencies: - '@types/mdast': 4.0.3 + '@types/mdast': 4.0.4 estree-util-is-identifier-name: 3.0.0 - estree-util-value-to-estree: 3.1.2 + estree-util-value-to-estree: 3.2.1 toml: 3.0.0 unified: 11.0.4 yaml: 2.4.2 @@ -33706,8 +33720,8 @@ snapshots: remark-parse@11.0.0: dependencies: - '@types/mdast': 4.0.3 - mdast-util-from-markdown: 2.0.0 + '@types/mdast': 4.0.4 + mdast-util-from-markdown: 2.0.2 micromark-util-types: 2.0.0 unified: 11.0.4 transitivePeerDependencies: @@ -33723,8 +33737,8 @@ snapshots: remark-rehype@11.1.0: dependencies: '@types/hast': 3.0.4 - '@types/mdast': 4.0.3 - mdast-util-to-hast: 13.1.0 + '@types/mdast': 4.0.4 + mdast-util-to-hast: 13.2.0 unified: 11.0.4 vfile: 6.0.1 @@ -33736,8 +33750,8 @@ snapshots: remark-stringify@11.0.0: dependencies: - '@types/mdast': 4.0.3 - mdast-util-to-markdown: 2.1.0 + '@types/mdast': 4.0.4 + mdast-util-to-markdown: 2.1.2 unified: 11.0.4 renderkid@3.0.0: @@ -34646,9 +34660,9 @@ snapshots: dependencies: inline-style-parser: 0.1.1 - style-to-object@1.0.6: + style-to-object@1.0.8: dependencies: - inline-style-parser: 0.2.3 + inline-style-parser: 0.2.4 styled-jsx@5.1.1(@babel/core@7.26.0)(react@18.3.1): dependencies: From 00348c9cc8586b48780f59f0688540f098684069 Mon Sep 17 00:00:00 2001 From: Darwin Ding Date: Wed, 22 Jan 2025 14:55:14 -0500 Subject: [PATCH 05/47] Add S3 as DBDocsDefinition cache for revalidate-all/loadWithUrl/finishRegister paths (#2048) --- .../fern-docs/bundle/src/server/DocsLoader.ts | 27 +++++++-- .../fdr-deploy/scripts/fdr-deploy-stack.ts | 58 +++++++++++++++++++ servers/fdr/src/__test__/local/s3.test.ts | 5 ++ servers/fdr/src/__test__/mock.ts | 5 ++ servers/fdr/src/app/FdrConfig.ts | 16 +++++ .../docs/v2/getDocsWriteV2Service.ts | 30 ++++++++++ servers/fdr/src/services/s3/S3Service.ts | 31 ++++++++++ 7 files changed, 166 insertions(+), 6 deletions(-) diff --git a/packages/fern-docs/bundle/src/server/DocsLoader.ts b/packages/fern-docs/bundle/src/server/DocsLoader.ts index 8dd91b034e..dac3e8b8b2 100644 --- a/packages/fern-docs/bundle/src/server/DocsLoader.ts +++ b/packages/fern-docs/bundle/src/server/DocsLoader.ts @@ -123,12 +123,27 @@ export class DocsLoader { DocsV2Read.LoadDocsForUrlResponse | undefined > { if (!this.#loadForDocsUrlResponse) { - const response = await loadWithUrl(this.domain); - - if (response.ok) { - this.#loadForDocsUrlResponse = response.body; - } else { - this.#error = response.error; + try { + const environmentType = process.env.NODE_ENV ?? "development"; + let dbDocsDefUrl = ""; + if (environmentType === "development") { + dbDocsDefUrl = `https://docs-definitions-dev2.buildwithfern.com/${this.domain}.json`; + } else if (environmentType === "production") { + dbDocsDefUrl = `https://docs-definitions.buildwithfern.com/${this.domain}.json`; + } + const response = await fetch(dbDocsDefUrl); + if (response.ok) { + const json = await response.json(); + return json as DocsV2Read.LoadDocsForUrlResponse; + } + } catch { + // Not served by cloudfront, fetch from Redis and then RDS + const response = await loadWithUrl(this.domain); + if (response.ok) { + this.#loadForDocsUrlResponse = response.body; + } else { + this.#error = response.error; + } } } return this.#loadForDocsUrlResponse; diff --git a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts index 0fd3e1f965..95aa266aa1 100644 --- a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts +++ b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts @@ -207,6 +207,53 @@ export class FdrDeployStack extends Stack { } ); + // for revalidate-all and finish-register workflow + const dbDocsDefinitionBucket = new Bucket( + this, + "fdr-docs-definitions-public", + { + bucketName: `fdr-${environmentType.toLowerCase()}-docs-definitions-public`, + cors: [ + { + allowedMethods: [ + HttpMethods.GET, + HttpMethods.POST, + HttpMethods.PUT, + ], + allowedOrigins: ["*"], + allowedHeaders: ["*"], + }, + ], + blockPublicAccess: { + blockPublicAcls: false, + blockPublicPolicy: false, + ignorePublicAcls: false, + restrictPublicBuckets: false, + }, + versioned: true, + } + ); + dbDocsDefinitionBucket.grantPublicAccess(); + + const dbDocsDefinitionDomainName = + environmentType === "PROD" + ? "docs-definitions.buildwithfern.com" + : "docs-definitions-dev2.buildwithfern.com"; + const dbDocsDefinitionDistribution = new cloudfront.Distribution( + this, + "DbDocsDefinitionDistribution", + { + defaultBehavior: { + origin: new origins.S3Origin(dbDocsDefinitionBucket), + viewerProtocolPolicy: + cloudfront.ViewerProtocolPolicy.REDIRECT_TO_HTTPS, + cachePolicy: cloudfront.CachePolicy.CACHING_OPTIMIZED, + }, + domainNames: [dbDocsDefinitionDomainName], + certificate, + } + ); + new route53.ARecord(this, "PublicDocsFilesRecord", { recordName: publicDocsFilesDomainName, target: route53.RecordTarget.fromAlias( @@ -215,6 +262,14 @@ export class FdrDeployStack extends Stack { zone: hostedZone, }); + new route53.ARecord(this, "DbDocsDefinitionRecord", { + recordName: dbDocsDefinitionDomainName, + target: route53.RecordTarget.fromAlias( + new targets.CloudFrontTarget(dbDocsDefinitionDistribution) + ), + zone: hostedZone, + }); + const fernDocsCacheEndpoint = this.constructElastiCacheInstance(this, { cacheName: options.cacheName, IVpc: vpc, @@ -265,6 +320,9 @@ export class FdrDeployStack extends Stack { PUBLIC_S3_BUCKET_REGION: publicDocsBucket.stack.region, PRIVATE_S3_BUCKET_NAME: privateDocsBucket.bucketName, PRIVATE_S3_BUCKET_REGION: privateDocsBucket.stack.region, + DB_DOCS_DEFINITION_BUCKET_NAME: dbDocsDefinitionBucket.bucketName, + DB_DOCS_DEFINITION_BUCKET_REGION: + dbDocsDefinitionBucket.stack.region, API_DEFINITION_SOURCE_BUCKET_NAME: privateApiDefinitionSourceBucket.bucketName, API_DEFINITION_SOURCE_BUCKET_REGION: diff --git a/servers/fdr/src/__test__/local/s3.test.ts b/servers/fdr/src/__test__/local/s3.test.ts index 70d985e486..c9bc3d6b06 100644 --- a/servers/fdr/src/__test__/local/s3.test.ts +++ b/servers/fdr/src/__test__/local/s3.test.ts @@ -19,6 +19,11 @@ describe("S3 Service", () => { bucketRegion: "us-east-1", urlOverride: undefined, }, + dbDocsDefinitionS3: { + bucketName: "fdr-dev2-db-docs-def-public", + bucketRegion: "us-east-1", + urlOverride: undefined, + }, privateApiDefinitionSourceS3: { bucketName: "fdr-source-files", bucketRegion: "us-east-1", diff --git a/servers/fdr/src/__test__/mock.ts b/servers/fdr/src/__test__/mock.ts index 866e685d3a..ae68844b22 100644 --- a/servers/fdr/src/__test__/mock.ts +++ b/servers/fdr/src/__test__/mock.ts @@ -150,6 +150,11 @@ export const baseMockFdrConfig: FdrConfig = { bucketRegion: "us-east-1", urlOverride: "http://s3-mock:9090", }, + dbDocsDefinitionS3: { + bucketName: "fdr", + bucketRegion: "us-east-1", + urlOverride: "http://s3-mock:9090", + }, privateApiDefinitionSourceS3: { bucketName: "fdr", bucketRegion: "us-east-1", diff --git a/servers/fdr/src/app/FdrConfig.ts b/servers/fdr/src/app/FdrConfig.ts index c083cf325e..60d2401654 100644 --- a/servers/fdr/src/app/FdrConfig.ts +++ b/servers/fdr/src/app/FdrConfig.ts @@ -10,6 +10,12 @@ const PRIVATE_S3_BUCKET_NAME_ENV_VAR = "PRIVATE_S3_BUCKET_NAME"; const PRIVATE_S3_BUCKET_REGION_ENV_VAR = "PRIVATE_S3_BUCKET_REGION"; const PRIVATE_S3_URL_OVERRIDE_ENV_VAR = "PRIVATE_S3_URL_OVERRIDE"; +const DB_DOCS_DEFINITION_BUCKET_NAME_ENV_VAR = "DB_DOCS_DEFINITION_BUCKET_NAME"; +const DB_DOCS_DEFINITION_BUCKET_REGION_ENV_VAR = + "DB_DOCS_DEFINITION_BUCKET_REGION"; +const DB_DOCS_DEFINITION_BUCKET_URL_OVERRIDE_ENV_VAR = + "DB_DOCS_DEFINITION_BUCKET_URL_OVERRIDE"; + const API_DEFINITION_SOURCE_BUCKET_NAME_ENV_VAR = "API_DEFINITION_SOURCE_BUCKET_NAME"; const API_DEFINITION_SOURCE_BUCKET_REGION_ENV_VAR = @@ -45,6 +51,7 @@ export interface FdrConfig { cdnPublicDocsUrl: string; publicDocsS3: S3Config; privateDocsS3: S3Config; + dbDocsDefinitionS3: S3Config; privateApiDefinitionSourceS3: S3Config; domainSuffix: string; algoliaAppId: string; @@ -80,6 +87,15 @@ export function getConfig(): FdrConfig { ), urlOverride: process.env[PRIVATE_S3_URL_OVERRIDE_ENV_VAR], }, + dbDocsDefinitionS3: { + bucketName: getEnvironmentVariableOrThrow( + DB_DOCS_DEFINITION_BUCKET_NAME_ENV_VAR + ), + bucketRegion: getEnvironmentVariableOrThrow( + DB_DOCS_DEFINITION_BUCKET_REGION_ENV_VAR + ), + urlOverride: process.env[DB_DOCS_DEFINITION_BUCKET_URL_OVERRIDE_ENV_VAR], + }, privateApiDefinitionSourceS3: { bucketName: getEnvironmentVariableOrThrow( API_DEFINITION_SOURCE_BUCKET_NAME_ENV_VAR diff --git a/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts b/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts index a24a8addd8..c3d4136fda 100644 --- a/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts +++ b/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts @@ -277,6 +277,36 @@ export function getDocsWriteV2Service(app: FdrApplication): DocsV2WriteService { indexSegments, }); + const readDocsDefinition = convertDocsDefinitionToRead({ + docsDbDefinition: dbDocsDefinition, + algoliaSearchIndex: undefined, + filesV2: {}, + apis: mapValues(apiDefinitionsById, (def) => + convertDbAPIDefinitionToRead(def) + ), + apisV2: mapValues(apiDefinitionsLatestById, (def) => def), + id: DocsV1Write.DocsConfigId(""), + search: getSearchInfoFromDocs({ + algoliaIndex: undefined, + indexSegmentIds: [], + activeIndexSegments: [], + docsDbDefinition: dbDocsDefinition, + app, + }), + }); + + try { + await app.services.s3.writeDBDocsDefinition({ + domain: docsRegistrationInfo.fernUrl.getFullUrl(), + readDocsDefinition, + }); + } catch (e) { + app.logger.error( + `Error while trying to write DB docs definition for ${docsRegistrationInfo.fernUrl}`, + e + ); + } + /** * IMPORTANT NOTE: * vercel cache is not shared between custom domains, so we need to revalidate on EACH custom domain individually diff --git a/servers/fdr/src/services/s3/S3Service.ts b/servers/fdr/src/services/s3/S3Service.ts index 5d6015645b..474f85ecf7 100644 --- a/servers/fdr/src/services/s3/S3Service.ts +++ b/servers/fdr/src/services/s3/S3Service.ts @@ -2,6 +2,7 @@ import { GetObjectCommand, PutObjectCommand, PutObjectCommandInput, + PutObjectCommandOutput, S3Client, } from "@aws-sdk/client-s3"; import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; @@ -37,6 +38,10 @@ export interface S3ApiDefinitionSourceFileInfo { } export interface S3Service { + writeDBDocsDefinition(arg0: { + domain: string; + readDocsDefinition: any; + }): Promise; getPresignedDocsAssetsUploadUrls({ domain, filepaths, @@ -79,6 +84,7 @@ export class S3ServiceImpl implements S3Service { private publicDocsS3: S3Client; private privateDocsS3: S3Client; private privateApiDefinitionSourceS3: S3Client; + private dbDocsDefinitionS3: S3Client; private presignedDownloadUrlCache = new Cache( 10_000, ONE_WEEK_IN_SECONDS @@ -106,6 +112,16 @@ export class S3ServiceImpl implements S3Service { secretAccessKey: config.awsSecretKey, }, }); + this.dbDocsDefinitionS3 = new S3Client({ + ...(config.dbDocsDefinitionS3.urlOverride != null + ? { endpoint: config.dbDocsDefinitionS3.urlOverride } + : {}), + region: config.dbDocsDefinitionS3.bucketRegion, + credentials: { + accessKeyId: config.awsAccessKey, + secretAccessKey: config.awsSecretKey, + }, + }); this.privateApiDefinitionSourceS3 = new S3Client({ ...(config.privateApiDefinitionSourceS3.urlOverride != null ? { endpoint: config.privateApiDefinitionSourceS3.urlOverride } @@ -315,6 +331,21 @@ export class S3ServiceImpl implements S3Service { }; } + async writeDBDocsDefinition({ + domain, + readDocsDefinition, + }: { + domain: string; + readDocsDefinition: any; + }): Promise { + const command = new PutObjectCommand({ + Bucket: this.config.dbDocsDefinitionS3.bucketName, + Key: `${domain}.json`, + Body: JSON.stringify(readDocsDefinition), + }); + return await this.dbDocsDefinitionS3.send(command); + } + constructS3DocsKey({ domain, time, From d4821148792f2dfb7cd0af7670ba5d576cb9a1af Mon Sep 17 00:00:00 2001 From: Deep Singhvi Date: Wed, 22 Jan 2025 16:41:16 -0500 Subject: [PATCH 06/47] Revert "fix(fdr): public s3 file uploads go through cloudfront" (#2052) --- servers/fdr/src/services/s3/S3Service.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/servers/fdr/src/services/s3/S3Service.ts b/servers/fdr/src/services/s3/S3Service.ts index 474f85ecf7..de34b98fa0 100644 --- a/servers/fdr/src/services/s3/S3Service.ts +++ b/servers/fdr/src/services/s3/S3Service.ts @@ -239,7 +239,7 @@ export class S3ServiceImpl implements S3Service { input.ContentType = "image/svg+xml"; } const command = new PutObjectCommand(input); - const result = { + return { url: await getSignedUrl( isPrivate ? this.privateDocsS3 : this.publicDocsS3, command, @@ -247,15 +247,6 @@ export class S3ServiceImpl implements S3Service { ), key, }; - if (!isPrivate) { - try { - const url = new URL(result.url); - result.url = result.url.replace(url.host, this.publicDocsCDNUrl); - } catch (error) { - console.error("Failed to replace S3 URL with CDN URL:", error); - } - } - return result; } async getPresignedApiDefinitionSourceDownloadUrl({ From 4ff17d6edd577777ebdc36bcf7c53be5587c7aa7 Mon Sep 17 00:00:00 2001 From: Darwin Ding Date: Wed, 22 Jan 2025 18:48:19 -0500 Subject: [PATCH 07/47] Fixing docs definition retrieval logic to use proper environment variables (#2053) --- .../fern-docs/bundle/src/server/DocsLoader.ts | 15 ++++++++------- packages/fern-docs/bundle/turbo.json | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/fern-docs/bundle/src/server/DocsLoader.ts b/packages/fern-docs/bundle/src/server/DocsLoader.ts index dac3e8b8b2..ecbb7aab82 100644 --- a/packages/fern-docs/bundle/src/server/DocsLoader.ts +++ b/packages/fern-docs/bundle/src/server/DocsLoader.ts @@ -119,18 +119,19 @@ export class DocsLoader { .load(); } + private getDocsDefinitionUrl() { + return ( + process.env.NEXT_PUBLIC_DOCS_DEFINITION_S3_URL ?? + "https://docs-definitions.buildwithfern.com" + ); + } + private async loadDocs(): Promise< DocsV2Read.LoadDocsForUrlResponse | undefined > { if (!this.#loadForDocsUrlResponse) { try { - const environmentType = process.env.NODE_ENV ?? "development"; - let dbDocsDefUrl = ""; - if (environmentType === "development") { - dbDocsDefUrl = `https://docs-definitions-dev2.buildwithfern.com/${this.domain}.json`; - } else if (environmentType === "production") { - dbDocsDefUrl = `https://docs-definitions.buildwithfern.com/${this.domain}.json`; - } + const dbDocsDefUrl = `${this.getDocsDefinitionUrl()}/${this.domain}.json`; const response = await fetch(dbDocsDefUrl); if (response.ok) { const json = await response.json(); diff --git a/packages/fern-docs/bundle/turbo.json b/packages/fern-docs/bundle/turbo.json index 5bff8627e3..b061612780 100644 --- a/packages/fern-docs/bundle/turbo.json +++ b/packages/fern-docs/bundle/turbo.json @@ -1,6 +1,7 @@ { "$schema": "https://turbo.build/schema.json", "extends": ["//"], + "globalEnv": ["FERN_ENVIRONMENT", "APPLICATION_ENVIRONMENT"], "tasks": { "docs:build": { "outputs": [".next/**", "!.next/cache/**"], From b252bfd2f3345e9e7419b7220eea6fc8f08c9d4e Mon Sep 17 00:00:00 2001 From: Darwin Ding Date: Wed, 22 Jan 2025 19:22:51 -0500 Subject: [PATCH 08/47] Use host instead of domain for revalidator logic (#2054) --- packages/fern-docs/bundle/src/server/DocsLoader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/fern-docs/bundle/src/server/DocsLoader.ts b/packages/fern-docs/bundle/src/server/DocsLoader.ts index ecbb7aab82..8c821dad19 100644 --- a/packages/fern-docs/bundle/src/server/DocsLoader.ts +++ b/packages/fern-docs/bundle/src/server/DocsLoader.ts @@ -131,7 +131,7 @@ export class DocsLoader { > { if (!this.#loadForDocsUrlResponse) { try { - const dbDocsDefUrl = `${this.getDocsDefinitionUrl()}/${this.domain}.json`; + const dbDocsDefUrl = `${this.getDocsDefinitionUrl()}/${this.host}.json`; const response = await fetch(dbDocsDefUrl); if (response.ok) { const json = await response.json(); From 705e1d57bbdf0776ebe4d872aaf89f52c2ab0d89 Mon Sep 17 00:00:00 2001 From: Andrew Jiang Date: Wed, 22 Jan 2025 19:58:25 -0500 Subject: [PATCH 09/47] refactor: hide filesv2 from static props (#2044) --- packages/fern-docs/bundle/next.config.mjs | 3 +- packages/fern-docs/bundle/package.json | 1 + .../fern-docs/auth/api-key-injection/route.ts | 3 +- packages/fern-docs/bundle/src/app/sitemap.ts | 2 +- packages/fern-docs/bundle/src/middleware.ts | 2 +- .../src/server/LoadDocsPerformanceTracker.ts | 2 +- .../bundle/src/server/auth/getAuthState.ts | 3 +- .../bundle/src/server/auth/workos-handler.ts | 2 +- .../src/server}/getBreadcrumbList.ts | 2 +- .../bundle/src/server/getMarkdownForPath.ts | 3 +- .../bundle/src/server/getSectionRoot.ts | 2 +- .../fern-docs/bundle/src/server/pageRoutes.ts | 3 +- .../bundle/src/server/queue-reindex.ts | 4 +- .../bundle/src/server/revalidator.ts | 2 +- .../bundle/src/server/withInitialProps.ts | 68 ++++-- .../src/server/withResolvedDocsContent.ts | 49 ++++- packages/fern-docs/bundle/tsconfig.json | 21 +- .../local-preview-bundle/next.config.mjs | 2 +- .../local-preview-bundle/package.json | 3 + .../src/utils/getDocsPageProps.ts | 95 ++++++++- packages/fern-docs/mdx/.depcheckrc.json | 10 +- packages/fern-docs/mdx/package.json | 2 + .../mdx/src/hast-utils/hast-to-string.ts | 1 + .../fern-docs/mdx/src/hast-utils/index.ts | 1 + packages/fern-docs/mdx/src/hast.ts | 10 + packages/fern-docs/mdx/src/index.ts | 12 +- packages/fern-docs/mdx/src/mdast.ts | 2 + packages/fern-docs/mdx/src/mdx-utils/index.ts | 1 + .../mdx-utils/mdx-jsx-attribute-to-string.ts | 30 +++ packages/fern-docs/mdx/src/strip-util.ts | 4 +- packages/fern-docs/mdx/src/toc.ts | 20 +- packages/fern-docs/mdx/src/types.ts | 4 + .../{next-seo => seo}/.stylelintrc.json | 0 .../fern-docs/{next-seo => seo}/README.md | 0 .../fern-docs/{next-seo => seo}/package.json | 6 +- .../src}/__test__/getBreadcrumbList.test.ts | 2 +- .../src}/__test__/getSeoProps.test.ts | 17 +- .../fern-docs/{next-seo => seo}/src/index.ts | 2 + .../src/jsonld/components/Breadcrumb.tsx | 0 .../{next-seo => seo}/src/jsonld/index.ts | 0 .../src/jsonld/types/breadcrumbs.ts | 0 .../{next-seo => seo}/src/meta/buildTags.tsx | 0 .../fern-docs/{next-seo => seo}/src/types.ts | 0 packages/fern-docs/seo/src/with-breadcrumb.ts | 55 +++++ .../seo/getSeoProp.ts => seo/src/with-seo.ts} | 52 +++-- .../{next-seo => seo}/tsconfig.eslint.json | 0 .../fern-docs/{next-seo => seo}/tsconfig.json | 0 packages/fern-docs/ui/.depcheckrc.json | 4 +- packages/fern-docs/ui/package.json | 8 +- packages/fern-docs/ui/src/atoms/docs.ts | 9 +- packages/fern-docs/ui/src/atoms/files.ts | 28 --- packages/fern-docs/ui/src/atoms/index.ts | 1 - packages/fern-docs/ui/src/atoms/logo.ts | 187 +++++++++++------ packages/fern-docs/ui/src/atoms/seo.ts | 2 +- packages/fern-docs/ui/src/atoms/types.ts | 38 +++- .../fern-docs/ui/src/components/FernImage.tsx | 196 ++++++++++-------- .../ui/src/components/JavascriptProvider.tsx | 14 +- .../src/components/__test__/FernImage.test.ts | 43 ---- .../ui/src/header/HeaderLogoImage.tsx | 67 ++---- .../ui/src/header/HeaderLogoSection.tsx | 4 +- .../ui/src/hooks/useInterceptNextDataHref.ts | 2 +- packages/fern-docs/ui/src/index.ts | 11 +- .../src/mdx/__test__/rehypeFernCode.test.ts | 4 +- .../mdx/bundlers/mdx-bundler-component.tsx | 9 +- .../ui/src/mdx/bundlers/mdx-bundler.ts | 16 +- .../ui/src/mdx/bundlers/next-mdx-remote.ts | 25 ++- .../fern-docs/ui/src/mdx/common/HastToJsx.tsx | 26 --- packages/fern-docs/ui/src/mdx/common/types.ts | 4 +- .../ui/src/mdx/components/html/embed.tsx | 33 --- .../ui/src/mdx/components/html/image.tsx | 60 ++---- .../ui/src/mdx/components/html/index.tsx | 1 - .../fern-docs/ui/src/mdx/components/index.tsx | 6 +- .../ui/src/mdx/plugins/rehype-files.ts | 125 +++++++++++ .../ui/src/mdx/plugins/rehypeExtractAsides.ts | 12 +- .../ui/src/mdx/plugins/rehypeFernCode.ts | 22 +- .../src/mdx/plugins/rehypeFernComponents.ts | 49 ++--- .../src/mdx/plugins/remark-extract-title.ts | 46 ++++ packages/fern-docs/ui/src/mdx/types.ts | 3 +- packages/fern-docs/ui/src/seo/NextSeo.tsx | 2 +- packages/fern-docs/ui/tsconfig.json | 2 +- .../fern-docs/utils/src/addLeadingSlash.ts | 3 - .../fern-docs/utils/src/getRedirectForPath.ts | 2 +- packages/fern-docs/utils/src/index.ts | 5 +- .../src/leading-slash.ts} | 4 + .../src/trailing-slash.ts} | 13 +- .../{withoutStaging.ts => without-staging.ts} | 0 pnpm-lock.yaml | 167 ++++++++------- 87 files changed, 1080 insertions(+), 676 deletions(-) rename packages/fern-docs/{ui/src/seo => bundle/src/server}/getBreadcrumbList.ts (97%) create mode 100644 packages/fern-docs/mdx/src/hast-utils/hast-to-string.ts create mode 100644 packages/fern-docs/mdx/src/hast.ts create mode 100644 packages/fern-docs/mdx/src/mdast.ts create mode 100644 packages/fern-docs/mdx/src/mdx-utils/mdx-jsx-attribute-to-string.ts create mode 100644 packages/fern-docs/mdx/src/types.ts rename packages/fern-docs/{next-seo => seo}/.stylelintrc.json (100%) rename packages/fern-docs/{next-seo => seo}/README.md (100%) rename packages/fern-docs/{next-seo => seo}/package.json (87%) rename packages/fern-docs/{ui/src/seo => seo/src}/__test__/getBreadcrumbList.test.ts (96%) rename packages/fern-docs/{ui/src/seo => seo/src}/__test__/getSeoProps.test.ts (66%) rename packages/fern-docs/{next-seo => seo}/src/index.ts (60%) rename packages/fern-docs/{next-seo => seo}/src/jsonld/components/Breadcrumb.tsx (100%) rename packages/fern-docs/{next-seo => seo}/src/jsonld/index.ts (100%) rename packages/fern-docs/{next-seo => seo}/src/jsonld/types/breadcrumbs.ts (100%) rename packages/fern-docs/{next-seo => seo}/src/meta/buildTags.tsx (100%) rename packages/fern-docs/{next-seo => seo}/src/types.ts (100%) create mode 100644 packages/fern-docs/seo/src/with-breadcrumb.ts rename packages/fern-docs/{ui/src/seo/getSeoProp.ts => seo/src/with-seo.ts} (88%) rename packages/fern-docs/{next-seo => seo}/tsconfig.eslint.json (100%) rename packages/fern-docs/{next-seo => seo}/tsconfig.json (100%) delete mode 100644 packages/fern-docs/ui/src/atoms/files.ts delete mode 100644 packages/fern-docs/ui/src/components/__test__/FernImage.test.ts delete mode 100644 packages/fern-docs/ui/src/mdx/common/HastToJsx.tsx delete mode 100644 packages/fern-docs/ui/src/mdx/components/html/embed.tsx create mode 100644 packages/fern-docs/ui/src/mdx/plugins/rehype-files.ts create mode 100644 packages/fern-docs/ui/src/mdx/plugins/remark-extract-title.ts delete mode 100644 packages/fern-docs/utils/src/addLeadingSlash.ts rename packages/fern-docs/{bundle/src/server/removeLeadingSlash.ts => utils/src/leading-slash.ts} (51%) rename packages/fern-docs/{bundle/src/server/trailingSlash.ts => utils/src/trailing-slash.ts} (61%) rename packages/fern-docs/utils/src/{withoutStaging.ts => without-staging.ts} (100%) diff --git a/packages/fern-docs/bundle/next.config.mjs b/packages/fern-docs/bundle/next.config.mjs index 83d1ab0397..5ed7efafca 100644 --- a/packages/fern-docs/bundle/next.config.mjs +++ b/packages/fern-docs/bundle/next.config.mjs @@ -10,6 +10,7 @@ const isTrailingSlashEnabled = process.env.TRAILING_SLASH === "1" || process.env.NEXT_PUBLIC_TRAILING_SLASH === "1"; +// TODO: move this to a shared location (this is copied in @fern-docs/ui FernImage.tsx) const DOCS_FILES_ALLOWLIST = [ { protocol: "https", @@ -66,7 +67,7 @@ const nextConfig = { "@fern-docs/components", "@fern-docs/edge-config", "@fern-docs/mdx", - "@fern-docs/next-seo", + "@fern-docs/seo", "@fern-docs/search-server", "@fern-docs/search-ui", "@fern-docs/search-utils", diff --git a/packages/fern-docs/bundle/package.json b/packages/fern-docs/bundle/package.json index fabf290a9a..15f302200c 100644 --- a/packages/fern-docs/bundle/package.json +++ b/packages/fern-docs/bundle/package.json @@ -39,6 +39,7 @@ "@fern-docs/search-server": "workspace:*", "@fern-docs/search-ui": "workspace:*", "@fern-docs/search-utils": "workspace:*", + "@fern-docs/seo": "workspace:*", "@fern-docs/syntax-highlighter": "workspace:*", "@fern-docs/ui": "workspace:*", "@fern-docs/utils": "workspace:*", diff --git a/packages/fern-docs/bundle/src/app/api/fern-docs/auth/api-key-injection/route.ts b/packages/fern-docs/bundle/src/app/api/fern-docs/auth/api-key-injection/route.ts index 4f84f7b539..4e715fcb50 100644 --- a/packages/fern-docs/bundle/src/app/api/fern-docs/auth/api-key-injection/route.ts +++ b/packages/fern-docs/bundle/src/app/api/fern-docs/auth/api-key-injection/route.ts @@ -6,8 +6,7 @@ import { getDocsDomainEdge, getHostEdge } from "@/server/xfernhost/edge"; import { withDefaultProtocol } from "@fern-api/ui-core-utils"; import { APIKeyInjectionConfig, OryAccessTokenSchema } from "@fern-docs/auth"; import { getAuthEdgeConfig } from "@fern-docs/edge-config"; -import { COOKIE_FERN_TOKEN } from "@fern-docs/utils"; -import { removeTrailingSlash } from "next/dist/shared/lib/router/utils/remove-trailing-slash"; +import { COOKIE_FERN_TOKEN, removeTrailingSlash } from "@fern-docs/utils"; import { cookies } from "next/headers"; import { NextRequest, NextResponse } from "next/server"; import urlJoin from "url-join"; diff --git a/packages/fern-docs/bundle/src/app/sitemap.ts b/packages/fern-docs/bundle/src/app/sitemap.ts index efe263e42c..0f7a476b24 100644 --- a/packages/fern-docs/bundle/src/app/sitemap.ts +++ b/packages/fern-docs/bundle/src/app/sitemap.ts @@ -1,9 +1,9 @@ import { DocsLoader } from "@/server/DocsLoader"; -import { conformTrailingSlash } from "@/server/trailingSlash"; import { withPrunedNavigation } from "@/server/withPrunedNavigation"; import { getDocsDomainApp, getHostApp } from "@/server/xfernhost/app"; import { NodeCollector } from "@fern-api/fdr-sdk/navigation"; import { withDefaultProtocol } from "@fern-api/ui-core-utils"; +import { conformTrailingSlash } from "@fern-docs/utils"; import type { MetadataRoute } from "next"; import urljoin from "url-join"; diff --git a/packages/fern-docs/bundle/src/middleware.ts b/packages/fern-docs/bundle/src/middleware.ts index 5b284303c8..9a1b7dd740 100644 --- a/packages/fern-docs/bundle/src/middleware.ts +++ b/packages/fern-docs/bundle/src/middleware.ts @@ -1,7 +1,7 @@ import { rewritePosthog } from "@/server/analytics/rewritePosthog"; import { extractNextDataPathname } from "@/server/extractNextDataPathname"; import { getLaunchDarklySettings } from "@fern-docs/edge-config"; -import { removeTrailingSlash } from "next/dist/shared/lib/router/utils/remove-trailing-slash"; +import { removeTrailingSlash } from "@fern-docs/utils"; import { NextResponse, type NextMiddleware } from "next/server"; import { MARKDOWN_PATTERN, RSS_PATTERN } from "./server/patterns"; import { withMiddlewareAuth } from "./server/withMiddlewareAuth"; diff --git a/packages/fern-docs/bundle/src/server/LoadDocsPerformanceTracker.ts b/packages/fern-docs/bundle/src/server/LoadDocsPerformanceTracker.ts index 0a6194bc3c..0ccb20c180 100644 --- a/packages/fern-docs/bundle/src/server/LoadDocsPerformanceTracker.ts +++ b/packages/fern-docs/bundle/src/server/LoadDocsPerformanceTracker.ts @@ -1,9 +1,9 @@ +import { track } from "@/server/analytics/posthog"; import { FernNavigation } from "@fern-api/fdr-sdk"; import { DocsPage } from "@fern-docs/ui"; import { TRACK_LOAD_DOCS_PERFORMANCE } from "@fern-docs/utils"; import { GetServerSidePropsResult } from "next/types"; import { ComponentProps } from "react"; -import { track } from "./analytics/posthog"; import type { LoadWithUrlResponse } from "./loadWithUrl"; export class LoadDocsPerformanceTracker { diff --git a/packages/fern-docs/bundle/src/server/auth/getAuthState.ts b/packages/fern-docs/bundle/src/server/auth/getAuthState.ts index 102fbf1852..1e03594251 100644 --- a/packages/fern-docs/bundle/src/server/auth/getAuthState.ts +++ b/packages/fern-docs/bundle/src/server/auth/getAuthState.ts @@ -5,8 +5,7 @@ import { getAuthEdgeConfig, getPreviewUrlAuthConfig, } from "@fern-docs/edge-config"; -import { withoutStaging } from "@fern-docs/utils"; -import { removeTrailingSlash } from "next/dist/shared/lib/router/utils/remove-trailing-slash"; +import { removeTrailingSlash, withoutStaging } from "@fern-docs/utils"; import urlJoin from "url-join"; import { safeVerifyFernJWTConfig } from "./FernJWT"; import { getAllowedRedirectUrls } from "./allowed-redirects"; diff --git a/packages/fern-docs/bundle/src/server/auth/workos-handler.ts b/packages/fern-docs/bundle/src/server/auth/workos-handler.ts index 5985ab7e99..f57fa1ab55 100644 --- a/packages/fern-docs/bundle/src/server/auth/workos-handler.ts +++ b/packages/fern-docs/bundle/src/server/auth/workos-handler.ts @@ -1,5 +1,5 @@ import { withDefaultProtocol } from "@fern-api/ui-core-utils"; -import { removeTrailingSlash } from "next/dist/shared/lib/router/utils/remove-trailing-slash"; +import { removeTrailingSlash } from "@fern-docs/utils"; import urlJoin from "url-join"; import { AuthState, getWorkosRbacRoles } from "./getAuthState"; import { getWorkosSSOAuthorizationUrl } from "./workos"; diff --git a/packages/fern-docs/ui/src/seo/getBreadcrumbList.ts b/packages/fern-docs/bundle/src/server/getBreadcrumbList.ts similarity index 97% rename from packages/fern-docs/ui/src/seo/getBreadcrumbList.ts rename to packages/fern-docs/bundle/src/server/getBreadcrumbList.ts index d2701e61bb..116e81504f 100644 --- a/packages/fern-docs/ui/src/seo/getBreadcrumbList.ts +++ b/packages/fern-docs/bundle/src/server/getBreadcrumbList.ts @@ -1,7 +1,7 @@ import type * as FernDocs from "@fern-api/fdr-sdk/docs"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { withDefaultProtocol } from "@fern-api/ui-core-utils"; -import { JsonLd } from "@fern-docs/next-seo"; +import { JsonLd } from "@fern-docs/seo"; import urljoin from "url-join"; function toUrl(domain: string, slug: FernNavigation.Slug): string { diff --git a/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts b/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts index 1071ea9552..70cc1095c5 100644 --- a/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts +++ b/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts @@ -6,12 +6,11 @@ import { } from "@fern-api/fdr-sdk/api-definition"; import { MarkdownText } from "@fern-api/fdr-sdk/docs"; import { isNonNullish } from "@fern-api/ui-core-utils"; -import { EdgeFlags } from "@fern-docs/utils"; +import { EdgeFlags, removeLeadingSlash } from "@fern-docs/utils"; import { isString } from "es-toolkit/predicate"; import { DocsLoader } from "./DocsLoader"; import { pascalCaseHeaderKey } from "./headerKeyCase"; import { convertToLlmTxtMarkdown } from "./llm-txt-md"; -import { removeLeadingSlash } from "./removeLeadingSlash"; export async function getMarkdownForPath( node: FernNavigation.NavigationNodePage, diff --git a/packages/fern-docs/bundle/src/server/getSectionRoot.ts b/packages/fern-docs/bundle/src/server/getSectionRoot.ts index 672e8b4748..052bcc3bef 100644 --- a/packages/fern-docs/bundle/src/server/getSectionRoot.ts +++ b/packages/fern-docs/bundle/src/server/getSectionRoot.ts @@ -1,6 +1,6 @@ import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { CONTINUE, STOP } from "@fern-api/fdr-sdk/traversers"; -import { removeLeadingSlash } from "./removeLeadingSlash"; +import { removeLeadingSlash } from "@fern-docs/utils"; export function getSectionRoot( root: FernNavigation.RootNode | undefined, diff --git a/packages/fern-docs/bundle/src/server/pageRoutes.ts b/packages/fern-docs/bundle/src/server/pageRoutes.ts index 01334a6fd8..51b6bb0851 100644 --- a/packages/fern-docs/bundle/src/server/pageRoutes.ts +++ b/packages/fern-docs/bundle/src/server/pageRoutes.ts @@ -1,6 +1,5 @@ -import { addLeadingSlash } from "@fern-docs/utils"; +import { addLeadingSlash, removeTrailingSlash } from "@fern-docs/utils"; import getAssetPathFromRoute from "next/dist/shared/lib/router/utils/get-asset-path-from-route"; -import { removeTrailingSlash } from "next/dist/shared/lib/router/utils/remove-trailing-slash"; import urlJoin from "url-join"; export function getPageRoute( diff --git a/packages/fern-docs/bundle/src/server/queue-reindex.ts b/packages/fern-docs/bundle/src/server/queue-reindex.ts index ae3f28d1d4..cb2308c9a0 100644 --- a/packages/fern-docs/bundle/src/server/queue-reindex.ts +++ b/packages/fern-docs/bundle/src/server/queue-reindex.ts @@ -5,9 +5,9 @@ import { HEADER_X_FERN_HOST, HEADER_X_VERCEL_PROTECTION_BYPASS, addLeadingSlash, + conformTrailingSlash, + removeTrailingSlash, } from "@fern-docs/utils"; -import { removeTrailingSlash } from "next/dist/shared/lib/router/utils/remove-trailing-slash"; -import { conformTrailingSlash } from "./trailingSlash"; const q = new Client({ token: qstashToken() }); diff --git a/packages/fern-docs/bundle/src/server/revalidator.ts b/packages/fern-docs/bundle/src/server/revalidator.ts index 0b6d75d14a..0644f13a33 100644 --- a/packages/fern-docs/bundle/src/server/revalidator.ts +++ b/packages/fern-docs/bundle/src/server/revalidator.ts @@ -1,7 +1,7 @@ +import { conformTrailingSlash } from "@fern-docs/utils"; import type { FernDocs } from "@fern-fern/fern-docs-sdk"; import type { NextApiResponse } from "next"; import urljoin from "url-join"; -import { conformTrailingSlash } from "./trailingSlash"; export class Revalidator implements Revalidator { constructor( diff --git a/packages/fern-docs/bundle/src/server/withInitialProps.ts b/packages/fern-docs/bundle/src/server/withInitialProps.ts index 3236ca66b1..49ae081d5d 100644 --- a/packages/fern-docs/bundle/src/server/withInitialProps.ts +++ b/packages/fern-docs/bundle/src/server/withInitialProps.ts @@ -7,20 +7,27 @@ import { getEdgeFlags, getSeoDisabled, } from "@fern-docs/edge-config"; +import { withSeo } from "@fern-docs/seo"; import { - DocsPage, - NavbarLink, + type DocsPage, + type ImageData, + type NavbarLink, getApiRouteSupplier, getGitHubInfo, getGitHubRepo, - getSeoProps, renderThemeStylesheet, + withLogo, } from "@fern-docs/ui"; import { serializeMdx } from "@fern-docs/ui/bundlers/mdx-bundler"; -import { addLeadingSlash, getRedirectForPath } from "@fern-docs/utils"; +import { + addLeadingSlash, + getRedirectForPath, + isTrailingSlashEnabled, +} from "@fern-docs/utils"; import { SidebarTab } from "@fern-platform/fdr-utils"; import { GetServerSidePropsResult, Redirect } from "next"; import { ComponentProps } from "react"; +import { UnreachableCaseError } from "ts-essentials"; import urlJoin from "url-join"; import { DocsLoader } from "./DocsLoader"; import { getAuthState } from "./auth/getAuthState"; @@ -28,12 +35,14 @@ import { getReturnToQueryParam } from "./auth/return-to"; import { handleLoadDocsError } from "./handleLoadDocsError"; import { withLaunchDarkly } from "./ld-adapter"; import type { LoadWithUrlResponse } from "./loadWithUrl"; -import { isTrailingSlashEnabled } from "./trailingSlash"; import { pruneNavigationPredicate, withPrunedNavigation, } from "./withPrunedNavigation"; -import { withResolvedDocsContent } from "./withResolvedDocsContent"; +import { + extractFrontmatterFromDocsContent, + withResolvedDocsContent, +} from "./withResolvedDocsContent"; import { withVersionSwitcherInfo } from "./withVersionSwitcherInfo"; interface WithInitialProps { @@ -192,12 +201,6 @@ export async function withInitialProps({ : undefined, }; - const logoHref = - docs.definition.config.logoHref ?? - (found.landingPage?.slug != null && !found.landingPage.hidden - ? encodeURI(addLeadingSlash(found.landingPage.slug)) - : undefined); - const navbarLinks: NavbarLink[] = []; docs.definition.config.navbarLinks?.forEach((link) => { @@ -295,6 +298,34 @@ export async function withInitialProps({ pruneNavigationPredicate(tab, pruneOpts) || tab === found.currentTab ); + function resolveFileSrc(src: string | undefined): ImageData | undefined { + if (src == null) { + return undefined; + } + + const fileId = FernNavigation.FileId( + src.startsWith("file:") ? src.slice(5) : src + ); + const file = docs.definition.filesV2[fileId]; + if (file == null) { + // the file is not found, so we return the src as the image data + return { src }; + } + + if (file.type === "image") { + return { + src: file.url, + width: file.width, + height: file.height, + blurDataURL: file.blurDataUrl, + }; + } else if (file.type === "url") { + return { src: file.url }; + } else { + throw new UnreachableCaseError(file); + } + } + const content = await withResolvedDocsContent({ domain: docs.baseUrl.domain, found, @@ -311,7 +342,9 @@ export async function withInitialProps({ tab: found?.currentTab?.title, }, }, + replaceSrc: resolveFileSrc, }); + const frontmatter = extractFrontmatterFromDocsContent(found.node.id, content); if (content == null) { return { notFound: true }; @@ -361,9 +394,7 @@ export async function withInitialProps({ colors, js: docs.definition.config.js, navbarLinks, - logoHeight: docs.definition.config.logoHeight, - logoHref: logoHref != null ? FernNavigation.Url(logoHref) : undefined, - files: docs.definition.filesV2, + logo: withLogo(docs.definition, found, frontmatter, resolveFileSrc), content, announcement: docs.definition.config.announcement != null @@ -382,15 +413,14 @@ export async function withInitialProps({ }, edgeFlags, apis: Object.keys(docs.definition.apis).map(FernNavigation.ApiDefinitionId), - seo: getSeoProps( + seo: withSeo( docs.baseUrl.domain, docs.definition.config, - docs.definition.pages, + frontmatter, docs.definition.filesV2, docs.definition.apis, found, - await getSeoDisabled(domain), - isTrailingSlashEnabled() + await getSeoDisabled(domain) ), user: authState.authed ? authState.user : undefined, fallback: {}, diff --git a/packages/fern-docs/bundle/src/server/withResolvedDocsContent.ts b/packages/fern-docs/bundle/src/server/withResolvedDocsContent.ts index ece2605e90..5c541c6cb1 100644 --- a/packages/fern-docs/bundle/src/server/withResolvedDocsContent.ts +++ b/packages/fern-docs/bundle/src/server/withResolvedDocsContent.ts @@ -1,6 +1,12 @@ import { DocsV1Read } from "@fern-api/fdr-sdk"; -import type * as FernNavigation from "@fern-api/fdr-sdk/navigation"; -import { resolveDocsContent, type DocsContent } from "@fern-docs/ui"; +import type * as FernDocs from "@fern-api/fdr-sdk/docs"; +import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; +import { getFrontmatter } from "@fern-docs/mdx"; +import { + resolveDocsContent, + type DocsContent, + type ImageData, +} from "@fern-docs/ui"; import { serializeMdx } from "@fern-docs/ui/bundlers/mdx-bundler"; import { EdgeFlags } from "@fern-docs/utils"; import { AuthState } from "./auth/getAuthState"; @@ -13,6 +19,7 @@ interface WithResolvedDocsContentOpts { definition: DocsV1Read.DocsDefinition; edgeFlags: EdgeFlags; scope?: Record; + replaceSrc?: (src: string) => ImageData | undefined; } export async function withResolvedDocsContent({ @@ -22,6 +29,7 @@ export async function withResolvedDocsContent({ definition, edgeFlags, scope, + replaceSrc, }: WithResolvedDocsContentOpts): Promise { const node = withPrunedNavigation(found.node, { visibleNodeIds: [found.node.id], @@ -67,9 +75,46 @@ export async function withResolvedDocsContent({ mdxOptions: { files: definition.jsFiles, scope, + + // inject the file url and dimensions for images and other embeddable files + replaceSrc, }, serializeMdx, domain, engine: "mdx-bundler", }); } + +export function extractFrontmatterFromDocsContent( + nodeId: FernNavigation.NodeId, + docsContent: DocsContent | undefined +): FernDocs.Frontmatter | undefined { + if (docsContent == null) { + return undefined; + } + switch (docsContent.type) { + case "markdown-page": + return getFrontmatterFromMarkdownText(docsContent.content); + case "changelog-entry": + return getFrontmatterFromMarkdownText(docsContent.page); + case "api-reference-page": { + const mdx = docsContent.mdxs[nodeId]; + if (mdx == null) { + return undefined; + } + return getFrontmatterFromMarkdownText(mdx.content); + } + default: + // TODO: handle changelog overview page and other pages + return undefined; + } +} + +function getFrontmatterFromMarkdownText( + markdownText: FernDocs.MarkdownText +): FernDocs.Frontmatter | undefined { + if (typeof markdownText === "string") { + return getFrontmatter(markdownText).data; + } + return markdownText.frontmatter; +} diff --git a/packages/fern-docs/bundle/tsconfig.json b/packages/fern-docs/bundle/tsconfig.json index 3d88b0649b..35dee12a4f 100644 --- a/packages/fern-docs/bundle/tsconfig.json +++ b/packages/fern-docs/bundle/tsconfig.json @@ -17,31 +17,40 @@ "path": "../../commons/fdr-utils" }, { - "path": "../search-utils" + "path": "../../fdr-sdk/tsconfig.build.json" }, { "path": "../auth" }, { - "path": "../edge-config" + "path": "../cache" }, { - "path": "../utils" + "path": "../components" }, { - "path": "../cache" + "path": "../edge-config" }, { "path": "../mdx" }, { - "path": "../../fdr-sdk/tsconfig.build.json" + "path": "../search-server" + }, + { + "path": "../search-utils" + }, + { + "path": "../seo" + }, + { + "path": "../syntax-highlighter" }, { "path": "../ui/tsconfig.build.json" }, { - "path": "../components" + "path": "../utils" } ] } diff --git a/packages/fern-docs/local-preview-bundle/next.config.mjs b/packages/fern-docs/local-preview-bundle/next.config.mjs index d1b2e9e13a..06c187348e 100644 --- a/packages/fern-docs/local-preview-bundle/next.config.mjs +++ b/packages/fern-docs/local-preview-bundle/next.config.mjs @@ -19,7 +19,7 @@ const nextConfig = { "@fern-docs/cache", "@fern-docs/components", "@fern-docs/mdx", - "@fern-docs/next-seo", + "@fern-docs/seo", "@fern-docs/search-server", "@fern-docs/search-ui", "@fern-docs/search-utils", diff --git a/packages/fern-docs/local-preview-bundle/package.json b/packages/fern-docs/local-preview-bundle/package.json index f62180d9f7..40f4cd05c1 100644 --- a/packages/fern-docs/local-preview-bundle/package.json +++ b/packages/fern-docs/local-preview-bundle/package.json @@ -36,6 +36,8 @@ "@fern-api/fdr-sdk": "workspace:*", "@fern-api/ui-core-utils": "workspace:*", "@fern-docs/components": "workspace:*", + "@fern-docs/mdx": "workspace:*", + "@fern-docs/seo": "workspace:*", "@fern-docs/syntax-highlighter": "workspace:*", "@fern-docs/ui": "workspace:*", "@fern-docs/utils": "workspace:*", @@ -48,6 +50,7 @@ "react": "^18", "react-dom": "^18", "styled-jsx": "^5.1.2", + "ts-essentials": "^10.0.1", "url-join": "5.0.0" }, "devDependencies": { diff --git a/packages/fern-docs/local-preview-bundle/src/utils/getDocsPageProps.ts b/packages/fern-docs/local-preview-bundle/src/utils/getDocsPageProps.ts index 0ee08581fd..329a4089d5 100644 --- a/packages/fern-docs/local-preview-bundle/src/utils/getDocsPageProps.ts +++ b/packages/fern-docs/local-preview-bundle/src/utils/getDocsPageProps.ts @@ -1,3 +1,4 @@ +import { FernDocs } from "@fern-api/fdr-sdk"; import { ApiDefinition, CodeSnippet, @@ -14,14 +15,18 @@ import { } from "@fern-api/fdr-sdk/client/types"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { visitDiscriminatedUnion } from "@fern-api/ui-core-utils"; +import { getFrontmatter } from "@fern-docs/mdx"; +import { withSeo } from "@fern-docs/seo"; import { + DocsContent, DocsPage, getGitHubInfo, getGitHubRepo, - getSeoProps, + ImageData, NavbarLink, renderThemeStylesheet, resolveDocsContent, + withLogo, } from "@fern-docs/ui"; import { serializeMdx } from "@fern-docs/ui/bundlers/next-mdx-remote"; import { @@ -32,6 +37,7 @@ import { import { SidebarTab } from "@fern-platform/fdr-utils"; import type { GetServerSidePropsResult } from "next"; import { ComponentProps } from "react"; +import { UnreachableCaseError } from "ts-essentials"; import urljoin from "url-join"; export async function getDocsPageProps( @@ -84,6 +90,34 @@ export async function getDocsPageProps( // TODO: get feature flags from the API const edgeFlags: EdgeFlags = DEFAULT_EDGE_FLAGS; + function resolveFileSrc(src: string | undefined): ImageData | undefined { + if (src == null) { + return undefined; + } + + const fileId = FernNavigation.FileId( + src.startsWith("file:") ? src.slice(5) : src + ); + const file = docs.definition.filesV2[fileId]; + if (file == null) { + // the file is not found, so we return the src as the image data + return { src }; + } + + if (file.type === "image") { + return { + src: file.url, + width: file.width, + height: file.height, + blurDataURL: file.blurDataUrl, + }; + } else if (file.type === "url") { + return { src: file.url }; + } else { + throw new UnreachableCaseError(file); + } + } + const content = await resolveDocsContent({ domain: docs.baseUrl.domain, node: node.node, @@ -109,11 +143,25 @@ export async function getDocsPageProps( edgeFlags, mdxOptions: { files: docs.definition.jsFiles, + scope: { + env: "development", + props: { + authed: false, + user: undefined, + version: node.currentVersion?.versionId, + tab: node.currentTab?.title, + }, + }, + + // inject the file url and dimensions for images and other embeddable files + replaceSrc: resolveFileSrc, }, serializeMdx, engine: "next-mdx-remote", }); + const frontmatter = extractFrontmatterFromDocsContent(node.node.id, content); + if (content == null) { console.error(`Failed to resolve path for ${slug}`); return { notFound: true }; @@ -190,9 +238,7 @@ export async function getDocsPageProps( colors, js: docs.definition.config.js, navbarLinks, - logoHeight: docs.definition.config.logoHeight, - logoHref: docs.definition.config.logoHref, - files: docs.definition.filesV2, + logo: withLogo(docs.definition, node, frontmatter, resolveFileSrc), content, announcement: docs.definition.config.announcement != null @@ -243,15 +289,14 @@ export async function getDocsPageProps( }, edgeFlags, apis: Object.keys(docs.definition.apis).map(FdrAPI.ApiDefinitionId), - seo: getSeoProps( + seo: withSeo( docs.baseUrl.domain, docs.definition.config, - docs.definition.pages, + frontmatter, docs.definition.filesV2, docs.definition.apis, node, - true, - false + true ), fallback: {}, analytics: undefined, @@ -362,3 +407,37 @@ const resolveExample = async ( return { ...example, snippets }; }; + +export function extractFrontmatterFromDocsContent( + nodeId: FernNavigation.NodeId, + docsContent: DocsContent | undefined +): FernDocs.Frontmatter | undefined { + if (docsContent == null) { + return undefined; + } + switch (docsContent.type) { + case "markdown-page": + return getFrontmatterFromMarkdownText(docsContent.content); + case "changelog-entry": + return getFrontmatterFromMarkdownText(docsContent.page); + case "api-reference-page": { + const mdx = docsContent.mdxs[nodeId]; + if (mdx == null) { + return undefined; + } + return getFrontmatterFromMarkdownText(mdx.content); + } + default: + // TODO: handle changelog overview page and other pages + return undefined; + } +} + +function getFrontmatterFromMarkdownText( + markdownText: FernDocs.MarkdownText +): FernDocs.Frontmatter | undefined { + if (typeof markdownText === "string") { + return getFrontmatter(markdownText).data; + } + return markdownText.frontmatter; +} diff --git a/packages/fern-docs/mdx/.depcheckrc.json b/packages/fern-docs/mdx/.depcheckrc.json index 9335a9197e..f8d31b353d 100644 --- a/packages/fern-docs/mdx/.depcheckrc.json +++ b/packages/fern-docs/mdx/.depcheckrc.json @@ -1,4 +1,12 @@ { - "ignores": ["@fern-platform/configs", "@types/node", "vite", "@types/react"], + "ignores": [ + "@fern-platform/configs", + "@types/node", + "@types/react", + "hast", + "mdast", + "mdx", + "vite" + ], "ignore-patterns": ["dist"] } diff --git a/packages/fern-docs/mdx/package.json b/packages/fern-docs/mdx/package.json index 57b768c725..146c9779ee 100644 --- a/packages/fern-docs/mdx/package.json +++ b/packages/fern-docs/mdx/package.json @@ -42,6 +42,7 @@ "@types/estree": "^1.0.6", "@types/hast": "^3.0.4", "@types/mdast": "^4.0.4", + "@types/mdx": "^2.0.13", "@types/unist": "^3.0.3", "collapse-white-space": "^2.1.0", "es-toolkit": "^1.27.0", @@ -67,6 +68,7 @@ "rehype-slug": "^6.0.0", "style-to-object": "^1.0.8", "ts-essentials": "^10.0.1", + "unified": "^11.0.4", "unist-util-visit": "^5.0.0", "vfile-message": "^4.0.2" }, diff --git a/packages/fern-docs/mdx/src/hast-utils/hast-to-string.ts b/packages/fern-docs/mdx/src/hast-utils/hast-to-string.ts new file mode 100644 index 0000000000..c5e02616c9 --- /dev/null +++ b/packages/fern-docs/mdx/src/hast-utils/hast-to-string.ts @@ -0,0 +1 @@ +export { toString as hastToString } from "hast-util-to-string"; diff --git a/packages/fern-docs/mdx/src/hast-utils/index.ts b/packages/fern-docs/mdx/src/hast-utils/index.ts index 7a026bddec..da9223c5e4 100644 --- a/packages/fern-docs/mdx/src/hast-utils/index.ts +++ b/packages/fern-docs/mdx/src/hast-utils/index.ts @@ -1,5 +1,6 @@ export * from "./hast-get-boolean-value"; export * from "./hast-mdx-to-props"; +export * from "./hast-to-string"; export * from "./is-hast-element"; export * from "./is-hast-text"; export * from "./parse-string-style"; diff --git a/packages/fern-docs/mdx/src/hast.ts b/packages/fern-docs/mdx/src/hast.ts new file mode 100644 index 0000000000..15d0e57149 --- /dev/null +++ b/packages/fern-docs/mdx/src/hast.ts @@ -0,0 +1,10 @@ +export type * from "hast"; +export type { MdxJsxElementHast as MdxJsxElement } from "./declarations"; + +declare module "hast" { + interface ElementData { + visited?: boolean; + meta?: string | null; + metastring?: string; + } +} diff --git a/packages/fern-docs/mdx/src/index.ts b/packages/fern-docs/mdx/src/index.ts index e8f87ec80a..9d9dccf7ec 100644 --- a/packages/fern-docs/mdx/src/index.ts +++ b/packages/fern-docs/mdx/src/index.ts @@ -1,5 +1,14 @@ export type { MdxJsxAttribute } from "mdast-util-mdx"; -export { visit } from "unist-util-visit"; +export type { MDXComponents } from "mdx/types"; +export type { PluggableList } from "unified"; +export { + CONTINUE, + EXIT, + SKIP, + visit, + type BuildVisitor, + type VisitorResult, +} from "unist-util-visit"; export * from "./declarations"; export * from "./frontmatter"; export * from "./handlers/index"; @@ -12,3 +21,4 @@ export * from "./sanitize/index"; export * from "./split-into-sections"; export * from "./strip-util"; export * from "./toc"; +export * from "./types"; diff --git a/packages/fern-docs/mdx/src/mdast.ts b/packages/fern-docs/mdx/src/mdast.ts new file mode 100644 index 0000000000..01a6fbbd59 --- /dev/null +++ b/packages/fern-docs/mdx/src/mdast.ts @@ -0,0 +1,2 @@ +export type * from "mdast"; +export type { MdxJsxElement } from "./declarations"; diff --git a/packages/fern-docs/mdx/src/mdx-utils/index.ts b/packages/fern-docs/mdx/src/mdx-utils/index.ts index 3b2a9cf5cb..094a8053ec 100644 --- a/packages/fern-docs/mdx/src/mdx-utils/index.ts +++ b/packages/fern-docs/mdx/src/mdx-utils/index.ts @@ -3,5 +3,6 @@ export * from "./is-mdx-element"; export * from "./is-mdx-expression"; export * from "./is-mdx-jsx-attr"; export * from "./markdown-to-string"; +export * from "./mdx-jsx-attribute-to-string"; export * from "./unknown-to-estree-expression"; export * from "./unknown-to-mdx-jsx-attr"; diff --git a/packages/fern-docs/mdx/src/mdx-utils/mdx-jsx-attribute-to-string.ts b/packages/fern-docs/mdx/src/mdx-utils/mdx-jsx-attribute-to-string.ts new file mode 100644 index 0000000000..37848bad9a --- /dev/null +++ b/packages/fern-docs/mdx/src/mdx-utils/mdx-jsx-attribute-to-string.ts @@ -0,0 +1,30 @@ +import type { MdxJsxAttribute } from "mdast-util-mdx-jsx"; + +export function mdxJsxAttributeToString( + attribute: MdxJsxAttribute +): string | undefined { + if (typeof attribute.value === "string") { + return attribute.value; + } + + if ( + typeof attribute.value === "object" && + attribute.value?.type === "mdxJsxAttributeValueExpression" + ) { + const expression = attribute.value.value; + + // if expression is wrapped in "" or '', then return the value inside + if (expression.startsWith('"') && expression.endsWith('"')) { + return expression.slice(1, -1); + } + + if (expression.startsWith("'") && expression.endsWith("'")) { + return expression.slice(1, -1); + } + + // if the expression is wrapped in backticks, we cannot parse it, because it may contain a variable. + // for now, we'll just return undefined. + } + + return undefined; +} diff --git a/packages/fern-docs/mdx/src/strip-util.ts b/packages/fern-docs/mdx/src/strip-util.ts index 90e462980c..86a05366d9 100644 --- a/packages/fern-docs/mdx/src/strip-util.ts +++ b/packages/fern-docs/mdx/src/strip-util.ts @@ -1,5 +1,5 @@ -import { toString } from "hast-util-to-string"; import { visit } from "unist-util-visit"; +import { hastToString } from "./hast-utils/hast-to-string.js"; import { isMdxJsxElementHast } from "./mdx-utils/is-mdx-element.js"; import { toTree } from "./parse.js"; @@ -35,5 +35,5 @@ export function stripUtil( }); // TODO: (andrew), this might have some issues with formatting new lines - return toString(hast); + return hastToString(hast); } diff --git a/packages/fern-docs/mdx/src/toc.ts b/packages/fern-docs/mdx/src/toc.ts index 63b5efb8ca..b826d6e3cd 100644 --- a/packages/fern-docs/mdx/src/toc.ts +++ b/packages/fern-docs/mdx/src/toc.ts @@ -1,8 +1,8 @@ import { slug } from "github-slugger"; import type { Doctype, ElementContent, Root } from "hast"; import { headingRank } from "hast-util-heading-rank"; -import { toString } from "hast-util-to-string"; import { SKIP, visit, type BuildVisitor } from "unist-util-visit"; +import { hastToString } from "./hast-utils"; import { hastGetBooleanValue } from "./hast-utils/hast-get-boolean-value"; import { isHastElement } from "./hast-utils/is-hast-element"; import { isMdxJsxElementHast } from "./mdx-utils/is-mdx-element"; @@ -81,7 +81,7 @@ export function makeToc( return; } - const title = toString(node); + const title = hastToString(node); headings.push({ depth: rank, id, title }); } @@ -99,9 +99,13 @@ export function makeToc( return; } - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const depth = parseInt(node.name[1]!); - const title = toString(node); + const depth = parseInt(node.name[1] ?? "0"); + if (depth < 1 || depth > 6) { + // depth must be between 1 and 6 + return; + } + + const title = hastToString(node); headings.push({ depth, id, title }); } @@ -179,8 +183,10 @@ function makeTree( const tree: TableOfContentsItem[] = []; while (headings.length > 0) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const firstToken = headings[0]!; + const firstToken = headings[0]; + if (!firstToken) { + break; + } // if the next heading is at a higher level if (firstToken.depth < depth) { diff --git a/packages/fern-docs/mdx/src/types.ts b/packages/fern-docs/mdx/src/types.ts new file mode 100644 index 0000000000..37646f4c08 --- /dev/null +++ b/packages/fern-docs/mdx/src/types.ts @@ -0,0 +1,4 @@ +import type * as Hast from "./hast"; +import type * as Mdast from "./mdast"; + +export type { Hast, Mdast }; diff --git a/packages/fern-docs/next-seo/.stylelintrc.json b/packages/fern-docs/seo/.stylelintrc.json similarity index 100% rename from packages/fern-docs/next-seo/.stylelintrc.json rename to packages/fern-docs/seo/.stylelintrc.json diff --git a/packages/fern-docs/next-seo/README.md b/packages/fern-docs/seo/README.md similarity index 100% rename from packages/fern-docs/next-seo/README.md rename to packages/fern-docs/seo/README.md diff --git a/packages/fern-docs/next-seo/package.json b/packages/fern-docs/seo/package.json similarity index 87% rename from packages/fern-docs/next-seo/package.json rename to packages/fern-docs/seo/package.json index 62e1362bee..46a5aff2fa 100644 --- a/packages/fern-docs/next-seo/package.json +++ b/packages/fern-docs/seo/package.json @@ -1,5 +1,5 @@ { - "name": "@fern-docs/next-seo", + "name": "@fern-docs/seo", "version": "0.0.0", "private": true, "repository": { @@ -30,11 +30,15 @@ }, "dependencies": { "@fern-api/fdr-sdk": "workspace:*", + "@fern-api/ui-core-utils": "workspace:*", + "@fern-docs/mdx": "workspace:*", + "@fern-docs/utils": "workspace:*", "next": "npm:@fern-api/next@14.2.9-fork.2", "react": "^18" }, "devDependencies": { "@fern-platform/configs": "workspace:*", + "@fern-platform/fdr-utils": "workspace:*", "@types/node": "^18.7.18", "@types/react": "^18", "depcheck": "^1.4.3", diff --git a/packages/fern-docs/ui/src/seo/__test__/getBreadcrumbList.test.ts b/packages/fern-docs/seo/src/__test__/getBreadcrumbList.test.ts similarity index 96% rename from packages/fern-docs/ui/src/seo/__test__/getBreadcrumbList.test.ts rename to packages/fern-docs/seo/src/__test__/getBreadcrumbList.test.ts index 0edd02fba9..480fca8f0a 100644 --- a/packages/fern-docs/ui/src/seo/__test__/getBreadcrumbList.test.ts +++ b/packages/fern-docs/seo/src/__test__/getBreadcrumbList.test.ts @@ -1,5 +1,5 @@ import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; -import { getBreadcrumbList } from "../getBreadcrumbList"; +import { getBreadcrumbList } from "../with-breadcrumb"; describe("getBreadcrumbList", () => { it("should override the title used in the breadcrumb's last item", () => { diff --git a/packages/fern-docs/ui/src/seo/__test__/getSeoProps.test.ts b/packages/fern-docs/seo/src/__test__/getSeoProps.test.ts similarity index 66% rename from packages/fern-docs/ui/src/seo/__test__/getSeoProps.test.ts rename to packages/fern-docs/seo/src/__test__/getSeoProps.test.ts index b193b4df38..73033bcf2f 100644 --- a/packages/fern-docs/ui/src/seo/__test__/getSeoProps.test.ts +++ b/packages/fern-docs/seo/src/__test__/getSeoProps.test.ts @@ -1,14 +1,13 @@ import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; -import { markdownToString } from "@fern-docs/mdx"; import { DefinitionObjectFactory } from "@fern-platform/fdr-utils"; -import { extractHeadline, getSeoProps } from "../getSeoProp"; +import { withSeo } from "../with-seo"; describe("getSeoProps", () => { it("seo disabled", () => { - const props = getSeoProps( + const props = withSeo( "host", DefinitionObjectFactory.createDocsDefinition().config, - {}, + undefined, {}, {}, { @@ -33,17 +32,9 @@ describe("getSeoProps", () => { slug: FernNavigation.Slug(""), } as FernNavigation.RootNode, }, - true, - false + true ); expect(props.noindex).toBe(true); expect(props.nofollow).toBe(true); }); - - it("extracts SEO title properly", () => { - expect(markdownToString(extractHeadline("#"))).toBe(""); - expect(markdownToString(extractHeadline("# goodcase"))).toBe("goodcase"); - expect(markdownToString(extractHeadline("## h2"))).toBe(undefined); - expect(markdownToString(extractHeadline("##nospaceh2"))).toBe(undefined); - }); }); diff --git a/packages/fern-docs/next-seo/src/index.ts b/packages/fern-docs/seo/src/index.ts similarity index 60% rename from packages/fern-docs/next-seo/src/index.ts rename to packages/fern-docs/seo/src/index.ts index d1a0332c18..ac4cb196fd 100644 --- a/packages/fern-docs/next-seo/src/index.ts +++ b/packages/fern-docs/seo/src/index.ts @@ -1,3 +1,5 @@ export * as JsonLd from "./jsonld"; export * from "./meta/buildTags"; export * from "./types"; +export * from "./with-breadcrumb"; +export * from "./with-seo"; diff --git a/packages/fern-docs/next-seo/src/jsonld/components/Breadcrumb.tsx b/packages/fern-docs/seo/src/jsonld/components/Breadcrumb.tsx similarity index 100% rename from packages/fern-docs/next-seo/src/jsonld/components/Breadcrumb.tsx rename to packages/fern-docs/seo/src/jsonld/components/Breadcrumb.tsx diff --git a/packages/fern-docs/next-seo/src/jsonld/index.ts b/packages/fern-docs/seo/src/jsonld/index.ts similarity index 100% rename from packages/fern-docs/next-seo/src/jsonld/index.ts rename to packages/fern-docs/seo/src/jsonld/index.ts diff --git a/packages/fern-docs/next-seo/src/jsonld/types/breadcrumbs.ts b/packages/fern-docs/seo/src/jsonld/types/breadcrumbs.ts similarity index 100% rename from packages/fern-docs/next-seo/src/jsonld/types/breadcrumbs.ts rename to packages/fern-docs/seo/src/jsonld/types/breadcrumbs.ts diff --git a/packages/fern-docs/next-seo/src/meta/buildTags.tsx b/packages/fern-docs/seo/src/meta/buildTags.tsx similarity index 100% rename from packages/fern-docs/next-seo/src/meta/buildTags.tsx rename to packages/fern-docs/seo/src/meta/buildTags.tsx diff --git a/packages/fern-docs/next-seo/src/types.ts b/packages/fern-docs/seo/src/types.ts similarity index 100% rename from packages/fern-docs/next-seo/src/types.ts rename to packages/fern-docs/seo/src/types.ts diff --git a/packages/fern-docs/seo/src/with-breadcrumb.ts b/packages/fern-docs/seo/src/with-breadcrumb.ts new file mode 100644 index 0000000000..aa671d7d2a --- /dev/null +++ b/packages/fern-docs/seo/src/with-breadcrumb.ts @@ -0,0 +1,55 @@ +import type * as FernDocs from "@fern-api/fdr-sdk/docs"; +import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; +import { withDefaultProtocol } from "@fern-api/ui-core-utils"; +import { addLeadingSlash, conformTrailingSlash } from "@fern-docs/utils"; +import * as JsonLd from "./jsonld"; + +function toUrl(domain: string, slug: FernNavigation.Slug): string { + return String( + new URL( + conformTrailingSlash(addLeadingSlash(slug)), + withDefaultProtocol(domain) + ) + ); +} + +export function getBreadcrumbList( + domain: string, + parents: readonly FernNavigation.NavigationNode[], + node: FernNavigation.NavigationNodePage, + title?: string +): FernDocs.JsonLdBreadcrumbList { + title ??= node.title; + + const elements: FernDocs.JsonLdBreadcrumbListElement[] = []; + const visitedSlugs = new Set(); + + parents.forEach((parent) => { + if (FernNavigation.hasMetadata(parent)) { + const slug = visitedSlugs.has(parent.slug) + ? FernNavigation.hasRedirect(parent) + ? parent.pointsTo != null && !visitedSlugs.has(parent.pointsTo) + ? parent.pointsTo + : undefined + : undefined + : parent.slug; + if (slug != null && slug !== node.slug) { + elements.push( + JsonLd.listItem( + elements.length + 1, + parent.title, + toUrl(domain, slug) + ) + ); + visitedSlugs.add(parent.slug); + } + } + }); + + // the current page is the last item in the breadcrumb + elements.push( + JsonLd.listItem(elements.length + 1, title, toUrl(domain, node.slug)) + ); + + return JsonLd.breadcrumbList(elements); +} diff --git a/packages/fern-docs/ui/src/seo/getSeoProp.ts b/packages/fern-docs/seo/src/with-seo.ts similarity index 88% rename from packages/fern-docs/ui/src/seo/getSeoProp.ts rename to packages/fern-docs/seo/src/with-seo.ts index 22835439fc..55a79a9a66 100644 --- a/packages/fern-docs/ui/src/seo/getSeoProp.ts +++ b/packages/fern-docs/seo/src/with-seo.ts @@ -1,5 +1,6 @@ import { APIV1Read, DocsV1Read } from "@fern-api/fdr-sdk/client/types"; -import { +import type * as FernDocs from "@fern-api/fdr-sdk/docs"; +import type { WithJsonLdBreadcrumbs, WithMetadataConfig, } from "@fern-api/fdr-sdk/docs"; @@ -7,12 +8,13 @@ import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { assertNonNullish, visitDiscriminatedUnion, + withDefaultProtocol, } from "@fern-api/ui-core-utils"; -import { getFrontmatter, markdownToString } from "@fern-docs/mdx"; -import type { LinkTag, MetaTag, NextSeoProps } from "@fern-docs/next-seo"; -import { getToHref } from "../hooks/useHref"; -import { getFontExtension } from "../themes/stylesheet/getFontVariables"; -import { getBreadcrumbList } from "./getBreadcrumbList"; +import { markdownToString } from "@fern-docs/mdx"; +import { addLeadingSlash, conformTrailingSlash } from "@fern-docs/utils"; + +import type { LinkTag, MetaTag, NextSeoProps } from "./types"; +import { getBreadcrumbList } from "./with-breadcrumb"; const EMPTY_METADATA_CONFIG: WithMetadataConfig & WithJsonLdBreadcrumbs = { "og:image": undefined, @@ -51,10 +53,10 @@ function getFile( }); } -export function getSeoProps( +export function withSeo( domain: string, { metadata, title, favicon, typographyV2: typography }: DocsV1Read.DocsConfig, - pages: Record, + frontmatter: FernDocs.Frontmatter | undefined, files: Record, apis: Record, { @@ -65,8 +67,7 @@ export function getSeoProps( FernNavigation.utils.Node.Found, "node" | "parents" | "currentVersion" | "root" >, - isSeoDisabled: boolean, - isTrailingSlashEnabled: boolean + isSeoDisabled: boolean ): NextSeoProps { const additionalMetaTags: MetaTag[] = []; const additionalLinkTags: LinkTag[] = []; @@ -85,10 +86,12 @@ export function getSeoProps( * Canonical slugs are computed upstream, where duplicated markdown pages, and multi-version docs are both handled. */ // TODO: set canonical domain in docs.yml - const toHref = getToHref(isTrailingSlashEnabled); - seo.canonical = toHref(node.canonicalSlug ?? node.slug, domain); - - const pageId = FernNavigation.getPageId(node); + seo.canonical = String( + new URL( + conformTrailingSlash(addLeadingSlash(node.canonicalSlug ?? node.slug)), + withDefaultProtocol(domain) + ) + ); let ogMetadata: WithMetadataConfig & WithJsonLdBreadcrumbs = { ...EMPTY_METADATA_CONFIG, @@ -104,9 +107,7 @@ export function getSeoProps( }); } - const page = pageId != null ? pages[pageId] : undefined; - if (page != null) { - const { data: frontmatter, content } = getFrontmatter(page.markdown); + if (frontmatter != null) { ogMetadata = { ...ogMetadata, ...frontmatter }; // add breadcrumb list if it exists, otherwise compute it @@ -150,9 +151,7 @@ export function getSeoProps( } } - seo.title = markdownToString( - frontmatter.headline ?? extractHeadline(content) ?? frontmatter.title - ); + seo.title = markdownToString(frontmatter.headline ?? frontmatter.title); seo.description = markdownToString( frontmatter.description ?? frontmatter.subtitle ?? frontmatter.excerpt ); @@ -343,13 +342,10 @@ function getPreloadedFont( }; } -// TODO: make this more robust and well-tested i.e. title over multiple lines -export function extractHeadline(markdownContent: string): string | undefined { - if ( - markdownContent.trim().startsWith("#") && - !markdownContent.trim().startsWith("##") - ) { - return markdownContent.trim().split("\n")[0]; +function getFontExtension(url: string): string { + const ext = url.split(".").pop(); + if (ext == null) { + throw new Error("No extension found for font: " + url); } - return; + return ext; } diff --git a/packages/fern-docs/next-seo/tsconfig.eslint.json b/packages/fern-docs/seo/tsconfig.eslint.json similarity index 100% rename from packages/fern-docs/next-seo/tsconfig.eslint.json rename to packages/fern-docs/seo/tsconfig.eslint.json diff --git a/packages/fern-docs/next-seo/tsconfig.json b/packages/fern-docs/seo/tsconfig.json similarity index 100% rename from packages/fern-docs/next-seo/tsconfig.json rename to packages/fern-docs/seo/tsconfig.json diff --git a/packages/fern-docs/ui/.depcheckrc.json b/packages/fern-docs/ui/.depcheckrc.json index c2c084b919..b6c988ccf8 100644 --- a/packages/fern-docs/ui/.depcheckrc.json +++ b/packages/fern-docs/ui/.depcheckrc.json @@ -14,7 +14,6 @@ "@tailwindcss/forms", "tailwindcss", "cssnano", - "unist-util-visit", "@chromatic-com/storybook", "@storybook/addon-essentials", "@storybook/addon-interactions", @@ -22,8 +21,7 @@ "@storybook/addon-onboarding", "@storybook/blocks", "@storybook/test", - "@emotion/is-prop-valid", - "mdx" + "@emotion/is-prop-valid" ], "ignore-patterns": ["dist"] } diff --git a/packages/fern-docs/ui/package.json b/packages/fern-docs/ui/package.json index cd0f6cad4b..023b552b69 100644 --- a/packages/fern-docs/ui/package.json +++ b/packages/fern-docs/ui/package.json @@ -49,9 +49,9 @@ "@fern-docs/cache": "workspace:*", "@fern-docs/components": "workspace:*", "@fern-docs/mdx": "workspace:*", - "@fern-docs/next-seo": "workspace:*", "@fern-docs/search-ui": "workspace:*", "@fern-docs/search-utils": "workspace:*", + "@fern-docs/seo": "workspace:*", "@fern-docs/syntax-highlighter": "workspace:*", "@fern-docs/utils": "workspace:*", "@fern-platform/fdr-utils": "workspace:*", @@ -84,8 +84,6 @@ "fastdom": "^1.0.12", "framer-motion": "^11.2.4", "github-slugger": "^2.0.0", - "hast-util-to-jsx-runtime": "^2.3.0", - "hast-util-to-string": "^3.0.0", "iconoir-react": "^7.7.0", "jose": "^5.2.3", "jotai": "^2.10.4", @@ -120,8 +118,6 @@ "swr": "^2.2.5", "three": "^0.171.0", "tinycolor2": "^1.6.0", - "unified": "^11.0.4", - "unist-util-visit": "^5.0.0", "url-join": "5.0.0", "use-memo-one": "^1.1.3", "webm-duration-fix": "^1.0.4", @@ -145,9 +141,7 @@ "@tailwindcss/typography": "^0.5.10", "@testing-library/jest-dom": "^6.4.2", "@testing-library/react": "^14.3.1", - "@types/hast": "^3.0.4", "@types/jsonpath": "^0.2.4", - "@types/mdx": "^2.0.13", "@types/node": "^18.7.18", "@types/numeral": "^2.0.5", "@types/react": "^18", diff --git a/packages/fern-docs/ui/src/atoms/docs.ts b/packages/fern-docs/ui/src/atoms/docs.ts index ac080a1f00..dffe3a5cc4 100644 --- a/packages/fern-docs/ui/src/atoms/docs.ts +++ b/packages/fern-docs/ui/src/atoms/docs.ts @@ -47,9 +47,12 @@ export const EMPTY_DOCS_STATE: DocsProps = { layout: undefined, js: undefined, navbarLinks: [], - logoHeight: undefined, - logoHref: undefined, - files: {}, + logo: { + height: undefined, + href: undefined, + light: undefined, + dark: undefined, + }, content: { type: "markdown-page", slug: FernNavigation.Slug(""), diff --git a/packages/fern-docs/ui/src/atoms/files.ts b/packages/fern-docs/ui/src/atoms/files.ts deleted file mode 100644 index df273a3b7b..0000000000 --- a/packages/fern-docs/ui/src/atoms/files.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { DocsV1Read } from "@fern-api/fdr-sdk/client/types"; -import { isEqual } from "es-toolkit/predicate"; -import { atom, useAtomValue } from "jotai"; -import { selectAtom } from "jotai/utils"; -import { useMemoOne } from "use-memo-one"; -import { DOCS_ATOM } from "./docs"; - -export const FILES_ATOM = selectAtom(DOCS_ATOM, (docs) => docs.files, isEqual); -FILES_ATOM.debugLabel = "FILES_ATOM"; - -export function useFile( - fileId: DocsV1Read.FileId -): DocsV1Read.File_ | undefined { - return useAtomValue( - useMemoOne( - () => - atom( - (get) => - get(FILES_ATOM)[ - DocsV1Read.FileId( - fileId.startsWith("file:") ? fileId.slice(5) : fileId - ) - ] - ), - [fileId] - ) - ); -} diff --git a/packages/fern-docs/ui/src/atoms/index.ts b/packages/fern-docs/ui/src/atoms/index.ts index feedd43abe..0eafc3d3f6 100644 --- a/packages/fern-docs/ui/src/atoms/index.ts +++ b/packages/fern-docs/ui/src/atoms/index.ts @@ -2,7 +2,6 @@ export * from "./announcement"; export * from "./apis"; export * from "./auth"; export * from "./docs"; -export * from "./files"; export * from "./flags"; export * from "./hooks"; export * from "./lang"; diff --git a/packages/fern-docs/ui/src/atoms/logo.ts b/packages/fern-docs/ui/src/atoms/logo.ts index 72dd12af84..8370185829 100644 --- a/packages/fern-docs/ui/src/atoms/logo.ts +++ b/packages/fern-docs/ui/src/atoms/logo.ts @@ -1,34 +1,117 @@ -import { - EMPTY_FRONTMATTER, - FileIdOrUrl, - Logo, - LogoConfiguration, -} from "@fern-api/fdr-sdk/docs"; -import { atom, useAtomValue } from "jotai"; +import type { FdrAPI, FernNavigation } from "@fern-api/fdr-sdk"; +import type { FileIdOrUrl, Frontmatter } from "@fern-api/fdr-sdk/docs"; +import { isPlainObject } from "@fern-api/ui-core-utils"; +import { addLeadingSlash, conformTrailingSlash } from "@fern-docs/utils"; +import { isEqual } from "es-toolkit/predicate"; +import { atom } from "jotai"; +import { selectAtom } from "jotai/utils"; import { DOCS_ATOM } from "./docs"; import { EDGE_FLAGS_ATOM } from "./flags"; +import { BASEPATH_ATOM } from "./navigation"; +import { ImageData } from "./types"; + +const DEFAULT_LOGO_HEIGHT = 20; export const LOGO_TEXT_ATOM = atom((get) => get(EDGE_FLAGS_ATOM).isDocsLogoTextEnabled ? "docs" : undefined ); -LOGO_TEXT_ATOM.debugLabel = "LOGO_TEXT_ATOM"; -export const LOGO_HREF_ATOM = atom( - (get) => get(DOCS_ATOM).logoHref -); -LOGO_HREF_ATOM.debugLabel = "LOGO_HREF_ATOM"; +const INTERNAL_LOGO_ATOM = selectAtom(DOCS_ATOM, (docs) => docs.logo, isEqual); -const DEFAULT_LOGO_HEIGHT = 20; -export const LOGO_HEIGHT_ATOM = atom( - (get) => get(DOCS_ATOM).logoHeight ?? DEFAULT_LOGO_HEIGHT -); -LOGO_HEIGHT_ATOM.debugLabel = "LOGO_HEIGHT_ATOM"; +interface LogoConfiguration { + href: string; + height: number; + light: ImageData | undefined; + dark: ImageData | undefined; +} + +export const LOGO_ATOM = atom((get) => { + const logo = get(INTERNAL_LOGO_ATOM); + const basepath = get(BASEPATH_ATOM); + return { + href: logo.href ?? basepath ?? "/", + height: logo.height && logo.height > 0 ? logo.height : DEFAULT_LOGO_HEIGHT, + light: logo.light, + dark: logo.dark, + }; +}); + +export function withLogo( + definition: FdrAPI.docs.v1.read.DocsDefinition, + found: FernNavigation.utils.Node.Found, + frontmatter: Frontmatter | undefined, + resolveFileSrc: (src: string | undefined) => ImageData | undefined +): { + height: number; + href: string; + light: ImageData | undefined; + dark: ImageData | undefined; +} { + const height = definition.config.logoHeight; + const href = + definition.config.logoHref ?? + encodeURI( + conformTrailingSlash( + addLeadingSlash( + found.landingPage?.canonicalSlug ?? + found.root.slug ?? + found.root.canonicalSlug ?? + found.root.slug + ) + ) + ); + + const frontmatterLogo = getLogoFromFrontmatter(frontmatter); + + const lightDocsYmlLogo = + definition.config.colorsV3?.type === "light" + ? definition.config.colorsV3.logo + : definition.config.colorsV3?.type === "darkAndLight" + ? definition.config.colorsV3.light.logo + : undefined; + const darkDocsYmlLogo = + definition.config.colorsV3?.type === "dark" + ? definition.config.colorsV3.logo + : definition.config.colorsV3?.type === "darkAndLight" + ? definition.config.colorsV3.dark.logo + : undefined; + + return { + height: height ?? DEFAULT_LOGO_HEIGHT, + href, + light: resolveFileSrc(frontmatterLogo.light ?? lightDocsYmlLogo), + dark: resolveFileSrc(frontmatterLogo.dark ?? darkDocsYmlLogo), + }; +} -export function useLogoHeight(): number { - return useAtomValue(LOGO_HEIGHT_ATOM); +/** + * Extracts the logo from the frontmatter, which will override the logo defined in docs.yml + * + * @param frontmatter - The frontmatter to extract the logo from. + * @returns The logo. + */ +function getLogoFromFrontmatter(frontmatter: Frontmatter | undefined): { + light?: string; + dark?: string; +} { + if (frontmatter == null) { + return { light: undefined, dark: undefined }; + } + const { logo } = frontmatter; + const defaultSrc = toSrcValue(logo); + const lightSrc = isPlainObject(logo) ? toSrcValue(logo.light) : undefined; + const darkSrc = isPlainObject(logo) ? toSrcValue(logo.dark) : undefined; + if (lightSrc && darkSrc) { + return { light: lightSrc, dark: darkSrc }; + } + const src = lightSrc ?? darkSrc ?? defaultSrc; + return { light: src, dark: src }; } -function isFileIdOrUrl(logo: Logo | undefined): logo is FileIdOrUrl { +/** + * Checks if a logo definition uses the legacy file ID or URL format. + */ +function isLegacyFileIdOrUrl(logo: unknown | undefined): logo is FileIdOrUrl { if (logo == null) { return false; } @@ -38,53 +121,25 @@ function isFileIdOrUrl(logo: Logo | undefined): logo is FileIdOrUrl { if (!("type" in logo && "value" in logo)) { return false; } + if (typeof logo.type !== "string" || typeof logo.value !== "string") { + return false; + } return logo.type === "fileId" || logo.type === "url"; } -export const LOGO_IMAGE_ATOM = atom((get) => { - const { content, colors } = get(DOCS_ATOM); - const markdownText = - content.type === "markdown-page" - ? content.content - : content.type === "changelog" && content.node.overviewPageId != null - ? content.pages[content.node.overviewPageId] - : content.type === "changelog-entry" - ? content.page - : undefined; - - const { logo } = - typeof markdownText === "object" - ? markdownText.frontmatter - : EMPTY_FRONTMATTER; - - if (logo != null && typeof logo === "object") { - if ( - "light" in logo && - "dark" in logo && - isFileIdOrUrl(logo.light) && - isFileIdOrUrl(logo.dark) - ) { - return { light: logo.light, dark: logo.dark }; - } - if ("light" in logo && isFileIdOrUrl(logo.light)) { - return { light: logo.light, dark: logo.light }; - } - if ("dark" in logo && isFileIdOrUrl(logo.dark)) { - return { light: logo.dark, dark: logo.dark }; - } - if (isFileIdOrUrl(logo)) { - return { light: logo, dark: logo }; - } +/** + * Converts a logo definition to a src value. + * + * @param unknown - The logo definition to convert. + * @returns The src value. + */ +function toSrcValue(unknown: unknown): string | undefined { + if (typeof unknown === "string") { + return unknown; } - - const light = - colors.light?.logo != null - ? { type: "fileId" as const, value: colors.light.logo } - : undefined; - const dark = - colors.dark?.logo != null - ? { type: "fileId" as const, value: colors.dark.logo } - : undefined; - - return { light: light ?? dark, dark: dark ?? light }; -}); + // note: this is a legacy implementation. + if (isLegacyFileIdOrUrl(unknown)) { + return unknown.value; + } + return undefined; +} diff --git a/packages/fern-docs/ui/src/atoms/seo.ts b/packages/fern-docs/ui/src/atoms/seo.ts index 1fd3cc14dd..601d5597d0 100644 --- a/packages/fern-docs/ui/src/atoms/seo.ts +++ b/packages/fern-docs/ui/src/atoms/seo.ts @@ -1,4 +1,4 @@ -import { NextSeoProps } from "@fern-docs/next-seo"; +import { NextSeoProps } from "@fern-docs/seo"; import { atom } from "jotai"; import { DOCS_ATOM } from "./docs"; import { THEME_BG_COLOR } from "./theme"; diff --git a/packages/fern-docs/ui/src/atoms/types.ts b/packages/fern-docs/ui/src/atoms/types.ts index 086189d9f7..9c835410e7 100644 --- a/packages/fern-docs/ui/src/atoms/types.ts +++ b/packages/fern-docs/ui/src/atoms/types.ts @@ -6,7 +6,7 @@ import type { import type * as FernDocs from "@fern-api/fdr-sdk/docs"; import type * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import type { FernUser } from "@fern-docs/auth"; -import { NextSeoProps } from "@fern-docs/next-seo"; +import { NextSeoProps } from "@fern-docs/seo"; import type { EdgeFlags } from "@fern-docs/utils"; import { ColorsConfig, @@ -79,11 +79,9 @@ export interface DocsProps { colors: ColorsConfig; announcement: AnnouncementConfig | undefined; layout: DocsV1Read.DocsLayoutConfig | undefined; - js: DocsV1Read.JsConfig | undefined; + js: JsConfig | undefined; navbarLinks: NavbarLink[]; - logoHeight: DocsV1Read.Height | undefined; - logoHref: DocsV1Read.Url | undefined; - files: Record; + logo: LogoConfiguration; content: DocsContent; edgeFlags: EdgeFlags; apis: FdrAPI.ApiDefinitionId[]; @@ -97,3 +95,33 @@ export interface DocsProps { stylesheet: string; featureFlagsConfig: FeatureFlagsConfig | undefined; } + +export interface ImageData { + src: string; + height?: number; + width?: number; + blurDataURL?: string; + blurWidth?: number; + blurHeight?: number; +} + +export interface LogoConfiguration { + height: number | undefined; + href: string | undefined; + light: ImageData | undefined; + dark: ImageData | undefined; +} + +export interface JsConfig { + remote: + | { + url: string; + strategy: + | "beforeInteractive" + | "afterInteractive" + | "lazyOnload" + | undefined; + }[] + | undefined; + inline: string[] | undefined; +} diff --git a/packages/fern-docs/ui/src/components/FernImage.tsx b/packages/fern-docs/ui/src/components/FernImage.tsx index b888b3f49f..52c71f8a75 100644 --- a/packages/fern-docs/ui/src/components/FernImage.tsx +++ b/packages/fern-docs/ui/src/components/FernImage.tsx @@ -1,86 +1,114 @@ /* eslint-disable @next/next/no-img-element */ -import type { DocsV1Read } from "@fern-api/fdr-sdk/client/types"; import Image from "next/image"; import { ComponentPropsWithoutRef, forwardRef } from "react"; +import { UnreachableCaseError } from "ts-essentials"; + +// TODO: move this to a shared location +const NEXT_IMAGE_HOSTS = [ + "fdr-prod-docs-files.s3.us-east-1.amazonaws.com", + "fdr-prod-docs-files-public.s3.amazonaws.com", + "fdr-dev2-docs-files.s3.us-east-1.amazonaws.com", + "fdr-dev2-docs-files-public.s3.amazonaws.com", + "files.buildwithfern.com", + "files-dev2.buildwithfern.com", +]; export const FernImage = forwardRef< HTMLImageElement, - Omit, "src" | "alt"> & { - /** - * FDR may return a URL or an image object depending on the version of the Fern CLI used to build the docs. - * If the file has width/height metadata, we can render it using next/image for optimized loading. - */ - src: DocsV1Read.File_ | undefined; - /** - * The alt text for the image. - * @default "" - */ - alt?: string; - } ->(({ src, ...props }, ref) => { + ComponentPropsWithoutRef +>((props, ref) => { + const { + src, + alt, + width, + height, + fill, + loader, + quality, + priority, + loading, + placeholder, + blurDataURL, + unoptimized, + overrideSrc, + onLoadingComplete, + layout, + objectFit, + objectPosition, + lazyBoundary, + lazyRoot, + ...rest + } = props; + if (src == null) { return null; } - const { width, height } = - src.type === "image" - ? getDimensions({ - intrinsicWidth: src.width, - intrinsicHeight: src.height, - width: props.width, - height: props.height, - }) - : props; - - const blurDataURL = - props.blurDataURL ?? (src.type === "image" ? src.blurDataUrl : undefined); + const originalSrc = getSrc(src); + const { host, pathname } = safeGetUrl(originalSrc); - const pathname = safeGetPathname(src.url); + const aspectRatio = withAspectRatio(withDimensions(props)); - if (src.type === "url") { + // nextjs requires a strict allowlist of hosts for + // so we'll fall back to if the host is not in the allowlist (or if no custom loader is provided) + if ((!host || !NEXT_IMAGE_HOSTS.includes(host)) && !loader) { return ( {props.alt} tags is `max-width: 100%; height: auto;` // which causes the image height to be ignored. we'll use the inline style prop to override this behavior: style={{ - width, - height, + aspectRatio, + width: props.style?.width == null ? "100%" : "auto", + height: "auto", ...props.style, }} /> ); } + // if we're here, we're using the component + // we'll use the inline style prop to override the aspect ratio + // and pass the rest of the props to the component return ( {alt} tags is `max-width: 100%; height: auto;` // which causes the image height to be ignored. we'll use the inline style prop to override this behavior: style={{ - width, - height, + aspectRatio, + width: props.style?.width == null ? "100%" : "auto", + height: "auto", ...props.style, }} /> @@ -89,51 +117,55 @@ export const FernImage = forwardRef< FernImage.displayName = "FernImage"; -function safeGetPathname(url: string): string | undefined { +function safeGetUrl(src: string): { + host: string | undefined; + pathname: string | undefined; +} { try { - return new URL(url, "https://n").pathname.toLowerCase(); + const url = new URL(src, "https://n"); + return { host: url.host, pathname: url.pathname.toLowerCase() }; } catch (_e) { - return undefined; + return { host: undefined, pathname: undefined }; } } -export function getDimensions({ - intrinsicWidth, - intrinsicHeight, - width, - height, -}: { - intrinsicWidth: number; - intrinsicHeight: number; - width?: number | `${number}`; - height?: number | `${number}`; -}): { width: number; height: number } { - const propWidth = asNumber(width); - const propHeight = asNumber(height); - - // If the user has explicitly set the width and height, use those values. - if (propWidth != null && propHeight != null) { - return { width: propWidth, height: propHeight }; +function getSrc(src: ComponentPropsWithoutRef["src"]): string { + if (typeof src === "string") { + return src; } - - const aspectRatio = intrinsicWidth / intrinsicHeight; - - // if the user has the width and height, use that to determine the aspect ratio - if (propWidth != null) { - return { width: propWidth, height: propWidth / aspectRatio }; - } else if (propHeight != null) { - return { width: propHeight * aspectRatio, height: propHeight }; + if (typeof src === "object" && "src" in src) { + return src.src; + } + if (typeof src === "object" && "default" in src) { + return src.default.src; } + throw new UnreachableCaseError(src); +} - return { width: intrinsicWidth, height: intrinsicHeight }; +function withDimensions( + props: ComponentPropsWithoutRef +): { width: number; height: number } | undefined { + if (props.width != null && props.height != null) { + return { width: Number(props.width), height: Number(props.height) }; + } + if ( + typeof props.src === "object" && + "width" in props.src && + "height" in props.src + ) { + return { width: props.src.width, height: props.src.height }; + } + if (typeof props.src === "object" && "default" in props.src) { + return { width: props.src.default.width, height: props.src.default.height }; + } + return undefined; } -function asNumber(value: number | `${number}` | undefined): number | undefined { - if (value == null) { +function withAspectRatio( + dimensions: { width: number; height: number } | undefined +): number | undefined { + if (dimensions == null) { return undefined; } - if (typeof value === "string") { - return parseInt(value, 10); - } - return value; + return dimensions.width / dimensions.height; } diff --git a/packages/fern-docs/ui/src/components/JavascriptProvider.tsx b/packages/fern-docs/ui/src/components/JavascriptProvider.tsx index 7d0e037849..1faa9f470a 100644 --- a/packages/fern-docs/ui/src/components/JavascriptProvider.tsx +++ b/packages/fern-docs/ui/src/components/JavascriptProvider.tsx @@ -1,12 +1,11 @@ import { atom, useAtomValue } from "jotai"; import Script from "next/script"; import { memo } from "react"; -import { DOCS_ATOM, FILES_ATOM } from "../atoms"; +import { DOCS_ATOM } from "../atoms"; const JS_ATOM = atom((get) => get(DOCS_ATOM).js); export const JavascriptProvider = memo(() => { - const files = useAtomValue(FILES_ATOM); const js = useAtomValue(JS_ATOM); return ( @@ -16,18 +15,15 @@ export const JavascriptProvider = memo(() => { {inline} ))} - {js?.files.map((file) => ( + {js?.remote?.map((remote) => (