From 78ae0c0b28eeccdfb0378797e5fb42a1118eda99 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:32:41 -0500 Subject: [PATCH] Extension improvements (#37) * extension-improvements override addFunctions method of NetlifyExtension class * extension-improvements add empty default for appName * extension-improvements add slack auth token to dbconfig * extension-improvements updating configurations, types --- libs/util/src/assertDbEnvVars.ts | 21 ++--- .../atlasClusterConnector.ts | 76 ------------------- .../databaseConnection/clusterConnector.ts | 19 +++++ .../clusterZeroConnector.ts | 27 +++---- .../databaseConnection/fetchDocsetsData.ts | 12 +-- .../fetchReposBranchesData.ts | 12 +-- .../src/databaseConnection/fetchSearchData.ts | 22 ++---- .../searchClusterConnector.ts | 27 +++++++ libs/util/src/databaseConnection/types.ts | 51 ++++++++----- libs/util/src/extension.ts | 35 +++++++-- libs/util/src/s3Connection/types.ts | 6 ++ 11 files changed, 150 insertions(+), 158 deletions(-) delete mode 100644 libs/util/src/databaseConnection/atlasClusterConnector.ts create mode 100644 libs/util/src/databaseConnection/clusterConnector.ts create mode 100644 libs/util/src/databaseConnection/searchClusterConnector.ts create mode 100644 libs/util/src/s3Connection/types.ts diff --git a/libs/util/src/assertDbEnvVars.ts b/libs/util/src/assertDbEnvVars.ts index 0c3f6ca86..4c278de08 100644 --- a/libs/util/src/assertDbEnvVars.ts +++ b/libs/util/src/assertDbEnvVars.ts @@ -2,35 +2,29 @@ export type Environments = 'dev' | 'stg' | 'dotcomstg' | 'prd' | 'dotcomprd'; export type CollectionName = 'repos_branches' | 'docsets' | 'documents'; -export type S3UploadParams = { - bucket: string; - prefix: string; - fileName: string; - manifest: string; -}; - -export type DbConfig = { +export type StaticEnvVars = { ATLAS_CLUSTER0_URI: string; ATLAS_SEARCH_URI: string; AWS_S3_ACCESS_KEY_ID: string; AWS_S3_SECRET_ACCESS_KEY: string; DOCSETS_COLLECTION: CollectionName; - DOCUMENTS_COLLECTION: CollectionName; + DOCUMENTS_COLLECTION: string; REPOS_BRANCHES_COLLECTION: CollectionName; + SLACK_AUTH_TOKEN: string; SLACK_SIGNING_SECRET: string; }; -const assertEnvVars = (vars: DbConfig) => { +const assertEnvVars = (vars: StaticEnvVars) => { const missingVars = Object.entries(vars) .filter(([, value]) => !value) .map(([key]) => `- ${key}`) .join('\n'); - if (missingVars) - throw new Error(`Missing env var(s) ${JSON.stringify(missingVars)}`); + if (missingVars) + throw new Error(`Missing env var(s) ${JSON.stringify(missingVars)}`); return vars; }; -export const getDbConfig = (): DbConfig => { +export const getDbConfig = (): StaticEnvVars => { const environmentVariables = assertEnvVars({ ATLAS_CLUSTER0_URI: `mongodb+srv://${process.env.MONGO_ATLAS_USERNAME}:${process.env.MONGO_ATLAS_PASSWORD}@${process.env.MONGO_ATLAS_CLUSTER0_HOST}/?retryWrites=true&w=majority`, ATLAS_SEARCH_URI: `mongodb+srv://${process.env.MONGO_ATLAS_USERNAME}:${process.env.MONGO_ATLAS_PASSWORD}@${process.env.MONGO_ATLAS_SEARCH_HOST}/?retryWrites=true&w=majority`, @@ -44,6 +38,7 @@ export const getDbConfig = (): DbConfig => { (process.env.REPOS_BRANCHES_COLLECTION as CollectionName) ?? 'repos_branches', SLACK_SIGNING_SECRET: process.env.SLACK_SIGNING_SECRET as string, + SLACK_AUTH_TOKEN: process.env.SLACK_AUTH_TOKEN as string, }); return environmentVariables; diff --git a/libs/util/src/databaseConnection/atlasClusterConnector.ts b/libs/util/src/databaseConnection/atlasClusterConnector.ts deleted file mode 100644 index b96752588..000000000 --- a/libs/util/src/databaseConnection/atlasClusterConnector.ts +++ /dev/null @@ -1,76 +0,0 @@ -import * as mongodb from 'mongodb'; -import type { CollectionName } from './types'; - -let clusterZeroClient: mongodb.MongoClient; -let searchClusterClient: mongodb.MongoClient; - -export type CollectionConnectionInfo = { - URI: string; - databaseName: string; - collectionName: CollectionName; - extensionName: string; -}; - -export const teardown = async (client: mongodb.MongoClient): Promise => { - await client.close(); -}; - -// Handles initial connection logic if needs to be initialized -const dbClient = async ({ - uri, - appName, -}: { uri: string; appName: string }): Promise => { - const client = new mongodb.MongoClient(uri, { appName }); - try { - await client.connect(); - return client; - } catch (error) { - throw new Error(`Error at client connection: ${error} `); - } -}; - -export const getPoolDb = async ({ - URI, - databaseName, - appName, -}: { - URI: string; - databaseName: string; - appName: string; -}): Promise => { - if (!clusterZeroClient) { - console.info('Creating new instance of Cluster Zero client'); - clusterZeroClient = await dbClient({ uri: URI, appName }); - } - return clusterZeroClient.db(databaseName); -}; - -export const getSearchDb = async ({ - URI, - databaseName, - appName, -}: { - URI: string; - databaseName: string; - appName: string; -}): Promise => { - if (!searchClusterClient) { - console.info('Creating new instance of Cluster Zero client'); - searchClusterClient = await dbClient({ uri: URI, appName }); - } - return searchClusterClient.db(databaseName); -}; - -export const closePoolDb = async () => { - if (clusterZeroClient) await teardown(clusterZeroClient); - else { - console.info('No client connection open to Cluster Zero client'); - } -}; - -export const closeSearchDb = async () => { - if (searchClusterClient) await teardown(searchClusterClient); - else { - console.info('No client connection open to Search Cluster client'); - } -}; diff --git a/libs/util/src/databaseConnection/clusterConnector.ts b/libs/util/src/databaseConnection/clusterConnector.ts new file mode 100644 index 000000000..1ef15ebf9 --- /dev/null +++ b/libs/util/src/databaseConnection/clusterConnector.ts @@ -0,0 +1,19 @@ +import * as mongodb from 'mongodb'; + +export const teardown = async (client: mongodb.MongoClient): Promise => { + await client.close(); +}; + +// Handles memoization of db object, and initial connection logic if needs to be initialized +export const dbClient = async ({ + uri, + appName, +}: { uri: string; appName: string }): Promise => { + const client = new mongodb.MongoClient(uri, { appName }); + try { + await client.connect(); + return client; + } catch (error) { + throw new Error(`Error at client connection: ${error} `); + } +}; diff --git a/libs/util/src/databaseConnection/clusterZeroConnector.ts b/libs/util/src/databaseConnection/clusterZeroConnector.ts index 31fde9a75..9588bb9a3 100644 --- a/libs/util/src/databaseConnection/clusterZeroConnector.ts +++ b/libs/util/src/databaseConnection/clusterZeroConnector.ts @@ -1,29 +1,20 @@ -import * as mongodb from 'mongodb'; +import type * as mongodb from 'mongodb'; +import { teardown, dbClient } from './clusterConnector'; let clusterZeroClient: mongodb.MongoClient; -export const teardown = async (client: mongodb.MongoClient): Promise => { - await client.close(); -}; - -// Handles memoization of db object, and initial connection logic if needs to be initialized -const dbClient = async (uri: string): Promise => { - const client = new mongodb.MongoClient(uri); - try { - await client.connect(); - return client; - } catch (error) { - throw new Error(`Error at client connection: ${error} `); - } -}; - export const getPoolDb = async ({ clusterZeroURI, databaseName, -}: { clusterZeroURI: string; databaseName: string }): Promise => { + appName, +}: { + clusterZeroURI: string; + databaseName: string; + appName: string; +}): Promise => { if (!clusterZeroClient) { console.info('Creating new instance of Cluster Zero client'); - clusterZeroClient = await dbClient(clusterZeroURI); + clusterZeroClient = await dbClient({ uri: clusterZeroURI, appName }); } return clusterZeroClient.db(databaseName); }; diff --git a/libs/util/src/databaseConnection/fetchDocsetsData.ts b/libs/util/src/databaseConnection/fetchDocsetsData.ts index 3a1b1ba2a..f2cf7a52e 100644 --- a/libs/util/src/databaseConnection/fetchDocsetsData.ts +++ b/libs/util/src/databaseConnection/fetchDocsetsData.ts @@ -1,22 +1,22 @@ import type * as mongodb from 'mongodb'; -import { getPoolDb } from './atlasClusterConnector'; +import { getPoolDb } from './clusterZeroConnector'; import type { DocsetsDocument } from './types'; export const getDocsetsCollection = async ({ - URI, + clusterZeroURI, databaseName, collectionName, extensionName, }: { - URI: string; + clusterZeroURI: string; databaseName: string; collectionName: string; - extensionName: string; + extensionName?: string; }): Promise> => { const dbSession = await getPoolDb({ - URI, + clusterZeroURI, databaseName, - appName: extensionName, + appName: extensionName ?? '', }); return dbSession.collection(collectionName); }; diff --git a/libs/util/src/databaseConnection/fetchReposBranchesData.ts b/libs/util/src/databaseConnection/fetchReposBranchesData.ts index c761b4242..a8428ccb1 100644 --- a/libs/util/src/databaseConnection/fetchReposBranchesData.ts +++ b/libs/util/src/databaseConnection/fetchReposBranchesData.ts @@ -1,22 +1,22 @@ import type * as mongodb from 'mongodb'; -import { getPoolDb } from './atlasClusterConnector'; +import { getPoolDb } from './clusterZeroConnector'; import type { ReposBranchesDocument } from './types'; export const getReposBranchesCollection = async ({ - URI, + clusterZeroURI, databaseName, collectionName, extensionName, }: { - URI: string; + clusterZeroURI: string; databaseName: string; collectionName: string; - extensionName: string; + extensionName?: string; }): Promise> => { const dbSession = await getPoolDb({ - URI, + clusterZeroURI, databaseName, - appName: extensionName, + appName: extensionName ?? '', }); return dbSession.collection(collectionName); }; diff --git a/libs/util/src/databaseConnection/fetchSearchData.ts b/libs/util/src/databaseConnection/fetchSearchData.ts index 1003a5cf4..65c132a19 100644 --- a/libs/util/src/databaseConnection/fetchSearchData.ts +++ b/libs/util/src/databaseConnection/fetchSearchData.ts @@ -1,30 +1,22 @@ import type * as mongodb from 'mongodb'; -import { getSearchDb } from './atlasClusterConnector'; - -export interface SearchDocument { - url: string; - slug: string; - lastModified: Date; - manifestRevisionId: string; - searchProperty: Array; - includeInGlobalSearch: boolean; -} +import { getSearchDb } from './searchClusterConnector'; +import type { SearchDocument } from './types'; export const getDocumentsCollection = async ({ - URI, + searchURI, databaseName, collectionName, extensionName, }: { - URI: string; + searchURI: string; databaseName: string; collectionName: string; - extensionName: string; + extensionName?: string; }): Promise> => { const dbSession = await getSearchDb({ - URI, + searchURI, databaseName, - appName: extensionName, + appName: extensionName ?? '', }); return dbSession.collection(collectionName); }; diff --git a/libs/util/src/databaseConnection/searchClusterConnector.ts b/libs/util/src/databaseConnection/searchClusterConnector.ts new file mode 100644 index 000000000..2bf740c09 --- /dev/null +++ b/libs/util/src/databaseConnection/searchClusterConnector.ts @@ -0,0 +1,27 @@ +import type * as mongodb from 'mongodb'; +import { teardown, dbClient } from './clusterConnector'; + +let searchClusterClient: mongodb.MongoClient; + +export const getSearchDb = async ({ + searchURI, + databaseName, + appName, +}: { + searchURI: string; + databaseName: string; + appName: string; +}): Promise => { + if (!searchClusterClient) { + console.info('Creating new instance of Cluster Zero client'); + searchClusterClient = await dbClient({ uri: searchURI, appName }); + } + return searchClusterClient.db(databaseName); +}; + +export const closeSearchDb = async () => { + if (searchClusterClient) await teardown(searchClusterClient); + else { + console.info('No client connection open to Search Cluster client'); + } +}; diff --git a/libs/util/src/databaseConnection/types.ts b/libs/util/src/databaseConnection/types.ts index 52e02ed24..9e66f0566 100644 --- a/libs/util/src/databaseConnection/types.ts +++ b/libs/util/src/databaseConnection/types.ts @@ -1,4 +1,4 @@ -export type PoolDbName = 'pool' | 'pool_test'; +export type Environments = 'dev' | 'stg' | 'dotcomstg' | 'prd' | 'dotcomprd'; export interface EnvironmentConfig { dev?: string; @@ -14,12 +14,19 @@ export interface DocsetsDocument { url: EnvironmentConfig; prefix: EnvironmentConfig; } -export type CollectionName = 'repos_branches' | 'docsets' | 'documents'; -export type CollectionConnectionInfo = { +export type clusterZeroConnectionInfo = { clusterZeroURI: string; - databaseName: PoolDbName; - collectionName: CollectionName; + databaseName: ClusterZeroDBName; + collectionName: string; + extensionName: string; +}; + +export type SearchClusterConnectionInfo = { + searchURI: string; + databaseName: SearchDBName; + collectionName: string; + extensionName: string; }; export interface BranchEntry { @@ -42,22 +49,30 @@ export interface ReposBranchesDocument { internalOnly: boolean; } -export type Environments = 'dev' | 'stg' | 'dotcomstg' | 'prd' | 'dotcomprd'; - export type S3UploadParams = { bucket: string; prefix: string; fileName: string; - manifest: string; + obj: string; }; -export type DbConfig = { - ATLAS_CLUSTER0_URI: string; - ATLAS_SEARCH_URI: string; - AWS_S3_ACCESS_KEY_ID: string; - AWS_S3_SECRET_ACCESS_KEY: string; - DOCSETS_COLLECTION: CollectionName; - DOCUMENTS_COLLECTION: CollectionName; - REPOS_BRANCHES_COLLECTION: CollectionName; - SLACK_SIGNING_SECRET: string; -}; +export interface SearchDocument { + url: string; + slug: string; + lastModified: Date; + manifestRevisionId: string; + searchProperty: Array; + includeInGlobalSearch: boolean; +} + +export type SearchDBName = 'search' | 'search-test' | 'search-stage'; + +export type PoolDBName = 'pool' | 'pool_test'; + +export type SnootyDBName = + | 'test' + | 'snooty_dev' + | 'snooty_dotcomstg' + | 'snooty_dotcomprd'; + +export type ClusterZeroDBName = PoolDBName | 'docs_metadata' | SnootyDBName; diff --git a/libs/util/src/extension.ts b/libs/util/src/extension.ts index d7e9a1553..4ca7badf3 100644 --- a/libs/util/src/extension.ts +++ b/libs/util/src/extension.ts @@ -8,8 +8,16 @@ import { NetlifyExtension, } from '@netlify/sdk'; import type z from 'zod'; -import type { DbConfig } from './databaseConnection/types'; -import { getDbConfig } from './assertDbEnvVars'; +import type { + BranchEntry, + DocsetsDocument, + Environments, + PoolDBName, + ReposBranchesDocument, + SearchDBName, + SnootyDBName, +} from './databaseConnection/types'; +import { getDbConfig, type StaticEnvVars } from './assertDbEnvVars'; type BuildHookWithEnvVars< DbConfig, @@ -65,19 +73,19 @@ export class Extension< z.ZodUnknown > { isEnabled: boolean; - dbEnvVars: DbConfig; + staticEnvVars: StaticEnvVars; constructor({ isEnabled }: ExtensionOptions) { super(); this.isEnabled = isEnabled; console.log(`Extension enabled: ${this.isEnabled}`); - this.dbEnvVars = getDbConfig(); + this.staticEnvVars = getDbConfig(); } addBuildEventHandler = async ( type: BuildHookType, func: BuildHookWithEnvVars< - DbConfig, + StaticEnvVars, Zod.infer, Zod.infer >, @@ -86,7 +94,7 @@ export class Extension< super.addBuildEventHandler( type, async (args) => { - const dbEnvVars = this.dbEnvVars; + const dbEnvVars = this.staticEnvVars; try { await func({ dbEnvVars, ...args }); } catch (e) { @@ -135,3 +143,18 @@ export class Extension< }); }; } + +export type ConfigEnvironmentVariables = Partial<{ + BRANCH: string; + SITE_NAME: string; + INCOMING_HOOK_URL: string; + INCOMING_HOOK_TITLE: string; + INCOMING_HOOK_BODY: string; + ENV: Environments; + REPO_ENTRY: ReposBranchesDocument; + DOCSET_ENTRY: DocsetsDocument; + BRANCH_ENTRY: BranchEntry; + POOL_DB_NAME: PoolDBName; + SEARCH_DB_NAME: SearchDBName; + SNOOTY_DB_NAME: SnootyDBName; +}>; diff --git a/libs/util/src/s3Connection/types.ts b/libs/util/src/s3Connection/types.ts new file mode 100644 index 000000000..c3cd839cd --- /dev/null +++ b/libs/util/src/s3Connection/types.ts @@ -0,0 +1,6 @@ +export type S3UploadParams = { + bucket: string; + prefix: string; + fileName: string; + manifest: string; +};