From 6adb6664b3d5f9d7d68ca2277756f9eaa0dd43a2 Mon Sep 17 00:00:00 2001 From: nigiri <168690269+0xnigir1@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:17:30 -0300 Subject: [PATCH] feat: revert multiple schemas setup --- apps/processing/README.md | 3 +-- apps/processing/src/config/env.ts | 1 - .../services/sharedDependencies.service.ts | 2 +- package.json | 6 ++--- scripts/migrations/README.md | 20 +++++++++------- scripts/migrations/package.json | 6 ++--- scripts/migrations/src/migrateDb.script.ts | 5 ++-- .../20241029T120000_initial.ts | 2 +- .../20241210T175001_strategy_registry.ts | 2 +- scripts/migrations/src/resetDb.script.ts | 5 ++-- scripts/migrations/src/utils/parsing.ts | 24 +++++++------------ 11 files changed, 32 insertions(+), 44 deletions(-) rename scripts/migrations/src/migrations/{chainData => }/20241029T120000_initial.ts (99%) rename scripts/migrations/src/migrations/{registries => }/20241210T175001_strategy_registry.ts (95%) diff --git a/apps/processing/README.md b/apps/processing/README.md index c405a5c..96f7b4f 100644 --- a/apps/processing/README.md +++ b/apps/processing/README.md @@ -28,8 +28,7 @@ Available options: |-----------------------------|--------------------------------------------------------------------------------------------------------------------------------|-----------|----------------------------------|-----------------------------------------------------------------| | `CHAINS` | JSON array of chain configurations | N/A | Yes | Each chain object requires: `id` (number), `name` (string), `rpcUrls` (string array). Optional: `fetchLimit` (default: 500), `fetchDelayMs` (default: 1000) | | `DATABASE_URL` | PostgreSQL Data Layer database connection URL | N/A | Yes | | -| `DATABASE_SCHEMA` | PostgreSQL Data Layer database chain-data schema name | public | Yes | | -| `DATABASE_REGISTRIES_SCHEMA` | PostgreSQL Data Layer database registries schema name | public | Yes | | +| `DATABASE_SCHEMA` | PostgreSQL Data Layer database schema name | public | Yes | | | `INDEXER_GRAPHQL_URL` | GraphQL endpoint for the indexer | N/A | Yes | | | `INDEXER_ADMIN_SECRET` | Admin secret for indexer authentication | N/A | Yes | | | `IPFS_GATEWAYS_URL` | Array of IPFS gateway URLs | N/A | Yes | Multiple gateways for redundancy | diff --git a/apps/processing/src/config/env.ts b/apps/processing/src/config/env.ts index 543ec86..4a201e2 100644 --- a/apps/processing/src/config/env.ts +++ b/apps/processing/src/config/env.ts @@ -30,7 +30,6 @@ const baseSchema = z.object({ }, "Chain IDs must be unique"), DATABASE_URL: z.string(), DATABASE_SCHEMA: z.string().default("public"), - DATABASE_REGISTRIES_SCHEMA: z.string().default("public"), INDEXER_GRAPHQL_URL: z.string().url(), INDEXER_ADMIN_SECRET: z.string(), PRICING_SOURCE: z.enum(["dummy", "coingecko"]).default("coingecko"), diff --git a/apps/processing/src/services/sharedDependencies.service.ts b/apps/processing/src/services/sharedDependencies.service.ts index 52e0c4e..612d8b6 100644 --- a/apps/processing/src/services/sharedDependencies.service.ts +++ b/apps/processing/src/services/sharedDependencies.service.ts @@ -76,7 +76,7 @@ export class SharedDependenciesService { ); const strategyRepository = new KyselyStrategyRepository( kyselyDatabase, - env.DATABASE_REGISTRIES_SCHEMA, + env.DATABASE_SCHEMA, ); const strategyRegistry = await InMemoryCachedStrategyRegistry.initialize( new Logger({ className: "InMemoryCachedStrategyRegistry" }), diff --git a/package.json b/package.json index 9679466..68e8a44 100644 --- a/package.json +++ b/package.json @@ -10,17 +10,15 @@ "build": "turbo run build", "check-types": "turbo run check-types", "clean": "turbo run clean", + "db:migrate": "pnpm run --filter @grants-stack-indexer/migrations db:migrate", + "db:reset": "pnpm run --filter @grants-stack-indexer/migrations db:reset", "dev": "turbo run dev", "format": "turbo run format", "format:fix": "turbo run format:fix", "preinstall": "npx only-allow pnpm", "lint": "turbo run lint", "lint:fix": "turbo run lint:fix", - "migrate:chain-data": "pnpm run --filter @grants-stack-indexer/migrations migrate:chain-data", - "migrate:registries": "pnpm run --filter @grants-stack-indexer/migrations migrate:registries", "prepare": "husky", - "reset:chain-data": "pnpm run --filter @grants-stack-indexer/migrations reset:chain-data", - "reset:registries": "pnpm run --filter @grants-stack-indexer/migrations reset:registries", "start": "turbo run start", "test": "turbo run test", "test:cov": "turbo run test:cov", diff --git a/scripts/migrations/README.md b/scripts/migrations/README.md index 1d06bcf..1d9736b 100644 --- a/scripts/migrations/README.md +++ b/scripts/migrations/README.md @@ -4,12 +4,10 @@ This package contains scripts for managing the database schema and migrations. ## Available Scripts -| Script | Description | -| -------------------- | ------------------------------------------------------------ | -| `migrate:chain-data` | Runs all pending database migrations on chainData tables | -| `migrate:registries` | Runs all pending database migrations on registries tables | -| `reset:chain-data` | Drops and recreates the database schema on chainData tables | -| `reset:registries` | Drops and recreates the database schema on registries tables | +| Script | Description | +| ------------ | --------------------------------------- | +| `db:migrate` | Runs all pending database migrations | +| `db:reset` | Drops and recreates the database schema | ## Environment Setup @@ -39,9 +37,13 @@ pnpm install To apply all pending migrations: ```bash -pnpm migrate:chain-data --schema=schema_name +pnpm db:migrate --schema=schema_name ``` +Optional arguments: + +- `--schema` or `-s`: Database schema name where migrations are applied. Defaults to `public`. + This will: 1. Load environment variables @@ -55,7 +57,7 @@ This will: To completely reset the database schema: ```bash -pnpm reset:chain-data --schema=schema_name +pnpm db:reset --schema=schema_name ``` **Warning**: This will: @@ -71,7 +73,7 @@ pnpm reset:chain-data --schema=schema_name 1. Create a new migration file in [`packages/repository/src/migrations`](../../packages//repository/migrations) 2. Name it using the format: `YYYYMMDDTHHmmss_description.ts` 3. Implement the `up` and `down` functions -4. Run `pnpm migrate:chain-data` to apply the new migration +4. Run `pnpm db:migrate` to apply the new migration Example migration file: diff --git a/scripts/migrations/package.json b/scripts/migrations/package.json index ffe8e67..c4c3180 100644 --- a/scripts/migrations/package.json +++ b/scripts/migrations/package.json @@ -16,14 +16,12 @@ "build": "tsc -p tsconfig.build.json", "check-types": "tsc --noEmit -p ./tsconfig.json", "clean": "rm -rf dist/", + "db:migrate": "tsx src/migrateDb.script.ts", + "db:reset": "tsx src/resetDb.script.ts", "format": "prettier --check \"{src,test}/**/*.{js,ts,json}\"", "format:fix": "prettier --write \"{src,test}/**/*.{js,ts,json}\"", "lint": "eslint \"{src,test}/**/*.{js,ts,json}\"", "lint:fix": "pnpm lint --fix", - "migrate:chain-data": "tsx src/migrateDb.script.ts --folder chainData", - "migrate:registries": "tsx src/migrateDb.script.ts --folder registries", - "reset:chain-data": "tsx src/resetDb.script.ts --folder chainData", - "reset:registries": "tsx src/resetDb.script.ts --folder registries", "test": "vitest run --config vitest.config.ts --passWithNoTests", "test:cov": "vitest run --config vitest.config.ts --coverage --passWithNoTests" }, diff --git a/scripts/migrations/src/migrateDb.script.ts b/scripts/migrations/src/migrateDb.script.ts index 90bc5fa..bef2362 100644 --- a/scripts/migrations/src/migrateDb.script.ts +++ b/scripts/migrations/src/migrateDb.script.ts @@ -22,7 +22,6 @@ configDotenv(); * - DATABASE_URL: PostgreSQL connection string * * Script arguments: - * - folder: Folder name to migrate (e.g. "chainData" or "registries") * - schema: Database schema name where migrations are applied * * The script will: @@ -34,7 +33,7 @@ configDotenv(); export const main = async (): Promise => { const { DATABASE_URL } = getDatabaseConfigFromEnv(); - const { folder, schema } = parseArguments(); + const { schema } = parseArguments(); const db = createKyselyDatabase({ connectionString: DATABASE_URL, @@ -46,7 +45,7 @@ export const main = async (): Promise => { const migrationResults = await migrateToLatest({ db, schema, - migrationsFolder: getMigrationsFolder(folder), + migrationsFolder: getMigrationsFolder(), }); if (migrationResults && migrationResults?.length > 0) { diff --git a/scripts/migrations/src/migrations/chainData/20241029T120000_initial.ts b/scripts/migrations/src/migrations/20241029T120000_initial.ts similarity index 99% rename from scripts/migrations/src/migrations/chainData/20241029T120000_initial.ts rename to scripts/migrations/src/migrations/20241029T120000_initial.ts index 904a139..778d0e8 100644 --- a/scripts/migrations/src/migrations/chainData/20241029T120000_initial.ts +++ b/scripts/migrations/src/migrations/20241029T120000_initial.ts @@ -1,6 +1,6 @@ import { Kysely, sql } from "kysely"; -import { getSchemaName } from "../../utils/index.js"; +import { getSchemaName } from "../utils/index.js"; /** * The up function is called when you update your database schema to the next version and down when you go back to previous version. diff --git a/scripts/migrations/src/migrations/registries/20241210T175001_strategy_registry.ts b/scripts/migrations/src/migrations/20241210T175001_strategy_registry.ts similarity index 95% rename from scripts/migrations/src/migrations/registries/20241210T175001_strategy_registry.ts rename to scripts/migrations/src/migrations/20241210T175001_strategy_registry.ts index 704a75d..85595e2 100644 --- a/scripts/migrations/src/migrations/registries/20241210T175001_strategy_registry.ts +++ b/scripts/migrations/src/migrations/20241210T175001_strategy_registry.ts @@ -1,6 +1,6 @@ import { Kysely } from "kysely"; -import { getSchemaName } from "../../utils/index.js"; +import { getSchemaName } from "../utils/index.js"; /** * The up function is called when you update your database schema to the next version and down when you go back to previous version. diff --git a/scripts/migrations/src/resetDb.script.ts b/scripts/migrations/src/resetDb.script.ts index b55f6a2..d5ca249 100644 --- a/scripts/migrations/src/resetDb.script.ts +++ b/scripts/migrations/src/resetDb.script.ts @@ -22,7 +22,6 @@ configDotenv(); * - DATABASE_URL: PostgreSQL connection string * * Script arguments: - * - folder: Folder name to migrate (e.g. "chainData" or "registries") * - schema: Database schema name where migrations are applied * * The script will: @@ -37,7 +36,7 @@ configDotenv(); const main = async (): Promise => { const { DATABASE_URL } = getDatabaseConfigFromEnv(); - const { folder, schema } = parseArguments(); + const { schema } = parseArguments(); const db = createKyselyDatabase({ connectionString: DATABASE_URL, @@ -49,7 +48,7 @@ const main = async (): Promise => { const resetResults = await resetDatabase({ db, schema, - migrationsFolder: getMigrationsFolder(folder), + migrationsFolder: getMigrationsFolder(), }); if (resetResults && resetResults?.length > 0) { diff --git a/scripts/migrations/src/utils/parsing.ts b/scripts/migrations/src/utils/parsing.ts index c062589..664bed0 100644 --- a/scripts/migrations/src/utils/parsing.ts +++ b/scripts/migrations/src/utils/parsing.ts @@ -4,28 +4,23 @@ import yargs from "yargs"; import { hideBin } from "yargs/helpers"; import { z } from "zod"; +const DEFAULT_SCHEMA = "public"; + const zodSchema = z.object({ - folder: z + schema: z .string() - .refine((value) => ["chainData", "registries"].includes(value), { - message: "Schema name must be either 'chainData' or 'registries'", - }) - .describe("Folder name to migrate"), - schema: z.string().describe("Database schema name where migrations are applied"), + .default(DEFAULT_SCHEMA) + .describe("Database schema name where migrations are applied"), }); export const parseArguments = (): z.infer => { return yargs(hideBin(process.argv)) - .option("folder", { - type: "string", - demandOption: true, - description: "Folder name to migrate", - }) .option("schema", { alias: "s", type: "string", demandOption: true, description: "Database schema name where migrations are applied", + default: DEFAULT_SCHEMA, }) .check((argv) => { zodSchema.parse(argv); @@ -34,15 +29,14 @@ export const parseArguments = (): z.infer => { .parseSync(); }; -export const getMigrationsFolder = (folder: string): string => { +export const getMigrationsFolder = (): string => { const migrationsFolder = path.join( path.dirname(new URL(import.meta.url).pathname), - `../migrations/${folder}`, + `../migrations`, ); - console.log("migrationsFolder", migrationsFolder); if (!existsSync(migrationsFolder)) { - throw new Error(`Migrations folder '${folder}' not found`); + throw new Error(`Migrations folder not found`); } return migrationsFolder;