From bbfde71af9a0bab3f9228c89b8776aa2715729c4 Mon Sep 17 00:00:00 2001 From: nigiri <168690269+0xnigir1@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:47:11 -0300 Subject: [PATCH] refactor: move migrations to scripts/migrations package (#36) --- package.json | 4 +- packages/repository/src/db/helpers.ts | 13 ------ packages/repository/src/db/index.ts | 1 - packages/repository/src/external.ts | 3 -- packages/repository/src/internal.ts | 1 - pnpm-lock.yaml | 36 ++++++++------- pnpm-workspace.yaml | 1 + .../migrations}/.env.example | 0 .../scripts => scripts/migrations}/README.md | 6 ++- .../migrations}/package.json | 3 +- .../migrations}/src/migrateDb.script.ts | 8 +++- .../src/migrations/20241029T120000_initial.ts | 2 +- .../migrations}/src/resetDb.script.ts | 8 +++- .../migrations}/src/schemas/index.ts | 0 scripts/migrations/src/utils/index.ts | 1 + .../migrations/src/utils/kysely.ts | 46 ++++++++++++------- .../migrations}/tsconfig.build.json | 0 .../migrations}/tsconfig.json | 0 .../migrations}/vitest.config.ts | 0 19 files changed, 75 insertions(+), 58 deletions(-) delete mode 100644 packages/repository/src/db/helpers.ts rename {apps/scripts => scripts/migrations}/.env.example (100%) rename {apps/scripts => scripts/migrations}/README.md (96%) rename {apps/scripts => scripts/migrations}/package.json (93%) rename {apps/scripts => scripts/migrations}/src/migrateDb.script.ts (87%) rename {packages/repository => scripts/migrations}/src/migrations/20241029T120000_initial.ts (99%) rename {apps/scripts => scripts/migrations}/src/resetDb.script.ts (87%) rename {apps/scripts => scripts/migrations}/src/schemas/index.ts (100%) create mode 100644 scripts/migrations/src/utils/index.ts rename packages/repository/src/db/provider.ts => scripts/migrations/src/utils/kysely.ts (69%) rename {apps/scripts => scripts/migrations}/tsconfig.build.json (100%) rename {apps/scripts => scripts/migrations}/tsconfig.json (100%) rename {apps/scripts => scripts/migrations}/vitest.config.ts (100%) diff --git a/package.json b/package.json index a0ecea6..0ec53b2 100644 --- a/package.json +++ b/package.json @@ -17,8 +17,8 @@ "lint": "turbo run lint", "lint:fix": "turbo run lint:fix", "prepare": "husky", - "script:db:migrate": "pnpm run --filter @grants-stack-indexer/scripts script:db:migrate", - "script:db:reset": "pnpm run --filter @grants-stack-indexer/scripts script:db:reset", + "script:db:migrate": "pnpm run --filter @grants-stack-indexer/migrations script:db:migrate", + "script:db:reset": "pnpm run --filter @grants-stack-indexer/migrations script:db:reset", "start": "turbo run start", "test": "turbo run test", "test:cov": "turbo run test:cov", diff --git a/packages/repository/src/db/helpers.ts b/packages/repository/src/db/helpers.ts deleted file mode 100644 index 119bea6..0000000 --- a/packages/repository/src/db/helpers.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { SchemaModule } from "kysely"; - -/** - * Since WithSchemaPlugin doesn't work with `sql.table`, we need to get the schema name manually. - * ref: https://github.com/kysely-org/kysely/issues/761 - */ -export const getSchemaName = (schema: SchemaModule): string => { - let name = "public"; - schema.createTable("test").$call((b) => { - name = b.toOperationNode().table.table.schema?.name ?? "public"; - }); - return name; -}; diff --git a/packages/repository/src/db/index.ts b/packages/repository/src/db/index.ts index f5909da..d1b2322 100644 --- a/packages/repository/src/db/index.ts +++ b/packages/repository/src/db/index.ts @@ -1,2 +1 @@ export * from "./connection.js"; -export * from "./provider.js"; diff --git a/packages/repository/src/external.ts b/packages/repository/src/external.ts index 5ddb658..8d10c3c 100644 --- a/packages/repository/src/external.ts +++ b/packages/repository/src/external.ts @@ -60,6 +60,3 @@ export { } from "./internal.js"; export { createKyselyPostgresDb as createKyselyDatabase } from "./internal.js"; - -export { migrateToLatest, resetDatabase } from "./db/index.js"; -export type { MigrationConfig } from "./db/index.js"; diff --git a/packages/repository/src/internal.ts b/packages/repository/src/internal.ts index df70aa9..bdbaa77 100644 --- a/packages/repository/src/internal.ts +++ b/packages/repository/src/internal.ts @@ -2,5 +2,4 @@ export * from "./types/index.js"; export * from "./interfaces/index.js"; export * from "./db/connection.js"; export * from "./repositories/kysely/index.js"; -export * from "./db/helpers.js"; export * from "./exceptions/index.js"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0bd975f..b47d90a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -136,22 +136,6 @@ importers: specifier: 4.19.2 version: 4.19.2 - apps/scripts: - dependencies: - "@grants-stack-indexer/repository": - specifier: workspace:* - version: link:../../packages/repository - dotenv: - specifier: 16.4.5 - version: 16.4.5 - zod: - specifier: 3.23.8 - version: 3.23.8 - devDependencies: - tsx: - specifier: 4.19.2 - version: 4.19.2 - packages/chain-providers: dependencies: "@grants-stack-indexer/shared": @@ -284,6 +268,25 @@ importers: specifier: 3.15.0 version: 3.15.0 + scripts/migrations: + dependencies: + "@grants-stack-indexer/repository": + specifier: workspace:* + version: link:../../packages/repository + dotenv: + specifier: 16.4.5 + version: 16.4.5 + kysely: + specifier: 0.27.4 + version: 0.27.4 + zod: + specifier: 3.23.8 + version: 3.23.8 + devDependencies: + tsx: + specifier: 4.19.2 + version: 4.19.2 + packages: "@adraffy/ens-normalize@1.10.0": resolution: @@ -2538,6 +2541,7 @@ packages: integrity: sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==, } engines: { node: ^12.22.0 || ^14.17.0 || >=16.0.0 } + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true espree@9.6.1: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 4af3162..4da8d09 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,4 @@ packages: - "apps/*" - "packages/*" + - "scripts/*" diff --git a/apps/scripts/.env.example b/scripts/migrations/.env.example similarity index 100% rename from apps/scripts/.env.example rename to scripts/migrations/.env.example diff --git a/apps/scripts/README.md b/scripts/migrations/README.md similarity index 96% rename from apps/scripts/README.md rename to scripts/migrations/README.md index 15aabeb..fb442af 100644 --- a/apps/scripts/README.md +++ b/scripts/migrations/README.md @@ -1,4 +1,4 @@ -# grants-stack-indexer: scripts +# grants-stack-indexer: migrations scripts This package contains scripts for managing the database schema and migrations. @@ -110,3 +110,7 @@ export async function down(db: Kysely): Promise { - Verify schema consistency TODO: add E2E tests for the scripts + +## References + +- [Kysely](https://kysely.dev/) diff --git a/apps/scripts/package.json b/scripts/migrations/package.json similarity index 93% rename from apps/scripts/package.json rename to scripts/migrations/package.json index bbb77c5..737e796 100644 --- a/apps/scripts/package.json +++ b/scripts/migrations/package.json @@ -1,5 +1,5 @@ { - "name": "@grants-stack-indexer/scripts", + "name": "@grants-stack-indexer/migrations", "version": "0.0.1", "private": true, "description": "", @@ -28,6 +28,7 @@ "dependencies": { "@grants-stack-indexer/repository": "workspace:*", "dotenv": "16.4.5", + "kysely": "0.27.4", "zod": "3.23.8" }, "devDependencies": { diff --git a/apps/scripts/src/migrateDb.script.ts b/scripts/migrations/src/migrateDb.script.ts similarity index 87% rename from apps/scripts/src/migrateDb.script.ts rename to scripts/migrations/src/migrateDb.script.ts index 9c413bc..3d7245b 100644 --- a/apps/scripts/src/migrateDb.script.ts +++ b/scripts/migrations/src/migrateDb.script.ts @@ -1,8 +1,10 @@ +import path from "path"; import { configDotenv } from "dotenv"; -import { createKyselyDatabase, migrateToLatest } from "@grants-stack-indexer/repository"; +import { createKyselyDatabase } from "@grants-stack-indexer/repository"; import { getDatabaseConfigFromEnv } from "./schemas/index.js"; +import { migrateToLatest } from "./utils/index.js"; configDotenv(); @@ -41,6 +43,10 @@ export const main = async (): Promise => { const migrationResults = await migrateToLatest({ db, schema: DATABASE_SCHEMA, + migrationsFolder: path.join( + path.dirname(new URL(import.meta.url).pathname), + "./migrations", + ), }); if (migrationResults && migrationResults?.length > 0) { diff --git a/packages/repository/src/migrations/20241029T120000_initial.ts b/scripts/migrations/src/migrations/20241029T120000_initial.ts similarity index 99% rename from packages/repository/src/migrations/20241029T120000_initial.ts rename to scripts/migrations/src/migrations/20241029T120000_initial.ts index cdbeaef..778d0e8 100644 --- a/packages/repository/src/migrations/20241029T120000_initial.ts +++ b/scripts/migrations/src/migrations/20241029T120000_initial.ts @@ -1,6 +1,6 @@ import { Kysely, sql } from "kysely"; -import { getSchemaName } from "../db/helpers.js"; +import { getSchemaName } from "../utils/index.js"; /** * The up function is called when you update your database schema to the next version and down when you go back to previous version. diff --git a/apps/scripts/src/resetDb.script.ts b/scripts/migrations/src/resetDb.script.ts similarity index 87% rename from apps/scripts/src/resetDb.script.ts rename to scripts/migrations/src/resetDb.script.ts index 37450f2..905c9eb 100644 --- a/apps/scripts/src/resetDb.script.ts +++ b/scripts/migrations/src/resetDb.script.ts @@ -1,8 +1,10 @@ +import path from "path"; import { configDotenv } from "dotenv"; -import { createKyselyDatabase, resetDatabase } from "@grants-stack-indexer/repository"; +import { createKyselyDatabase } from "@grants-stack-indexer/repository"; import { getDatabaseConfigFromEnv } from "./schemas/index.js"; +import { resetDatabase } from "./utils/index.js"; configDotenv(); @@ -44,6 +46,10 @@ const main = async (): Promise => { const resetResults = await resetDatabase({ db, schema: DATABASE_SCHEMA, + migrationsFolder: path.join( + path.dirname(new URL(import.meta.url).pathname), + "./migrations", + ), }); if (resetResults && resetResults?.length > 0) { diff --git a/apps/scripts/src/schemas/index.ts b/scripts/migrations/src/schemas/index.ts similarity index 100% rename from apps/scripts/src/schemas/index.ts rename to scripts/migrations/src/schemas/index.ts diff --git a/scripts/migrations/src/utils/index.ts b/scripts/migrations/src/utils/index.ts new file mode 100644 index 0000000..7b157b3 --- /dev/null +++ b/scripts/migrations/src/utils/index.ts @@ -0,0 +1 @@ +export * from "./kysely.js"; diff --git a/packages/repository/src/db/provider.ts b/scripts/migrations/src/utils/kysely.ts similarity index 69% rename from packages/repository/src/db/provider.ts rename to scripts/migrations/src/utils/kysely.ts index 0ecc705..1e757f0 100644 --- a/packages/repository/src/db/provider.ts +++ b/scripts/migrations/src/utils/kysely.ts @@ -1,14 +1,32 @@ import { promises as fs } from "fs"; import * as path from "path"; -import { FileMigrationProvider, Kysely, MigrationResult, Migrator, NO_MIGRATIONS } from "kysely"; +import { + FileMigrationProvider, + Kysely, + MigrationResult, + Migrator, + NO_MIGRATIONS, + SchemaModule, +} from "kysely"; -import { Database } from "./connection.js"; - -export interface MigrationConfig { - db: Kysely; +export interface MigrationConfig { + db: Kysely; schema: string; + migrationsFolder: string; } +/** + * Since WithSchemaPlugin doesn't work with `sql.table`, we need to get the schema name manually. + * ref: https://github.com/kysely-org/kysely/issues/761 + */ +export const getSchemaName = (schema: SchemaModule): string => { + let name = "public"; + schema.createTable("test").$call((b) => { + name = b.toOperationNode().table.table.schema?.name ?? "public"; + }); + return name; +}; + /** * Applies all migrations to the database up to the latest version. * @@ -17,8 +35,8 @@ export interface MigrationConfig { * @param config.schema - The schema to use for the migrations. Should be the same as the schema used in the Kysely database instance. * @returns The migration results. */ -export async function migrateToLatest( - config: MigrationConfig, +export async function migrateToLatest( + config: MigrationConfig, ): Promise { await config.db.schema.createSchema(config.schema).ifNotExists().execute(); @@ -27,10 +45,7 @@ export async function migrateToLatest( provider: new FileMigrationProvider({ fs, path, - migrationFolder: path.join( - path.dirname(new URL(import.meta.url).pathname), - "../migrations", - ), + migrationFolder: config.migrationsFolder, }), migrationTableSchema: config.schema, }); @@ -62,18 +77,15 @@ export async function migrateToLatest( * @param config.schema - The schema to use for the migrations. Should be the same as the schema used in the Kysely database instance. * @returns The migration results. */ -export async function resetDatabase( - config: MigrationConfig, +export async function resetDatabase( + config: MigrationConfig, ): Promise { const migrator = new Migrator({ db: config.db, provider: new FileMigrationProvider({ fs, path, - migrationFolder: path.join( - path.dirname(new URL(import.meta.url).pathname), - "../migrations", - ), + migrationFolder: config.migrationsFolder, }), }); diff --git a/apps/scripts/tsconfig.build.json b/scripts/migrations/tsconfig.build.json similarity index 100% rename from apps/scripts/tsconfig.build.json rename to scripts/migrations/tsconfig.build.json diff --git a/apps/scripts/tsconfig.json b/scripts/migrations/tsconfig.json similarity index 100% rename from apps/scripts/tsconfig.json rename to scripts/migrations/tsconfig.json diff --git a/apps/scripts/vitest.config.ts b/scripts/migrations/vitest.config.ts similarity index 100% rename from apps/scripts/vitest.config.ts rename to scripts/migrations/vitest.config.ts