From 72b0048fb5b2c2faa467bdce8b4bda9992ccceee Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Wed, 2 Oct 2024 23:28:25 +0200 Subject: [PATCH 01/20] run postgres locally via docker --- .gitignore | 1 + api/docker-compose.yml | 10 ++++++++++ api/package.json | 4 +++- 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 api/docker-compose.yml diff --git a/.gitignore b/.gitignore index f6f04c8ef..2224ad7d0 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ coverage api/oracle-cloud/build api/fetch_cache api/sqlite_db +api/postgres_db api/nodemon.json # web diff --git a/api/docker-compose.yml b/api/docker-compose.yml new file mode 100644 index 000000000..f8126dba8 --- /dev/null +++ b/api/docker-compose.yml @@ -0,0 +1,10 @@ +version: "3" +services: + postgress: + image: postgres + ports: + - "5432:5432" + volumes: + - ./postgres_db:/var/lib/postgresql/data + environment: + POSTGRES_HOST_AUTH_METHOD: trust diff --git a/api/package.json b/api/package.json index 358fdcb6f..91d8f4311 100644 --- a/api/package.json +++ b/api/package.json @@ -78,7 +78,9 @@ "lint:ts-prune": "tsx ../packages/tooling/setup-ts-prune.ts && ts-prune --error", "lint:tsc": "tspc --noEmit", "start": "node dist/app/index.js", - "start:dev": "tsx ../packages/tooling/nodemon.ts \"@dzcode.io/api\" && nodemon dist/app/index.js", + "start:nodemon": "nodemon dist/app/index.js", + "start:dev": "tsx ../packages/tooling/nodemon.ts \"@dzcode.io/api\" && npm-run-all --parallel start:nodemon db:server", + "db:server": "docker compose up", "test": "npm run build && npm run test:alone", "test:alone": "jest --config ../packages/tooling/jest.config.ts --rootDir .", "test:watch": "npm-run-all build --parallel build:watch \"test:alone --watch {@}\" --" From 6083a799efd513bb8b531813d68f93e92f8c6777 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Thu, 3 Oct 2024 16:38:28 +0200 Subject: [PATCH 02/20] replaced sqlite with postgres --- .gitignore | 1 - api/db/migrations/0000_bouncy_micromax.sql | 84 ++++++++ api/db/migrations/0000_melodic_shotgun.sql | 60 ------ api/db/migrations/meta/0000_snapshot.json | 211 +++++++++------------ api/db/migrations/meta/_journal.json | 8 +- api/docker-compose.yml | 1 + api/drizzle.config.ts | 2 +- api/package.json | 7 +- api/src/app/index.ts | 4 +- api/src/config/types.ts | 3 +- api/src/postgres/service.ts | 25 +++ api/src/sqlite/service.ts | 26 --- package-lock.json | 76 +++++++- package.json | 4 +- 14 files changed, 281 insertions(+), 231 deletions(-) create mode 100644 api/db/migrations/0000_bouncy_micromax.sql delete mode 100644 api/db/migrations/0000_melodic_shotgun.sql create mode 100644 api/src/postgres/service.ts delete mode 100644 api/src/sqlite/service.ts diff --git a/.gitignore b/.gitignore index 2224ad7d0..3d15193ae 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,6 @@ coverage # api api/oracle-cloud/build api/fetch_cache -api/sqlite_db api/postgres_db api/nodemon.json diff --git a/api/db/migrations/0000_bouncy_micromax.sql b/api/db/migrations/0000_bouncy_micromax.sql new file mode 100644 index 000000000..68aa3cb1f --- /dev/null +++ b/api/db/migrations/0000_bouncy_micromax.sql @@ -0,0 +1,84 @@ +CREATE TABLE IF NOT EXISTS "contributions" ( + "id" text PRIMARY KEY NOT NULL, + "record_imported_at" text DEFAULT CURRENT_TIMESTAMP NOT NULL, + "title" text NOT NULL, + "updated_at" text NOT NULL, + "url" text NOT NULL, + "type" text NOT NULL, + "run_id" text NOT NULL, + "activity_count" integer NOT NULL, + "repository_id" text NOT NULL, + "contributor_id" text NOT NULL, + CONSTRAINT "contributions_url_unique" UNIQUE("url") +); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "contributor_repository_relation" ( + "contributor_id" text NOT NULL, + "repository_id" text NOT NULL, + "record_imported_at" text DEFAULT CURRENT_TIMESTAMP NOT NULL, + "run_id" text DEFAULT 'initial-run-id' NOT NULL, + "score" integer NOT NULL, + CONSTRAINT "contributor_repository_relation_pk" PRIMARY KEY("contributor_id","repository_id") +); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "contributors" ( + "id" text PRIMARY KEY NOT NULL, + "record_imported_at" text DEFAULT CURRENT_TIMESTAMP NOT NULL, + "run_id" text DEFAULT 'initial-run-id' NOT NULL, + "name" text NOT NULL, + "username" text NOT NULL, + "url" text NOT NULL, + "avatar_url" text NOT NULL, + CONSTRAINT "contributors_url_unique" UNIQUE("url") +); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "projects" ( + "id" text PRIMARY KEY NOT NULL, + "record_imported_at" text DEFAULT CURRENT_TIMESTAMP NOT NULL, + "name" text NOT NULL, + "slug" text NOT NULL, + "run_id" text DEFAULT 'initial-run-id' NOT NULL, + CONSTRAINT "projects_slug_unique" UNIQUE("slug") +); +--> statement-breakpoint +CREATE TABLE IF NOT EXISTS "repositories" ( + "id" text PRIMARY KEY NOT NULL, + "record_imported_at" text DEFAULT CURRENT_TIMESTAMP NOT NULL, + "provider" text NOT NULL, + "owner" text NOT NULL, + "name" text NOT NULL, + "run_id" text DEFAULT 'initial-run-id' NOT NULL, + "project_id" text NOT NULL, + "stars" integer NOT NULL, + CONSTRAINT "repositories_provider_owner_name_unique" UNIQUE("provider","owner","name") +); +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "contributions" ADD CONSTRAINT "contributions_repository_id_repositories_id_fk" FOREIGN KEY ("repository_id") REFERENCES "public"."repositories"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "contributions" ADD CONSTRAINT "contributions_contributor_id_contributors_id_fk" FOREIGN KEY ("contributor_id") REFERENCES "public"."contributors"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "contributor_repository_relation" ADD CONSTRAINT "contributor_repository_relation_contributor_id_contributors_id_fk" FOREIGN KEY ("contributor_id") REFERENCES "public"."contributors"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "contributor_repository_relation" ADD CONSTRAINT "contributor_repository_relation_repository_id_repositories_id_fk" FOREIGN KEY ("repository_id") REFERENCES "public"."repositories"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; +--> statement-breakpoint +DO $$ BEGIN + ALTER TABLE "repositories" ADD CONSTRAINT "repositories_project_id_projects_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."projects"("id") ON DELETE no action ON UPDATE no action; +EXCEPTION + WHEN duplicate_object THEN null; +END $$; diff --git a/api/db/migrations/0000_melodic_shotgun.sql b/api/db/migrations/0000_melodic_shotgun.sql deleted file mode 100644 index 7bc8d2efc..000000000 --- a/api/db/migrations/0000_melodic_shotgun.sql +++ /dev/null @@ -1,60 +0,0 @@ -CREATE TABLE `contributions` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, - `title` text NOT NULL, - `updated_at` text NOT NULL, - `url` text NOT NULL, - `type` text NOT NULL, - `run_id` text NOT NULL, - `activity_count` integer NOT NULL, - `repository_id` integer NOT NULL, - `contributor_id` integer NOT NULL, - FOREIGN KEY (`repository_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action, - FOREIGN KEY (`contributor_id`) REFERENCES `contributors`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE TABLE `contributor_repository_relation` ( - `contributor_id` integer NOT NULL, - `repository_id` integer NOT NULL, - `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, - `run_id` text DEFAULT 'initial-run-id' NOT NULL, - `score` integer NOT NULL, - PRIMARY KEY(`contributor_id`, `repository_id`), - FOREIGN KEY (`contributor_id`) REFERENCES `contributors`(`id`) ON UPDATE no action ON DELETE no action, - FOREIGN KEY (`repository_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE TABLE `contributors` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, - `run_id` text DEFAULT 'initial-run-id' NOT NULL, - `name` text NOT NULL, - `username` text NOT NULL, - `url` text NOT NULL, - `avatar_url` text NOT NULL -); ---> statement-breakpoint -CREATE TABLE `projects` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, - `name` text NOT NULL, - `slug` text NOT NULL, - `run_id` text DEFAULT 'initial-run-id' NOT NULL -); ---> statement-breakpoint -CREATE TABLE `repositories` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, - `provider` text NOT NULL, - `owner` text NOT NULL, - `name` text NOT NULL, - `run_id` text DEFAULT 'initial-run-id' NOT NULL, - `project_id` integer NOT NULL, - `stars` integer NOT NULL, - FOREIGN KEY (`project_id`) REFERENCES `projects`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE UNIQUE INDEX `contributions_url_unique` ON `contributions` (`url`);--> statement-breakpoint -CREATE UNIQUE INDEX `contributors_url_unique` ON `contributors` (`url`);--> statement-breakpoint -CREATE UNIQUE INDEX `projects_slug_unique` ON `projects` (`slug`);--> statement-breakpoint -CREATE UNIQUE INDEX `repositories_provider_owner_name_unique` ON `repositories` (`provider`,`owner`,`name`); \ No newline at end of file diff --git a/api/db/migrations/meta/0000_snapshot.json b/api/db/migrations/meta/0000_snapshot.json index ccfd4f149..e70ed8f92 100644 --- a/api/db/migrations/meta/0000_snapshot.json +++ b/api/db/migrations/meta/0000_snapshot.json @@ -1,93 +1,76 @@ { - "version": "6", - "dialect": "sqlite", - "id": "2fd49a6b-a3d6-4f53-a4f5-ac67492fc5dd", + "id": "09cef8e3-1a79-48fb-b36f-3273afd27a03", "prevId": "00000000-0000-0000-0000-000000000000", + "version": "7", + "dialect": "postgresql", "tables": { - "contributions": { + "public.contributions": { "name": "contributions", + "schema": "", "columns": { "id": { "name": "id", - "type": "integer", + "type": "text", "primaryKey": true, - "notNull": true, - "autoincrement": true + "notNull": true }, "record_imported_at": { "name": "record_imported_at", "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false, "default": "CURRENT_TIMESTAMP" }, "title": { "name": "title", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "updated_at": { "name": "updated_at", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "url": { "name": "url", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "type": { "name": "type", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "run_id": { "name": "run_id", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "activity_count": { "name": "activity_count", "type": "integer", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "repository_id": { "name": "repository_id", - "type": "integer", + "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "contributor_id": { "name": "contributor_id", - "type": "integer", + "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "contributions_url_unique": { - "name": "contributions_url_unique", - "columns": [ - "url" - ], - "isUnique": true + "notNull": true } }, + "indexes": {}, "foreignKeys": { "contributions_repository_id_repositories_id_fk": { "name": "contributions_repository_id_repositories_id_fk", @@ -117,31 +100,37 @@ } }, "compositePrimaryKeys": {}, - "uniqueConstraints": {} + "uniqueConstraints": { + "contributions_url_unique": { + "name": "contributions_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + } }, - "contributor_repository_relation": { + "public.contributor_repository_relation": { "name": "contributor_repository_relation", + "schema": "", "columns": { "contributor_id": { "name": "contributor_id", - "type": "integer", + "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "repository_id": { "name": "repository_id", - "type": "integer", + "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "record_imported_at": { "name": "record_imported_at", "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false, "default": "CURRENT_TIMESTAMP" }, "run_id": { @@ -149,15 +138,13 @@ "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false, "default": "'initial-run-id'" }, "score": { "name": "score", "type": "integer", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true } }, "indexes": {}, @@ -191,31 +178,30 @@ }, "compositePrimaryKeys": { "contributor_repository_relation_pk": { + "name": "contributor_repository_relation_pk", "columns": [ "contributor_id", "repository_id" - ], - "name": "contributor_repository_relation_pk" + ] } }, "uniqueConstraints": {} }, - "contributors": { + "public.contributors": { "name": "contributors", + "schema": "", "columns": { "id": { "name": "id", - "type": "integer", + "type": "text", "primaryKey": true, - "notNull": true, - "autoincrement": true + "notNull": true }, "record_imported_at": { "name": "record_imported_at", "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false, "default": "CURRENT_TIMESTAMP" }, "run_id": { @@ -223,178 +209,152 @@ "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false, "default": "'initial-run-id'" }, "name": { "name": "name", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "username": { "name": "username", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "url": { "name": "url", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "avatar_url": { "name": "avatar_url", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true } }, - "indexes": { + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { "contributors_url_unique": { "name": "contributors_url_unique", + "nullsNotDistinct": false, "columns": [ "url" - ], - "isUnique": true + ] } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} + } }, - "projects": { + "public.projects": { "name": "projects", + "schema": "", "columns": { "id": { "name": "id", - "type": "integer", + "type": "text", "primaryKey": true, - "notNull": true, - "autoincrement": true + "notNull": true }, "record_imported_at": { "name": "record_imported_at", "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false, "default": "CURRENT_TIMESTAMP" }, "name": { "name": "name", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "slug": { "name": "slug", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "run_id": { "name": "run_id", "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false, "default": "'initial-run-id'" } }, - "indexes": { + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { "projects_slug_unique": { "name": "projects_slug_unique", + "nullsNotDistinct": false, "columns": [ "slug" - ], - "isUnique": true + ] } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} + } }, - "repositories": { + "public.repositories": { "name": "repositories", + "schema": "", "columns": { "id": { "name": "id", - "type": "integer", + "type": "text", "primaryKey": true, - "notNull": true, - "autoincrement": true + "notNull": true }, "record_imported_at": { "name": "record_imported_at", "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false, "default": "CURRENT_TIMESTAMP" }, "provider": { "name": "provider", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "owner": { "name": "owner", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "name": { "name": "name", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "run_id": { "name": "run_id", "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false, "default": "'initial-run-id'" }, "project_id": { "name": "project_id", - "type": "integer", + "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "stars": { "name": "stars", "type": "integer", "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "repositories_provider_owner_name_unique": { - "name": "repositories_provider_owner_name_unique", - "columns": [ - "provider", - "owner", - "name" - ], - "isUnique": true + "notNull": true } }, + "indexes": {}, "foreignKeys": { "repositories_project_id_projects_id_fk": { "name": "repositories_project_id_projects_id_fk", @@ -411,16 +371,25 @@ } }, "compositePrimaryKeys": {}, - "uniqueConstraints": {} + "uniqueConstraints": { + "repositories_provider_owner_name_unique": { + "name": "repositories_provider_owner_name_unique", + "nullsNotDistinct": false, + "columns": [ + "provider", + "owner", + "name" + ] + } + } } }, "enums": {}, + "schemas": {}, + "sequences": {}, "_meta": { + "columns": {}, "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} + "tables": {} } } \ No newline at end of file diff --git a/api/db/migrations/meta/_journal.json b/api/db/migrations/meta/_journal.json index 6a87978fb..1fe6d13ba 100644 --- a/api/db/migrations/meta/_journal.json +++ b/api/db/migrations/meta/_journal.json @@ -1,12 +1,12 @@ { "version": "7", - "dialect": "sqlite", + "dialect": "postgresql", "entries": [ { "idx": 0, - "version": "6", - "when": 1727027402330, - "tag": "0000_melodic_shotgun", + "version": "7", + "when": 1727955597397, + "tag": "0000_bouncy_micromax", "breakpoints": true } ] diff --git a/api/docker-compose.yml b/api/docker-compose.yml index f8126dba8..b6cb8bfd2 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -8,3 +8,4 @@ services: - ./postgres_db:/var/lib/postgresql/data environment: POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: db diff --git a/api/drizzle.config.ts b/api/drizzle.config.ts index e4f7ef132..f5194eab1 100644 --- a/api/drizzle.config.ts +++ b/api/drizzle.config.ts @@ -3,5 +3,5 @@ import { defineConfig } from "drizzle-kit"; export default defineConfig({ schema: "src/**/*table.ts", out: "./db/migrations", - dialect: "sqlite", + dialect: "postgresql", }); diff --git a/api/package.json b/api/package.json index 91d8f4311..bbb9ba8e5 100644 --- a/api/package.json +++ b/api/package.json @@ -13,7 +13,6 @@ "@sentry/node": "^8.28.0", "@sentry/profiling-node": "^8.28.0", "@types/make-fetch-happen": "^10.0.4", - "better-sqlite3": "^11.2.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "cors": "^2.8.5", @@ -27,6 +26,7 @@ "lodash": "^4.17.21", "make-fetch-happen": "^13.0.1", "morgan": "^1.10.0", + "postgres": "^3.4.4", "reflect-metadata": "^0.2.2", "routing-controllers": "^0.10.4", "typedi": "^0.10.0", @@ -34,7 +34,6 @@ }, "devDependencies": { "@dzcode.io/tooling": "*", - "@types/better-sqlite3": "^7.6.11", "@types/body-parser": "^1.19.0", "@types/cors": "^2.8.9", "@types/express": "^4.17.9", @@ -65,6 +64,7 @@ "clean": "lerna run clean:alone --scope=@dzcode.io/api --include-dependencies --stream", "clean:alone": "del dist coverage fetch_cache oracle-cloud/build", "db:generate-migration": "drizzle-kit generate", + "db:server": "docker compose up", "deploy": "del ./oracle-cloud/build && tsx oracle-cloud/deploy.ts production", "deploy:stg": "del ./oracle-cloud/build && tsx oracle-cloud/deploy.ts staging", "generate:bundle-info": "tsx ../packages/tooling/bundle-info.ts", @@ -78,9 +78,8 @@ "lint:ts-prune": "tsx ../packages/tooling/setup-ts-prune.ts && ts-prune --error", "lint:tsc": "tspc --noEmit", "start": "node dist/app/index.js", - "start:nodemon": "nodemon dist/app/index.js", "start:dev": "tsx ../packages/tooling/nodemon.ts \"@dzcode.io/api\" && npm-run-all --parallel start:nodemon db:server", - "db:server": "docker compose up", + "start:nodemon": "nodemon dist/app/index.js", "test": "npm run build && npm run test:alone", "test:alone": "jest --config ../packages/tooling/jest.config.ts --rootDir .", "test:watch": "npm-run-all build --parallel build:watch \"test:alone --watch {@}\" --" diff --git a/api/src/app/index.ts b/api/src/app/index.ts index ff935a1b5..0924fb0c1 100644 --- a/api/src/app/index.ts +++ b/api/src/app/index.ts @@ -15,7 +15,7 @@ import { GithubController } from "src/github/controller"; import { LoggerService } from "src/logger/service"; import { MilestoneController } from "src/milestone/controller"; import { ProjectController } from "src/project/controller"; -import { SQLiteService } from "src/sqlite/service"; +import { PostgresService } from "src/postgres/service"; import Container from "typedi"; import { LoggerMiddleware } from "./middlewares/logger"; @@ -26,7 +26,7 @@ import { SecurityMiddleware } from "./middlewares/security"; useContainer(Container); // eslint-disable-line react-hooks/rules-of-hooks // Initialize Database -Container.get(SQLiteService); +Container.get(PostgresService); const { NODE_ENV, PORT } = Container.get(ConfigService).env(); diff --git a/api/src/config/types.ts b/api/src/config/types.ts index 05cfc1aed..0e2399e7a 100644 --- a/api/src/config/types.ts +++ b/api/src/config/types.ts @@ -19,7 +19,8 @@ export class EnvRecord { FETCH_CACHE_PATH = "./fetch_cache"; @IsString() - SQLITE_DB_PATH = "./sqlite_db"; + // TODO-ZM: localhost to postgres for non-development environments + POSTGRES_URI = "postgres://postgres@localhost:5432/db"; @IsString() @IsOptional() diff --git a/api/src/postgres/service.ts b/api/src/postgres/service.ts new file mode 100644 index 000000000..0b6670a0b --- /dev/null +++ b/api/src/postgres/service.ts @@ -0,0 +1,25 @@ +import postgres from "postgres"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { migrate } from "drizzle-orm/postgres-js/migrator"; +import { join } from "path"; +import { ConfigService } from "src/config/service"; +import { LoggerService } from "src/logger/service"; +import { Service } from "typedi"; + +@Service() +export class PostgresService { + public db; + + constructor( + private readonly configService: ConfigService, + private readonly loggerService: LoggerService, + ) { + this.loggerService.info({ message: "Initializing Postgres database" }); + const { POSTGRES_URI } = this.configService.env(); + + const queryClient = postgres(POSTGRES_URI); + this.db = drizzle(queryClient); + migrate(this.db, { migrationsFolder: join(__dirname, "../../db/migrations") }); + this.loggerService.info({ message: "Database migration complete" }); + } +} diff --git a/api/src/sqlite/service.ts b/api/src/sqlite/service.ts deleted file mode 100644 index c1865d03f..000000000 --- a/api/src/sqlite/service.ts +++ /dev/null @@ -1,26 +0,0 @@ -import Database from "better-sqlite3"; -import { drizzle } from "drizzle-orm/better-sqlite3"; -import { migrate } from "drizzle-orm/better-sqlite3/migrator"; -import { mkdirSync } from "fs"; -import { join } from "path"; -import { ConfigService } from "src/config/service"; -import { LoggerService } from "src/logger/service"; -import { Service } from "typedi"; - -@Service() -export class SQLiteService { - public db; - - constructor( - private readonly configService: ConfigService, - private readonly loggerService: LoggerService, - ) { - this.loggerService.info({ message: "Initializing SQLite database" }); - const { SQLITE_DB_PATH } = this.configService.env(); - mkdirSync(SQLITE_DB_PATH, { recursive: true }); - const sqlite = new Database(join(SQLITE_DB_PATH, "main.sqlite")); - this.db = drizzle(sqlite); - migrate(this.db, { migrationsFolder: join(__dirname, "../../db/migrations") }); - this.loggerService.info({ message: "Database migration complete" }); - } -} diff --git a/package-lock.json b/package-lock.json index 390c7a2d7..d2969ff29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -68,7 +68,6 @@ "@sentry/node": "^8.28.0", "@sentry/profiling-node": "^8.28.0", "@types/make-fetch-happen": "^10.0.4", - "better-sqlite3": "^11.2.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "cors": "^2.8.5", @@ -82,6 +81,7 @@ "lodash": "^4.17.21", "make-fetch-happen": "^13.0.1", "morgan": "^1.10.0", + "postgres": "^3.4.4", "reflect-metadata": "^0.2.2", "routing-controllers": "^0.10.4", "typedi": "^0.10.0", @@ -89,7 +89,6 @@ }, "devDependencies": { "@dzcode.io/tooling": "*", - "@types/better-sqlite3": "^7.6.11", "@types/body-parser": "^1.19.0", "@types/cors": "^2.8.9", "@types/express": "^4.17.9", @@ -6224,8 +6223,9 @@ "version": "7.6.11", "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.11.tgz", "integrity": "sha512-i8KcD3PgGtGBLl3+mMYA8PdKkButvPyARxA7IQAd6qeslht13qxb1zzO8dRCtE7U3IoJS782zDBAeoKiM695kg==", - "devOptional": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "@types/node": "*" } @@ -8102,6 +8102,8 @@ "integrity": "sha512-iHt9j8NPYF3oKCNOO5ZI4JwThjt3Z6J6XrcwG85VNMVzv1ByqrHWv5VILEbCMFWDsoHhXvQ7oC8vgRXFAKgl9w==", "hasInstallScript": true, "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "bindings": "^1.5.0", "prebuild-install": "^7.1.1" @@ -8155,6 +8157,8 @@ "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "file-uri-to-path": "1.0.0" } @@ -8163,6 +8167,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "devOptional": true, "license": "MIT", "dependencies": { "buffer": "^5.5.0", @@ -8319,6 +8324,7 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "devOptional": true, "funding": [ { "type": "github", @@ -11243,6 +11249,8 @@ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -11293,6 +11301,8 @@ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=4.0.0" } @@ -12374,6 +12384,7 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "devOptional": true, "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -13220,6 +13231,8 @@ "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", "license": "(MIT OR WTFPL)", + "optional": true, + "peer": true, "engines": { "node": ">=6" } @@ -13649,7 +13662,9 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "node_modules/filelist": { "version": "1.0.4", @@ -13991,6 +14006,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "devOptional": true, "license": "MIT" }, "node_modules/fs-extra": { @@ -14766,7 +14782,9 @@ "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "node_modules/glob": { "version": "7.2.3", @@ -15540,6 +15558,7 @@ "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "devOptional": true, "license": "ISC" }, "node_modules/init-package-json": { @@ -19660,6 +19679,8 @@ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=10" }, @@ -19732,6 +19753,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "devOptional": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -19927,7 +19949,9 @@ "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "node_modules/modify-values": { "version": "1.0.1", @@ -20181,7 +20205,9 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "node_modules/natural-compare": { "version": "1.4.0", @@ -21119,6 +21145,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "devOptional": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -22755,6 +22782,19 @@ "dev": true, "license": "MIT" }, + "node_modules/postgres": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.4.tgz", + "integrity": "sha512-IbyN+9KslkqcXa8AO9fxpk97PA4pzewvpi2B3Dwy9u4zpV32QicaEdgmF3eSQUzdRk7ttDHQejNgAEr4XoeH4A==", + "license": "Unlicense", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/porsager" + } + }, "node_modules/postgres-array": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", @@ -22799,6 +22839,8 @@ "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz", "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "detect-libc": "^2.0.0", "expand-template": "^2.0.3", @@ -23100,6 +23142,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "devOptional": true, "license": "MIT", "dependencies": { "end-of-stream": "^1.1.0", @@ -23239,6 +23282,8 @@ "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "optional": true, + "peer": true, "dependencies": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -23254,6 +23299,8 @@ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", "license": "MIT", + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -24760,7 +24807,9 @@ "url": "https://feross.org/support" } ], - "license": "MIT" + "license": "MIT", + "optional": true, + "peer": true }, "node_modules/simple-get": { "version": "4.0.1", @@ -24781,6 +24830,8 @@ } ], "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "decompress-response": "^6.0.0", "once": "^1.3.1", @@ -26306,6 +26357,8 @@ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", "license": "MIT", + "optional": true, + "peer": true, "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -26317,12 +26370,15 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "license": "ISC" + "license": "ISC", + "optional": true, + "peer": true }, "node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "devOptional": true, "license": "MIT", "dependencies": { "bl": "^4.0.3", @@ -27120,6 +27176,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "devOptional": true, "license": "Apache-2.0", "dependencies": { "safe-buffer": "^5.0.1" @@ -28915,6 +28972,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "devOptional": true, "license": "ISC" }, "node_modules/write-file-atomic": { diff --git a/package.json b/package.json index 90379bc65..abe51eb60 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "private": true, "scripts": { "build": "lerna run build:alone --stream", - "build:watch": "lerna run build:alone:watch --parallel", + "build:watch": "lerna run build:alone:watch --parallel --stream", "clean": "lerna run clean:alone --stream", "deploy": "lerna run deploy --parallel", "deploy:stg": "lerna run deploy:stg --parallel", @@ -76,7 +76,7 @@ "postinstall": "npm run patch-package && (husky install && husky set .husky/pre-commit \"npm run lint:staged\") || exit 0", "pre-deploy": "lerna run pre-deploy --parallel", "prepare": "ts-patch install -s", - "start:dev": "lerna run start:dev --parallel", + "start:dev": "lerna run start:dev --parallel --stream", "test": "npm run build && npm run test:alone", "test:alone": "lerna run test:alone --stream", "version:apply": "tsx packages/tooling/version-apply.ts", From 075a2d22be463c03695c906438caaa56675e6a6b Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Thu, 3 Oct 2024 16:41:31 +0200 Subject: [PATCH 03/20] changed primary keys types from integer to text --- api/src/contribution/table.ts | 10 +++++----- api/src/contributor/table.ts | 12 ++++++------ api/src/digest/cron.ts | 13 +++++++++++-- api/src/github/types.ts | 2 ++ api/src/project/table.ts | 7 +++---- api/src/repository/table.ts | 8 ++++---- packages/models/src/_base/index.ts | 2 +- web/src/pages/projects/index.tsx | 8 ++++---- 8 files changed, 36 insertions(+), 26 deletions(-) diff --git a/api/src/contribution/table.ts b/api/src/contribution/table.ts index e8501a1a4..07b49ea65 100644 --- a/api/src/contribution/table.ts +++ b/api/src/contribution/table.ts @@ -1,11 +1,11 @@ import { ContributionEntity } from "@dzcode.io/models/dist/contribution"; import { sql } from "drizzle-orm"; -import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { integer, pgTable, text } from "drizzle-orm/pg-core"; import { contributorsTable } from "src/contributor/table"; import { repositoriesTable } from "src/repository/table"; -export const contributionsTable = sqliteTable("contributions", { - id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }), +export const contributionsTable = pgTable("contributions", { + id: text("id").notNull().primaryKey(), recordImportedAt: text("record_imported_at") .notNull() .default(sql`CURRENT_TIMESTAMP`), @@ -15,10 +15,10 @@ export const contributionsTable = sqliteTable("contributions", { type: text("type").notNull().$type(), runId: text("run_id").notNull(), activityCount: integer("activity_count").notNull(), - repositoryId: integer("repository_id") + repositoryId: text("repository_id") .notNull() .references(() => repositoriesTable.id), - contributorId: integer("contributor_id") + contributorId: text("contributor_id") .notNull() .references(() => contributorsTable.id), }); diff --git a/api/src/contributor/table.ts b/api/src/contributor/table.ts index 4a6cd366d..35b81336f 100644 --- a/api/src/contributor/table.ts +++ b/api/src/contributor/table.ts @@ -1,10 +1,10 @@ import { ContributorEntity } from "@dzcode.io/models/dist/contributor"; import { sql } from "drizzle-orm"; -import { integer, primaryKey, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { integer, primaryKey, pgTable, text } from "drizzle-orm/pg-core"; import { repositoriesTable } from "src/repository/table"; -export const contributorsTable = sqliteTable("contributors", { - id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }), +export const contributorsTable = pgTable("contributors", { + id: text("id").notNull().primaryKey(), recordImportedAt: text("record_imported_at") .notNull() .default(sql`CURRENT_TIMESTAMP`), @@ -19,13 +19,13 @@ contributorsTable.$inferSelect satisfies ContributorEntity; export type ContributorRow = typeof contributorsTable.$inferInsert; -export const contributorRepositoryRelationTable = sqliteTable( +export const contributorRepositoryRelationTable = pgTable( "contributor_repository_relation", { - contributorId: integer("contributor_id") + contributorId: text("contributor_id") .notNull() .references(() => contributorsTable.id), - repositoryId: integer("repository_id") + repositoryId: text("repository_id") .notNull() .references(() => repositoriesTable.id), recordImportedAt: text("record_imported_at") diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 6b93cefbb..a98d9fbf6 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -67,7 +67,11 @@ export class DigestCron { const projectsFromDataFolder = await this.dataService.listProjects(); for (const project of projectsFromDataFolder) { - const [{ id: projectId }] = await this.projectsRepository.upsert({ ...project, runId }); + const [{ id: projectId }] = await this.projectsRepository.upsert({ + ...project, + runId, + id: project.slug, + }); let addedRepositoryCount = 0; try { @@ -79,13 +83,15 @@ export class DigestCron { repo: repository.name, }); + const provider = "github"; const [{ id: repositoryId }] = await this.repositoriesRepository.upsert({ - provider: "github", + provider, name: repoInfo.name, owner: repoInfo.owner.login, runId, projectId, stars: repoInfo.stargazers_count, + id: `${provider}-${repoInfo.id}`, }); addedRepositoryCount++; @@ -105,6 +111,7 @@ export class DigestCron { url: githubUser.html_url, avatarUrl: githubUser.avatar_url, runId, + id: `${provider}-${githubUser.login}`, }); await this.contributorsRepository.upsertRelationWithRepository({ @@ -124,6 +131,7 @@ export class DigestCron { url: type === "PULL_REQUEST" ? issue.pull_request.html_url : issue.html_url, repositoryId, contributorId, + id: `${provider}-${issue.id}`, }); } @@ -143,6 +151,7 @@ export class DigestCron { url: repoContributor.html_url, avatarUrl: repoContributor.avatar_url, runId, + id: `${provider}-${repoContributor.login}`, }); await this.contributorsRepository.upsertRelationWithRepository({ diff --git a/api/src/github/types.ts b/api/src/github/types.ts index d4b104d92..d78b07e89 100644 --- a/api/src/github/types.ts +++ b/api/src/github/types.ts @@ -100,12 +100,14 @@ export interface GetRateLimitResponse extends GeneralResponse { } export interface GetRepositoryResponse { + id: number; name: string; owner: GithubUser; stargazers_count: number; } interface GithubIssue { + id: number; title: string; user: GithubUser; labels: string[]; diff --git a/api/src/project/table.ts b/api/src/project/table.ts index 6301943a3..01aad9c1c 100644 --- a/api/src/project/table.ts +++ b/api/src/project/table.ts @@ -1,14 +1,13 @@ import { ProjectEntity } from "@dzcode.io/models/dist/project"; import { sql } from "drizzle-orm"; -import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { pgTable, text } from "drizzle-orm/pg-core"; -export const projectsTable = sqliteTable("projects", { - id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }), +export const projectsTable = pgTable("projects", { + id: text("id").notNull().primaryKey(), recordImportedAt: text("record_imported_at") .notNull() .default(sql`CURRENT_TIMESTAMP`), name: text("name").notNull(), - slug: text("slug").notNull().unique(), runId: text("run_id").notNull().default("initial-run-id"), }); diff --git a/api/src/repository/table.ts b/api/src/repository/table.ts index 79824cd72..5fa278614 100644 --- a/api/src/repository/table.ts +++ b/api/src/repository/table.ts @@ -1,12 +1,12 @@ import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; import { sql } from "drizzle-orm"; -import { integer, sqliteTable, text, unique } from "drizzle-orm/sqlite-core"; +import { integer, pgTable, text, unique } from "drizzle-orm/pg-core"; import { projectsTable } from "src/project/table"; -export const repositoriesTable = sqliteTable( +export const repositoriesTable = pgTable( "repositories", { - id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }), + id: text("id").notNull().primaryKey(), recordImportedAt: text("record_imported_at") .notNull() .default(sql`CURRENT_TIMESTAMP`), @@ -14,7 +14,7 @@ export const repositoriesTable = sqliteTable( owner: text("owner").notNull(), name: text("name").notNull(), runId: text("run_id").notNull().default("initial-run-id"), - projectId: integer("project_id") + projectId: text("project_id") .notNull() .references(() => projectsTable.id), stars: integer("stars").notNull(), diff --git a/packages/models/src/_base/index.ts b/packages/models/src/_base/index.ts index 276e9dfa3..e4c2511d6 100644 --- a/packages/models/src/_base/index.ts +++ b/packages/models/src/_base/index.ts @@ -1,4 +1,4 @@ export type BaseEntity = { - id: number; + id: string; runId: string; }; diff --git a/web/src/pages/projects/index.tsx b/web/src/pages/projects/index.tsx index 1290f6678..b080c3406 100644 --- a/web/src/pages/projects/index.tsx +++ b/web/src/pages/projects/index.tsx @@ -67,7 +67,7 @@ export default function Page(): JSX.Element { d="M11.48 3.499a.562.562 0 0 1 1.04 0l2.125 5.111a.563.563 0 0 0 .475.345l5.518.442c.499.04.701.663.321.988l-4.204 3.602a.563.563 0 0 0-.182.557l1.285 5.385a.562.562 0 0 1-.84.61l-4.725-2.885a.562.562 0 0 0-.586 0L6.982 20.54a.562.562 0 0 1-.84-.61l1.285-5.386a.562.562 0 0 0-.182-.557l-4.204-3.602a.562.562 0 0 1 .321-.988l5.518-.442a.563.563 0 0 0 .475-.345L11.48 3.5Z" /> - {project.stars} + {project.totalRepoStars}
@@ -82,10 +82,10 @@ export default function Page(): JSX.Element { - {project.activityCount} + {project.totalRepoScore}
@@ -103,7 +103,7 @@ export default function Page(): JSX.Element { d="M18 18.72a9.094 9.094 0 0 0 3.741-.479 3 3 0 0 0-4.682-2.72m.94 3.198.001.031c0 .225-.012.447-.037.666A11.944 11.944 0 0 1 12 21c-2.17 0-4.207-.576-5.963-1.584A6.062 6.062 0 0 1 6 18.719m12 0a5.971 5.971 0 0 0-.941-3.197m0 0A5.995 5.995 0 0 0 12 12.75a5.995 5.995 0 0 0-5.058 2.772m0 0a3 3 0 0 0-4.681 2.72 8.986 8.986 0 0 0 3.74.477m.94-3.197a5.971 5.971 0 0 0-.94 3.197M15 6.75a3 3 0 1 1-6 0 3 3 0 0 1 6 0Zm6 3a2.25 2.25 0 1 1-4.5 0 2.25 2.25 0 0 1 4.5 0Zm-13.5 0a2.25 2.25 0 1 1-4.5 0 2.25 2.25 0 0 1 4.5 0Z" /> - {project.contributorCount} + {project.totalRepoContributorCount}
From a3c7942bdc0261104dff96d40d71764a1ab4c7d6 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Thu, 3 Oct 2024 16:44:04 +0200 Subject: [PATCH 04/20] migrate projects endpoint to use postgres --- api/src/project/repository.ts | 60 +++++++++++++++------------- api/src/project/types.ts | 11 ++--- packages/models/src/project/index.ts | 1 - web/src/utils/project.ts | 4 +- 4 files changed, 41 insertions(+), 35 deletions(-) diff --git a/api/src/project/repository.ts b/api/src/project/repository.ts index 0e9e34313..aa1adc571 100644 --- a/api/src/project/repository.ts +++ b/api/src/project/repository.ts @@ -3,14 +3,14 @@ import { camelCaseObject } from "src/_utils/case"; import { unStringifyDeep } from "src/_utils/unstringify-deep"; import { contributorRepositoryRelationTable } from "src/contributor/table"; import { repositoriesTable } from "src/repository/table"; -import { SQLiteService } from "src/sqlite/service"; +import { PostgresService } from "src/postgres/service"; import { Service } from "typedi"; import { ProjectRow, projectsTable } from "./table"; @Service() export class ProjectRepository { - constructor(private readonly sqliteService: SQLiteService) {} + constructor(private readonly postgresService: PostgresService) {} public async findName(projectId: number) { const statement = sql` @@ -62,34 +62,38 @@ export class ProjectRepository { public async findForList() { const statement = sql` SELECT - p.id as id, - p.name as name, - p.slug as slug, - sum(rs.repo_contributor_count) as contributor_count, - sum(rs.stars) as stars, - sum(rs.repo_score) as activity_count, - 100 * sum(rs.repo_contributor_count) + 100 * sum(rs.stars) + max(rs.repo_score) - sum(rs.repo_score) / sum(rs.repo_contributor_count) as score + id, + name, + sum(repo_with_stats.contributor_count)::int as total_repo_contributor_count, + sum(repo_with_stats.stars)::int as total_repo_stars, + sum(repo_with_stats.score)::int as total_repo_score, + ROUND( 100 * sum(repo_with_stats.contributor_count) + 100 * sum(repo_with_stats.stars) + max(repo_with_stats.score) - sum(repo_with_stats.score) / sum(repo_with_stats.contributor_count) )::int as ranking FROM - (SELECT - *, - sum(crr.score) as repo_score, - count(*) as repo_contributor_count, - sum(r.stars) as stars + ( + SELECT + repository_id, + project_id, + sum(score) as score, + count(*) as contributor_count, + stars FROM - ${contributorRepositoryRelationTable} crr + ${contributorRepositoryRelationTable} JOIN - ${repositoriesTable} r ON crr.repository_id = r.id + ${repositoriesTable} ON ${contributorRepositoryRelationTable.repositoryId} = ${repositoriesTable.id} GROUP BY - r.id) as rs + ${contributorRepositoryRelationTable.repositoryId}, ${repositoriesTable.projectId}, ${repositoriesTable.stars} + ) as repo_with_stats JOIN - ${projectsTable} p ON rs.project_id = p.id + ${projectsTable} ON ${projectsTable.id} = repo_with_stats.project_id GROUP BY - p.id + ${projectsTable.id} ORDER BY - score DESC + ranking DESC `; - const raw = this.sqliteService.db.all(statement); - const unStringifiedRaw = unStringifyDeep(raw); + + const raw = await this.postgresService.db.execute(statement); + const entries = Array.from(raw); + const unStringifiedRaw = unStringifyDeep(entries); const camelCased = camelCaseObject(unStringifiedRaw); return camelCased; } @@ -126,21 +130,23 @@ export class ProjectRepository { } public async upsert(project: ProjectRow) { - return await this.sqliteService.db + return await this.postgresService.db .insert(projectsTable) .values(project) .onConflictDoUpdate({ - target: projectsTable.slug, + target: projectsTable.id, set: project, }) .returning({ id: projectsTable.id }); } - public async deleteById(id: number) { - return await this.sqliteService.db.delete(projectsTable).where(eq(projectsTable.id, id)); + public async deleteById(id: string) { + return await this.postgresService.db.delete(projectsTable).where(eq(projectsTable.id, id)); } public async deleteAllButWithRunId(runId: string) { - return await this.sqliteService.db.delete(projectsTable).where(ne(projectsTable.runId, runId)); + return await this.postgresService.db + .delete(projectsTable) + .where(ne(projectsTable.runId, runId)); } } diff --git a/api/src/project/types.ts b/api/src/project/types.ts index 7efd3a9e3..af7db316b 100644 --- a/api/src/project/types.ts +++ b/api/src/project/types.ts @@ -5,17 +5,18 @@ import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; import { GeneralResponse } from "src/app/types"; export interface GetProjectsForSitemapResponse extends GeneralResponse { + // @ts-expect-error hola projects: Array>; } export interface GetProjectsResponse extends GeneralResponse { projects: Array< - Pick & { + Pick & { repositories: Array>; - contributorCount: number; - activityCount: number; - score: number; - stars: number; + totalRepoContributorCount: number; + totalRepoScore: number; + totalRepoStars: number; + ranking: number; } >; } diff --git a/packages/models/src/project/index.ts b/packages/models/src/project/index.ts index a647ebb6f..92973da32 100644 --- a/packages/models/src/project/index.ts +++ b/packages/models/src/project/index.ts @@ -1,6 +1,5 @@ import { BaseEntity } from "src/_base"; export type ProjectEntity = BaseEntity & { - slug: string; name: string; }; diff --git a/web/src/utils/project.ts b/web/src/utils/project.ts index 3ce9628a2..606a4827f 100644 --- a/web/src/utils/project.ts +++ b/web/src/utils/project.ts @@ -1,5 +1,5 @@ import { ProjectEntity } from "@dzcode.io/models/dist/project"; -export function getProjectURL({ id, slug }: Pick): string { - return `/projects/${slug}-${id}`; +export function getProjectURL({ id }: Pick): string { + return `/projects/${id}`; } From ead0d6a71d155779973e9cf3d77c83ce5508549a Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Thu, 3 Oct 2024 19:14:36 +0200 Subject: [PATCH 05/20] migrate project detail endpoint to use postgres --- ...bouncy_micromax.sql => 0000_oval_zemo.sql} | 4 +- api/db/migrations/meta/0000_snapshot.json | 18 +------ api/db/migrations/meta/_journal.json | 4 +- api/package.json | 4 +- api/src/contribution/repository.ts | 29 +++++----- api/src/contributor/repository.ts | 54 +++++++++---------- api/src/project/controller.ts | 10 ++-- api/src/project/repository.ts | 50 ++++++++++------- api/src/project/types.ts | 13 +++-- api/src/repository/repository.ts | 38 ++++++++----- package-lock.json | 21 +++++++- package.json | 3 +- web/src/pages/projects/project/index.tsx | 14 ++--- 13 files changed, 141 insertions(+), 121 deletions(-) rename api/db/migrations/{0000_bouncy_micromax.sql => 0000_oval_zemo.sql} (96%) diff --git a/api/db/migrations/0000_bouncy_micromax.sql b/api/db/migrations/0000_oval_zemo.sql similarity index 96% rename from api/db/migrations/0000_bouncy_micromax.sql rename to api/db/migrations/0000_oval_zemo.sql index 68aa3cb1f..8cca3daf2 100644 --- a/api/db/migrations/0000_bouncy_micromax.sql +++ b/api/db/migrations/0000_oval_zemo.sql @@ -36,9 +36,7 @@ CREATE TABLE IF NOT EXISTS "projects" ( "id" text PRIMARY KEY NOT NULL, "record_imported_at" text DEFAULT CURRENT_TIMESTAMP NOT NULL, "name" text NOT NULL, - "slug" text NOT NULL, - "run_id" text DEFAULT 'initial-run-id' NOT NULL, - CONSTRAINT "projects_slug_unique" UNIQUE("slug") + "run_id" text DEFAULT 'initial-run-id' NOT NULL ); --> statement-breakpoint CREATE TABLE IF NOT EXISTS "repositories" ( diff --git a/api/db/migrations/meta/0000_snapshot.json b/api/db/migrations/meta/0000_snapshot.json index e70ed8f92..e9cba0834 100644 --- a/api/db/migrations/meta/0000_snapshot.json +++ b/api/db/migrations/meta/0000_snapshot.json @@ -1,5 +1,5 @@ { - "id": "09cef8e3-1a79-48fb-b36f-3273afd27a03", + "id": "3685bc1c-7589-45e6-bc50-a8a76537f753", "prevId": "00000000-0000-0000-0000-000000000000", "version": "7", "dialect": "postgresql", @@ -272,12 +272,6 @@ "primaryKey": false, "notNull": true }, - "slug": { - "name": "slug", - "type": "text", - "primaryKey": false, - "notNull": true - }, "run_id": { "name": "run_id", "type": "text", @@ -289,15 +283,7 @@ "indexes": {}, "foreignKeys": {}, "compositePrimaryKeys": {}, - "uniqueConstraints": { - "projects_slug_unique": { - "name": "projects_slug_unique", - "nullsNotDistinct": false, - "columns": [ - "slug" - ] - } - } + "uniqueConstraints": {} }, "public.repositories": { "name": "repositories", diff --git a/api/db/migrations/meta/_journal.json b/api/db/migrations/meta/_journal.json index 1fe6d13ba..53909155f 100644 --- a/api/db/migrations/meta/_journal.json +++ b/api/db/migrations/meta/_journal.json @@ -5,8 +5,8 @@ { "idx": 0, "version": "7", - "when": 1727955597397, - "tag": "0000_bouncy_micromax", + "when": 1727968406636, + "tag": "0000_oval_zemo", "breakpoints": true } ] diff --git a/api/package.json b/api/package.json index bbb9ba8e5..dd4930c70 100644 --- a/api/package.json +++ b/api/package.json @@ -64,7 +64,7 @@ "clean": "lerna run clean:alone --scope=@dzcode.io/api --include-dependencies --stream", "clean:alone": "del dist coverage fetch_cache oracle-cloud/build", "db:generate-migration": "drizzle-kit generate", - "db:server": "docker compose up", + "db:server": "docker compose down && docker compose up", "deploy": "del ./oracle-cloud/build && tsx oracle-cloud/deploy.ts production", "deploy:stg": "del ./oracle-cloud/build && tsx oracle-cloud/deploy.ts staging", "generate:bundle-info": "tsx ../packages/tooling/bundle-info.ts", @@ -79,7 +79,7 @@ "lint:tsc": "tspc --noEmit", "start": "node dist/app/index.js", "start:dev": "tsx ../packages/tooling/nodemon.ts \"@dzcode.io/api\" && npm-run-all --parallel start:nodemon db:server", - "start:nodemon": "nodemon dist/app/index.js", + "start:nodemon": "wait-port localhost:5432 && nodemon dist/app/index.js", "test": "npm run build && npm run test:alone", "test:alone": "jest --config ../packages/tooling/jest.config.ts --rootDir .", "test:watch": "npm-run-all build --parallel build:watch \"test:alone --watch {@}\" --" diff --git a/api/src/contribution/repository.ts b/api/src/contribution/repository.ts index d87a173a8..98ebff3af 100644 --- a/api/src/contribution/repository.ts +++ b/api/src/contribution/repository.ts @@ -5,40 +5,39 @@ import { unStringifyDeep } from "src/_utils/unstringify-deep"; import { contributorsTable } from "src/contributor/table"; import { projectsTable } from "src/project/table"; import { repositoriesTable } from "src/repository/table"; -import { SQLiteService } from "src/sqlite/service"; +import { PostgresService } from "src/postgres/service"; import { Service } from "typedi"; import { ContributionRow, contributionsTable } from "./table"; @Service() export class ContributionRepository { - constructor(private readonly sqliteService: SQLiteService) {} + constructor(private readonly postgresService: PostgresService) {} - public async findForProject(projectId: number) { + public async findForProject(projectId: string) { const statement = sql` SELECT - c.id as id, - c.title as title, - c.type as type + ${contributionsTable.id}, + ${contributionsTable.title} FROM - ${contributionsTable} c + ${contributionsTable} INNER JOIN - ${repositoriesTable} r ON c.repository_id = r.id + ${repositoriesTable} ON ${contributionsTable.repositoryId} = ${repositoriesTable.id} WHERE - r.project_id = ${projectId} + ${repositoriesTable.projectId} = ${projectId} ORDER BY - c.updated_at DESC + ${contributionsTable.updatedAt} DESC `; - const raw = this.sqliteService.db.all(statement); - const unStringifiedRaw = unStringifyDeep(raw); + const raw = await this.postgresService.db.execute(statement); + const entries = Array.from(raw); + const unStringifiedRaw = unStringifyDeep(entries); const camelCased = camelCaseObject(unStringifiedRaw); - return camelCased; } public async upsert(contribution: ContributionRow) { - return await this.sqliteService.db + return await this.postgresService.db .insert(contributionsTable) .values(contribution) .onConflictDoUpdate({ @@ -49,7 +48,7 @@ export class ContributionRepository { } public async deleteAllButWithRunId(runId: string) { - return await this.sqliteService.db + return await this.postgresService.db .delete(contributionsTable) .where(ne(contributionsTable.runId, runId)); } diff --git a/api/src/contributor/repository.ts b/api/src/contributor/repository.ts index 50f90e987..956e56e3d 100644 --- a/api/src/contributor/repository.ts +++ b/api/src/contributor/repository.ts @@ -3,7 +3,7 @@ import { camelCaseObject } from "src/_utils/case"; import { unStringifyDeep } from "src/_utils/unstringify-deep"; import { projectsTable } from "src/project/table"; import { repositoriesTable } from "src/repository/table"; -import { SQLiteService } from "src/sqlite/service"; +import { PostgresService } from "src/postgres/service"; import { Service } from "typedi"; import { @@ -15,37 +15,33 @@ import { @Service() export class ContributorRepository { - constructor(private readonly sqliteService: SQLiteService) {} + constructor(private readonly postgresService: PostgresService) {} - public async findForProject(projectId: number) { + public async findForProject(projectId: string) { const statement = sql` SELECT - cr.id as id, - cr.name as name, - cr.username as username, - cr.avatar_url as avatar_url + ${contributorsTable.id}, + ${contributorsTable.name}, + ${contributorsTable.avatarUrl}, + sum(${contributorRepositoryRelationTable.score}) as ranking FROM - (SELECT - crr.contributor_id as id, - sum (crr.score) as score - FROM - ${contributorRepositoryRelationTable} crr - INNER JOIN - ${repositoriesTable} r ON crr.repository_id = r.id - WHERE - r.project_id = ${projectId} - GROUP BY - crr.contributor_id - ORDER BY - score DESC) as c - INNER JOIN - ${contributorsTable} cr ON c.id = cr.id + ${contributorRepositoryRelationTable} + JOIN + ${repositoriesTable} ON ${contributorRepositoryRelationTable.repositoryId} = ${repositoriesTable.id} + JOIN + ${contributorsTable} ON ${contributorRepositoryRelationTable.contributorId} = ${contributorsTable.id} + WHERE + ${repositoriesTable.projectId} = ${projectId} + GROUP BY + ${contributorsTable.id} + ORDER BY + ranking DESC `; - const raw = this.sqliteService.db.all(statement); - const unStringifiedRaw = unStringifyDeep(raw); + const raw = await this.postgresService.db.execute(statement); + const entries = Array.from(raw); + const unStringifiedRaw = unStringifyDeep(entries); const camelCased = camelCaseObject(unStringifiedRaw); - return camelCased; } @@ -126,7 +122,7 @@ export class ContributorRepository { } public async upsert(contributor: ContributorRow) { - return await this.sqliteService.db + return await this.postgresService.db .insert(contributorsTable) .values(contributor) .onConflictDoUpdate({ @@ -139,7 +135,7 @@ export class ContributorRepository { public async upsertRelationWithRepository( contributorRelationWithRepository: ContributorRepositoryRelationRow, ) { - return await this.sqliteService.db + return await this.postgresService.db .insert(contributorRepositoryRelationTable) .values(contributorRelationWithRepository) .onConflictDoUpdate({ @@ -156,13 +152,13 @@ export class ContributorRepository { } public async deleteAllRelationWithRepositoryButWithRunId(runId: string) { - return await this.sqliteService.db + return await this.postgresService.db .delete(contributorRepositoryRelationTable) .where(ne(contributorRepositoryRelationTable.runId, runId)); } public async deleteAllButWithRunId(runId: string) { - return await this.sqliteService.db + return await this.postgresService.db .delete(contributorsTable) .where(ne(contributorsTable.runId, runId)); } diff --git a/api/src/project/controller.ts b/api/src/project/controller.ts index 051313b09..55e335a65 100644 --- a/api/src/project/controller.ts +++ b/api/src/project/controller.ts @@ -41,12 +41,12 @@ export class ProjectController { } @Get("/:id") - public async getProject(@Param("id") id: number): Promise { + public async getProject(@Param("id") id: string): Promise { const [project, repositories, contributors, contributions] = await Promise.all([ - await this.projectRepository.findWithStats(id), - await this.repositoryRepository.findForProject(id), - await this.contributorRepository.findForProject(id), - await this.contributionRepository.findForProject(id), + this.projectRepository.findWithStats(id), + this.repositoryRepository.findForProject(id), + this.contributorRepository.findForProject(id), + this.contributionRepository.findForProject(id), ]); if (!project) throw new NotFoundError("Project not found"); diff --git a/api/src/project/repository.ts b/api/src/project/repository.ts index aa1adc571..f9b708754 100644 --- a/api/src/project/repository.ts +++ b/api/src/project/repository.ts @@ -29,32 +29,44 @@ export class ProjectRepository { return camelCased; } - public async findWithStats(projectId: number) { + public async findWithStats(projectId: string) { const statement = sql` SELECT - p.id as id, - p.name as name, - p.slug as slug, - count(DISTINCT r.id) as repository_count, - sum(crr.score) as activity_count, - count(DISTINCT crr.contributor_id) as contributor_count, - -- @TODO-ZM: this is wrong, but works for now, please group by repository id - sum(DISTINCT r.stars) as stars + id, + name, + sum(repo_with_stats.contributor_count)::int as total_repo_contributor_count, + sum(repo_with_stats.stars)::int as total_repo_stars, + sum(repo_with_stats.score)::int as total_repo_score, + count(*) as repo_count, + ROUND( 100 * sum(repo_with_stats.contributor_count) + 100 * sum(repo_with_stats.stars) + max(repo_with_stats.score) - sum(repo_with_stats.score) / sum(repo_with_stats.contributor_count) )::int as ranking FROM - ${repositoriesTable} r - JOIN - ${contributorRepositoryRelationTable} crr ON r.id = crr.repository_id + ( + SELECT + repository_id, + project_id, + sum(score) as score, + count(*) as contributor_count, + stars + FROM + ${contributorRepositoryRelationTable} + JOIN + ${repositoriesTable} ON ${contributorRepositoryRelationTable.repositoryId} = ${repositoriesTable.id} + WHERE + ${repositoriesTable.projectId} = ${projectId} + GROUP BY + ${contributorRepositoryRelationTable.repositoryId}, ${repositoriesTable.projectId}, ${repositoriesTable.stars} + ) as repo_with_stats JOIN - ${projectsTable} p ON r.project_id = p.id - WHERE - project_id = ${projectId} + ${projectsTable} ON ${projectsTable.id} = repo_with_stats.project_id GROUP BY - r.project_id + ${projectsTable.id} + ORDER BY + ranking DESC `; - const raw = this.sqliteService.db.get(statement); - if (!raw) return null; - const unStringifiedRaw = unStringifyDeep(raw); + const raw = await this.postgresService.db.execute(statement); + const entries = Array.from(raw); + const unStringifiedRaw = unStringifyDeep(entries[0]); const camelCased = camelCaseObject(unStringifiedRaw); return camelCased; } diff --git a/api/src/project/types.ts b/api/src/project/types.ts index af7db316b..20a453b2b 100644 --- a/api/src/project/types.ts +++ b/api/src/project/types.ts @@ -5,14 +5,12 @@ import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; import { GeneralResponse } from "src/app/types"; export interface GetProjectsForSitemapResponse extends GeneralResponse { - // @ts-expect-error hola - projects: Array>; + projects: Array>; } export interface GetProjectsResponse extends GeneralResponse { projects: Array< Pick & { - repositories: Array>; totalRepoContributorCount: number; totalRepoScore: number; totalRepoStars: number; @@ -36,10 +34,11 @@ export interface GetProjectResponse extends GeneralResponse { } >; contributions: Array>; - contributorCount: number; - repositoryCount: number; - activityCount: number; - stars: number; + + totalRepoContributorCount: number; + repoCount: number; + totalRepoScore: number; + totalRepoStars: number; }; } diff --git a/api/src/repository/repository.ts b/api/src/repository/repository.ts index b1fbc51d9..5141fa0ab 100644 --- a/api/src/repository/repository.ts +++ b/api/src/repository/repository.ts @@ -1,45 +1,55 @@ import { ne, sql } from "drizzle-orm"; -import { SQLiteService } from "src/sqlite/service"; +import { PostgresService } from "src/postgres/service"; import { Service } from "typedi"; import { repositoriesTable, RepositoryRow } from "./table"; import { unStringifyDeep } from "src/_utils/unstringify-deep"; import { camelCaseObject } from "src/_utils/case"; +import { contributorRepositoryRelationTable } from "src/contributor/table"; @Service() export class RepositoryRepository { - constructor(private readonly sqliteService: SQLiteService) {} - public async findForProject(projectId: number) { + constructor(private readonly postgresService: PostgresService) {} + public async findForProject(projectId: string) { const statement = sql` SELECT - r.id as id, - r.owner as owner, - r.name as name, - r.provider as provider + id, + owner, + name, + provider, + ROUND( 100 * sum(score) + 100 * stars + max(score) - sum(score) / count(*) )::int as ranking FROM - ${repositoriesTable} r + ${contributorRepositoryRelationTable} + JOIN + ${repositoriesTable} ON ${contributorRepositoryRelationTable.repositoryId} = ${repositoriesTable.id} WHERE - r.project_id = ${projectId} + ${repositoriesTable.projectId} = ${projectId} + GROUP BY + ${repositoriesTable.id}, ${repositoriesTable.stars} + ORDER BY + ranking DESC `; - const raw = this.sqliteService.db.all(statement); - const unStringifiedRaw = unStringifyDeep(raw); + + const raw = await this.postgresService.db.execute(statement); + const entries = Array.from(raw); + const unStringifiedRaw = unStringifyDeep(entries); const camelCased = camelCaseObject(unStringifiedRaw); return camelCased; } public async upsert(repository: RepositoryRow) { - return await this.sqliteService.db + return await this.postgresService.db .insert(repositoriesTable) .values(repository) .onConflictDoUpdate({ - target: [repositoriesTable.provider, repositoriesTable.owner, repositoriesTable.name], + target: [repositoriesTable.id], set: repository, }) .returning({ id: repositoriesTable.id }); } public async deleteAllButWithRunId(runId: string) { - return await this.sqliteService.db + return await this.postgresService.db .delete(repositoriesTable) .where(ne(repositoriesTable.runId, runId)); } diff --git a/package-lock.json b/package-lock.json index d2969ff29..b19568f10 100644 --- a/package-lock.json +++ b/package-lock.json @@ -49,7 +49,8 @@ "tsx": "^4.19.1", "typescript": "5.5.3", "typescript-eslint": "^8.5.0", - "typescript-transform-paths": "^3.5.0" + "typescript-transform-paths": "^3.5.0", + "wait-port": "^1.1.0" }, "engines": { "node": ">=20", @@ -27901,6 +27902,24 @@ "node": ">=12.0.0" } }, + "node_modules/wait-port": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/wait-port/-/wait-port-1.1.0.tgz", + "integrity": "sha512-3e04qkoN3LxTMLakdqeWth8nih8usyg+sf1Bgdf9wwUkp05iuK1eSY/QpLvscT/+F/gA89+LpUmmgBtesbqI2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2", + "commander": "^9.3.0", + "debug": "^4.3.4" + }, + "bin": { + "wait-port": "bin/wait-port.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/walk-up-path": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/walk-up-path/-/walk-up-path-3.0.1.tgz", diff --git a/package.json b/package.json index abe51eb60..3a2c66106 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,8 @@ "tsx": "^4.19.1", "typescript": "5.5.3", "typescript-eslint": "^8.5.0", - "typescript-transform-paths": "^3.5.0" + "typescript-transform-paths": "^3.5.0", + "wait-port": "^1.1.0" }, "engines": { "node": ">=20", diff --git a/web/src/pages/projects/project/index.tsx b/web/src/pages/projects/project/index.tsx index 31a1ebb3c..534b0643b 100644 --- a/web/src/pages/projects/project/index.tsx +++ b/web/src/pages/projects/project/index.tsx @@ -99,9 +99,9 @@ export default function Page(): JSX.Element {
-
{project.stars}
+
{project.totalRepoStars}
- {project.repositoryCount}{" "} + {project.repoCount}{" "}
@@ -119,16 +119,16 @@ export default function Page(): JSX.Element {
-
{project.activityCount}
+
{project.totalRepoScore}
- {project.contributorCount}{" "} + {project.totalRepoContributorCount}{" "}
@@ -149,14 +149,14 @@ export default function Page(): JSX.Element { ))} -
{project.contributorCount}
+
{project.totalRepoContributorCount}
{remainingContributorsCount > 0 ? `${localize("project-show-top-n")} ${topMax}` - : `${localize("project-from-n-repositories-pre")} ${project.repositoryCount} ${localize("project-from-n-repositories-post")}`} + : `${localize("project-from-n-repositories-pre")} ${project.repoCount} ${localize("project-from-n-repositories-post")}`}
From 8d27003ef2fd152698a0a831088ced19fb91f29c Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Thu, 3 Oct 2024 20:01:04 +0200 Subject: [PATCH 06/20] migrate project controller to use postgres --- api/src/project/controller.ts | 2 +- api/src/project/repository.ts | 55 ++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/api/src/project/controller.ts b/api/src/project/controller.ts index 55e335a65..bf9cb922f 100644 --- a/api/src/project/controller.ts +++ b/api/src/project/controller.ts @@ -62,7 +62,7 @@ export class ProjectController { } @Get("/:id/name") - public async getProjectName(@Param("id") id: number): Promise { + public async getProjectName(@Param("id") id: string): Promise { const project = await this.projectRepository.findName(id); if (!project) throw new NotFoundError("Project not found"); diff --git a/api/src/project/repository.ts b/api/src/project/repository.ts index f9b708754..7e64e7dfe 100644 --- a/api/src/project/repository.ts +++ b/api/src/project/repository.ts @@ -12,19 +12,23 @@ import { ProjectRow, projectsTable } from "./table"; export class ProjectRepository { constructor(private readonly postgresService: PostgresService) {} - public async findName(projectId: number) { + public async findName(projectId: string) { const statement = sql` SELECT - name + ${projectsTable.id} FROM ${projectsTable} WHERE - id = ${projectId} + ${projectsTable.id} = ${projectId} `; - const raw = this.sqliteService.db.get(statement); - if (!raw) return null; - const unStringifiedRaw = unStringifyDeep(raw); + const raw = await this.postgresService.db.execute(statement); + const entries = Array.from(raw); + const entry = entries[0]; + + if (!entry) return null; + + const unStringifiedRaw = unStringifyDeep(entry); const camelCased = camelCaseObject(unStringifiedRaw); return camelCased; } @@ -66,7 +70,8 @@ export class ProjectRepository { const raw = await this.postgresService.db.execute(statement); const entries = Array.from(raw); - const unStringifiedRaw = unStringifyDeep(entries[0]); + const entry = entries[0]; + const unStringifiedRaw = unStringifyDeep(entry); const camelCased = camelCaseObject(unStringifiedRaw); return camelCased; } @@ -113,30 +118,34 @@ export class ProjectRepository { public async findForSitemap() { const statement = sql` SELECT - p.id as id, - p.slug as slug, - 100 * sum(rs.repo_contributor_count) + 100 * sum(rs.stars) + max(rs.repo_score) - sum(rs.repo_score) / sum(rs.repo_contributor_count) as score + id, + ROUND( 100 * sum(repo_with_stats.contributor_count) + 100 * sum(repo_with_stats.stars) + max(repo_with_stats.score) - sum(repo_with_stats.score) / sum(repo_with_stats.contributor_count) )::int as ranking FROM - (SELECT - *, - sum(crr.score) as repo_score, - count(*) as repo_contributor_count, - sum(r.stars) as stars + ( + SELECT + repository_id, + project_id, + sum(score) as score, + count(*) as contributor_count, + stars FROM - ${contributorRepositoryRelationTable} crr + ${contributorRepositoryRelationTable} JOIN - ${repositoriesTable} r ON crr.repository_id = r.id + ${repositoriesTable} ON ${contributorRepositoryRelationTable.repositoryId} = ${repositoriesTable.id} GROUP BY - r.id) as rs + ${contributorRepositoryRelationTable.repositoryId}, ${repositoriesTable.projectId}, ${repositoriesTable.stars} + ) as repo_with_stats JOIN - ${projectsTable} p ON rs.project_id = p.id + ${projectsTable} ON ${projectsTable.id} = repo_with_stats.project_id GROUP BY - p.id + ${projectsTable.id} ORDER BY - score DESC + ranking DESC `; - const raw = this.sqliteService.db.all(statement); - const unStringifiedRaw = unStringifyDeep(raw); + + const raw = await this.postgresService.db.execute(statement); + const entries = Array.from(raw); + const unStringifiedRaw = unStringifyDeep(entries); const camelCased = camelCaseObject(unStringifiedRaw); return camelCased; } From 6b2e7466bf61fa57e11cc76983eb849a30d294ff Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Thu, 3 Oct 2024 20:31:03 +0200 Subject: [PATCH 07/20] migrate contributions controller to use postgres --- api/src/contribution/repository.ts | 101 +++++++++++++++-------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/api/src/contribution/repository.ts b/api/src/contribution/repository.ts index 98ebff3af..634bcc145 100644 --- a/api/src/contribution/repository.ts +++ b/api/src/contribution/repository.ts @@ -41,7 +41,7 @@ export class ContributionRepository { .insert(contributionsTable) .values(contribution) .onConflictDoUpdate({ - target: [contributionsTable.url], + target: contributionsTable.url, set: contribution, }) .returning({ id: contributionsTable.id }); @@ -56,60 +56,61 @@ export class ContributionRepository { public async findForList() { const statement = sql` SELECT - p.id as id, - p.name as name, - json_group_array( - json_object('id', r.id, 'name', r.name, 'owner', r.owner, 'contributions', r.contributions) - ) AS repositories + p.id as id, + p.name as name, + json_agg( + json_build_object('id', r.id, 'name', r.name, 'owner', r.owner, 'contributions', r.contributions) + ) AS repositories FROM - (SELECT - r.id as id, - r.owner as owner, - r.name as name, - r.project_id as project_id, - json_group_array( - json_object( - 'id', - c.id, - 'title', - c.title, - 'type', - c.type, - 'url', - c.url, - 'updated_at', - c.updated_at, - 'activity_count', - c.activity_count, - 'contributor', - json_object( - 'id', - cr.id, - 'name', - cr.name, - 'username', - cr.username, - 'avatar_url', - cr.avatar_url - ) - ) - ) AS contributions - FROM - ${contributionsTable} c - INNER JOIN - ${repositoriesTable} r ON c.repository_id = r.id - INNER JOIN - ${contributorsTable} cr ON c.contributor_id = cr.id - GROUP BY - c.id) AS r + (SELECT + r.id as id, + r.owner as owner, + r.name as name, + r.project_id as project_id, + json_agg( + json_build_object( + 'id', + c.id, + 'title', + c.title, + 'type', + c.type, + 'url', + c.url, + 'updated_at', + c.updated_at, + 'activity_count', + c.activity_count, + 'contributor', + json_build_object( + 'id', + cr.id, + 'name', + cr.name, + 'username', + cr.username, + 'avatar_url', + cr.avatar_url + ) + ) + ) AS contributions + FROM + ${contributionsTable} c + INNER JOIN + ${repositoriesTable} r ON c.repository_id = r.id + INNER JOIN + ${contributorsTable} cr ON c.contributor_id = cr.id + GROUP BY + r.id) AS r INNER JOIN - ${projectsTable} p ON r.project_id = p.id + ${projectsTable} p ON r.project_id = p.id GROUP BY - p.id + p.id `; - const raw = this.sqliteService.db.all(statement); - const unStringifiedRaw = unStringifyDeep(raw); + const raw = await this.postgresService.db.execute(statement); + const entries = Array.from(raw); + const unStringifiedRaw = unStringifyDeep(entries); const reversed = reverseHierarchy(unStringifiedRaw, [ { from: "repositories", setParentAs: "project" }, From b0faae77e258582eb0cdb3d61f4a0099f4036c29 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Thu, 3 Oct 2024 20:46:05 +0200 Subject: [PATCH 08/20] migrate contributors controller to use postgres --- api/src/contributor/repository.ts | 113 ++++++++++++++---------------- 1 file changed, 53 insertions(+), 60 deletions(-) diff --git a/api/src/contributor/repository.ts b/api/src/contributor/repository.ts index 956e56e3d..7a4054e3c 100644 --- a/api/src/contributor/repository.ts +++ b/api/src/contributor/repository.ts @@ -47,74 +47,67 @@ export class ContributorRepository { public async findForList() { const statement = sql` - SELECT - sum(c.score) as score, - cr.id as id, - cr.name as name, - cr.username as username, - cr.url as url, - cr.avatar_url as avatar_url, - json_group_array( - json_object( - 'id', - p.id, - 'name', - p.name, - 'score', - c.score, - 'repositories', - c.repositories - ) + SELECT + SUM(c.score) AS score, + cr.id AS id, + cr.name AS name, + cr.username AS username, + cr.url AS url, + cr.avatar_url AS avatar_url, + JSON_AGG( + JSON_BUILD_OBJECT( + 'id', p.id, + 'name', p.name, + 'score', c.score, + 'repositories', c.repositories + ) ) AS projects - FROM - (SELECT - sum(crr.score) as score, - crr.contributor_id as contributor_id, - crr.project_id as project_id, - json_group_array( - json_object( - 'id', - r.id, - 'owner', - r.owner, - 'name', - r.name, - 'score', - crr.score - ) - ) AS repositories - FROM - (SELECT - contributor_id, - repository_id, - score, - r.project_id as project_id - FROM - ${contributorRepositoryRelationTable} crr - INNER JOIN - ${repositoriesTable} r ON crr.repository_id = r.id - ORDER BY - crr.score DESC - ) as crr - INNER JOIN + FROM ( + SELECT + SUM(crr.score) AS score, + crr.contributor_id AS contributor_id, + crr.project_id AS project_id, + JSON_AGG( + JSON_BUILD_OBJECT( + 'id', r.id, + 'owner', r.owner, + 'name', r.name, + 'score', crr.score + ) + ) AS repositories + FROM ( + SELECT + contributor_id, + repository_id, + score, + r.project_id AS project_id + FROM + ${contributorRepositoryRelationTable} crr + INNER JOIN ${repositoriesTable} r ON crr.repository_id = r.id + ORDER BY + crr.score ASC + ) AS crr + INNER JOIN + ${repositoriesTable} r ON crr.repository_id = r.id GROUP BY - crr.contributor_id, crr.project_id + crr.contributor_id, crr.project_id ORDER BY - crr.score DESC - ) as c - INNER JOIN + SUM(crr.score) ASC + ) AS c + INNER JOIN ${contributorsTable} cr ON c.contributor_id = cr.id - INNER JOIN + INNER JOIN ${projectsTable} p ON c.project_id = p.id - GROUP BY - c.contributor_id - ORDER BY + GROUP BY + c.contributor_id, cr.id, cr.name, cr.username, cr.url, cr.avatar_url + ORDER BY score DESC `; - const raw = this.sqliteService.db.all(statement); - const unStringifiedRaw = unStringifyDeep(raw); + const raw = await this.postgresService.db.execute(statement); + const entries = Array.from(raw); + const unStringifiedRaw = unStringifyDeep(entries); const camelCased = camelCaseObject(unStringifiedRaw); @@ -126,7 +119,7 @@ export class ContributorRepository { .insert(contributorsTable) .values(contributor) .onConflictDoUpdate({ - target: contributorsTable.url, + target: contributorsTable.id, set: contributor, }) .returning({ id: contributorsTable.id }); From 11749790d9b6ecf03dd2b3cb52b44ad3c5fb957a Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Thu, 3 Oct 2024 21:32:43 +0200 Subject: [PATCH 09/20] simplified contributors endpoint --- api/src/contribution/repository.ts | 2 +- api/src/contributor/repository.ts | 74 ++++++------------------------ api/src/contributor/types.ts | 11 +---- web/src/pages/team/index.tsx | 37 ++++++--------- web/src/utils/contributor.ts | 7 +-- web/src/utils/repository.test.ts | 12 +---- web/src/utils/repository.ts | 4 -- 7 files changed, 35 insertions(+), 112 deletions(-) diff --git a/api/src/contribution/repository.ts b/api/src/contribution/repository.ts index 634bcc145..8b293f414 100644 --- a/api/src/contribution/repository.ts +++ b/api/src/contribution/repository.ts @@ -22,7 +22,7 @@ export class ContributionRepository { FROM ${contributionsTable} INNER JOIN - ${repositoriesTable} ON ${contributionsTable.repositoryId} = ${repositoriesTable.id} + ${repositoriesTable} ON ${contributionsTable.repositoryId} = ${repositoriesTable.id} WHERE ${repositoriesTable.projectId} = ${projectId} ORDER BY diff --git a/api/src/contributor/repository.ts b/api/src/contributor/repository.ts index 7a4054e3c..15823e619 100644 --- a/api/src/contributor/repository.ts +++ b/api/src/contributor/repository.ts @@ -1,7 +1,6 @@ import { ne, sql } from "drizzle-orm"; import { camelCaseObject } from "src/_utils/case"; import { unStringifyDeep } from "src/_utils/unstringify-deep"; -import { projectsTable } from "src/project/table"; import { repositoriesTable } from "src/repository/table"; import { PostgresService } from "src/postgres/service"; import { Service } from "typedi"; @@ -47,70 +46,27 @@ export class ContributorRepository { public async findForList() { const statement = sql` - SELECT - SUM(c.score) AS score, - cr.id AS id, - cr.name AS name, - cr.username AS username, - cr.url AS url, - cr.avatar_url AS avatar_url, - JSON_AGG( - JSON_BUILD_OBJECT( - 'id', p.id, - 'name', p.name, - 'score', c.score, - 'repositories', c.repositories - ) - ) AS projects - FROM ( - SELECT - SUM(crr.score) AS score, - crr.contributor_id AS contributor_id, - crr.project_id AS project_id, - JSON_AGG( - JSON_BUILD_OBJECT( - 'id', r.id, - 'owner', r.owner, - 'name', r.name, - 'score', crr.score - ) - ) AS repositories - FROM ( - SELECT - contributor_id, - repository_id, - score, - r.project_id AS project_id - FROM - ${contributorRepositoryRelationTable} crr - INNER JOIN - ${repositoriesTable} r ON crr.repository_id = r.id - ORDER BY - crr.score ASC - ) AS crr - INNER JOIN - ${repositoriesTable} r ON crr.repository_id = r.id - GROUP BY - crr.contributor_id, crr.project_id - ORDER BY - SUM(crr.score) ASC - ) AS c - INNER JOIN - ${contributorsTable} cr ON c.contributor_id = cr.id - INNER JOIN - ${projectsTable} p ON c.project_id = p.id - GROUP BY - c.contributor_id, cr.id, cr.name, cr.username, cr.url, cr.avatar_url - ORDER BY - score DESC + SELECT + ${contributorsTable.id}, + ${contributorsTable.name}, + ${contributorsTable.avatarUrl}, + sum(${contributorRepositoryRelationTable.score}) as ranking + FROM + ${contributorRepositoryRelationTable} + JOIN + ${repositoriesTable} ON ${contributorRepositoryRelationTable.repositoryId} = ${repositoriesTable.id} + JOIN + ${contributorsTable} ON ${contributorRepositoryRelationTable.contributorId} = ${contributorsTable.id} + GROUP BY + ${contributorsTable.id} + ORDER BY + ranking DESC `; const raw = await this.postgresService.db.execute(statement); const entries = Array.from(raw); const unStringifiedRaw = unStringifyDeep(entries); - const camelCased = camelCaseObject(unStringifiedRaw); - return camelCased; } diff --git a/api/src/contributor/types.ts b/api/src/contributor/types.ts index f0aede29f..85f15f8e6 100644 --- a/api/src/contributor/types.ts +++ b/api/src/contributor/types.ts @@ -1,17 +1,10 @@ import { ContributorEntity } from "@dzcode.io/models/dist/contributor"; -import { ProjectEntity } from "@dzcode.io/models/dist/project"; -import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; import { GeneralResponse } from "src/app/types"; export interface GetContributorsResponse extends GeneralResponse { contributors: Array< - Pick & { - projects: Array< - Pick & { - repositories: Array>; - } - >; - score: number; + Pick & { + ranking: number; } >; } diff --git a/web/src/pages/team/index.tsx b/web/src/pages/team/index.tsx index 5deb87b3e..8d3df6427 100644 --- a/web/src/pages/team/index.tsx +++ b/web/src/pages/team/index.tsx @@ -1,12 +1,13 @@ import React from "react"; import { useEffect } from "react"; import { Helmet } from "react-helmet-async"; +import { Link } from "src/components/link"; import { Loading } from "src/components/loading"; import { Locale, useLocale } from "src/components/locale"; import { TryAgain } from "src/components/try-again"; import { fetchContributorsListAction } from "src/redux/actions/contributors"; import { useAppDispatch, useAppSelector } from "src/redux/store"; -import { getRepositoryName } from "src/utils/repository"; +import { getContributorURL } from "src/utils/contributor"; // ts-prune-ignore-next export default function Page(): JSX.Element { @@ -43,28 +44,18 @@ export default function Page(): JSX.Element { ) : (
{contributorsList.map((contributor, contributorIndex) => ( -
-
- {contributor.name} -

{contributor.name}

-
- {contributor.projects.map((project, projectIndex) => ( -
- {project.name} - {project.repositories.map((repository, repositoryIndex) => ( - - {getRepositoryName(repository)} - - ))} -
- ))} -
-
-
+ + {contributor.name} + {contributor.name} + ))}
)} diff --git a/web/src/utils/contributor.ts b/web/src/utils/contributor.ts index d56527d1f..b36ce114e 100644 --- a/web/src/utils/contributor.ts +++ b/web/src/utils/contributor.ts @@ -1,8 +1,5 @@ import { ContributorEntity } from "@dzcode.io/models/dist/contributor"; -export function getContributorURL({ - id, - username, -}: Pick): string { - return `/team/${username}-${id}`; +export function getContributorURL({ id }: Pick): string { + return `/team/${id}`; } diff --git a/web/src/utils/repository.test.ts b/web/src/utils/repository.test.ts index cf61d31e2..51582dad1 100644 --- a/web/src/utils/repository.test.ts +++ b/web/src/utils/repository.test.ts @@ -1,14 +1,4 @@ -import { getRepositoryName, getRepositoryURL } from "./repository"; - -describe("getRepositoryName", () => { - it("should return the repository name", () => { - const repository = { - owner: "dzcode.io", - name: "dzcode.io", - } as const; - expect(getRepositoryName(repository)).toBe("dzcode.io/dzcode.io"); - }); -}); +import { getRepositoryURL } from "./repository"; describe("getRepositoryURL", () => { it("should return the repository URL", () => { diff --git a/web/src/utils/repository.ts b/web/src/utils/repository.ts index 264e4c247..16df67689 100644 --- a/web/src/utils/repository.ts +++ b/web/src/utils/repository.ts @@ -1,9 +1,5 @@ import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; -export function getRepositoryName(repository: Pick): string { - return `${repository.owner}/${repository.name}`; -} - export const getRepositoryURL = ({ provider, owner, From c94048b840c8404f2bb0d78c964401d7e34647d1 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 4 Oct 2024 20:27:41 +0200 Subject: [PATCH 10/20] Refactor unStringifyDeep function to handle errors gracefully --- api/src/_utils/unstringify-deep.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/_utils/unstringify-deep.ts b/api/src/_utils/unstringify-deep.ts index 40e0300d9..feb5133be 100644 --- a/api/src/_utils/unstringify-deep.ts +++ b/api/src/_utils/unstringify-deep.ts @@ -24,8 +24,9 @@ export function unStringifyDeep(obj: any): any { } else { result[key] = value; } + // eslint-disable-next-line @typescript-eslint/no-unused-vars } catch (error) { - console.error(error); + // Do nothing } } } From 59e8183cc39df07886cd43d70e489d7ae081b3a8 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 4 Oct 2024 20:28:35 +0200 Subject: [PATCH 11/20] Refactor EnvRecord class to dynamically generate POSTGRES_URI based on NODE_ENV --- api/src/config/types.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/api/src/config/types.ts b/api/src/config/types.ts index 0e2399e7a..93870e8c3 100644 --- a/api/src/config/types.ts +++ b/api/src/config/types.ts @@ -1,4 +1,5 @@ import { Environment, environments } from "@dzcode.io/utils/dist/config/environment"; +import { Expose } from "class-transformer"; import { IsOptional, IsString, Matches } from "class-validator"; import { readFileSync } from "fs-extra"; @@ -18,9 +19,12 @@ export class EnvRecord { @IsString() FETCH_CACHE_PATH = "./fetch_cache"; - @IsString() - // TODO-ZM: localhost to postgres for non-development environments - POSTGRES_URI = "postgres://postgres@localhost:5432/db"; + @Expose() + get POSTGRES_URI() { + return this.NODE_ENV === "development" + ? "postgres://postgres@localhost:5432/db" + : "postgres://postgres@postgres:5432/db"; + } @IsString() @IsOptional() From d871bd2de5a9e6468684ba783f800bddbe292a83 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 4 Oct 2024 20:28:48 +0200 Subject: [PATCH 12/20] include Docker installation requirement --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3aae036a4..3d2c6a738 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Make sure you have: - [Git](https://git-scm.com/) - [Nodejs](https://nodejs.org/) version 20 or higher (we recommend using [volta](https://docs.volta.sh/guide/getting-started) over plain install or [nvm](https://github.com/nvm-sh/nvm)) +- [Docker](https://www.docker.com/) installed and running. ### Run it locally From 7561758a95df85f71547db3cf7590ecc83e49f83 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 4 Oct 2024 20:32:34 +0200 Subject: [PATCH 13/20] add Postgres service in staging and prod instances --- api/oracle-cloud/docker-compose.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/api/oracle-cloud/docker-compose.yml b/api/oracle-cloud/docker-compose.yml index 98f1b187d..0647016dc 100644 --- a/api/oracle-cloud/docker-compose.yml +++ b/api/oracle-cloud/docker-compose.yml @@ -11,7 +11,6 @@ services: - ./nginx.conf:/etc/nginx/nginx.conf api: build: "." - # restart: always ports: - "7070:7070" env_file: @@ -19,6 +18,16 @@ services: volumes: - /home/ubuntu/app-data/api/fetch_cache:/usr/src/repo/api/fetch_cache - /home/ubuntu/app-data/api/sqlite_db:/usr/src/repo/api/sqlite_db + postgress: + image: postgres + ports: + - "5432:5432" + volumes: + - /home/ubuntu/app-data/api/postgres_db:/var/lib/postgresql/data + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: db + lhserver: image: patrickhulce/lhci-server restart: always From ad09566f99fdc9eff52fa9be6eb8418e322b19da Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 4 Oct 2024 21:03:58 +0200 Subject: [PATCH 14/20] Refactor PostgresService to handle database migration asynchronously --- api/src/app/index.ts | 69 +++++++++++++++++++------------------ api/src/postgres/service.ts | 23 +++++++++++-- 2 files changed, 56 insertions(+), 36 deletions(-) diff --git a/api/src/app/index.ts b/api/src/app/index.ts index 0924fb0c1..607959751 100644 --- a/api/src/app/index.ts +++ b/api/src/app/index.ts @@ -25,36 +25,39 @@ import { SecurityMiddleware } from "./middlewares/security"; // Use typedi container useContainer(Container); // eslint-disable-line react-hooks/rules-of-hooks -// Initialize Database -Container.get(PostgresService); - -const { NODE_ENV, PORT } = Container.get(ConfigService).env(); - -// Add crons to DI container -const CronServices = [DigestCron]; -CronServices.forEach((service) => Container.get(service)); - -// Create the app: -const routingControllersOptions: RoutingControllersOptions = { - controllers: [ - ContributionController, - GithubController, - MilestoneController, - ProjectController, - ContributorController, - RobotsController, - ], - middlewares: [SecurityMiddleware, LoggerMiddleware], - cors: Container.get(SecurityMiddleware).cors(), -}; -const app: Application = createExpressServer(routingControllersOptions); - -const logger = Container.get(LoggerService); - -Sentry.setupExpressErrorHandler(app); - -// Start it -app.listen(PORT, () => { - const commonConfig = fsConfig(NODE_ENV); - logger.info({ message: `API Server up on: ${commonConfig.api.url}/` }); -}); +(async () => { + // Initialize Database + const postgresService = Container.get(PostgresService); + await postgresService.migrate(); + + const { NODE_ENV, PORT } = Container.get(ConfigService).env(); + + // Add crons to DI container + const CronServices = [DigestCron]; + CronServices.forEach((service) => Container.get(service)); + + // Create the app: + const routingControllersOptions: RoutingControllersOptions = { + controllers: [ + ContributionController, + GithubController, + MilestoneController, + ProjectController, + ContributorController, + RobotsController, + ], + middlewares: [SecurityMiddleware, LoggerMiddleware], + cors: Container.get(SecurityMiddleware).cors(), + }; + const app: Application = createExpressServer(routingControllersOptions); + + const logger = Container.get(LoggerService); + + Sentry.setupExpressErrorHandler(app); + + // Start it + app.listen(PORT, () => { + const commonConfig = fsConfig(NODE_ENV); + logger.info({ message: `API Server up on: ${commonConfig.api.url}/` }); + }); +})(); diff --git a/api/src/postgres/service.ts b/api/src/postgres/service.ts index 0b6670a0b..c270849a5 100644 --- a/api/src/postgres/service.ts +++ b/api/src/postgres/service.ts @@ -8,7 +8,15 @@ import { Service } from "typedi"; @Service() export class PostgresService { - public db; + private isReady = false; + private drizzleDB: ReturnType; + + public get db() { + if (!this.isReady) { + throw new Error("Database is not ready yet"); + } + return this.drizzleDB; + } constructor( private readonly configService: ConfigService, @@ -18,8 +26,17 @@ export class PostgresService { const { POSTGRES_URI } = this.configService.env(); const queryClient = postgres(POSTGRES_URI); - this.db = drizzle(queryClient); - migrate(this.db, { migrationsFolder: join(__dirname, "../../db/migrations") }); + this.drizzleDB = drizzle(queryClient); + this.loggerService.info({ message: "Database migration started" }); + } + + public async migrate() { + if (this.isReady) throw new Error("Database is already ready"); + + this.loggerService.info({ message: "Database migration started" }); + await migrate(this.drizzleDB, { migrationsFolder: join(__dirname, "../../db/migrations") }); this.loggerService.info({ message: "Database migration complete" }); + + this.isReady = true; } } From 67d7964ee34984e49955db6f4f94d0b5016dd98f Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 4 Oct 2024 21:04:22 +0200 Subject: [PATCH 15/20] correct typo in service name in docker-compose.yml --- api/oracle-cloud/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/oracle-cloud/docker-compose.yml b/api/oracle-cloud/docker-compose.yml index 0647016dc..1357051a6 100644 --- a/api/oracle-cloud/docker-compose.yml +++ b/api/oracle-cloud/docker-compose.yml @@ -18,7 +18,7 @@ services: volumes: - /home/ubuntu/app-data/api/fetch_cache:/usr/src/repo/api/fetch_cache - /home/ubuntu/app-data/api/sqlite_db:/usr/src/repo/api/sqlite_db - postgress: + postgres: image: postgres ports: - "5432:5432" From 48d2e36d0caaca6dde5c1e70b4d406233a7c55d4 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 4 Oct 2024 21:04:38 +0200 Subject: [PATCH 16/20] wait for Postgres service before starting the application --- api/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/package.json b/api/package.json index dd4930c70..c090b9b67 100644 --- a/api/package.json +++ b/api/package.json @@ -77,7 +77,7 @@ "lint:prettier": "prettier --config ../packages/tooling/.prettierrc --ignore-path ../packages/tooling/.prettierignore --log-level warn", "lint:ts-prune": "tsx ../packages/tooling/setup-ts-prune.ts && ts-prune --error", "lint:tsc": "tspc --noEmit", - "start": "node dist/app/index.js", + "start": "wait-port postgres:5432 && node dist/app/index.js", "start:dev": "tsx ../packages/tooling/nodemon.ts \"@dzcode.io/api\" && npm-run-all --parallel start:nodemon db:server", "start:nodemon": "wait-port localhost:5432 && nodemon dist/app/index.js", "test": "npm run build && npm run test:alone", From ca7633a6e0c4b6bba870358031315c52fcc707fc Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 4 Oct 2024 21:27:47 +0200 Subject: [PATCH 17/20] Refactor project ID handling in project pages --- web/cloudflare/handler/project.ts | 4 ++-- web/src/_entry/app.tsx | 2 +- web/src/pages/projects/project/index.tsx | 8 ++++---- web/src/redux/actions/project.ts | 3 +-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/web/cloudflare/handler/project.ts b/web/cloudflare/handler/project.ts index 87070564e..b0c73d5a9 100644 --- a/web/cloudflare/handler/project.ts +++ b/web/cloudflare/handler/project.ts @@ -31,8 +31,8 @@ export const handleProjectRequest: PagesFunction = async (context) => { "en") as LanguageEntity["code"]; const notFound = language === "ar" ? notFoundAr : notFoundEn; - const projectIdRegex = /projects\/(.*)-(.\d+)/; - const projectId = pathName?.match(projectIdRegex)?.[2]; + const projectIdRegex = /projects\/(.*)/; + const projectId = pathName?.match(projectIdRegex)?.[1]; if (!projectId) return new Response(notFound, { diff --git a/web/src/_entry/app.tsx b/web/src/_entry/app.tsx index 7a9fd21e5..946a9adaa 100644 --- a/web/src/_entry/app.tsx +++ b/web/src/_entry/app.tsx @@ -26,7 +26,7 @@ let routes: Array< }, { pageName: "projects/project", - path: "/projects/:projectSlugWithId", + path: "/projects/:projectId", }, { pageName: "faq", diff --git a/web/src/pages/projects/project/index.tsx b/web/src/pages/projects/project/index.tsx index 534b0643b..298e6cb52 100644 --- a/web/src/pages/projects/project/index.tsx +++ b/web/src/pages/projects/project/index.tsx @@ -20,7 +20,7 @@ export default function Page(): JSX.Element { const { localize } = useLocale(); const { project } = useAppSelector((state) => state.projectPage); const dispatch = useAppDispatch(); - const { projectSlugWithId } = useParams<{ projectSlugWithId: string }>(); + const { projectId } = useParams<{ projectId: string }>(); const topMax = 3; const { topContributors, remainingContributorsCount } = useMemo(() => { if (project === null || project === "ERROR" || project === "404") @@ -34,8 +34,8 @@ export default function Page(): JSX.Element { }, [project]); useEffect(() => { - dispatch(fetchProjectAction(projectSlugWithId)); - }, [dispatch, projectSlugWithId]); + dispatch(fetchProjectAction(projectId)); + }, [dispatch, projectId]); if (project === "404") { return ; @@ -70,7 +70,7 @@ export default function Page(): JSX.Element { error={localize("global-generic-error")} action={localize("global-try-again")} onClick={() => { - dispatch(fetchProjectAction(projectSlugWithId)); + dispatch(fetchProjectAction(projectId)); }} /> ) : project === null ? ( diff --git a/web/src/redux/actions/project.ts b/web/src/redux/actions/project.ts index c05f501e5..d434c341a 100644 --- a/web/src/redux/actions/project.ts +++ b/web/src/redux/actions/project.ts @@ -5,9 +5,8 @@ import { AppState } from "src/redux/store"; import { fetchV2 } from "src/utils/fetch"; export const fetchProjectAction = - (projectSlugWithId?: string): ThunkAction => + (id?: string): ThunkAction => async (dispatch) => { - const id = projectSlugWithId?.split("-").pop(); if (!id) { dispatch(projectPageSlice.actions.set({ project: "404" })); return; From a448d5d36c8989f2dd954501d9432eb7ec7e2a25 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 4 Oct 2024 21:55:20 +0200 Subject: [PATCH 18/20] return project name in /project/:id/name --- api/src/project/repository.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/project/repository.ts b/api/src/project/repository.ts index 7e64e7dfe..57213e062 100644 --- a/api/src/project/repository.ts +++ b/api/src/project/repository.ts @@ -15,7 +15,8 @@ export class ProjectRepository { public async findName(projectId: string) { const statement = sql` SELECT - ${projectsTable.id} + ${projectsTable.id}, + ${projectsTable.name} FROM ${projectsTable} WHERE From c1e1305e9c13216616e07f76b6b73352effb263c Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 4 Oct 2024 22:11:47 +0200 Subject: [PATCH 19/20] disabled lh-server for now --- .github/workflows/ci.reusable.lighthouse.yml | 3 ++- api/oracle-cloud/docker-compose.yml | 8 -------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.reusable.lighthouse.yml b/.github/workflows/ci.reusable.lighthouse.yml index a0fb85da0..51e3c37d9 100644 --- a/.github/workflows/ci.reusable.lighthouse.yml +++ b/.github/workflows/ci.reusable.lighthouse.yml @@ -108,4 +108,5 @@ jobs: path: ./web/.lighthouseci - name: "Get current time" run: echo "LHCI_BUILD_CONTEXT__COMMIT_TIME=$(date '+%Y-%m-%d %H:%M:%S %z')" >> $GITHUB_ENV - - run: npx lerna run lh:upload --scope "@dzcode.io/web" + # @TODO-ZM: upload to grafana instead of lhci-server + # - run: npx lerna run lh:upload --scope "@dzcode.io/web" diff --git a/api/oracle-cloud/docker-compose.yml b/api/oracle-cloud/docker-compose.yml index 1357051a6..9dbc55779 100644 --- a/api/oracle-cloud/docker-compose.yml +++ b/api/oracle-cloud/docker-compose.yml @@ -27,11 +27,3 @@ services: environment: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_DB: db - - lhserver: - image: patrickhulce/lhci-server - restart: always - ports: - - "9001:9001" - volumes: - - /home/ubuntu/app-data/web/lh_data:/data From 70275a4ee3b16229ab128b11014c4970515580b0 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 4 Oct 2024 22:20:14 +0200 Subject: [PATCH 20/20] remove redundant proxy_pass rules --- api/oracle-cloud/nginx.conf | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/api/oracle-cloud/nginx.conf b/api/oracle-cloud/nginx.conf index 11a84ceab..fbbaf0bdd 100644 --- a/api/oracle-cloud/nginx.conf +++ b/api/oracle-cloud/nginx.conf @@ -6,18 +6,8 @@ http { server { client_max_body_size 10M; listen 80; - # redirect traffic from lh-stage.dzcode.io, lh_stage.dzcode.io, lh.dzcode.io to lhserver:9001 - # and from api-stage.dzcode.io, api_stage.dzcode.io, api.dzcode.io to api:7070 + location / { - if ($host ~* ^lh-stage.dzcode.io$) { - proxy_pass http://lhserver:9001; - } - if ($host ~* ^lh_stage.dzcode.io$) { - proxy_pass http://lhserver:9001; - } - if ($host ~* ^lh.dzcode.io$) { - proxy_pass http://lhserver:9001; - } if ($host ~* ^api-stage.dzcode.io$) { proxy_pass http://api:7070; }