Import and present repo metadata from GitHub and GitLab

.vscode/launch.json

@@ -35,6 +35,17 @@
                 "run",
                 "update"
             ],
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Fullcheck",
+            "runtimeExecutable": "yarn",
+            "cwd": "${workspaceFolder}",
+            "runtimeArgs": [
+                "run",
+                "fullcheck"
+            ],
         }
     ]
 }

api/package.json

@@ -12,7 +12,7 @@
         "migrate": "dotenv -e ../.env -- kysely migrate",
         "migrate:down": "dotenv -e ../.env -- kysely migrate down",
         "db:up": "yarn migrate latest",
-        "test": "vitest --watch=false --no-file-parallelism",
+        "test": "dotenv -e ../.env -- vitest --watch=false --no-file-parallelism",
         "dev": "yarn build && yarn start",
         "build": "tsc",
         "start": "yarn db:up && dotenv -e ../.env -- node dist/src/entrypoints/start-api.js",
@@ -70,6 +70,7 @@
         "vitest": "^1.2.2"
     },
     "dependencies": {
+        "@gitbeaker/core": "^42.1.0",
         "@octokit/graphql": "^7.0.2",
         "@trpc/server": "^10.18.0",
         "@types/pg": "^8.11.6",

api/src/core/adapters/GitHub/api/repo.ts

@@ -0,0 +1,78 @@
+import { Octokit } from "@octokit/rest";
+
+import { env } from "../../../../env";
+
+export const repoGitHubEndpointMaker = (repoUrl: string | URL) => {
+    const octokit = new Octokit({
+        auth: env.githubPersonalAccessTokenForApiRateLimit
+    });
+    let repoUrlObj = typeof repoUrl === "string" ? URL.parse(repoUrl) : repoUrl;
+    if (!repoUrlObj) return undefined;
+
+    // Case .git at the end
+    if (repoUrlObj.pathname.endsWith("/")) repoUrlObj.pathname = repoUrlObj.pathname.slice(0, -1);
+    if (repoUrlObj.pathname.endsWith(".git")) repoUrlObj.pathname = repoUrlObj.pathname.slice(0, -4);
+
+    const parsed = repoUrlObj.pathname.split("/").filter(text => text);
+
+    const repo = parsed[1];
+    const owner = parsed[0];
+
+    return {
+        issues: {
+            getLastClosedIssue: async () => {
+                try {
+                    const resIssues = await octokit.request("GET /repos/{owner}/{repo}/issues", {
+                        owner,
+                        repo,
+                        headers: {
+                            "X-GitHub-Api-Version": "2022-11-28"
+                        },
+                        direction: "desc",
+                        state: "closed"
+                    });
+
+                    return resIssues.data[0];
+                } catch (error) {
+                    return undefined;
+                }
+            }
+        },
+        commits: {
+            getLastCommit: async () => {
+                try {
+                    const resCommit = await octokit.request("GET /repos/{owner}/{repo}/commits", {
+                        owner,
+                        repo,
+                        headers: {
+                            "X-GitHub-Api-Version": "2022-11-28"
+                        },
+                        direction: "desc"
+                    });
+                    return resCommit.data[0];
+                } catch (error) {
+                    return undefined;
+                }
+            }
+        },
+        mergeRequests: {
+            getLast: async () => {
+                try {
+                    const resPull = await octokit.request("GET /repos/{owner}/{repo}/pulls", {
+                        owner,
+                        repo,
+                        headers: {
+                            "X-GitHub-Api-Version": "2022-11-28"
+                        },
+                        direction: "desc",
+                        state: "closed"
+                    });
+
+                    return resPull.data[0];
+                } catch (error) {
+                    return undefined;
+                }
+            }
+        }
+    };
+};

api/src/core/adapters/GitLab/api/project.ts

@@ -0,0 +1,54 @@
+import { CommitSchema, IssueSchema, MergeRequestSchema } from "@gitbeaker/core";
+import { repoUrlToAPIUrl } from "./utils";
+
+const getApiCallTakeFirst = async <T>(url: string): Promise<T | undefined> => {
+    const res = await fetch(url, {
+        signal: AbortSignal.timeout(10000)
+    }).catch(err => {
+        console.error(url, err);
+    });
+
+    if (!res) {
+        return undefined;
+    }
+    if (res.status === 404) {
+        console.error("Ressource not available");
+        return undefined;
+    }
+    if (res.status === 403) {
+        console.info(`You don't seems to be allowed on ${url}`);
+        return undefined;
+    }
+
+    const result: T[] = await res.json();
+
+    return result[0];
+};
+
+const getLastClosedIssue = async (projectUrl: string) => {
+    return getApiCallTakeFirst<IssueSchema>(`${projectUrl}/issues?sort=desc&state=closed`);
+};
+
+const getLastCommit = async (projectUrl: string) => {
+    return getApiCallTakeFirst<CommitSchema>(`${projectUrl}/repository/commits?sort=desc`);
+};
+
+const getLastMergeRequest = async (projectUrl: string) => {
+    return getApiCallTakeFirst<MergeRequestSchema>(`${projectUrl}/merge_requests?state=closed&sort=desc`);
+};
+
+export const projectGitLabApiMaker = (repoUrl: string | URL) => {
+    const apiProjectEndpoint = repoUrlToAPIUrl(repoUrl);
+
+    return {
+        issues: {
+            getLastClosedIssue: () => getLastClosedIssue(apiProjectEndpoint)
+        },
+        commits: {
+            getLastCommit: () => getLastCommit(apiProjectEndpoint)
+        },
+        mergeRequests: {
+            getLast: () => getLastMergeRequest(apiProjectEndpoint)
+        }
+    };
+};

api/src/core/adapters/GitLab/api/utils.ts

@@ -0,0 +1,30 @@
+export const repoUrlToAPIUrl = (projectUrl: string | URL): string => {
+    let url = projectUrl;
+
+    if (typeof url === "string") {
+        // Case git+ at the beging
+        if (url.startsWith("git+")) url = url.substring(4);
+
+        // Case ssh protocol
+        if (url.startsWith("git@")) url = url.replace(":", "/").replace("git@", "https://");
+
+        // Case .git at the end
+        if (url.endsWith(".git")) url = url.slice(0, -4);
+    }
+
+    const urlObj = typeof projectUrl === "string" ? URL.parse(url) : projectUrl;
+
+    if (!urlObj) {
+        throw new Error("Bad URL");
+    }
+
+    const base = urlObj.origin;
+
+    let projectPath = urlObj.pathname.substring(1);
+    if (projectPath.includes("/-/")) projectPath = projectPath.split("-")[0];
+    // Case / at the end
+    if (projectPath.endsWith("/")) projectPath = projectPath.slice(0, -1);
+    projectPath = projectPath.replaceAll("/", "%2F");
+
+    return `${base}/api/v4/projects/${projectPath}`;
+};

api/src/core/adapters/dbApi/kysely/createPgSoftwareExternalDataRepository.ts

@@ -13,6 +13,7 @@ export const createPgSoftwareExternalDataRepository = (db: Kysely<Database>): So
             programmingLanguages: JSON.stringify(softwareExternalData.programmingLanguages),
             referencePublications: JSON.stringify(softwareExternalData.referencePublications),
             identifiers: JSON.stringify(softwareExternalData.identifiers),
+            repoMetadata: JSON.stringify(softwareExternalData.repoMetadata),
             description: JSON.stringify(softwareExternalData.description)
         };
 

api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts

@@ -5,6 +5,7 @@ import { SoftwareRepository } from "../../../ports/DbApiV2";
 import { Software } from "../../../usecases/readWriteSillData";
 import { Database } from "./kysely.database";
 import { stripNullOrUndefinedValues, jsonBuildObject } from "./kysely.utils";
+import { SILL } from "../../../../types/SILL";
 
 const dateParser = (str: string | Date | undefined | null) => {
     if (str && typeof str === "string") {
@@ -16,6 +17,19 @@ const dateParser = (str: string | Date | undefined | null) => {
     }
 };
 
+const computeRepoMetadata = (repoMetadata: SILL.RepoMetadata | undefined | null): SILL.RepoMetadata | undefined => {
+    const newMedata = repoMetadata;
+    if (!newMedata || !newMedata.healthCheck) return undefined;
+
+    let score = 0;
+    if (repoMetadata.healthCheck?.lastClosedIssue) score += 1;
+    if (repoMetadata.healthCheck?.lastClosedIssuePullRequest) score += 1;
+    if (repoMetadata.healthCheck?.lastCommit) score += 1;
+    newMedata.healthCheck.score = score / 3;
+
+    return newMedata;
+};
+
 export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareRepository => {
     const getBySoftwareId = makeGetSoftwareById(db);
     return {
@@ -184,6 +198,7 @@ export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareReposi
                         programmingLanguages: softwareExternalData?.programmingLanguages ?? [],
                         referencePublications: softwareExternalData?.referencePublications,
                         identifiers: softwareExternalData?.identifiers,
+                        repoMetadata: computeRepoMetadata(softwareExternalData?.repoMetadata),
                         applicationCategories: software.categories.concat(
                             softwareExternalData?.applicationCategories ?? []
                         ),
@@ -283,7 +298,8 @@ export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareReposi
                             categories: undefined, // merged in applicationCategories, set to undefined to remove it
                             programmingLanguages: softwareExternalData?.programmingLanguages ?? [],
                             referencePublications: softwareExternalData?.referencePublications,
-                            identifiers: softwareExternalData?.identifiers
+                            identifiers: softwareExternalData?.identifiers,
+                            repoMetadata: computeRepoMetadata(softwareExternalData?.repoMetadata)
                         });
                     }
                 );
@@ -402,6 +418,7 @@ const makeGetSoftwareBuilder = (db: Kysely<Database>) =>
                     applicationCategories: ref("ext.applicationCategories"),
                     referencePublications: ref("ext.referencePublications"),
                     identifiers: ref("ext.identifiers"),
+                    repoMetadata: ref("ext.repoMetadata"),
                     keywords: ref("ext.keywords"),
                     softwareVersion: ref("ext.softwareVersion"),
                     publicationTime: ref("ext.publicationTime")
@@ -534,6 +551,7 @@ const makeGetSoftwareById =
                     programmingLanguages: softwareExternalData?.programmingLanguages ?? [],
                     referencePublications: softwareExternalData?.referencePublications,
                     identifiers: softwareExternalData?.identifiers,
+                    repoMetadata: computeRepoMetadata(softwareExternalData?.repoMetadata),
                     applicationCategories: filterDuplicate(
                         software.categories.concat(softwareExternalData?.applicationCategories ?? [])
                     ),

api/src/core/adapters/dbApi/kysely/kysely.database.ts

@@ -97,6 +97,7 @@ type SoftwareExternalDatasTable = {
     referencePublications: JSONColumnType<SILL.ScholarlyArticle[]> | null;
     publicationTime: Date | null;
     identifiers: JSONColumnType<SILL.Identification[]> | null;
+    repoMetadata: JSONColumnType<SILL.RepoMetadata> | null;
 };
 
 type SoftwareType =

api/src/core/adapters/dbApi/kysely/migrations/1741091782480_add-repo-metadata.ts

@@ -0,0 +1,9 @@
+import { Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+    await db.schema.alterTable("software_external_datas").addColumn("repoMetadata", "jsonb").execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+    await db.schema.alterTable("software_external_datas").dropColumn("repoMetadata").execute();
+}

api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts

@@ -428,6 +428,7 @@ describe("pgDbApi", () => {
                     applicationCategories: JSON.stringify(softExtData.applicationCategories),
                     programmingLanguages: JSON.stringify(softExtData.programmingLanguages),
                     identifiers: JSON.stringify(softExtData.identifiers),
+                    repoMetadata: JSON.stringify(softExtData.repoMetadata),
                     referencePublications: JSON.stringify(softExtData.referencePublications)
                 }))
             )

api/src/core/adapters/fetchExternalData.test.ts

@@ -186,6 +186,7 @@ describe("fetches software extra data (from different providers)", () => {
                     programmingLanguages: [],
                     referencePublications: null,
                     identifiers: [],
+                    repoMetadata: null,
                     softwareVersion: "5.0.1",
                     publicationTime: new Date("2022-04-12T00:00:00.000Z")
                 },
@@ -215,6 +216,7 @@ describe("fetches software extra data (from different providers)", () => {
                     programmingLanguages: ["JavaScript"],
                     referencePublications: null,
                     identifiers: [],
+                    repoMetadata: null,
                     softwareVersion: expect.any(String),
                     publicationTime: expect.any(Date)
                 }
@@ -270,6 +272,7 @@ describe("fetches software extra data (from different providers)", () => {
                     websiteUrl: "https://httpd.apache.org/",
                     referencePublications: null,
                     identifiers: [],
+                    repoMetadata: null,
                     programmingLanguages: ["C"],
                     softwareVersion: "2.5.0-alpha",
                     publicationTime: new Date("2017-11-08T00:00:00.000Z")

api/src/core/adapters/hal/getHalSoftware.test.ts

@@ -38,6 +38,13 @@ describe("HAL", () => {
                 "programmingLanguages": undefined,
                 "applicationCategories": ["Computer Science [cs]"],
                 "referencePublications": undefined,
+                "repoMetadata": {
+                    "healthCheck": {
+                        "lastClosedIssue": undefined,
+                        "lastClosedIssuePullRequest": undefined,
+                        "lastCommit": 1729459216000
+                    }
+                },
                 "identifiers": [
                     {
                         "@type": "PropertyValue",