feat: ipfs metadata provider

packages/metadata/README.md

@@ -0,0 +1,52 @@
+# Grants Stack Indexer v2: Metadata package
+
+This package exposes a metadata provider that can be used to retrieved metadata from IPFS.
+
+## Setup
+
+1. Install dependencies running `pnpm install`
+
+## Available Scripts
+
+Available scripts that can be run using `pnpm`:
+
+| Script        | Description                                             |
+| ------------- | ------------------------------------------------------- |
+| `build`       | Build library using tsc                                 |
+| `check-types` | Check types issues using tsc                            |
+| `clean`       | Remove `dist` folder                                    |
+| `lint`        | Run ESLint to check for coding standards                |
+| `lint:fix`    | Run linter and automatically fix code formatting issues |
+| `format`      | Check code formatting and style using Prettier          |
+| `format:fix`  | Run formatter and automatically fix issues              |
+| `test`        | Run tests using vitest                                  |
+| `test:cov`    | Run tests with coverage report                          |
+
+## Usage
+
+### Importing the Package
+
+You can import the package in your TypeScript or JavaScript files as follows:
+
+```typescript
+import { IpfsProvider } from "@grants-stack-indexer/metadata";
+```
+
+### Example
+
+```typescript
+const provider = new IpfsProvider(["https://ipfs.io", "https://cloudflare-ipfs.com"]);
+const metadata = await provider.getMetadata("QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ");
+```
+
+## API
+
+### [IMetadataProvider](./src/interfaces/metadata.interface.ts)
+
+Available methods
+
+-   `getMetadata<T>(ipfsCid: string, validateContent?: z.ZodSchema<T>): Promise<T | undefined>`
+
+## References
+
+-   [IPFS](https://docs.ipfs.tech/reference/http-api/)

packages/metadata/package.json

@@ -0,0 +1,38 @@
+{
+    "name": "@grants-stack-indexer/metadata",
+    "version": "0.0.1",
+    "private": true,
+    "description": "",
+    "license": "MIT",
+    "author": "Wonderland",
+    "type": "module",
+    "main": "./dist/src/index.js",
+    "types": "./dist/src/index.d.ts",
+    "directories": {
+        "src": "src"
+    },
+    "files": [
+        "dist/*",
+        "package.json",
+        "!**/*.tsbuildinfo"
+    ],
+    "scripts": {
+        "build": "tsc -p tsconfig.build.json",
+        "check-types": "tsc --noEmit -p ./tsconfig.json",
+        "clean": "rm -rf dist/",
+        "format": "prettier --check \"{src,test}/**/*.{js,ts,json}\"",
+        "format:fix": "prettier --write \"{src,test}/**/*.{js,ts,json}\"",
+        "lint": "eslint \"{src,test}/**/*.{js,ts,json}\"",
+        "lint:fix": "pnpm lint --fix",
+        "test": "vitest run --config vitest.config.ts --passWithNoTests",
+        "test:cov": "vitest run --config vitest.config.ts --coverage"
+    },
+    "dependencies": {
+        "@grants-stack-indexer/shared": "workspace:0.0.1",
+        "axios": "1.7.7",
+        "zod": "3.23.8"
+    },
+    "devDependencies": {
+        "axios-mock-adapter": "2.0.0"
+    }
+}

packages/metadata/src/exceptions/emptyGateways.exception.ts

@@ -0,0 +1,5 @@
+export class EmptyGatewaysUrlsException extends Error {
+    constructor() {
+        super("Gateways array cannot be empty");
+    }
+}

packages/metadata/src/exceptions/index.ts

@@ -0,0 +1,3 @@
+export * from "./emptyGateways.exception.js";
+export * from "./invalidCid.exception.js";
+export * from "./invalidContent.exception.js";

packages/metadata/src/exceptions/invalidCid.exception.ts

@@ -0,0 +1,6 @@
+export class InvalidCidException extends Error {
+    constructor(cid: string) {
+        super(`Invalid CID: ${cid}`);
+        this.name = "InvalidCidException";
+    }
+}

packages/metadata/src/exceptions/invalidContent.exception.ts

@@ -0,0 +1,5 @@
+export class InvalidContentException extends Error {
+    constructor(message: string) {
+        super(message);
+    }
+}

packages/metadata/src/external.ts

@@ -0,0 +1,9 @@
+export { IpfsProvider } from "./internal.js";
+
+export {
+    EmptyGatewaysUrlsException,
+    InvalidCidException,
+    InvalidContentException,
+} from "./internal.js";
+
+export type { IMetadataProvider } from "./internal.js";

packages/metadata/src/index.ts

@@ -0,0 +1 @@
+export * from "./external.js";

packages/metadata/src/interfaces/index.ts

@@ -0,0 +1 @@
+export * from "./metadata.interface.js";

packages/metadata/src/interfaces/metadata.interface.ts

@@ -0,0 +1,15 @@
+import z from "zod";
+
+/**
+ * Metadata provider interface
+ */
+export interface IMetadataProvider {
+    /**
+     * Get metadata from IPFS
+     * @param ipfsCid - IPFS CID
+     * @returns - Metadata
+     * @throws - InvalidCidException if the CID is invalid
+     * @throws - InvalidContentException if the retrieved content is invalid
+     */
+    getMetadata<T>(ipfsCid: string, validateContent?: z.ZodSchema<T>): Promise<T | undefined>;
+}

packages/metadata/src/internal.ts

@@ -0,0 +1,4 @@
+export * from "./exceptions/index.js";
+export * from "./interfaces/index.js";
+export * from "./utils/index.js";
+export * from "./providers/index.js";

packages/metadata/src/providers/index.ts

@@ -0,0 +1 @@
+export * from "./ipfs.provider.js";

packages/metadata/src/providers/ipfs.provider.ts

@@ -0,0 +1,75 @@
+import axios, { AxiosInstance } from "axios";
+import { z } from "zod";
+
+import type { IMetadataProvider } from "../internal.js";
+import {
+    EmptyGatewaysUrlsException,
+    InvalidCidException,
+    InvalidContentException,
+    isValidCid,
+} from "../internal.js";
+
+export class IpfsProvider implements IMetadataProvider {
+    private readonly axiosInstance: AxiosInstance;
+
+    constructor(private readonly gateways: string[]) {
+        if (gateways.length === 0) {
+            throw new EmptyGatewaysUrlsException();
+        }
+
+        this.gateways = gateways;
+        this.axiosInstance = axios.create();
+    }
+
+    /* @inheritdoc */
+    async getMetadata<T>(
+        ipfsCid: string,
+        validateContent?: z.ZodSchema<T>,
+    ): Promise<T | undefined> {
+        if (!isValidCid(ipfsCid)) {
+            throw new InvalidCidException(ipfsCid);
+        }
+
+        for (const gateway of this.gateways) {
+            const url = `${gateway}/ipfs/${ipfsCid}`;
+            try {
+                //TODO: retry policy for each gateway
+                const { data } = await this.axiosInstance.get<T>(url);
+                return this.validateData(data, validateContent);
+            } catch (error: unknown) {
+                if (error instanceof InvalidContentException) throw error;
+
+                if (axios.isAxiosError(error)) {
+                    console.warn(`Failed to fetch from ${url}: ${error.message}`);
+                } else {
+                    console.error(`Failed to fetch from ${url}: ${error}`);
+                }
+            }
+        }
+
+        console.error(`Failed to fetch IPFS data for CID ${ipfsCid} from all gateways.`);
+        return undefined;
+    }
+
+    /**
+     * Validates the data using the provided schema.
+     *
+     * @param data - The data to validate.
+     * @param validateContent (optional) - The schema to validate the data against.
+     * @returns The validated data.
+     * @throws InvalidContentException if the data does not match the schema.
+     */
+    private validateData<T>(data: T, validateContent?: z.ZodSchema<T>): T {
+        if (validateContent) {
+            const parsedData = validateContent.safeParse(data);
+            if (parsedData.success) {
+                return parsedData.data;
+            } else {
+                throw new InvalidContentException(
+                    parsedData.error.issues.map((issue) => JSON.stringify(issue)).join("\n"),
+                );
+            }
+        }
+        return data;
+    }
+}

packages/metadata/src/utils/index.ts

@@ -0,0 +1,4 @@
+export const isValidCid = (cid: string): boolean => {
+    const cidRegex = /^(Qm[1-9A-HJ-NP-Za-km-z]{44}|baf[0-9A-Za-z]{50,})$/;
+    return cidRegex.test(cid);
+};

packages/metadata/test/providers/ipfs.provider.spec.ts

@@ -0,0 +1,93 @@
+import MockAdapter from "axios-mock-adapter";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { z } from "zod";
+
+import {
+    EmptyGatewaysUrlsException,
+    InvalidCidException,
+    InvalidContentException,
+    IpfsProvider,
+} from "../../src/external.js";
+
+describe("IpfsProvider", () => {
+    let mock: MockAdapter;
+    let provider: IpfsProvider;
+    const gateways = ["https://ipfs.io", "https://cloudflare-ipfs.com"];
+    const validCid = "QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ";
+
+    beforeEach(() => {
+        provider = new IpfsProvider(gateways);
+        mock = new MockAdapter(provider["axiosInstance"]);
+    });
+
+    afterEach(() => {
+        mock.reset();
+    });
+
+    describe("constructor", () => {
+        it("throw EmptyGatewaysUrlsException when initialized with empty gateways array", () => {
+            expect(() => new IpfsProvider([])).toThrow(EmptyGatewaysUrlsException);
+        });
+    });
+
+    describe("getMetadata", () => {
+        it("throw InvalidCidException for invalid CID", async () => {
+            await expect(() => provider.getMetadata("invalid-cid")).rejects.toThrow(
+                InvalidCidException,
+            );
+        });
+
+        it("fetch metadata successfully from the first working gateway", async () => {
+            const mockData = { name: "Test Data" };
+            mock.onGet(`${gateways[0]}/ipfs/${validCid}`).reply(200, mockData);
+
+            const result = await provider.getMetadata(validCid);
+            expect(result).toEqual(mockData);
+        });
+
+        it("try the next gateway if the first one fails", async () => {
+            const mockData = { name: "Test Data" };
+            mock.onGet(`${gateways[0]}/ipfs/${validCid}`).networkError();
+            mock.onGet(`${gateways[1]}/ipfs/${validCid}`).reply(200, mockData);
+
+            const result = await provider.getMetadata(validCid);
+            expect(result).toEqual(mockData);
+        });
+
+        it("return undefined if all gateways fail", async () => {
+            gateways.forEach((gateway) => {
+                mock.onGet(`${gateway}/ipfs/${validCid}`).networkError();
+            });
+
+            const result = await provider.getMetadata(validCid);
+            expect(result).toBeUndefined();
+        });
+
+        it("validate content with provided schema", async () => {
+            const mockData = { name: "Test Data", age: 30 };
+            mock.onGet(`${gateways[0]}/ipfs/${validCid}`).reply(200, mockData);
+
+            const schema = z.object({
+                name: z.string(),
+                age: z.number(),
+            });
+
+            const result = await provider.getMetadata(validCid, schema);
+            expect(result).toEqual(mockData);
+        });
+
+        it("throw InvalidContentException when content does not match schema", async () => {
+            const mockData = { name: "Test Data", age: "thirty" };
+            mock.onGet(`${gateways[0]}/ipfs/${validCid}`).reply(200, mockData);
+
+            const schema = z.object({
+                name: z.string(),
+                age: z.number(),
+            });
+
+            await expect(() => provider.getMetadata(validCid, schema)).rejects.toThrow(
+                InvalidContentException,
+            );
+        });
+    });
+});

packages/metadata/test/utils/index.test.ts

@@ -0,0 +1,39 @@
+import { describe, expect, it } from "vitest";
+
+import { isValidCid } from "../../src/utils/index.js";
+
+describe("isValidCid", () => {
+    it("return true for valid CIDs", () => {
+        const validCids = [
+            "QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ",
+            "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi",
+        ];
+
+        validCids.forEach((cid) => {
+            expect(isValidCid(cid)).toBe(true);
+        });
+    });
+
+    it("return false for invalid CIDs", () => {
+        const invalidCids = [
+            "",
+            "QmInvalidCID",
+            "bafInvalidCID",
+            "Qm1234567890123456789012345678901234567890123",
+            "baf123456789012345678901234567890123456789012345678901",
+            "not a CID at all",
+        ];
+
+        invalidCids.forEach((cid) => {
+            expect(isValidCid(cid)).toBe(false);
+        });
+    });
+
+    it("return false for non-string inputs", () => {
+        const nonStringInputs = [null, undefined, 123, {}, []];
+
+        nonStringInputs.forEach((input) => {
+            expect(isValidCid(input as unknown as string)).toBe(false);
+        });
+    });
+});

packages/metadata/tsconfig.build.json

@@ -0,0 +1,13 @@
+/* Based on total-typescript no-dom library config */
+/* https://github.com/total-typescript/tsconfig */
+{
+    "extends": "../../tsconfig.build.json",
+    "compilerOptions": {
+        "composite": true,
+        "declarationMap": true,
+        "declaration": true,
+        "outDir": "dist"
+    },
+    "include": ["src/**/*"],
+    "exclude": ["node_modules", "dist", "test"]
+}

packages/metadata/tsconfig.json

@@ -0,0 +1,4 @@
+{
+    "extends": "../../tsconfig.json",
+    "include": ["src/**/*", "test/utils/index.test.ts"]
+}