From f442eda797c41b317bcd53018735152c22d19396 Mon Sep 17 00:00:00 2001 From: nigiri <168690269+0xnigir1@users.noreply.github.com> Date: Wed, 9 Oct 2024 10:51:30 -0300 Subject: [PATCH] feat: ipfs metadata provider (#5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # 🤖 Linear Closes GIT-65 ## Description - add `metadata` package with IpfsProvider ## Checklist before requesting a review - [x] I have conducted a self-review of my code. - [x] I have conducted a QA. - [x] If it is a core feature, I have included comprehensive tests. --- packages/metadata/README.md | 52 +++++++++++ packages/metadata/package.json | 38 ++++++++ .../src/exceptions/emptyGateways.exception.ts | 5 + packages/metadata/src/exceptions/index.ts | 3 + .../src/exceptions/invalidCid.exception.ts | 6 ++ .../exceptions/invalidContent.exception.ts | 5 + packages/metadata/src/external.ts | 9 ++ packages/metadata/src/index.ts | 1 + packages/metadata/src/interfaces/index.ts | 1 + .../src/interfaces/metadata.interface.ts | 15 +++ packages/metadata/src/internal.ts | 4 + packages/metadata/src/providers/index.ts | 1 + .../metadata/src/providers/ipfs.provider.ts | 75 +++++++++++++++ packages/metadata/src/utils/index.ts | 4 + .../test/providers/ipfs.provider.spec.ts | 93 +++++++++++++++++++ packages/metadata/test/utils/index.test.ts | 39 ++++++++ packages/metadata/tsconfig.build.json | 13 +++ packages/metadata/tsconfig.json | 4 + packages/metadata/vitest.config.ts | 22 +++++ pnpm-lock.yaml | 19 +++- 20 files changed, 407 insertions(+), 2 deletions(-) create mode 100644 packages/metadata/README.md create mode 100644 packages/metadata/package.json create mode 100644 packages/metadata/src/exceptions/emptyGateways.exception.ts create mode 100644 packages/metadata/src/exceptions/index.ts create mode 100644 packages/metadata/src/exceptions/invalidCid.exception.ts create mode 100644 packages/metadata/src/exceptions/invalidContent.exception.ts create mode 100644 packages/metadata/src/external.ts create mode 100644 packages/metadata/src/index.ts create mode 100644 packages/metadata/src/interfaces/index.ts create mode 100644 packages/metadata/src/interfaces/metadata.interface.ts create mode 100644 packages/metadata/src/internal.ts create mode 100644 packages/metadata/src/providers/index.ts create mode 100644 packages/metadata/src/providers/ipfs.provider.ts create mode 100644 packages/metadata/src/utils/index.ts create mode 100644 packages/metadata/test/providers/ipfs.provider.spec.ts create mode 100644 packages/metadata/test/utils/index.test.ts create mode 100644 packages/metadata/tsconfig.build.json create mode 100644 packages/metadata/tsconfig.json create mode 100644 packages/metadata/vitest.config.ts diff --git a/packages/metadata/README.md b/packages/metadata/README.md new file mode 100644 index 0000000..65e714a --- /dev/null +++ b/packages/metadata/README.md @@ -0,0 +1,52 @@ +# Grants Stack Indexer v2: Metadata package + +This package exposes a metadata provider that can be used to retrieved metadata from IPFS. + +## Setup + +1. Install dependencies running `pnpm install` + +## Available Scripts + +Available scripts that can be run using `pnpm`: + +| Script | Description | +| ------------- | ------------------------------------------------------- | +| `build` | Build library using tsc | +| `check-types` | Check types issues using tsc | +| `clean` | Remove `dist` folder | +| `lint` | Run ESLint to check for coding standards | +| `lint:fix` | Run linter and automatically fix code formatting issues | +| `format` | Check code formatting and style using Prettier | +| `format:fix` | Run formatter and automatically fix issues | +| `test` | Run tests using vitest | +| `test:cov` | Run tests with coverage report | + +## Usage + +### Importing the Package + +You can import the package in your TypeScript or JavaScript files as follows: + +```typescript +import { IpfsProvider } from "@grants-stack-indexer/metadata"; +``` + +### Example + +```typescript +const provider = new IpfsProvider(["https://ipfs.io", "https://cloudflare-ipfs.com"]); +const metadata = await provider.getMetadata("QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ"); +``` + +## API + +### [IMetadataProvider](./src/interfaces/metadata.interface.ts) + +Available methods + +- `getMetadata(ipfsCid: string, validateContent?: z.ZodSchema): Promise` + +## References + +- [IPFS](https://docs.ipfs.tech/reference/http-api/) diff --git a/packages/metadata/package.json b/packages/metadata/package.json new file mode 100644 index 0000000..b9ca88d --- /dev/null +++ b/packages/metadata/package.json @@ -0,0 +1,38 @@ +{ + "name": "@grants-stack-indexer/metadata", + "version": "0.0.1", + "private": true, + "description": "", + "license": "MIT", + "author": "Wonderland", + "type": "module", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "directories": { + "src": "src" + }, + "files": [ + "dist/*", + "package.json", + "!**/*.tsbuildinfo" + ], + "scripts": { + "build": "tsc -p tsconfig.build.json", + "check-types": "tsc --noEmit -p ./tsconfig.json", + "clean": "rm -rf dist/", + "format": "prettier --check \"{src,test}/**/*.{js,ts,json}\"", + "format:fix": "prettier --write \"{src,test}/**/*.{js,ts,json}\"", + "lint": "eslint \"{src,test}/**/*.{js,ts,json}\"", + "lint:fix": "pnpm lint --fix", + "test": "vitest run --config vitest.config.ts --passWithNoTests", + "test:cov": "vitest run --config vitest.config.ts --coverage" + }, + "dependencies": { + "@grants-stack-indexer/shared": "workspace:0.0.1", + "axios": "1.7.7", + "zod": "3.23.8" + }, + "devDependencies": { + "axios-mock-adapter": "2.0.0" + } +} diff --git a/packages/metadata/src/exceptions/emptyGateways.exception.ts b/packages/metadata/src/exceptions/emptyGateways.exception.ts new file mode 100644 index 0000000..d70b599 --- /dev/null +++ b/packages/metadata/src/exceptions/emptyGateways.exception.ts @@ -0,0 +1,5 @@ +export class EmptyGatewaysUrlsException extends Error { + constructor() { + super("Gateways array cannot be empty"); + } +} diff --git a/packages/metadata/src/exceptions/index.ts b/packages/metadata/src/exceptions/index.ts new file mode 100644 index 0000000..93607ba --- /dev/null +++ b/packages/metadata/src/exceptions/index.ts @@ -0,0 +1,3 @@ +export * from "./emptyGateways.exception.js"; +export * from "./invalidCid.exception.js"; +export * from "./invalidContent.exception.js"; diff --git a/packages/metadata/src/exceptions/invalidCid.exception.ts b/packages/metadata/src/exceptions/invalidCid.exception.ts new file mode 100644 index 0000000..437cd25 --- /dev/null +++ b/packages/metadata/src/exceptions/invalidCid.exception.ts @@ -0,0 +1,6 @@ +export class InvalidCidException extends Error { + constructor(cid: string) { + super(`Invalid CID: ${cid}`); + this.name = "InvalidCidException"; + } +} diff --git a/packages/metadata/src/exceptions/invalidContent.exception.ts b/packages/metadata/src/exceptions/invalidContent.exception.ts new file mode 100644 index 0000000..8da5363 --- /dev/null +++ b/packages/metadata/src/exceptions/invalidContent.exception.ts @@ -0,0 +1,5 @@ +export class InvalidContentException extends Error { + constructor(message: string) { + super(message); + } +} diff --git a/packages/metadata/src/external.ts b/packages/metadata/src/external.ts new file mode 100644 index 0000000..78fd970 --- /dev/null +++ b/packages/metadata/src/external.ts @@ -0,0 +1,9 @@ +export { IpfsProvider } from "./internal.js"; + +export { + EmptyGatewaysUrlsException, + InvalidCidException, + InvalidContentException, +} from "./internal.js"; + +export type { IMetadataProvider } from "./internal.js"; diff --git a/packages/metadata/src/index.ts b/packages/metadata/src/index.ts new file mode 100644 index 0000000..a5a2748 --- /dev/null +++ b/packages/metadata/src/index.ts @@ -0,0 +1 @@ +export * from "./external.js"; diff --git a/packages/metadata/src/interfaces/index.ts b/packages/metadata/src/interfaces/index.ts new file mode 100644 index 0000000..c9f5d31 --- /dev/null +++ b/packages/metadata/src/interfaces/index.ts @@ -0,0 +1 @@ +export * from "./metadata.interface.js"; diff --git a/packages/metadata/src/interfaces/metadata.interface.ts b/packages/metadata/src/interfaces/metadata.interface.ts new file mode 100644 index 0000000..b4552a9 --- /dev/null +++ b/packages/metadata/src/interfaces/metadata.interface.ts @@ -0,0 +1,15 @@ +import z from "zod"; + +/** + * Metadata provider interface + */ +export interface IMetadataProvider { + /** + * Get metadata from IPFS + * @param ipfsCid - IPFS CID + * @returns - Metadata + * @throws - InvalidCidException if the CID is invalid + * @throws - InvalidContentException if the retrieved content is invalid + */ + getMetadata(ipfsCid: string, validateContent?: z.ZodSchema): Promise; +} diff --git a/packages/metadata/src/internal.ts b/packages/metadata/src/internal.ts new file mode 100644 index 0000000..05d2905 --- /dev/null +++ b/packages/metadata/src/internal.ts @@ -0,0 +1,4 @@ +export * from "./exceptions/index.js"; +export * from "./interfaces/index.js"; +export * from "./utils/index.js"; +export * from "./providers/index.js"; diff --git a/packages/metadata/src/providers/index.ts b/packages/metadata/src/providers/index.ts new file mode 100644 index 0000000..f575273 --- /dev/null +++ b/packages/metadata/src/providers/index.ts @@ -0,0 +1 @@ +export * from "./ipfs.provider.js"; diff --git a/packages/metadata/src/providers/ipfs.provider.ts b/packages/metadata/src/providers/ipfs.provider.ts new file mode 100644 index 0000000..2114fbd --- /dev/null +++ b/packages/metadata/src/providers/ipfs.provider.ts @@ -0,0 +1,75 @@ +import axios, { AxiosInstance } from "axios"; +import { z } from "zod"; + +import type { IMetadataProvider } from "../internal.js"; +import { + EmptyGatewaysUrlsException, + InvalidCidException, + InvalidContentException, + isValidCid, +} from "../internal.js"; + +export class IpfsProvider implements IMetadataProvider { + private readonly axiosInstance: AxiosInstance; + + constructor(private readonly gateways: string[]) { + if (gateways.length === 0) { + throw new EmptyGatewaysUrlsException(); + } + + this.gateways = gateways; + this.axiosInstance = axios.create(); + } + + /* @inheritdoc */ + async getMetadata( + ipfsCid: string, + validateContent?: z.ZodSchema, + ): Promise { + if (!isValidCid(ipfsCid)) { + throw new InvalidCidException(ipfsCid); + } + + for (const gateway of this.gateways) { + const url = `${gateway}/ipfs/${ipfsCid}`; + try { + //TODO: retry policy for each gateway + const { data } = await this.axiosInstance.get(url); + return this.validateData(data, validateContent); + } catch (error: unknown) { + if (error instanceof InvalidContentException) throw error; + + if (axios.isAxiosError(error)) { + console.warn(`Failed to fetch from ${url}: ${error.message}`); + } else { + console.error(`Failed to fetch from ${url}: ${error}`); + } + } + } + + console.error(`Failed to fetch IPFS data for CID ${ipfsCid} from all gateways.`); + return undefined; + } + + /** + * Validates the data using the provided schema. + * + * @param data - The data to validate. + * @param validateContent (optional) - The schema to validate the data against. + * @returns The validated data. + * @throws InvalidContentException if the data does not match the schema. + */ + private validateData(data: T, validateContent?: z.ZodSchema): T { + if (validateContent) { + const parsedData = validateContent.safeParse(data); + if (parsedData.success) { + return parsedData.data; + } else { + throw new InvalidContentException( + parsedData.error.issues.map((issue) => JSON.stringify(issue)).join("\n"), + ); + } + } + return data; + } +} diff --git a/packages/metadata/src/utils/index.ts b/packages/metadata/src/utils/index.ts new file mode 100644 index 0000000..f3d2faa --- /dev/null +++ b/packages/metadata/src/utils/index.ts @@ -0,0 +1,4 @@ +export const isValidCid = (cid: string): boolean => { + const cidRegex = /^(Qm[1-9A-HJ-NP-Za-km-z]{44}|baf[0-9A-Za-z]{50,})$/; + return cidRegex.test(cid); +}; diff --git a/packages/metadata/test/providers/ipfs.provider.spec.ts b/packages/metadata/test/providers/ipfs.provider.spec.ts new file mode 100644 index 0000000..07d48ac --- /dev/null +++ b/packages/metadata/test/providers/ipfs.provider.spec.ts @@ -0,0 +1,93 @@ +import MockAdapter from "axios-mock-adapter"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { + EmptyGatewaysUrlsException, + InvalidCidException, + InvalidContentException, + IpfsProvider, +} from "../../src/external.js"; + +describe("IpfsProvider", () => { + let mock: MockAdapter; + let provider: IpfsProvider; + const gateways = ["https://ipfs.io", "https://cloudflare-ipfs.com"]; + const validCid = "QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ"; + + beforeEach(() => { + provider = new IpfsProvider(gateways); + mock = new MockAdapter(provider["axiosInstance"]); + }); + + afterEach(() => { + mock.reset(); + }); + + describe("constructor", () => { + it("throw EmptyGatewaysUrlsException when initialized with empty gateways array", () => { + expect(() => new IpfsProvider([])).toThrow(EmptyGatewaysUrlsException); + }); + }); + + describe("getMetadata", () => { + it("throw InvalidCidException for invalid CID", async () => { + await expect(() => provider.getMetadata("invalid-cid")).rejects.toThrow( + InvalidCidException, + ); + }); + + it("fetch metadata successfully from the first working gateway", async () => { + const mockData = { name: "Test Data" }; + mock.onGet(`${gateways[0]}/ipfs/${validCid}`).reply(200, mockData); + + const result = await provider.getMetadata(validCid); + expect(result).toEqual(mockData); + }); + + it("try the next gateway if the first one fails", async () => { + const mockData = { name: "Test Data" }; + mock.onGet(`${gateways[0]}/ipfs/${validCid}`).networkError(); + mock.onGet(`${gateways[1]}/ipfs/${validCid}`).reply(200, mockData); + + const result = await provider.getMetadata(validCid); + expect(result).toEqual(mockData); + }); + + it("return undefined if all gateways fail", async () => { + gateways.forEach((gateway) => { + mock.onGet(`${gateway}/ipfs/${validCid}`).networkError(); + }); + + const result = await provider.getMetadata(validCid); + expect(result).toBeUndefined(); + }); + + it("validate content with provided schema", async () => { + const mockData = { name: "Test Data", age: 30 }; + mock.onGet(`${gateways[0]}/ipfs/${validCid}`).reply(200, mockData); + + const schema = z.object({ + name: z.string(), + age: z.number(), + }); + + const result = await provider.getMetadata(validCid, schema); + expect(result).toEqual(mockData); + }); + + it("throw InvalidContentException when content does not match schema", async () => { + const mockData = { name: "Test Data", age: "thirty" }; + mock.onGet(`${gateways[0]}/ipfs/${validCid}`).reply(200, mockData); + + const schema = z.object({ + name: z.string(), + age: z.number(), + }); + + await expect(() => provider.getMetadata(validCid, schema)).rejects.toThrow( + InvalidContentException, + ); + }); + }); +}); diff --git a/packages/metadata/test/utils/index.test.ts b/packages/metadata/test/utils/index.test.ts new file mode 100644 index 0000000..f6394c3 --- /dev/null +++ b/packages/metadata/test/utils/index.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; + +import { isValidCid } from "../../src/utils/index.js"; + +describe("isValidCid", () => { + it("return true for valid CIDs", () => { + const validCids = [ + "QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ", + "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + ]; + + validCids.forEach((cid) => { + expect(isValidCid(cid)).toBe(true); + }); + }); + + it("return false for invalid CIDs", () => { + const invalidCids = [ + "", + "QmInvalidCID", + "bafInvalidCID", + "Qm1234567890123456789012345678901234567890123", + "baf123456789012345678901234567890123456789012345678901", + "not a CID at all", + ]; + + invalidCids.forEach((cid) => { + expect(isValidCid(cid)).toBe(false); + }); + }); + + it("return false for non-string inputs", () => { + const nonStringInputs = [null, undefined, 123, {}, []]; + + nonStringInputs.forEach((input) => { + expect(isValidCid(input as unknown as string)).toBe(false); + }); + }); +}); diff --git a/packages/metadata/tsconfig.build.json b/packages/metadata/tsconfig.build.json new file mode 100644 index 0000000..a9bfa3d --- /dev/null +++ b/packages/metadata/tsconfig.build.json @@ -0,0 +1,13 @@ +/* Based on total-typescript no-dom library config */ +/* https://github.com/total-typescript/tsconfig */ +{ + "extends": "../../tsconfig.build.json", + "compilerOptions": { + "composite": true, + "declarationMap": true, + "declaration": true, + "outDir": "dist" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "test"] +} diff --git a/packages/metadata/tsconfig.json b/packages/metadata/tsconfig.json new file mode 100644 index 0000000..77bf752 --- /dev/null +++ b/packages/metadata/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*", "test/utils/index.test.ts"] +} diff --git a/packages/metadata/vitest.config.ts b/packages/metadata/vitest.config.ts new file mode 100644 index 0000000..8e1bbf4 --- /dev/null +++ b/packages/metadata/vitest.config.ts @@ -0,0 +1,22 @@ +import path from "path"; +import { configDefaults, defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, // Use Vitest's global API without importing it in each file + environment: "node", // Use the Node.js environment + include: ["test/**/*.spec.ts"], // Include test files + exclude: ["node_modules", "dist"], // Exclude certain directories + coverage: { + provider: "v8", + reporter: ["text", "json", "html"], // Coverage reporters + exclude: ["node_modules", "dist", "src/index.ts", ...configDefaults.exclude], // Files to exclude from coverage + }, + }, + resolve: { + alias: { + // Setup path alias based on tsconfig paths + "@": path.resolve(__dirname, "src"), + }, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 194e1d9..439813c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -96,6 +96,22 @@ importers: specifier: 5.2.2 version: 5.2.2 + packages/metadata: + dependencies: + "@grants-stack-indexer/shared": + specifier: workspace:0.0.1 + version: link:../shared + axios: + specifier: 1.7.7 + version: 1.7.7 + zod: + specifier: 3.23.8 + version: 3.23.8 + devDependencies: + axios-mock-adapter: + specifier: 2.0.0 + version: 2.0.0(axios@1.7.7) + packages/pricing: dependencies: "@grants-stack-indexer/shared": @@ -6087,5 +6103,4 @@ snapshots: yocto-queue@1.1.1: {} - zod@3.23.8: - optional: true + zod@3.23.8: {}