Merge branch 'dev' into feat/chain-data-repository

.env.example

@@ -1,10 +1,31 @@
+######################################################
+############### DATALAYER POSTGRES ###################
+######################################################
+DATALAYER_POSTGRES_PASSWORD=testing
+DATALAYER_PG_USER=postgres
+DATALAYER_PG_DATABASE=datalayer-postgres
+DATALAYER_POSTGRES_EXPOSED_PORT=5434
+
+
+############################################################
+############### ENVIO POSTGRES & INDEXER ###################
+############################################################
+ENVIO_POSTGRES_PASSWORD=testing
+ENVIO_PG_USER=postgres
+ENVIO_PG_DATABASE=envio-dev
+
 ############################################
 ############### POSTGRES ###################
 ############################################
-POSTGRES_EXPOSED_PORT=5433
-POSTGRES_PASSWORD=testing
-POSTGRES_USER=postgres
-POSTGRES_DB=envio-dev
+ENVIO_POSTGRES_EXPOSED_PORT=5433
+
+############################################
+############### INDEXER ####################
+############################################
+ENVIO_PG_HOST=envio-postgres
+ENVIO_PG_PORT=5432 
+HASURA_GRAPHQL_ENDPOINT=http://graphql-engine:8080/v1/metadata
+TUI_OFF=true
 
 ############################################
 ############### HASURA #####################
@@ -17,15 +38,4 @@ HASURA_GRAPHQL_NO_OF_RETRIES=10
 HASURA_GRAPHQL_ADMIN_SECRET=testing
 HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES="true"
 PORT=8080
-HASURA_GRAPHQL_UNAUTHORIZED_ROLE=public
-
-############################################
-############### INDEXER ####################
-############################################
-ENVIO_PG_HOST=envio-postgres
-ENVIO_PG_PORT=5432 
-ENVIO_POSTGRES_PASSWORD=testing
-ENVIO_PG_USER=postgres
-ENVIO_PG_DATABASE=envio-dev
-HASURA_GRAPHQL_ENDPOINT=http://graphql-engine:8080/v1/metadata
-TUI_OFF=true
+HASURA_GRAPHQL_UNAUTHORIZED_ROLE=public

docker-compose.yaml

@@ -1,13 +1,32 @@
 services:
+    datalayer-postgres:
+        image: postgres:16
+        restart: always
+        ports:
+            - "${DATALAYER_POSTGRES_EXPOSED_PORT:-5432}:5432"
+        volumes:
+            - db_data:/var/lib/datalayer-postgresql/data
+        env_file:
+            - .env
+        environment:
+            POSTGRES_DB: ${DATALAYER_PG_DATABASE}
+            POSTGRES_USER: ${DATALAYER_PG_USER}
+            POSTGRES_PASSWORD: ${DATALAYER_POSTGRES_PASSWORD}
+        networks:
+            - datalayer
     envio-postgres:
         image: postgres:16
         restart: always
         ports:
-            - "${POSTGRES_PORT:-5433}:5432"
+            - "${ENVIO_POSTGRES_EXPOSED_PORT:-5433}:5432"
         volumes:
-            - db_data:/var/lib/postgresql/data
+            - db_data:/var/lib/envio-postgresql/data
         env_file:
             - .env
+        environment:
+            POSTGRES_DB: ${ENVIO_PG_DATABASE}
+            POSTGRES_USER: ${ENVIO_PG_USER}
+            POSTGRES_PASSWORD: ${ENVIO_POSTGRES_PASSWORD}
         networks:
             - indexer-service
     graphql-engine:
@@ -46,3 +65,5 @@ volumes:
 networks:
     indexer-service:
         name: indexer_test_network
+    datalayer:
+        name: datalayer_test_network

packages/data-flow/README.md

@@ -0,0 +1,45 @@
+# @grants-stack-indexer/data-flow
+
+Is a library that provides the core components of the processing pipeline for gitcoin grants-stack-indexer.
+
+## Available Scripts
+
+Available scripts that can be run using `pnpm`:
+
+| Script        | Description                                             |
+| ------------- | ------------------------------------------------------- |
+| `build`       | Build library using tsc                                 |
+| `check-types` | Check types issues using tsc                            |
+| `clean`       | Remove `dist` folder                                    |
+| `lint`        | Run ESLint to check for coding standards                |
+| `lint:fix`    | Run linter and automatically fix code formatting issues |
+| `format`      | Check code formatting and style using Prettier          |
+| `format:fix`  | Run formatter and automatically fix issues              |
+| `test`        | Run tests using vitest                                  |
+| `test:cov`    | Run tests with coverage report                          |
+
+## Usage
+
+### Importing the Package
+
+You can import the package in your TypeScript or JavaScript files as follows:
+
+```typescript
+import { EventsFetcher } from "@grants-stack-indexer/data-flow";
+```
+
+### Example
+
+```typescript
+const eventsFetcher = new EventsFetcher(indexerClient);
+
+const chainId = 1;
+const blockNumber = 1000;
+const logIndex = 0;
+
+const result = await eventsFetcher.fetcEventsByBlockNumberAndLogIndex(
+    chainId,
+    blockNumber,
+    logIndex,
+);
+```

packages/data-flow/package.json

@@ -0,0 +1,35 @@
+{
+    "name": "@grants-stack-indexer/data-flow",
+    "version": "0.0.1",
+    "private": true,
+    "description": "",
+    "license": "MIT",
+    "author": "Wonderland",
+    "type": "module",
+    "main": "./dist/src/index.js",
+    "types": "./dist/src/index.d.ts",
+    "directories": {
+        "src": "src"
+    },
+    "files": [
+        "dist/*",
+        "package.json",
+        "!**/*.tsbuildinfo"
+    ],
+    "scripts": {
+        "build": "tsc -p tsconfig.build.json",
+        "check-types": "tsc --noEmit -p ./tsconfig.json",
+        "clean": "rm -rf dist/",
+        "format": "prettier --check \"{src,test}/**/*.{js,ts,json}\"",
+        "format:fix": "prettier --write \"{src,test}/**/*.{js,ts,json}\"",
+        "lint": "eslint \"{src,test}/**/*.{js,ts,json}\"",
+        "lint:fix": "pnpm lint --fix",
+        "test": "vitest run --config vitest.config.ts --passWithNoTests",
+        "test:cov": "vitest run --config vitest.config.ts --coverage"
+    },
+    "dependencies": {
+        "@grants-stack-indexer/indexer-client": "workspace:*",
+        "@grants-stack-indexer/shared": "workspace:*",
+        "viem": "2.21.19"
+    }
+}

packages/data-flow/src/eventsFetcher.ts

@@ -0,0 +1,22 @@
+import { IIndexerClient } from "@grants-stack-indexer/indexer-client";
+import { AnyProtocolEvent } from "@grants-stack-indexer/shared";
+
+import { IEventsFetcher } from "./interfaces/index.js";
+
+export class EventsFetcher implements IEventsFetcher {
+    constructor(private indexerClient: IIndexerClient) {}
+    /* @inheritdoc */
+    async fetchEventsByBlockNumberAndLogIndex(
+        chainId: bigint,
+        blockNumber: bigint,
+        logIndex: number,
+        limit: number = 100,
+    ): Promise<AnyProtocolEvent[]> {
+        return await this.indexerClient.getEventsAfterBlockNumberAndLogIndex(
+            chainId,
+            blockNumber,
+            logIndex,
+            limit,
+        );
+    }
+}

packages/data-flow/src/external.ts

@@ -0,0 +1 @@
+export { EventsFetcher } from "./internal.js";

packages/data-flow/src/index.ts

@@ -0,0 +1 @@
+export * from "./external.js";

packages/data-flow/src/interfaces/index.ts

@@ -0,0 +1,20 @@
+import { AnyProtocolEvent } from "@grants-stack-indexer/shared";
+
+/**
+ * Interface for the events fetcher
+ */
+export interface IEventsFetcher {
+    /**
+     * Fetch the events by block number and log index for a chain
+     * @param chainId id of the chain
+     * @param blockNumber block number to fetch events from
+     * @param logIndex log index in the block to fetch events from
+     * @param limit limit of events to fetch
+     */
+    fetchEventsByBlockNumberAndLogIndex(
+        chainId: bigint,
+        blockNumber: bigint,
+        logIndex: number,
+        limit?: number,
+    ): Promise<AnyProtocolEvent[]>;
+}

packages/data-flow/src/internal.ts

@@ -0,0 +1 @@
+export * from "./eventsFetcher.js";

packages/data-flow/test/unit/eventsFetcher.spec.ts

@@ -0,0 +1,78 @@
+import { beforeEach, describe, expect, it, Mocked, vi } from "vitest";
+
+import { IIndexerClient } from "@grants-stack-indexer/indexer-client";
+import { AnyProtocolEvent } from "@grants-stack-indexer/shared";
+
+import { EventsFetcher } from "../../src/eventsFetcher.js";
+
+describe("EventsFetcher", () => {
+    let indexerClientMock: Mocked<IIndexerClient>;
+    let eventsFetcher: EventsFetcher;
+
+    beforeEach(() => {
+        indexerClientMock = {
+            getEventsAfterBlockNumberAndLogIndex: vi.fn(),
+        };
+
+        eventsFetcher = new EventsFetcher(indexerClientMock);
+    });
+
+    it("should fetch events by block number and log index", async () => {
+        const mockEvents: AnyProtocolEvent[] = [
+            {
+                chain_id: 1,
+                block_number: 12345,
+                block_timestamp: 123123123,
+                contract_name: "Allo",
+                event_name: "PoolCreated",
+                src_address: "0x1234567890123456789012345678901234567890",
+                log_index: 0,
+                params: { contractAddress: "0x1234" },
+            },
+            {
+                chain_id: 1,
+                block_number: 12345,
+                block_timestamp: 123123123,
+                contract_name: "Allo",
+                event_name: "PoolCreated",
+                src_address: "0x1234567890123456789012345678901234567890",
+                log_index: 0,
+                params: { contractAddress: "0x1234" },
+            },
+        ];
+        const chainId = 1n;
+        const blockNumber = 1000n;
+        const logIndex = 0;
+        const limit = 100;
+
+        indexerClientMock.getEventsAfterBlockNumberAndLogIndex.mockResolvedValue(mockEvents);
+
+        const result = await eventsFetcher.fetchEventsByBlockNumberAndLogIndex(
+            chainId,
+            blockNumber,
+            logIndex,
+        );
+
+        expect(indexerClientMock.getEventsAfterBlockNumberAndLogIndex).toHaveBeenCalledWith(
+            chainId,
+            blockNumber,
+            logIndex,
+            limit,
+        );
+        expect(result).toEqual(mockEvents);
+    });
+
+    it("should handle errors thrown by indexer client", async () => {
+        const chainId = 1n;
+        const blockNumber = 1000n;
+        const logIndex = 0;
+
+        indexerClientMock.getEventsAfterBlockNumberAndLogIndex.mockRejectedValue(
+            new Error("Network error"),
+        );
+
+        await expect(
+            eventsFetcher.fetchEventsByBlockNumberAndLogIndex(chainId, blockNumber, logIndex),
+        ).rejects.toThrow("Network error");
+    });
+});

packages/data-flow/tsconfig.build.json

@@ -0,0 +1,13 @@
+/* Based on total-typescript no-dom library config */
+/* https://github.com/total-typescript/tsconfig */
+{
+    "extends": "../../tsconfig.build.json",
+    "compilerOptions": {
+        "composite": true,
+        "declarationMap": true,
+        "declaration": true,
+        "outDir": "dist"
+    },
+    "include": ["src/**/*"],
+    "exclude": ["node_modules", "dist", "tests"]
+}

packages/data-flow/tsconfig.json

@@ -0,0 +1,4 @@
+{
+    "extends": "../../tsconfig.json",
+    "include": ["src/**/*"]
+}

packages/data-flow/vitest.config.ts

@@ -0,0 +1,22 @@
+import path from "path";
+import { configDefaults, defineConfig } from "vitest/config";
+
+export default defineConfig({
+    test: {
+        globals: true, // Use Vitest's global API without importing it in each file
+        environment: "node", // Use the Node.js environment
+        include: ["test/**/*.spec.ts"], // Include test files
+        exclude: ["node_modules", "dist"], // Exclude certain directories
+        coverage: {
+            provider: "v8",
+            reporter: ["text", "json", "html"], // Coverage reporters
+            exclude: ["node_modules", "dist", "src/index.ts", ...configDefaults.exclude], // Files to exclude from coverage
+        },
+    },
+    resolve: {
+        alias: {
+            // Setup path alias based on tsconfig paths
+            "@": path.resolve(__dirname, "src"),
+        },
+    },
+});

packages/indexer-client/README.md

@@ -0,0 +1,36 @@
+# @grants-stack-indexer/indexer-client
+
+Is library for interacting with blockchain event indexing services.
+
+## Available Scripts
+
+Available scripts that can be run using `pnpm`:
+
+| Script        | Description                                             |
+| ------------- | ------------------------------------------------------- |
+| `build`       | Build library using tsc                                 |
+| `check-types` | Check types issues using tsc                            |
+| `clean`       | Remove `dist` folder                                    |
+| `lint`        | Run ESLint to check for coding standards                |
+| `lint:fix`    | Run linter and automatically fix code formatting issues |
+| `format`      | Check code formatting and style using Prettier          |
+| `format:fix`  | Run formatter and automatically fix issues              |
+| `test`        | Run tests using vitest                                  |
+| `test:cov`    | Run tests with coverage report                          |
+
+## Usage
+
+### Importing the Package
+
+You can import the package in your TypeScript or JavaScript files as follows:
+
+```typescript
+import { EnvioIndexerClient } from "@grants-stack-indexer/indexer-client";
+```
+
+### Example
+
+```typescript
+const envioIndexerClient = new EnvioIndexerClient("http://example.com/graphql", "secret");
+await envioIndexerClient.getEventsByBlockNumberAndLogIndex(1, 12345, 0);
+```