chore: changes to adjust pagination for fetching accounts

src/mappings/constants.ts

@@ -48,6 +48,7 @@ export const IBC_MESSAGE_TRANSFER_VALUE = b64encode('/ibc.applications.transfer.
 export const IBC_MESSAGE_RECEIVE_PACKET_VALUE = b64encode('/ibc.core.channel.v1.MsgRecvPacket');
 export const RECEPIENT_KEY = b64encode('recipient');
 export const SENDER_KEY = b64encode('sender');
+export const SPENDER_KEY = b64encode('spender');
 export const RECEIVER_KEY = b64encode('receiver');
 export const AMOUNT_KEY = b64encode('amount');
 export const TRANSFER_PORT_VALUE = 'transfer';

src/mappings/custom-types.ts

@@ -3,10 +3,10 @@ export type Balance = {
   amount: string;
 };
 
-interface PubKey {
+type PubKey = {
   '@type': string;
   key: string;
-}
+};
 
 export type BaseAccount = {
   '@type': string;

src/mappings/events/balances.ts

@@ -2,6 +2,7 @@ import { Account, Balances } from '../../types';
 import { BALANCE_FIELDS } from '../constants';
 import { b64decode } from '../utils';
 import { CosmosEvent } from '@subql/types-cosmos';
+import { AMOUNT_KEY, RECEIVER_KEY, SPENDER_KEY } from '../constants';
 
 interface Attribute {
   key: string;
@@ -33,11 +34,11 @@ export const balancesEventKit = () => {
   }
 
   function getData(cosmosEvent: CosmosEvent) {
-    let dataAlreadyDecoded = true;
-    const value = getAttributeValue(cosmosEvent.event, BALANCE_FIELDS.amount);
+    let dataAlreadyDecoded = false;
+    const value = getAttributeValue(cosmosEvent.event, AMOUNT_KEY);
 
     if (!value) {
-      dataAlreadyDecoded = false;
+      dataAlreadyDecoded = true;
     }
 
     const data = dataAlreadyDecoded

src/mappings/mappingHandlers.ts

@@ -35,7 +35,14 @@ import { boardAuxEventKit } from './events/boardAux';
 import { priceFeedEventKit } from './events/priceFeed';
 import { vaultsEventKit } from './events/vaults';
 import { reservesEventKit } from './events/reserves';
-import { AccountsResponse, BaseAccount, ModuleAccount, BalancesResponse, Balance, VestingAccount } from './custom-types';
+import {
+  AccountsResponse,
+  BaseAccount,
+  ModuleAccount,
+  BalancesResponse,
+  Balance,
+  VestingAccount,
+} from './custom-types';
 import { Operation, balancesEventKit } from './events/balances';
 import crossFetch from 'cross-fetch';
 
@@ -305,16 +312,26 @@ export const handleBalanceEvent = async (cosmosEvent: CosmosEvent): Promise<void
   }
 };
 
-const fetchAccounts = async (): Promise<AccountsResponse | void> => {
+const fetchAccounts = async (
+  offset: string,
+  limit: string,
+): Promise<[(BaseAccount | ModuleAccount | VestingAccount)[], string | null]> => {
   try {
-    const response = await crossFetch(FETCH_ACCOUNTS_URL);
-    const accounts: AccountsResponse = await response.json();
-    return accounts;
+    const url = new URL(FETCH_ACCOUNTS_URL);
+    url.searchParams.append('pagination.limit', limit);
+    url.searchParams.append('pagination.offset', offset);
+
+    const response = await crossFetch(url.toString());
+    const parsedResponse: AccountsResponse = await response.json();
+
+    const accounts: (BaseAccount | ModuleAccount | VestingAccount)[] = parsedResponse.accounts;
+
+    return [accounts, parsedResponse.pagination.next_key];
   } catch (error) {
     logger.error(`Error fetching accounts: ${error}`);
+    return [[], ''];
   }
 };
-
 const extractAddresses = (accounts: (BaseAccount | ModuleAccount | VestingAccount)[]): Array<string> => {
   return accounts
     .map((account) => {
@@ -348,20 +365,39 @@ const saveAccountBalances = async (address: string, balances: Balance[]) => {
     newBalance.denom = denom;
 
     await newBalance.save();
+    logger.info(`Save Operation Successful`);
   }
 };
 
-export const initiateBalancesTable = async (block: CosmosBlock): Promise<void> => {
-  const response = await fetchAccounts();
+const fetchAndSaveBalances = async (accountAddresses: string[]) => {
+  const fetchPromises = accountAddresses.map(async (address) => {
+    const response = await fetchBalance(address);
+    if (response) {
+      return saveAccountBalances(address, response.balances);
+    }
+  });
 
-  if (response) {
-    const accountAddresses = extractAddresses(response.accounts).filter(Boolean);
-    for (let address of accountAddresses) {
-      const response = await fetchBalance(address);
+  await Promise.all(fetchPromises);
+};
 
-      if (response) {
-        await saveAccountBalances(address, response.balances);
-      }
+export const initiateBalancesTable = async (block: CosmosBlock): Promise<void> => {
+  let nextKey: string | null = null;
+  let offset: number = 0;
+  const limit: number = 1000;
+
+  logger.info('Initiate Balances Table');
+  do {
+    logger.info('Fetching Accounts');
+    const [accounts, nextPaginationKey] = await fetchAccounts(String(offset), String(limit));
+    logger.info(`Accounts Fetched: ${accounts.length}`);
+    logger.info(`Pagination Key ${nextKey}`);
+    nextKey = nextPaginationKey;
+    if (accounts) {
+      logger.info('Storing Balances');
+      const accountAddresses = extractAddresses(accounts).filter(Boolean);
+      await fetchAndSaveBalances(accountAddresses);
+      logger.info('Balances Stored Successfully');
+      offset += accounts.length; 
     }
-  }
+  } while (nextKey);
 };