From c2a0dc34d8d72f21a710b8b2bd514d23b02bbd4f Mon Sep 17 00:00:00 2001 From: SibghatUllah1997 <163976001+SibghatCodora@users.noreply.github.com> Date: Sat, 30 Mar 2024 02:51:50 +0500 Subject: [PATCH] indexer added for trading volume --- src/config.ts | 9 +- src/controllers/snapHodlConfigController.ts | 309 ++++--- src/cronJobs.ts | 131 +-- src/index.ts | 8 +- src/models/SnapHodlConfig.ts | 170 +++- src/models/SnapHodlConfigBalance.ts | 39 +- src/models/TradingSnapshot.ts | 42 + src/openStaking.ts | 38 +- src/services/stakingService.ts | 201 ++--- src/services/tradingService.ts | 114 +++ src/types.ts | 29 +- src/utils/helpers.ts | 852 +++++++++++++++----- 12 files changed, 1420 insertions(+), 522 deletions(-) create mode 100644 src/models/TradingSnapshot.ts create mode 100644 src/services/tradingService.ts diff --git a/src/config.ts b/src/config.ts index 063cce3..9cc26d6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,7 +11,12 @@ export const DB_COLLECTION = process.env.DB_COLLECTION; export const DB_COLLECTION_STAKING_SNAPSHOT = process.env.DB_COLLECTION_STAKING_SNAPSHOT; export const DB_COLLECTION_SNAP_CONFIG_BALANCE = process.env.DB_COLLECTION_SNAP_CONFIG_BALANCE; export const DB_COLLECTION_SNAP_HODL_CONFIG = process.env.DB_COLLECTION_SNAP_HODL_CONFIG; +export const DB_COLLECTION_TRADING_SNAP_HODL_CONFIG = + process.env.DB_COLLECTION_TRADING_SNAP_HODL_CONFIG; +export const DB_COLLECTION_TRADING_SNAPSHOT = + process.env.DB_COLLECTION_TRADING_SNAPSHOT; export const CRON_SCHEDULE = process.env.CRON_SCHEDULE; +export const VOLUME_CRON_SCHEDULE = process.env.VOLUME_CRON_SCHEDULE; export const PORT = process.env.PORT; // An array to hold the environment variables @@ -22,7 +27,9 @@ const envVariables = [ { name: 'DB_COLLECTION', value: DB_COLLECTION }, { name: 'DB_COLLECTION_STAKING_SNAPSHOT', value: DB_COLLECTION_STAKING_SNAPSHOT }, { name: 'DB_COLLECTION_SNAP_CONFIG_BALANCE', value: DB_COLLECTION_SNAP_CONFIG_BALANCE }, - { name: 'CRON_SCHEDULE', value: CRON_SCHEDULE } + { name: 'CRON_SCHEDULE', value: CRON_SCHEDULE }, + { name: 'VOLUME_CRON_SCHEDULE', value: VOLUME_CRON_SCHEDULE }, + { name: 'PORT', value: PORT }, ]; // Check if each environment variable is set diff --git a/src/controllers/snapHodlConfigController.ts b/src/controllers/snapHodlConfigController.ts index 0dacef1..d901a62 100644 --- a/src/controllers/snapHodlConfigController.ts +++ b/src/controllers/snapHodlConfigController.ts @@ -1,153 +1,256 @@ // src/controllers/snapHodlConfigController.ts import { Request, Response } from 'express'; +import TradingSnapHodlModel from '../models/TradingSnapshot'; import SnapHodlConfigModel from '../models/SnapHodlConfig'; import SnapHodlConfigBalanceModel from '../models/SnapHodlConfigBalance'; import { SnapHodlConfig } from '../types'; +import BigNumber from 'bignumber.js'; import mongoose from 'mongoose'; function sortStakingContractData(data: any[]) { - return data.sort((a, b) => a.stakingPoolName.localeCompare(b.stakingPoolName)); + return data.sort((a, b) => a.stakingPoolName.localeCompare(b.stakingPoolName)); } function toLowerCaseStakingContractData(data: any[]) { - return data.map(item => ({ - ...item, - stakingContractAddress: item.stakingContractAddress.toLowerCase(), - tokenContractAddress: item.tokenContractAddress.toLowerCase(), - stakingPoolType: item.stakingPoolType.toLowerCase(), - })); + return data.map((item) => ({ + ...item, + stakingContractAddress: item.stakingContractAddress.toLowerCase(), + tokenContractAddress: item.tokenContractAddress.toLowerCase(), + stakingPoolType: item.stakingPoolType.toLowerCase(), + })); } - -export const retrieveSnapHodlConfigs = async (): Promise => { +export const getChainTradingSnapShotsTotalBySnapHodlConfigIdAndChainId = async ( + req: Request, + res: Response +) => { + if (req.query.snapHodlConfigId && req.query.chainId) { try { - return await SnapHodlConfigModel.find(); - } catch (err) { - if (err instanceof Error) { - throw new Error(err.message); - } else { - throw new Error('An error occurred when attempting to fetch SnapHodlConfigs'); + const snapHodlConfig: SnapHodlConfig | null = + await SnapHodlConfigModel.findOne({ + _id: { $eq: req.query.snapHodlConfigId }, + }); + if ( + snapHodlConfig && + snapHodlConfig.tradingVolumeContractData && + snapHodlConfig.tradingVolumeContractData?.length > 0 + ) { + let total = new BigNumber("0"); + let result: any[] = []; + for ( + let i = 0; + i < snapHodlConfig.tradingVolumeContractData.length; + i++ + ) { + if ( + snapHodlConfig.tradingVolumeContractData[i].chainId === + req.query.chainId + ) { + const tradingSnapshot = await TradingSnapHodlModel.findOne({ + chainId: { $eq: req.query.chainId }, + liquidityPoolAddress: { + $eq: snapHodlConfig.tradingVolumeContractData[i] + .liquidityPoolAddress, + }, + tokenContractAddress: { + $eq: snapHodlConfig.tradingVolumeContractData[i] + .tokenContractAddress, + }, + }); + + if (tradingSnapshot && tradingSnapshot?.tradesVolumeBalances) { + console.log(tradingSnapshot?.tradesVolumeBalances); + Object.entries(tradingSnapshot?.tradesVolumeBalances).forEach( + ([address, balance]) => { + console.log(balance) + total = total.plus(balance); + }); + } + } } + return res.json({totalTradingByChainId:total}); + } + } catch (error) { + if (error instanceof Error) { + return res.status(500).json({ message: error.message }); + } else { + return res.status(500).json({ message: "An unexpected error occurred." }); + } } + } + return res.json({total : 0}); +}; + +export const retrieveSnapHodlConfigs = async (): Promise => { + try { + return await SnapHodlConfigModel.find(); + } catch (err) { + if (err instanceof Error) { + throw new Error(err.message); + } else { + throw new Error('An error occurred when attempting to fetch SnapHodlConfigs'); + } + } }; export const getSnapHodlConfigs = async (req: Request, res: Response) => { - try { - const snapHodlConfigs = await retrieveSnapHodlConfigs(); - res.json(snapHodlConfigs); - } catch (err) { - if (err instanceof Error) { - res.status(500).json({ message: err.message }); - } else { - res.status(500).json({ message: 'An error occurred when attempting to request SnapHodlConfigs to be retrieved' }); - } + try { + const snapHodlConfigs = await retrieveSnapHodlConfigs(); + res.json(snapHodlConfigs); + } catch (err) { + if (err instanceof Error) { + res.status(500).json({ message: err.message }); + } else { + res.status(500).json({ message: 'An error occurred when attempting to request SnapHodlConfigs to be retrieved' }); } + } }; export const createSnapHodlConfig = async (req: Request, res: Response) => { - const { snapShotConfigName, isActive, stakingContractData } = req.body; + const { snapShotConfigName, isActive, stakingContractData } = req.body; - // Convert stakingContractData values to lower case - const lowerCaseStakingContractData = toLowerCaseStakingContractData(stakingContractData); + // Convert stakingContractData values to lower case + const lowerCaseStakingContractData = toLowerCaseStakingContractData(stakingContractData); - // Sort stakingContractData by stakingPoolName for consistent comparison - const sortedStakingContractData = sortStakingContractData(lowerCaseStakingContractData); + // Sort stakingContractData by stakingPoolName for consistent comparison + const sortedStakingContractData = sortStakingContractData(lowerCaseStakingContractData); - // Check for duplicate stakingContractData - const duplicateConfig = await SnapHodlConfigModel.findOne({ - stakingContractData: { $eq: sortedStakingContractData }, - }); + // Check for duplicate stakingContractData + const duplicateConfig = await SnapHodlConfigModel.findOne({ + stakingContractData: { $eq: sortedStakingContractData }, + }); - if (duplicateConfig) { - return res.status(400).json({ - message: `Duplicate stakingContractData detected. Please use the ${duplicateConfig.snapShotConfigName} config, which has the same stakingContractData, or change the stakingContractData for the new config.` - }); - } - - const newSnapHodlConfig = new SnapHodlConfigModel({ - snapShotConfigName, - isActive, - stakingContractData: sortedStakingContractData, + if (duplicateConfig) { + return res.status(400).json({ + message: `Duplicate stakingContractData detected. Please use the ${duplicateConfig.snapShotConfigName} config, which has the same stakingContractData, or change the stakingContractData for the new config.` }); + } - try { - const savedSnapHodlConfig = await newSnapHodlConfig.save(); - res.json(savedSnapHodlConfig); - } catch (err) { - if (err instanceof Error) { - res.status(500).json({ message: err.message }); - } else { - res.status(500).json({ message: 'An error occurred' }); - } + const newSnapHodlConfig = new SnapHodlConfigModel({ + snapShotConfigName, + isActive, + stakingContractData: sortedStakingContractData, + }); + + try { + const savedSnapHodlConfig = await newSnapHodlConfig.save(); + res.json(savedSnapHodlConfig); + } catch (err) { + if (err instanceof Error) { + res.status(500).json({ message: err.message }); + } else { + res.status(500).json({ message: 'An error occurred' }); } + } }; export const getSnapShotBySnapShotIdAndAddress = async (req: Request, res: Response) => { - try { - let { snapShotId, address } = req.params; - const { raw } = req.query; + try { + let { snapShotId, address } = req.params; + const { raw } = req.query; - address = address.toLowerCase(); + address = address.toLowerCase(); - const snapHodlConfigBalance = await SnapHodlConfigBalanceModel.findOne({ snapHodlConfigId: snapShotId }); + const snapHodlConfigBalance = await SnapHodlConfigBalanceModel.findOne({ snapHodlConfigId: snapShotId }); - if (!snapHodlConfigBalance) { - return res.status(404).json({ message: 'SnapShot not found' }); - } + if (!snapHodlConfigBalance) { +return res.status(404).json({ message: 'SnapShot not found' }); + } - const snapShotBalance = snapHodlConfigBalance.totalStakedBalance.get(address); + const snapShotBalance = snapHodlConfigBalance.totalStakedBalance.get(address); - if (!snapShotBalance) { - return res.status(404).json({ message: 'Address not found in SnapShot' }); - } + if (!snapShotBalance) { + return res.status(404).json({ message: 'Address not found in SnapShot' }); + } - if (raw === 'true') { - return res.send(snapShotBalance.toString()); - } else { - const result = { - snapShotConfigName: snapHodlConfigBalance.snapShotConfigName, - address: address, - snapShotBalance: snapShotBalance, - updatedAt: snapHodlConfigBalance.updatedAt, - }; - - return res.json(result); - } - } catch (error) { - if (error instanceof Error) { - return res.status(500).json({ message: error.message }); - } else { - return res.status(500).json({ message: "An unexpected error occurred." }); - } + if (raw === 'true') { + return res.send(snapShotBalance.toString()); + } else { + const result = { + snapShotConfigName: snapHodlConfigBalance.snapShotConfigName, + address: address, + snapShotBalance: snapShotBalance, + updatedAt: snapHodlConfigBalance.updatedAt, + }; + + return res.json(result); + } + } catch (error) { + if (error instanceof Error) { + return res.status(500).json({ message: error.message }); + } else { + return res.status(500).json({ message: "An unexpected error occurred." }); } + } }; // New function to retrieve all documents from DB_COLLECTION_SNAP_CONFIG_BALANCE export const getAllSnapShots = async (req: Request, res: Response) => { - try { - const query: { [key: string]: any } = {}; - if(req.query.snapHodlConfigId){ - query.snapHodlConfigId = req.query.snapHodlConfigId; - } - const page = parseInt(req.query.page as string) || 1; // defaults to 1 if not provided - const limit = parseInt(req.query.limit as string) || 10; // defaults to 10 if not provided - const skip = (page - 1) * limit; - - // Find all documents in the collection with pagination - const snapHodlConfigBalances = await SnapHodlConfigBalanceModel.find(query).skip(skip).limit(limit); - - // Send the result as a JSON response - return res.json(snapHodlConfigBalances); - } catch (error) { - if (error instanceof Error) { - return res.status(500).json({ message: error.message }); - } else { - return res.status(500).json({ message: "An unexpected error occurred." }); - } + try { + const query: { [key: string]: any } = {}; + if(req.query.snapHodlConfigId){ + query.snapHodlConfigId = req.query.snapHodlConfigId; } -}; \ No newline at end of file + const page = parseInt(req.query.page as string) || 1; // defaults to 1 if not provided + const limit = parseInt(req.query.limit as string) || 10; // defaults to 10 if not provided + const skip = (page - 1) * limit; + + // Find all documents in the collection with pagination + const snapHodlConfigBalances = await SnapHodlConfigBalanceModel.find(query).skip(skip).limit(limit); + + // Send the result as a JSON response + return res.json(snapHodlConfigBalances); + } catch (error) { + if (error instanceof Error) { + return res.status(500).json({ message: error.message }); + } else { + return res.status(500).json({ message: "An unexpected error occurred." }); + } + } +}; + +// export async function saveBulkUserTransactionVolume( +// data: any +// ): Promise { +// // const client = new MongoClient(connectionString); +// // let leaderboard = null; +// try { +// // await client.connect(); +// // const database = client.db(dbName); +// // console.log("database: ", database); +// // const collection = database.collection(dbCollection); + +// // console.log("collection: ", collection); + +// const bulkOperations: any = []; + +// // Prepare update operations for existing documents +// Object.keys(data).forEach((item) => { +// bulkOperations.push({ +// updateOne: { +// filter: { walletAddress: item }, // Filter by unique key +// update: { +// $set: { walletAddress: item, totalVolume: data[item] }, +// }, // Update all fields in the document +// upsert: true, // Insert a new document if no match is found +// }, +// }); +// }); + +// // You can add additional insertOne operations for new documents here +// console.log("bulkOperations: ", bulkOperations); +// const result = await LeaderboardModel.bulkWrite(bulkOperations); +// console.log("result: ", result); +// } catch (err) { +// console.error( +// "Error while saving the data against address the database:", +// err +// ); +// } +// // return leaderboard; +// } diff --git a/src/cronJobs.ts b/src/cronJobs.ts index 6d0e5d1..61eb171 100644 --- a/src/cronJobs.ts +++ b/src/cronJobs.ts @@ -3,59 +3,92 @@ import cron from 'node-cron'; import axios from "axios"; import _ from 'lodash'; -import { SnapHodlConfig, StakingContractDataItem } from "./types"; +import { SnapHodlConfig, StakingContractDataItem, TradingVolumeContractDataItem, } from "./types"; import { retrieveSnapHodlConfigs } from './controllers/snapHodlConfigController'; -import { processStakingContractDataItem, getSnapHodlConfigBalance } from "./utils/helpers"; +import { processStakingContractDataItem, getSnapHodlConfigBalance, processTradingContractDataItem, getSnapHodlConfigTradingVolumeBalance, getSnapShotBySnapShotUserVolumeAndReward } from "./utils/helpers"; import { - APP_NAME, - DB_CONNECTION_STRING, - DB_NAME, - DB_COLLECTION_STAKING_SNAPSHOT, - CRON_SCHEDULE + APP_NAME, + DB_CONNECTION_STRING, + DB_NAME, + DB_COLLECTION_STAKING_SNAPSHOT, + CRON_SCHEDULE } from './config'; export const scheduleJobs = () => { - // Schedule cron job - cron.schedule(CRON_SCHEDULE!, async () => { - console.log('Running the job every 5 minutes'); - try { - // Fetch data from the API - - const snapHodlConfigs: SnapHodlConfig[] = await retrieveSnapHodlConfigs(); - - let uniqueStakingContractDataItems: StakingContractDataItem[] = []; - for (const item of snapHodlConfigs) { - const { stakingContractData, isActive } = item; - if (isActive) { - uniqueStakingContractDataItems = [ - ...uniqueStakingContractDataItems, - ...stakingContractData - ]; - } - } - - // Filter unique stakingContractData based on stakingContractAddress, tokenContractAddress, and chainId - uniqueStakingContractDataItems = _.uniqBy(uniqueStakingContractDataItems, ({ stakingContractAddress, tokenContractAddress, chainId }) => { - return `${stakingContractAddress}-${tokenContractAddress}-${chainId}`; - }); - - // Start processing uniqueStakingContractDataItems concurrently - await Promise.all(uniqueStakingContractDataItems.map(item => processStakingContractDataItem( - item, - DB_NAME!, - DB_COLLECTION_STAKING_SNAPSHOT!, - DB_CONNECTION_STRING!, - APP_NAME! - ))); - - // After processStakingContractDataItem function calls - await Promise.all(snapHodlConfigs.map(getSnapHodlConfigBalance)); - - const utcStr = new Date().toUTCString(); - console.log(`Cron finished at:`, utcStr); - - } catch (error) { - console.error("Error fetching data from the API or processing data:", error); + // Schedule cron job + cron.schedule(CRON_SCHEDULE!, async () => { + console.log('Running the job every 5 minutes'); + try { + // Fetch data from the API + + const snapHodlConfigs: SnapHodlConfig[] = await retrieveSnapHodlConfigs(); + + let uniqueStakingContractDataItems: StakingContractDataItem[] = []; + let uniqueTradingContractDataItems: TradingVolumeContractDataItem[] = []; + for (const item of snapHodlConfigs) { + const { stakingContractData, tradingVolumeContractData, isActive } = item; + if (isActive) { + uniqueStakingContractDataItems = [ + ...uniqueStakingContractDataItems, + ...stakingContractData, + ]; + uniqueTradingContractDataItems = [ + ...uniqueTradingContractDataItems, + ...tradingVolumeContractData, + ]; + } + } + + // Filter unique stakingContractData based on stakingContractAddress, tokenContractAddress, and chainId + uniqueStakingContractDataItems = _.uniqBy(uniqueStakingContractDataItems, ({ stakingContractAddress, tokenContractAddress, chainId }) => { + return `${stakingContractAddress}-${tokenContractAddress}-${chainId}`; + }); + + // Filter unique tradingVolumeContractData based on liquidityPoolAddress, tokenContractAddress, and chainId + uniqueTradingContractDataItems = _.uniqBy( + uniqueTradingContractDataItems, + ({ liquidityPoolAddress, tokenContractAddress, chainId }) => { + return `${liquidityPoolAddress}-${tokenContractAddress}-${chainId}`; } - }); + ); + + // console.log({ uniqueTradingContractDataItems }); + + console.log('1============processTradingContractDataItem==============='); + await Promise.all( + uniqueTradingContractDataItems.map((item) => + processTradingContractDataItem(item) + ) + ); + console.log( + '2============getSnapHodlConfigTradingVolumeBalance==============' + ); + await Promise.all( + snapHodlConfigs.map(getSnapHodlConfigTradingVolumeBalance) + ); + console.log('3============processStakingContractDataItem============='); + // Start processing uniqueStakingContractDataItems concurrently + await Promise.all( uniqueStakingContractDataItems.map((item) => processStakingContractDataItem( + item, + DB_NAME!, + DB_COLLECTION_STAKING_SNAPSHOT!, + DB_CONNECTION_STRING!, + APP_NAME! + ))); + console.log('4========getSnapHodlConfigBalance============'); + // After processStakingContractDataItem function calls + await Promise.all(snapHodlConfigs.map(getSnapHodlConfigBalance)); + console.log('5=========Reward========='); + // sum the volume of user and their reward + await Promise.all( + snapHodlConfigs.map(getSnapShotBySnapShotUserVolumeAndReward) + ); + + const utcStr = new Date().toUTCString(); + console.log(`Cron finished at:`, utcStr); + + } catch (error) { + console.error("Error fetching data from the API or processing data:", error); + } + }); }; diff --git a/src/index.ts b/src/index.ts index fb30958..4d4e679 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,7 @@ import dotenv from 'dotenv'; import mongoose from 'mongoose'; import _ from 'lodash'; import { scheduleJobs } from './cronJobs'; -import { getSnapHodlConfigs, createSnapHodlConfig, getSnapShotBySnapShotIdAndAddress, getAllSnapShots } from './controllers/snapHodlConfigController'; +import { getSnapHodlConfigs, createSnapHodlConfig, getSnapShotBySnapShotIdAndAddress, getAllSnapShots, getChainTradingSnapShotsTotalBySnapHodlConfigIdAndChainId } from './controllers/snapHodlConfigController'; import cors from 'cors'; @@ -18,12 +18,13 @@ app.use(express.json()); app.use(cors()); mongoose.connect(DB_CONNECTION_STRING as string, { - dbName: DB_NAME as string -}) + dbName: DB_NAME as string + }) .then(() => console.log('MongoDB connection established')) .catch(err => console.log('MongoDB connection error:', err)); scheduleJobs(); +// blockToBlockVolumeScheduleJobs(); app.get('/', async (req, res) => { res.send('Server running'); @@ -35,6 +36,7 @@ app.post('/snapHodlConfig', createSnapHodlConfig); app.get('/getSnapShotBySnapShotIdAndAddress/:snapShotId/:address', getSnapShotBySnapShotIdAndAddress); app.get('/getSnapShotBySnapShotIdAndAddress/:snapShotId/:address/raw', getSnapShotBySnapShotIdAndAddress); +app.get('/getChainTradingSnapShotsTotalBySnapHodlConfigIdAndChainId', getChainTradingSnapShotsTotalBySnapHodlConfigIdAndChainId); app.get('/getAllSnapShots', getAllSnapShots); diff --git a/src/models/SnapHodlConfig.ts b/src/models/SnapHodlConfig.ts index 5b591cb..7b7aa55 100644 --- a/src/models/SnapHodlConfig.ts +++ b/src/models/SnapHodlConfig.ts @@ -8,72 +8,154 @@ import dotenv from 'dotenv'; dotenv.config(); export interface IStakingContractData { - _id?: boolean; - stakingPoolName: { type: string, required: boolean }; - stakingContractAddress: { type: string, match: RegExp, validate: { validator: (value: any) => boolean, message: string }, required: boolean }; - stakingPoolType: { type: string, enum: string[], required: boolean }; - tokenContractAddress: { type: string, match: RegExp, validate: { validator: (value: any) => boolean, message: string }, required: boolean }; - chainId: { type: string, required: boolean }; - fromBlock: { type: number, min: number, required: boolean }; - toBlock: { type: string | number, validate: { validator: (value: any) => boolean, message: string }, required: boolean }; - blockIterationSize: { type: number, min: number, required: boolean }; + _id?: boolean; + stakingPoolName: { type: string, required: boolean }; + stakingContractAddress: { type: string, match: RegExp, validate: { validator: (value: any) => boolean, message: string }, required: boolean }; + stakingPoolType: { type: string, enum: string[], required: boolean }; + tokenContractAddress: { type: string, match: RegExp, validate: { validator: (value: any) => boolean, message: string }, required: boolean }; + chainId: { type: string, required: boolean }; + fromBlock: { type: number, min: number, required: boolean }; + toBlock: { type: string | number, validate: { validator: (value: any) => boolean, message: string }, required: boolean }; + blockIterationSize: { type: number, min: number, required: boolean }; + excludedWalletAddresses: string[] +} + +export interface ITradingVolumeContractData { + _id?: boolean; + tradingPoolName: { type: string; required: boolean }; + tradingPoolType: { type: string; enum: string[]; required: boolean }; + tokenContractAddress: { + type: string; + match: RegExp; + validate: { validator: (value: any) => boolean; message: string }; + required: boolean; + }; + chainId: { type: string; required: boolean }; + fromBlock: { type: number; min: number; required: boolean }; + toBlock: { + type: string | number; + validate: { validator: (value: any) => boolean; message: string }; + required: boolean; + }; + blockIterationSize: { type: number; min: number; required: boolean }; + liquidityPoolAddress: { + type: string; + match: RegExp; + validate: { validator: (value: any) => boolean; message: string }; + required: boolean; + }; + minimumTradingBalance: { type: number; min: number; required: boolean }; + excludedWalletAddresses: string[]; } export interface ISnapHodlConfig extends Document { - snapShotConfigName: { type: string, required: boolean }; - isActive: { type: boolean, required: boolean }; - stakingContractData: IStakingContractData[]; + snapShotConfigName: { type: string, required: boolean }; + isActive: { type: boolean, required: boolean }; + stakingContractData: IStakingContractData[]; + tradingVolumeContractData: ITradingVolumeContractData[]; } const ethereumAddressRegex = /^(0x)?[0-9a-f]{40}$/i; export const stakingContractDataSchema = new Schema({ - _id: false, - stakingPoolName: { type: String, required: true }, - stakingContractAddress: { - type: String, - required: true, - validate: { - validator: function (value: string) { - return ethereumAddressRegex.test(value); - }, - message: 'A valid EVM address is required for stakingContractAddress.' + _id: false, + stakingPoolName: { type: String, required: true }, + stakingContractAddress: { + type: String, + required: true, + validate: { + validator: function (value: string) { + return ethereumAddressRegex.test(value); + }, + message: 'A valid EVM address is required for stakingContractAddress.' + } + }, + stakingPoolType: { type: String, enum: ['standard', 'open'], required: true }, + tokenContractAddress: { + type: String, + required: true, + validate: { + validator: function (value: string) { + return ethereumAddressRegex.test(value); + }, + message: 'A valid EVM address is required for tokenContractAddress.' + } + }, + chainId: { type: String, required: true }, + fromBlock: { type: Number, min: 0, required: true }, + toBlock: { + type: Schema.Types.Mixed, + required: true, + validate: { + validator: function (value: string | number) { + if (typeof value === 'string') { + return value === 'latest'; } + return Number.isInteger(value) && value >= 0; + }, + message: 'Invalid value for `toBlock`. Must be a non-negative integer or "latest".' + } + }, + blockIterationSize: { type: Number, min: 0, required: true }, + excludedWalletAddresses: [{type: String, required: false}], +}); + +export const tradingVolumeContractDataSchema = + new Schema({ + _id: false, + tradingPoolName: { type: String, required: true }, + tradingPoolType: { + type: String, + enum: ["standard", "open"], + required: true, }, - stakingPoolType: { type: String, enum: ['standard', 'open'], required: true }, tokenContractAddress: { - type: String, - required: true, - validate: { - validator: function (value: string) { - return ethereumAddressRegex.test(value); - }, - message: 'A valid EVM address is required for tokenContractAddress.' - } + type: String, + required: true, + validate: { + validator: function (value: string) { + return ethereumAddressRegex.test(value); + }, + message: "A valid EVM address is required for tokenContractAddress.", + }, }, chainId: { type: String, required: true }, fromBlock: { type: Number, min: 0, required: true }, toBlock: { - type: Schema.Types.Mixed, - required: true, - validate: { - validator: function (value: string | number) { - if (typeof value === 'string') { - return value === 'latest'; - } - return Number.isInteger(value) && value >= 0; - }, - message: 'Invalid value for `toBlock`. Must be a non-negative integer or "latest".' - } + type: Schema.Types.Mixed, + required: true, + validate: { + validator: function (value: string | number) { + if (typeof value === "string") { + return value === "latest"; + } + return Number.isInteger(value) && value >= 0; + }, + message: + 'Invalid value for `toBlock`. Must be a non-negative integer or "latest".', + }, }, blockIterationSize: { type: Number, min: 0, required: true }, + liquidityPoolAddress: { + type: String, + required: true, + validate: { + validator: function (value: string) { + return ethereumAddressRegex.test(value); + }, + message: "A valid EVM address is required for tokenContractAddress.", + }, + }, + minimumTradingBalance: { type: Number, min: 0, required: true }, + excludedWalletAddresses: [{type: String, required: false}], }); const SnapHodlConfigSchema = new Schema({ snapShotConfigName: { type: String, required: true }, isActive: { type: Boolean, required: true }, - stakingContractData: [stakingContractDataSchema] -}, { collection: DB_COLLECTION_SNAP_HODL_CONFIG }); + stakingContractData: [stakingContractDataSchema], + tradingVolumeContractData: [tradingVolumeContractDataSchema], + }, { collection: DB_COLLECTION_SNAP_HODL_CONFIG }); const SnapHodlConfigModel = mongoose.model('SnapHodlConfigModel', SnapHodlConfigSchema); diff --git a/src/models/SnapHodlConfigBalance.ts b/src/models/SnapHodlConfigBalance.ts index 3bd78c6..7570826 100644 --- a/src/models/SnapHodlConfigBalance.ts +++ b/src/models/SnapHodlConfigBalance.ts @@ -1,34 +1,53 @@ import mongoose, { Document, Schema, Types } from 'mongoose'; -import { IStakingContractData, stakingContractDataSchema } from './SnapHodlConfig'; +import { IStakingContractData, stakingContractDataSchema, ITradingVolumeContractData, tradingVolumeContractDataSchema} from './SnapHodlConfig'; import { DB_COLLECTION_SNAP_CONFIG_BALANCE } from '../config'; interface IStakingContractDataBalance extends IStakingContractData { - totalStakedBalance: string; + totalStakedBalance: string; +} + +interface ITotalTradingVolumeBalance extends ITradingVolumeContractData { + totalTradingVolumeBalance: string; } interface ISnapHodlConfigBalance extends Document { - snapHodlConfigId: mongoose.Types.ObjectId; - snapShotConfigName: string; - stakingContractDataBalances: IStakingContractDataBalance[]; - totalStakedBalance: Map; - createdAt: Date; - updatedAt: Date; + snapHodlConfigId: mongoose.Types.ObjectId; + snapShotConfigName: string; + stakingContractDataBalances: IStakingContractDataBalance[]; + totalTradingVolumeBalance: Object; + totalStakedBalance: Map; + totalTradingVolume: Map; + totalUserVolume: Map; + totalUserReward: Map; + totalVolume: string; + createdAt: Date; + updatedAt: Date; } +const tradingContractDataBalanceSchema = new Schema( { + ...tradingVolumeContractDataSchema.obj, + totalTradingVolumeBalance: { type: String, required: true }, + } +); const stakingContractDataBalanceSchema = new Schema({ ...stakingContractDataSchema.obj, totalStakedBalance: { type: String, required: true }, -}); + }); const SnapHodlConfigBalanceSchema = new Schema({ snapHodlConfigId: { type: Schema.Types.ObjectId, required: true }, // Changed here snapShotConfigName: { type: String, required: true }, stakingContractDataBalances: [stakingContractDataBalanceSchema], totalStakedBalance: { type: Map, of: String }, + totalTradingVolumeBalance: [tradingContractDataBalanceSchema], + totalTradingVolume: { type: Map, of: String }, + totalUserVolume: { type: Map, of: String }, + totalVolume: { type: String, required: true }, + totalUserReward: { type: Map, of: String }, createdAt: { type: Date, required: true }, updatedAt: { type: Date, required: true }, - + }, { collection: DB_COLLECTION_SNAP_CONFIG_BALANCE }); const SnapHodlConfigBalanceModel = mongoose.model('SnapHodlConfigBalanceModel', SnapHodlConfigBalanceSchema); diff --git a/src/models/TradingSnapshot.ts b/src/models/TradingSnapshot.ts new file mode 100644 index 0000000..c9e34f5 --- /dev/null +++ b/src/models/TradingSnapshot.ts @@ -0,0 +1,42 @@ +// src/SnapHodlConfig.ts + +import mongoose, { Document, Schema } from "mongoose"; + +import dotenv from "dotenv"; + +dotenv.config(); + +export interface ITradingSnapshotSchema extends Document { + _id?: number; + chainId: string; + liquidityPoolAddress: string; + tokenContractAddress: string; + latestBlockCaptured: number; + stakingPoolName: string; + stakingPoolType: string; + timestamp: Date; + uniqueTraders: any[]; + tradesVolumeBalances: any[]; +} + +const TradingSnapshotSchema = new Schema( + { + chainId: { type: String, required: true }, + liquidityPoolAddress: { type: String, required: true }, + tokenContractAddress: { type: String, required: true }, + latestBlockCaptured: Number, + stakingPoolName: { type: String, required: true }, + stakingPoolType: { type: String, required: true }, + timestamp: Date, + uniqueTraders: Array, + tradesVolumeBalances: Object, + }, + { collection: "tradingSnapshot" } +); + +const TradingSnapHodlModel = mongoose.model( + "tradingSnapshot", + TradingSnapshotSchema +); + +export default TradingSnapHodlModel; diff --git a/src/openStaking.ts b/src/openStaking.ts index 094f2a2..d9793ef 100644 --- a/src/openStaking.ts +++ b/src/openStaking.ts @@ -37,6 +37,7 @@ export async function getUniqueStakersFromOpenStaking( existingSnapshot ? existingSnapshot.uniqueStakers : [] ); + // Convert block identifiers to actual block numbers const currentBlockNumber = await web3Instance.eth.getBlockNumber(); const resolvedFromBlock = fromBlock === "latest" ? currentBlockNumber : fromBlock; const resolvedToBlock = toBlock === "latest" ? currentBlockNumber : toBlock; @@ -59,17 +60,38 @@ export async function getUniqueStakersFromOpenStaking( address: tokenContractAddress, topics: [transferEventSignature, null, web3Instance.eth.abi.encodeParameter("address", stakingContractAddress)], }; + try { const logs = await web3Instance.eth.getPastLogs(transferEventFilter); - - for (const log of logs) { - const transactionReceipt = await web3Instance.eth.getTransactionReceipt(log.transactionHash); - const transactionSender = transactionReceipt.from.toLowerCase(); - - // Add the transaction sender to the set of unique stakers - uniqueStakers.add(transactionSender); - } + console.log("Fetched logs:", logs); + + logs.forEach((log) => { + console.log("Log:", log); + + const eventInterface = tokenContract.options.jsonInterface.find( + (i: any) => i.signature === log.topics[0] + ); + + if (!eventInterface) { + console.error("Event interface not found for signature:", log.topics[0]); + return; + } + + const inputs = eventInterface.inputs as AbiInput[]; + + const event = web3Instance.eth.abi.decodeLog( + inputs, + log.data, + log.topics.slice(1) + ); + console.log("Decoded event:", event); + + // Check if the destination address matches the staking contract address + if (event.dst.toLowerCase() === stakingContractAddress.toLowerCase()) { + uniqueStakers.add(event.src.toLowerCase()); + } + }); console.log("Unique stakers fetched for blocks", currentBlock, "to", endBlock); await saveStakingSnapshot( diff --git a/src/services/stakingService.ts b/src/services/stakingService.ts index fde62e9..5e29f9a 100644 --- a/src/services/stakingService.ts +++ b/src/services/stakingService.ts @@ -3,122 +3,123 @@ import { MongoClient } from "mongodb"; export async function getLatestStakingSnapshot( - stakingContractAddress: string, - tokenContractAddress: string, - chainId: string, - dbName: string, - dbCollection: string, - connectionString: string + stakingContractAddress: string, + tokenContractAddress: string, + chainId: string, + dbName: string, + dbCollection: string, + connectionString: string ): Promise { - const client = new MongoClient(connectionString); - let snapshot = null; - try { - await client.connect(); - const database = client.db(dbName); - const collection = database.collection(dbCollection); - - const filter = { - stakingContractAddress, - tokenContractAddress, - chainId - }; - - snapshot = await collection.findOne(filter); - } catch (err) { - console.error("Error fetching the latest staking snapshot from the database:", err); - } finally { - await client.close(); - } - return snapshot; + const client = new MongoClient(connectionString); + let snapshot = null; + try { + await client.connect(); + const database = client.db(dbName); + const collection = database.collection(dbCollection); + console.log("collection: ", collection); + const filter = { + stakingContractAddress, + tokenContractAddress, + chainId + }; + console.log("filter: ", filter); + snapshot = await collection.findOne(filter); + console.log("snapshot: ", snapshot); + } catch (err) { + console.error("Error fetching the latest staking snapshot from the database:", err); + } finally { + await client.close(); + } + return snapshot; } export async function saveStakingSnapshot( - stakingPoolName: string, - stakingContractAddress: string, - stakingPoolType: string, - tokenContractAddress: string, - chainId: string, - latestBlockCaptured: number, - uniqueStakers: string[], - dbName: string, - dbCollection: string, - connectionString: string + stakingPoolName: string, + stakingContractAddress: string, + stakingPoolType: string, + tokenContractAddress: string, + chainId: string, + latestBlockCaptured: number, + uniqueStakers: string[], + dbName: string, + dbCollection: string, + connectionString: string ) { - const client = new MongoClient(connectionString); - try { - await client.connect(); - const database = client.db(dbName); - const collection = database.collection(dbCollection); + const client = new MongoClient(connectionString); + try { + await client.connect(); + const database = client.db(dbName); + const collection = database.collection(dbCollection); - const snapshot = { - stakingPoolName, - stakingContractAddress, - stakingPoolType, - tokenContractAddress, - chainId, - latestBlockCaptured, - uniqueStakers, - timestamp: new Date() - }; + const snapshot = { + stakingPoolName, + stakingContractAddress, + stakingPoolType, + tokenContractAddress, + chainId, + latestBlockCaptured, + uniqueStakers, + timestamp: new Date() + }; - const filter = { - stakingContractAddress, - tokenContractAddress, - chainId - }; + const filter = { + stakingContractAddress, + tokenContractAddress, + chainId + }; - const update = { - $set: snapshot - }; + const update = { + $set: snapshot + }; - const options = { - upsert: true - }; + const options = { + upsert: true + }; - await collection.updateOne(filter, update, options); - } catch (err) { - console.error("Error saving staking snapshot to the database:", err); - } finally { - await client.close(); - } + await collection.updateOne(filter, update, options); + } catch (err) { + console.error("Error saving staking snapshot to the database:", err); + } finally { + await client.close(); + } } export async function saveStakedBalances( - stakingContractAddress: string, - tokenContractAddress: string, - chainId: string, - stakedBalances: { [stakerAddress: string]: string }, - dbName: string, - dbCollection: string, - connectionString: string + stakingContractAddress: string, + tokenContractAddress: string, + chainId: string, + stakedBalances: { [stakerAddress: string]: string }, + dbName: string, + dbCollection: string, + connectionString: string ) { - const client = new MongoClient(connectionString); - try { - await client.connect(); - const database = client.db(dbName); - const collection = database.collection(dbCollection); + const client = new MongoClient(connectionString); + try { + await client.connect(); + const database = client.db(dbName); + const collection = database.collection(dbCollection); - const filter = { - stakingContractAddress, - tokenContractAddress, - chainId - }; + const filter = { + stakingContractAddress, + tokenContractAddress, + chainId + }; - const update = { - $set: { - stakedBalances, - timestamp: new Date() - } - }; + const update = { + $set: { + stakedBalances, + timestamp: new Date() + } + }; - const options = { - upsert: true - }; + const options = { + upsert: true + }; - await collection.updateOne(filter, update, options); - } catch (err) { - console.error("Error saving staked balances to the database:", err); - } finally { - await client.close(); - } + await collection.updateOne(filter, update, options); + } catch (err) { + console.error("Error saving staked balances to the database:", err); + } finally { + await client.close(); + } } diff --git a/src/services/tradingService.ts b/src/services/tradingService.ts new file mode 100644 index 0000000..b256a1d --- /dev/null +++ b/src/services/tradingService.ts @@ -0,0 +1,114 @@ +// src/services/stakingService.ts + +import TradingSnapshot from "../models/TradingSnapshot"; +import SnapHodlConfigBalanceModel from "../models/SnapHodlConfigBalance"; + +export async function getLatestTradingSnapshot( + liquidityPoolAddress: string, + tokenContractAddress: string, + chainId: string +): Promise { + let snapshot = null; + try { + console.log( + "Fetching the latest trading snapshot from the database", + liquidityPoolAddress, + tokenContractAddress, + chainId + ); + + const filter = { + liquidityPoolAddress, + tokenContractAddress, + chainId, + }; + + snapshot = await TradingSnapshot.findOne(filter); + } catch (err) { + console.error( + "Error fetching the latest trading snapshot from the database:", + err + ); + } + return snapshot; +} + +export async function saveTradingSnapshot( + tradingPoolName: string, + liquidityPoolAddress: string, + tradingPoolType: string, + tokenContractAddress: string, + chainId: string, + latestBlockCaptured: number, + uniqueTraders: string[], + tradesVolumeBalances: { [key: string]: any } +) { + console.log("saveTradingSnapshot : start") + // console.log("tradesVolumeBalances : ",tradesVolumeBalances) + + + try { + const snapshot = { + tradingPoolName, + liquidityPoolAddress, + tradingPoolType, + tokenContractAddress, + chainId, + latestBlockCaptured, + uniqueTraders, + timestamp: new Date(), + tradesVolumeBalances, + }; + + const filter = { + liquidityPoolAddress, + tokenContractAddress, + chainId, + }; + + // console.log("filter : ",filter) + + const update = { + $set: snapshot, + }; + + const options = { + upsert: true, + }; + + await TradingSnapshot.updateOne(filter, update, options); + } catch (err) { + console.error("Error saving staking snapshot to the database:", err); + } + console.log("saveTradingSnapshot : completed") +} + +export async function saveTradedBalances( + liquidityPoolAddress: string, + tokenContractAddress: string, + chainId: string, + totalTradingVolumeBalance: { [stakerAddress: string]: string } +) { + try { + const filter = { + liquidityPoolAddress, + tokenContractAddress, + chainId, + }; + + const update = { + $set: { + totalTradingVolumeBalance, + timestamp: new Date(), + }, + }; + + const options = { + upsert: true, + }; + + await SnapHodlConfigBalanceModel.updateOne(filter, update, options); + } catch (err) { + console.error("Error saving staked balances to the database:", err); + } +} diff --git a/src/types.ts b/src/types.ts index 300f9e4..153649b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -10,13 +10,28 @@ export type StakingContractDataItem = { chainId: string; fromBlock: number | "latest"; toBlock: number | "latest"; - blockIterationSize: number + blockIterationSize: number; + excludedWalletAddresses: string[]; +}; + +export type TradingVolumeContractDataItem = { + tradingPoolName: string; + tradingPoolType: string; + tokenContractAddress: string; + chainId: string; + fromBlock: number | "latest"; + toBlock: number | "latest"; + blockIterationSize: number; + liquidityPoolAddress: string; + minimumTradingBalance: number; + excludedWalletAddresses: string[]; }; export type SnapHodlConfig = { _id: string; snapShotConfigName: string; stakingContractData: StakingContractDataItem[]; + tradingVolumeContractData: TradingVolumeContractDataItem[]; __v?: number; isActive: boolean; }; @@ -31,18 +46,24 @@ export type SnapHodlConfigFullDb = { stakingPoolType: string; uniqueStakers: string[]; stakedBalances: { [address: string]: string }; -} +}; export type SnapHodlConfigBalance = { snapHodlConfigId: ObjectId; snapShotConfigName: string; - stakingContractDataBalances: { + stakingContractDataBalances?: { stakingContractAddress: string; tokenContractAddress: string; chainId: string; totalStakedBalance: string; }[]; - totalStakedBalance: Record; + totalTradingVolumeBalance?: { + tokenContractAddress: string; + chainId: string; + totalTradingVolume: string; + }[]; + totalTradingVolume?: Record; + totalStakedBalance?: Record; createdAt: Date; updatedAt: Date; }; diff --git a/src/utils/helpers.ts b/src/utils/helpers.ts index a10ec2e..e9360d4 100644 --- a/src/utils/helpers.ts +++ b/src/utils/helpers.ts @@ -1,233 +1,685 @@ // src/utils/helpers.ts -import { SnapHodlConfig, SnapHodlConfigBalance, StakingContractDataItem } from "../types"; -import { APP_NAME, DB_CONNECTION_STRING, DB_NAME, DB_COLLECTION_STAKING_SNAPSHOT, DB_COLLECTION, DB_COLLECTION_SNAP_CONFIG_BALANCE } from '../config'; +import { SnapHodlConfig, SnapHodlConfigBalance, StakingContractDataItem, TradingVolumeContractDataItem } from "../types"; +import { APP_NAME, DB_CONNECTION_STRING, DB_NAME, DB_COLLECTION_STAKING_SNAPSHOT, DB_COLLECTION, DB_COLLECTION_SNAP_CONFIG_BALANCE, DB_COLLECTION_TRADING_SNAPSHOT } from '../config'; import { MongoClient, ObjectId } from 'mongodb'; import web3 from "web3"; import BigNumber from "bignumber.js"; import { getRpcUrl } from "./getRpcUrl"; +import { AbiItem } from "web3-utils"; import { getUniqueStakersFromOpenStaking, getOpenStakingStakedBalances } from "../openStaking"; import { getUniqueStakers, getStakedBalances } from "../standardStaking"; import { getTokenDecimals } from "./getTokenDecimals"; import { updateTotalStakedBalances } from "./updateTotalStakedBalances"; +import tokenContractAbi from "../../ABI/tokenContractAbi.json"; +import { + getLatestTradingSnapshot, + saveTradedBalances, + saveTradingSnapshot, +} from "../services/tradingService"; +import { add } from "lodash"; export const getWeb3Instance = (rpcUrl: string | undefined): web3 => { - if (!rpcUrl) { - throw new Error("RPC URL is undefined."); - } - return new web3(rpcUrl); + if (!rpcUrl) { + throw new Error("RPC URL is undefined."); + } + return new web3(rpcUrl); }; -export const processStakingContractDataItem = async ( - item: StakingContractDataItem, - dbName: string, - collectionName: string, - connectionString: string, - appName: string +export const processTradingContractDataItem = async ( + item: TradingVolumeContractDataItem ) => { - let totalStakedBalances: { [address: string]: string } = {}; - let finalResults: any[] = []; - - const stakingPoolName = item.stakingPoolName; - const stakingContractAddress = item.stakingContractAddress; - const stakingPoolType = item.stakingPoolType; - const tokenContractAddress = item.tokenContractAddress; - const chainId = item.chainId; - const fromBlock = item.fromBlock; - const toBlock = item.toBlock; - const blockIterationSize = item.blockIterationSize; - const rpcUrl = await getRpcUrl(chainId, APP_NAME, DB_CONNECTION_STRING!, DB_NAME!, DB_COLLECTION!); - - try { - const web3Instance = getWeb3Instance(rpcUrl); - const decimals = await getTokenDecimals(tokenContractAddress, web3Instance); - console.log("Token decimals:", decimals); - - if (stakingPoolType === "standard") { - const stakers = await getUniqueStakers( - stakingPoolName, - stakingContractAddress, - stakingPoolType, - tokenContractAddress, - chainId, - web3Instance, - fromBlock, - toBlock, - blockIterationSize, - DB_NAME!, - DB_COLLECTION_STAKING_SNAPSHOT!, - DB_CONNECTION_STRING! - ); - console.log("Unique staker addresses:", stakers); - - const stakedBalances = await getStakedBalances( - stakingPoolName, - stakingContractAddress, - stakingPoolType, - tokenContractAddress, - chainId, - stakers, - decimals, - web3Instance, - DB_NAME!, - DB_COLLECTION_STAKING_SNAPSHOT!, - DB_CONNECTION_STRING! - ); - // console.log("Staked balances:", stakedBalances); - - const result = { - stakingPoolName: stakingPoolName, - stakedBalances: stakedBalances, - }; - - // console.log("Result:", JSON.stringify(result, null, 2)); - - // Update the totalStakedBalances object - updateTotalStakedBalances(stakedBalances, totalStakedBalances); - - // Add the result to the finalResults array - finalResults.push(result); - } else if (stakingPoolType === "open") { - const uniqueStakers = await getUniqueStakersFromOpenStaking( - stakingPoolName, - stakingContractAddress, - stakingPoolType, - tokenContractAddress, - chainId, - web3Instance, - fromBlock, - toBlock, - blockIterationSize, - DB_NAME!, - DB_COLLECTION_STAKING_SNAPSHOT!, - DB_CONNECTION_STRING! - ); - // console.log("Unique staker addresses from open staking:", uniqueStakers); - - const stakedBalances = await getOpenStakingStakedBalances( - stakingPoolName, - stakingContractAddress, - tokenContractAddress, - chainId, - uniqueStakers, - decimals, - web3Instance, - DB_NAME!, - DB_COLLECTION_STAKING_SNAPSHOT!, - DB_CONNECTION_STRING! - ); - // console.log("Staked balances from open staking:", stakedBalances); - - const result = { - stakingPoolName: stakingPoolName, - stakedBalances: stakedBalances, - }; - - // console.log("Result:", JSON.stringify(result, null, 2)); - - // Update the totalStakedBalances object - updateTotalStakedBalances(stakedBalances, totalStakedBalances); - - // Add the result to the finalResults array - finalResults.push(result); + console.log("processTradingContractDataItem : start") + // console.log({ item }); + const rpcUrl = await getRpcUrl( + item.chainId, + APP_NAME, + DB_CONNECTION_STRING!, + DB_NAME!, + DB_COLLECTION! + ); + // console.log({ chainId: item.chainId, rpcUrl }); + const web3Instance = getWeb3Instance(rpcUrl); + const tokenContract = new web3Instance.eth.Contract( + tokenContractAbi as unknown as AbiItem[], + item.tokenContractAddress + ); + const decimals = await tokenContract.methods.decimals().call(); + const existingTradingSnapshot = await getLatestTradingSnapshot( + item.liquidityPoolAddress, + item.tokenContractAddress, + item.chainId + ); + // console.log({ existingTradingSnapshot }); + let uniqueTraders = new Set( + existingTradingSnapshot ? existingTradingSnapshot.uniqueTraders : [] + ); + const step = item.blockIterationSize; + + // Convert block identifiers to actual block numbers + const currentBlockNumber = await web3Instance.eth.getBlockNumber(); + const resolvedFromBlock = + item.fromBlock === "latest" ? currentBlockNumber : item.fromBlock; + const resolvedToBlock = + item.toBlock === "latest" ? currentBlockNumber : item.toBlock; + + let startBlock = resolvedFromBlock; + if ( + existingTradingSnapshot && + existingTradingSnapshot.latestBlockCaptured >= resolvedFromBlock + ) { + startBlock = existingTradingSnapshot.latestBlockCaptured + 1; + } + + let buyByWalletAddress: { [key: string]: any } = {}; + + // console.log(existingTradingSnapshot); + + if(existingTradingSnapshot && + existingTradingSnapshot.tradesVolumeBalances){ + buyByWalletAddress = existingTradingSnapshot.tradesVolumeBalances; + } + + // console.log(buyByWalletAddress) + + let alreadyProcessedBuyTransactionHashes:string[] = []; + let alreadyProcessedSellTransactionHashes:string[] = []; + + for ( + let currentBlock = startBlock; + currentBlock < resolvedToBlock; + currentBlock += step + ) { + const endBlock = Math.min(currentBlock + step - 1, resolvedToBlock); + + // console.log(currentBlock); + // console.log(endBlock); + + const transferEventSignature = (tokenContract as any)._jsonInterface.find( + (event: any) => { + return event.name === "Transfer"; + } + ).signature; + const buyTransferEventFilter = { + fromBlock: currentBlock, + toBlock: endBlock, + address: item.tokenContractAddress, + topics: [ + transferEventSignature, + web3Instance.eth.abi.encodeParameter( + "address", + item.liquidityPoolAddress + ), + null, + ], + }; + + + // console.log("Buy Transfer event filter:", buyTransferEventFilter); + const buyLogs = await web3Instance.eth.getPastLogs(buyTransferEventFilter); + // console.log(buyLogs); + console.log("buyLogs : "+buyLogs.length); + for(let i=0;i { + // console.log("buy log: ", log); + const walletAddress = web3Instance.eth.abi.decodeParameter( + "address", + log.topics[2] + ); + uniqueTraders.add(walletAddress.toString()); + // console.log(`Wallet Address: ${walletAddress}`); + const value = web3.utils.hexToNumberString(log.data); + // console.log(`Transfer Amount: ${value}`); + const humanReadableBalance = new BigNumber(value) + .dividedBy(new BigNumber(10).pow(decimals)) + .toString(); + buyByWalletAddress[`${walletAddress}`] = buyByWalletAddress[ + `${walletAddress}` + ] + ? typeof buyByWalletAddress[`${walletAddress}`] === "string" || buyByWalletAddress[`${walletAddress}`] instanceof String + ? buyByWalletAddress[`${walletAddress}`] = new BigNumber(buyByWalletAddress[`${walletAddress}`]).plus(humanReadableBalance) + : buyByWalletAddress[`${walletAddress}`].plus(humanReadableBalance) + : new BigNumber(humanReadableBalance); + // console.log(buyByWalletAddress[`${walletAddress}`]); + }); + } + } + + const sellTransferEventFilter = { + fromBlock: currentBlock, + toBlock: endBlock, + address: item.tokenContractAddress, + topics: [ + transferEventSignature, + null, + web3Instance.eth.abi.encodeParameter( + "address", + "0x20dDbFd14F316D417f5B1a981B5Dc926a4dFd4D1" + ), + ], + }; + // console.log("Transfer event filter:", sellTransferEventFilter); + const sellLogs = await web3Instance.eth.getPastLogs( + sellTransferEventFilter + ); + // console.log(sellLogs); + console.log("sellLogs : "+sellLogs.length); + for(let i=0;i { + // console.log("sell log: ", log); + const walletAddress = web3Instance.eth.abi.decodeParameter( + "address", + log.topics[1] + ); + // console.log(`Wallet Address: ${walletAddress}`); + uniqueTraders.add(walletAddress.toString()); + const value = web3.utils.hexToNumberString(log.data); + const humanReadableBalance = new BigNumber(value) + .dividedBy(new BigNumber(10).pow(decimals)) + .toString(); + buyByWalletAddress[`${walletAddress}`] = buyByWalletAddress[ + `${walletAddress}` + ] + ? typeof buyByWalletAddress[`${walletAddress}`] === "string" || buyByWalletAddress[`${walletAddress}`] instanceof String + ? buyByWalletAddress[`${walletAddress}`] = new BigNumber(buyByWalletAddress[`${walletAddress}`]).plus(humanReadableBalance) + : buyByWalletAddress[`${walletAddress}`].plus(humanReadableBalance) + : new BigNumber(humanReadableBalance); + // console.log(buyByWalletAddress[`${walletAddress}`].toFixed()); + }); + } + } + // console.log(buyByWalletAddress, "buyByWalletAddress"); + // uniqueTraders = new Set([...Object.keys(buyByWalletAddress)]); + Object.keys(buyByWalletAddress).forEach((address) => { + buyByWalletAddress[address] = buyByWalletAddress[address].toString(); + }); + // console.log("Unique traders:", uniqueTraders); + // console.log("Buy and sell value:", buyByWalletAddress); + + // console.log("endBlock : ",endBlock) + // console.log("currentBlock : ",currentBlock) + // console.log("buyByWalletAddress : ",buyByWalletAddress); + await saveTradingSnapshot( + item.tradingPoolName, + item.liquidityPoolAddress, + item.tradingPoolType, + item.tokenContractAddress, + item.chainId, + endBlock, + Array.from(uniqueTraders), + buyByWalletAddress + ); + + } + console.log("processTradingContractDataItem : completed") +}; - } catch (error) { - console.error("Error processing data item:", error); +export const processStakingContractDataItem = async ( + item: StakingContractDataItem, + dbName: string, + collectionName: string, + connectionString: string, + appName: string +) => { + let totalStakedBalances: { [address: string]: string } = {}; + let finalResults: any[] = []; + + const stakingPoolName = item.stakingPoolName; + const stakingContractAddress = item.stakingContractAddress; + const stakingPoolType = item.stakingPoolType; + const tokenContractAddress = item.tokenContractAddress; + const chainId = item.chainId; + const fromBlock = item.fromBlock; + const toBlock = item.toBlock; + const blockIterationSize = item.blockIterationSize; + const rpcUrl = await getRpcUrl(chainId, APP_NAME, DB_CONNECTION_STRING!, DB_NAME!, DB_COLLECTION!); + + try { + const web3Instance = getWeb3Instance(rpcUrl); + const decimals = await getTokenDecimals(tokenContractAddress, web3Instance); + console.log("Token decimals:", decimals); + + if (stakingPoolType === "standard") { + const stakers = await getUniqueStakers( + stakingPoolName, + stakingContractAddress, + stakingPoolType, + tokenContractAddress, + chainId, + web3Instance, + fromBlock, + toBlock, + blockIterationSize, + DB_NAME!, + DB_COLLECTION_STAKING_SNAPSHOT!, + DB_CONNECTION_STRING! + ); + // console.log("Unique staker addresses:", stakers); + + const stakedBalances = await getStakedBalances( + stakingPoolName, + stakingContractAddress, + stakingPoolType, + tokenContractAddress, + chainId, + stakers, + decimals, + web3Instance, + DB_NAME!, + DB_COLLECTION_STAKING_SNAPSHOT!, + DB_CONNECTION_STRING! + ); + // console.log("Staked balances:", stakedBalances); + + const result = { + stakingPoolName: stakingPoolName, + stakedBalances: stakedBalances, + }; + + // console.log("Result:", JSON.stringify(result, null, 2)); + + // Update the totalStakedBalances object + updateTotalStakedBalances(stakedBalances, totalStakedBalances); + + // Add the result to the finalResults array + finalResults.push(result); + } else if (stakingPoolType === "open") { + const uniqueStakers = await getUniqueStakersFromOpenStaking( + stakingPoolName, + stakingContractAddress, + stakingPoolType, + tokenContractAddress, + chainId, + web3Instance, + fromBlock, + toBlock, + blockIterationSize, + DB_NAME!, + DB_COLLECTION_STAKING_SNAPSHOT!, + DB_CONNECTION_STRING! + ); + // console.log("Unique staker addresses from open staking:", uniqueStakers); + + const stakedBalances = await getOpenStakingStakedBalances( + stakingPoolName, + stakingContractAddress, + tokenContractAddress, + chainId, + uniqueStakers, + decimals, + web3Instance, + DB_NAME!, + DB_COLLECTION_STAKING_SNAPSHOT!, + DB_CONNECTION_STRING! + ); + // console.log("Staked balances from open staking:", stakedBalances); + + const result = { + stakingPoolName: stakingPoolName, + stakedBalances: stakedBalances, + }; + + // console.log("Result:", JSON.stringify(result, null, 2)); + + // Update the totalStakedBalances object + updateTotalStakedBalances(stakedBalances, totalStakedBalances); + + // Add the result to the finalResults array + finalResults.push(result); } - // Add the total staked balances to the finalResults array - finalResults.push({ stakingPoolName: "totalStakedBalances", stakedBalances: totalStakedBalances }); + // Add logic for other staking pool types here if needed + + } catch (error) { + console.error("Error processing data item:", error); + } - // console.log("Final Results:", JSON.stringify(finalResults, null, 2)); + // Add the total staked balances to the finalResults array + finalResults.push({ stakingPoolName: "totalStakedBalances", stakedBalances: totalStakedBalances }); - return finalResults; + // console.log("Final Results:", JSON.stringify(finalResults, null, 2)); + + return finalResults; }; export const getSnapHodlConfigBalance = async (snapHodlConfig: SnapHodlConfig) => { - if (!snapHodlConfig.isActive) { - return; - } - const client = new MongoClient(DB_CONNECTION_STRING!); - await client.connect(); - - console.log(`${snapHodlConfig.snapShotConfigName} isActive:`, snapHodlConfig.isActive); - const stakingContractObjects = snapHodlConfig.stakingContractData; - const stakingContractDataBalances = []; - const totalStakedBalance: { [address: string]: BigNumber } = {}; - - for (const stakingContractObject of stakingContractObjects) { - const { stakingContractAddress, tokenContractAddress, chainId } = stakingContractObject; - - const query = { - stakingContractAddress, - tokenContractAddress, - chainId, - }; - - const snapshots = await client.db(DB_NAME).collection(DB_COLLECTION_STAKING_SNAPSHOT!).find(query).toArray(); - let totalBalance = new BigNumber(0); - - snapshots.forEach((snapshot) => { - Object.entries(snapshot.stakedBalances).forEach(([address, balance]) => { - const balanceBN = new BigNumber(balance); - totalBalance = totalBalance.plus(balanceBN); - - if (totalStakedBalance[address]) { - totalStakedBalance[address] = totalStakedBalance[address].plus(balanceBN); - } else { - totalStakedBalance[address] = balanceBN; - } - }); - }); + if (!snapHodlConfig.isActive) { + return; + } + const client = new MongoClient(DB_CONNECTION_STRING!); + await client.connect(); + + console.log(`${snapHodlConfig.snapShotConfigName} isActive:`, snapHodlConfig.isActive); + const stakingContractObjects = snapHodlConfig.stakingContractData; + const stakingContractDataBalances = []; + const totalStakedBalance: { [address: string]: BigNumber } = {}; + + for (const stakingContractObject of stakingContractObjects) { + const { stakingContractAddress, tokenContractAddress, chainId, excludedWalletAddresses } = stakingContractObject; + + const query = { + stakingContractAddress, + tokenContractAddress, + chainId, + }; - stakingContractDataBalances.push({ - stakingContractAddress, - tokenContractAddress, - chainId, - totalStakedBalance: totalBalance.toString(), - }); - } + const snapshots = await client.db(DB_NAME).collection(DB_COLLECTION_STAKING_SNAPSHOT!).find(query).toArray(); + let totalBalance = new BigNumber(0); - await client.close(); + snapshots.forEach((snapshot) => { + Object.entries(snapshot.stakedBalances).forEach(([address, balance]) => { + if(!excludedWalletAddresses || excludedWalletAddresses.length === 0 || !excludedWalletAddresses.includes(address)){ + const balanceBN = new BigNumber(balance); + totalBalance = totalBalance.plus(balanceBN); - const result: SnapHodlConfigBalance = { - snapHodlConfigId: new ObjectId(snapHodlConfig._id), - snapShotConfigName: snapHodlConfig.snapShotConfigName, - stakingContractDataBalances, - totalStakedBalance: Object.fromEntries( - Object.entries(totalStakedBalance).map(([address, balance]) => [address, balance.toString()]) - ), - createdAt: new Date(), // add this - updatedAt: new Date(), // add this + if (totalStakedBalance[address]) { + totalStakedBalance[address] = totalStakedBalance[address].plus(balanceBN); + } else { + totalStakedBalance[address] = balanceBN; + } + } + }); + }); + + stakingContractDataBalances.push({ + stakingContractAddress, + tokenContractAddress, + chainId, + totalStakedBalance: totalBalance.toString(), + }); + } + + await client.close(); + + const result: SnapHodlConfigBalance = { + snapHodlConfigId: new ObjectId(snapHodlConfig._id), + snapShotConfigName: snapHodlConfig.snapShotConfigName, + stakingContractDataBalances, + totalStakedBalance: Object.fromEntries( + Object.entries(totalStakedBalance).map(([address, balance]) => [address, balance.toString()]) + ), + createdAt: new Date(), // add this + updatedAt: new Date(), // add this + }; + + // Reconnect to the database to insert the result + await client.connect(); + const collection = client.db(DB_NAME).collection(DB_COLLECTION_SNAP_CONFIG_BALANCE!); + const existingDoc = await collection.findOne({ snapHodlConfigId: new ObjectId(snapHodlConfig._id) }); + + if (existingDoc) { + await collection.updateOne( + { snapHodlConfigId: new ObjectId(snapHodlConfig._id) }, + { + $set: { + ...result, + updatedAt: new Date(), + createdAt: existingDoc.createdAt ? existingDoc.createdAt : new Date(), + }, + } + ); + } else { + await collection.insertOne({ + ...result, + createdAt: new Date(), + updatedAt: new Date(), + }); + } + + await client.close(); + return result; +}; + +export const getSnapHodlConfigTradingVolumeBalance = async ( + snapHodlConfig: SnapHodlConfig +) => { + if (!snapHodlConfig.isActive) { + return; + } + const client = new MongoClient(DB_CONNECTION_STRING!); + await client.connect(); + + console.log( + `${snapHodlConfig.snapShotConfigName} isActive:`, + snapHodlConfig.isActive + ); + const tradingContractObjects = snapHodlConfig.tradingVolumeContractData; + const totalTradingVolumeBalance: any = []; + const totalTradingVolume: { [address: string]: BigNumber } = {}; + let totalBalance = new BigNumber(0); + + for (const tradingContractObject of tradingContractObjects) { + const { liquidityPoolAddress, tokenContractAddress, chainId , excludedWalletAddresses, minimumTradingBalance } = + tradingContractObject; + + const query = { + liquidityPoolAddress, + tokenContractAddress, + chainId, }; - // Reconnect to the database to insert the result - await client.connect(); - const collection = client.db(DB_NAME).collection(DB_COLLECTION_SNAP_CONFIG_BALANCE!); - const existingDoc = await collection.findOne({ snapHodlConfigId: new ObjectId(snapHodlConfig._id) }); - - if (existingDoc) { - await collection.updateOne( - { snapHodlConfigId: new ObjectId(snapHodlConfig._id) }, - { - $set: { - ...result, - updatedAt: new Date(), - createdAt: existingDoc.createdAt ? existingDoc.createdAt : new Date(), - }, + const snapshots = await client + .db(DB_NAME) + .collection(DB_COLLECTION_TRADING_SNAPSHOT!) + .find(query) + .toArray(); + + console.log("excludedWalletAddresses : ", excludedWalletAddresses) + + snapshots.forEach((snapshot) => { + Object.entries(snapshot.tradesVolumeBalances).forEach( + ([address, balance]) => { + if(!excludedWalletAddresses || excludedWalletAddresses.length === 0 || !excludedWalletAddresses.includes(address)){ + const balanceBN = new BigNumber(balance); + + + if(!minimumTradingBalance || minimumTradingBalance === 0 || balanceBN.isGreaterThanOrEqualTo(minimumTradingBalance)){ + totalBalance = totalBalance.plus(balanceBN); + if (totalTradingVolume[address]) { + totalTradingVolume[address] = + totalTradingVolume[address].plus(balanceBN); + } else { + totalTradingVolume[address] = balanceBN; + } } - ); - } else { - await collection.insertOne({ - ...result, - createdAt: new Date(), - updatedAt: new Date(), - }); - } + }else{ + console.log("address was avoided because it was excluded : ", address) + } + } + ); + }); + + Object.keys(totalTradingVolume).forEach((address) => { + (totalTradingVolume as any)[address] = + totalTradingVolume[address].toString(); + }); + + totalTradingVolumeBalance.push({ + liquidityPoolAddress, + tokenContractAddress, + chainId, + totalTradingVolume: totalBalance.toString(), + }); + } + + await client.close(); + // console.log("Total Trading Volume Balance:", totalTradingVolumeBalance); + // console.log("Total Trading Volume:", totalTradingVolume); + let result: any = { + snapHodlConfigId: new ObjectId(snapHodlConfig._id), + snapShotConfigName: snapHodlConfig.snapShotConfigName, + totalTradingVolumeBalance, + totalTradingVolume: totalTradingVolume, + createdAt: new Date(), // add this + updatedAt: new Date(), // add this + }; + + // Reconnect to the database to insert the result + await client.connect(); + const collection = client + .db(DB_NAME) + .collection(DB_COLLECTION_SNAP_CONFIG_BALANCE!); + const existingDoc = await collection.findOne({ + snapHodlConfigId: new ObjectId(snapHodlConfig._id), + }); + + if (existingDoc) { + await collection.updateOne( + { snapHodlConfigId: new ObjectId(snapHodlConfig._id) }, + { + $set: { + ...result, + updatedAt: new Date(), + createdAt: existingDoc.createdAt ? existingDoc.createdAt : new Date(), + }, + } + ); + } else { + await collection.insertOne({ + ...result, + createdAt: new Date(), + updatedAt: new Date(), + }); + } + + await client.close(); + return result; +}; - await client.close(); - return result; +export const getSnapShotBySnapShotUserVolumeAndReward = async ( + snapHodlConfig: SnapHodlConfig +) => { + const client = new MongoClient(DB_CONNECTION_STRING!); + await client.connect(); + const collection = client + .db(DB_NAME) + .collection(DB_COLLECTION_SNAP_CONFIG_BALANCE!); + const snapHodlConfigBalance = await collection.findOne({ + snapHodlConfigId: new ObjectId(snapHodlConfig._id), + }); + const reward = 25000; + let totalVolume = 0; + let totalUserVolume: any = {}; + let totalUserReward: any = {}; + if (snapHodlConfigBalance) { + console.log({ + snapHodlConfigBalance: snapHodlConfigBalance, + }); + Object.entries(snapHodlConfigBalance.totalStakedBalance).forEach( + ([address, balance]) => { + totalVolume = totalVolume + Number(balance); + if (totalUserVolume[address]) { + totalUserVolume[address] = totalUserVolume[address].plus(balance); + } else { + totalUserVolume[address] = balance; + } + } + ); + + Object.entries(snapHodlConfigBalance.totalTradingVolume).forEach( + ([address, balance]) => { + totalVolume = totalVolume + Number(balance); + if (totalUserVolume[address]) { + totalUserVolume[address] = totalUserVolume[address].plus(balance); + } else { + totalUserVolume[address] = balance; + } + } + ); + + Object.entries(totalUserVolume).forEach(([address, balance]: any) => { + + let userVolumePercentageOutOfTotalVolumne = new BigNumber(balance).dividedBy(totalVolume).multipliedBy(100); + let userReward = new BigNumber(reward).multipliedBy(userVolumePercentageOutOfTotalVolumne.dividedBy(100)); + totalUserReward[address] = userReward.toString(); + }); + // console.log("totalUserVolume:", totalUserVolume); + // console.log("totalVolume:", totalVolume.toString()); + console.log( + "length of totalUserVolume:", + Object.keys(totalUserVolume).length + ); + console.log( + "length of totalUserReward:", + Object.keys(totalUserReward).length + ); + console.log( + "length of totalTradingVolume", + Object.keys(snapHodlConfigBalance.totalTradingVolume).length + ); + console.log( + "length of totalStakedBalance", + Object.keys(snapHodlConfigBalance.totalStakedBalance).length + ); + let result = { + ...snapHodlConfigBalance, + totalUserVolume, + totalVolume: totalVolume.toString(), + totalUserReward, + }; + await collection.updateOne( + { snapHodlConfigId: new ObjectId(snapHodlConfig._id) }, + { + $set: { + ...result, + updatedAt: new Date(), + createdAt: snapHodlConfigBalance.createdAt + ? snapHodlConfigBalance.createdAt + : new Date(), + }, + } + ); + } }; +// export function updateTotalTradesVolumeBalances( +// balances: { [address: string]: string }, +// totalStakedBalances: { [address: string]: string } +// ): void { +// for (const key in balances) { +// const value = balances[key]; +// if (totalStakedBalances[key]) { +// const existingBalance = new BigNumber(totalStakedBalances[key]); +// const newBalance = existingBalance.plus(value); +// totalStakedBalances[key] = newBalance.toString(); +// } else { +// totalStakedBalances[key] = value; +// } +// } +// }