diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d5249e..4cbbebd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: - name: Build Docker image run: | - docker build -t ${{ secrets.DOCKER_HUB_USERNAME }}/stockdog:alpha-${{ github.event.number }} . + docker build -t ${{ secrets.DOCKER_HUB_USERNAME }}/stockdog:alpha-${{ github.sha }} . - name: Login to DockerHub uses: docker/login-action@v3 @@ -53,12 +53,12 @@ jobs: password: ${{ secrets.DOCKER_HUB_TOKEN }} - name: Push Docker image - run: docker push ${{ secrets.DOCKER_HUB_USERNAME }}/stockdog:alpha-${{ github.event.number }} + run: docker push ${{ secrets.DOCKER_HUB_USERNAME }}/stockdog:alpha-${{ github.sha }} - name: Update image metadata uses: docker/metadata-action@v5 with: - images: ${{ secrets.DOCKER_HUB_USERNAME }}/stockdog:alpha-${{ github.event.number }} + images: ${{ secrets.DOCKER_HUB_USERNAME }}/stockdog:alpha-${{ github.sha }} tags: | type=sha labels: | diff --git a/docker-compose.yml b/docker-compose.yml index c68a96b..83eab3c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,9 +13,19 @@ services: - db-data:/var/lib/postgresql/data networks: - stockdog - + grafana: + image: grafana/grafana:latest + ports: + - '3300:3000' + volumes: + - grafana-storage:/var/lib/grafana + networks: + - stockdog + depends_on: + - db volumes: db-data: + grafana-storage: networks: stockdog: diff --git a/src/asset-management/entities/asset-exchange.entity.ts b/src/asset-management/entities/asset-exchange.entity.ts index 37abfc3..640a5c3 100644 --- a/src/asset-management/entities/asset-exchange.entity.ts +++ b/src/asset-management/entities/asset-exchange.entity.ts @@ -1,10 +1,17 @@ -import { Entity, PrimaryGeneratedColumn, ManyToOne, OneToMany } from 'typeorm'; +import { + Entity, + PrimaryGeneratedColumn, + ManyToOne, + OneToMany, + Unique, +} from 'typeorm'; import { Asset } from './asset.entity'; import { Exchange } from './exchange.entity'; import { TradingData } from './trading-data.entity'; import { DeliveryData } from './delivery-data.entity'; @Entity() +@Unique(['asset', 'exchange']) export class AssetExchange { @PrimaryGeneratedColumn() id: number; @@ -15,9 +22,9 @@ export class AssetExchange { @ManyToOne(() => Exchange, (exchange) => exchange.assetExchanges) exchange: Exchange; - @OneToMany(() => TradingData, (tradingData) => tradingData) + @OneToMany(() => TradingData, (tradingData) => tradingData.assetExchange) tradingData: TradingData[]; - @OneToMany(() => DeliveryData, (deliveryData) => deliveryData) + @OneToMany(() => DeliveryData, (deliveryData) => deliveryData.assetExchange) deliveryData: DeliveryData[]; } diff --git a/src/asset-management/entities/delivery-data.entity.ts b/src/asset-management/entities/delivery-data.entity.ts index 47cfe5d..56ab8b0 100644 --- a/src/asset-management/entities/delivery-data.entity.ts +++ b/src/asset-management/entities/delivery-data.entity.ts @@ -1,7 +1,14 @@ -import { Entity, Column, ManyToOne, PrimaryGeneratedColumn } from 'typeorm'; +import { + Entity, + Column, + ManyToOne, + PrimaryGeneratedColumn, + Unique, +} from 'typeorm'; import { AssetExchange } from './asset-exchange.entity'; @Entity() +@Unique(['date', 'assetExchange']) export class DeliveryData { @PrimaryGeneratedColumn() id: number; @@ -17,7 +24,7 @@ export class DeliveryData { @ManyToOne( () => AssetExchange, - (assetExchange) => assetExchange.tradingData, + (assetExchange) => assetExchange.deliveryData, { onDelete: 'CASCADE', }, diff --git a/src/asset-management/entities/trading-data.entity.ts b/src/asset-management/entities/trading-data.entity.ts index 91a2738..e3c16aa 100644 --- a/src/asset-management/entities/trading-data.entity.ts +++ b/src/asset-management/entities/trading-data.entity.ts @@ -1,7 +1,14 @@ -import { Entity, Column, ManyToOne, PrimaryGeneratedColumn } from 'typeorm'; +import { + Entity, + Column, + ManyToOne, + PrimaryGeneratedColumn, + Unique, +} from 'typeorm'; import { AssetExchange } from './asset-exchange.entity'; @Entity() +@Unique(['date', 'assetExchange']) export class TradingData { @PrimaryGeneratedColumn() id: number; diff --git a/src/asset-management/services/delivery-data.service.ts b/src/asset-management/services/delivery-data.service.ts index 7398c58..c7745d8 100644 --- a/src/asset-management/services/delivery-data.service.ts +++ b/src/asset-management/services/delivery-data.service.ts @@ -3,6 +3,7 @@ import { DeliveryDataDTO } from '../dto'; import { DeliveryData } from '../entities'; import { DeliveryDataRepository } from '../repositories'; import { validateAndThrowError } from '../utils/validate-dto-error'; +import { InsertResult } from 'typeorm'; @Injectable() export class DeliveryDataService { @@ -18,8 +19,10 @@ export class DeliveryDataService { return deliveryData; } - async saveDeliveryData(deliveryData: DeliveryDataDTO): Promise { + async saveDeliveryData(deliveryData: DeliveryDataDTO): Promise { await validateAndThrowError(deliveryData, 'deliveryDataDTO'); - return this.deliveryDataRepository.create(deliveryData as DeliveryData); + return this.deliveryDataRepository.upsert(deliveryData as DeliveryData, { + conflictPaths: { date: true, assetExchange: true }, + }); } } diff --git a/src/asset-management/services/trading-data.service.ts b/src/asset-management/services/trading-data.service.ts index f3a9476..acc59e1 100644 --- a/src/asset-management/services/trading-data.service.ts +++ b/src/asset-management/services/trading-data.service.ts @@ -3,6 +3,7 @@ import { TradingDataDTO } from '../dto'; import { TradingData } from '../entities'; import { TradingDataRepository } from '../repositories'; import { validateAndThrowError } from '../utils/validate-dto-error'; +import { InsertResult } from 'typeorm'; @Injectable() export class TradingDataService { @@ -16,8 +17,10 @@ export class TradingDataService { return tradingData; } - async saveTradingData(tradingData: TradingDataDTO): Promise { + async saveTradingData(tradingData: TradingDataDTO): Promise { await validateAndThrowError(tradingData, 'TradingDataDTO'); - return this.tradingDataRepository.create(tradingData as TradingData); + return this.tradingDataRepository.upsert(tradingData as TradingData, { + conflictPaths: { date: true, assetExchange: true }, + }); } } diff --git a/src/data-sync/bse.service.ts b/src/data-sync/bse.service.ts index 43d859c..863411e 100644 --- a/src/data-sync/bse.service.ts +++ b/src/data-sync/bse.service.ts @@ -5,6 +5,7 @@ import { AssetDto, DeliveryDataDTO, TradingDataDTO } from './dto'; import parseCSV from './utils/csv-parser'; import { Stream } from 'stream'; import { Exchange } from 'src/asset-management/types/enums'; +import { AxiosHeaders } from 'axios'; enum STOCK_DATA_CSV_HEADERS { SYMBOL = 'Security Id', NAME_OF_COMPANY = 'Issuer Name', @@ -41,6 +42,8 @@ enum DELIVERY_DATA_CSV_HEADERS { export class BseService { constructor(private readonly AM: AssetManagement) {} + private logger = new Logger(BseService.name); + async handleAssetData(csvData: Stream) { const bseExchange = await this.AM.exchangeService.findOrCreateExchange( Exchange.BSE, @@ -68,14 +71,14 @@ export class BseService { ); const parser = await parseCSV(streamData, CSV_SEPARATOR.PIPE); for await (const record of parser) { - const assetExchangeCode = record[TRADE_DATA_CSV_HEADERS.SYMBOL]; + const assetExchangeCode = record[DELIVERY_DATA_CSV_HEADERS.SYMBOL]; const assetExchange = await this.AM.assetExchangeService.findBySymbol( assetExchangeCode, bseExchange, ['asset'], ); if (!assetExchange) { - Logger.warn( + this.logger.warn( `Asset Details not found for stock: ${ record[TRADE_DATA_CSV_HEADERS.NAME] }`, @@ -114,7 +117,7 @@ export class BseService { ['asset'], ); if (!assetExchange) { - Logger.warn( + this.logger.warn( `Asset Details not found for stock: ${ record[TRADE_DATA_CSV_HEADERS.NAME] } AssetExchangeCode: ${assetExchangeCode}`, @@ -161,14 +164,18 @@ export class BseService { const year = date.getFullYear(); const month = ('0' + (date.getMonth() + 1)).slice(-2); // Months are 0-based in JavaScript const day = ('0' + date.getDate()).slice(-2); + const headers = new AxiosHeaders({ + 'User-Agent': + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11', + Referer: 'https://www.bseindia.com/markets/marketinfo/BhavCopy.aspx', + encoding: null, + }); return { - userAgent: - 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11', - referer: 'https://www.bseindia.com/markets/marketinfo/BhavCopy.aspx', assetUrl: `https://api.bseindia.com/BseIndiaAPI/api/LitsOfScripCSVDownload/w?segment=Equity&status=Active&industry=&Group=&Scripcode=`, deliveryURL: `https://www.bseindia.com/BSEDATA/gross/${year}/SCBSEALL${day}${month}.zip`, tradingURL: `https://www.bseindia.com/download/BhavCopy/Equity/BhavCopy_BSE_CM_0_0_0_${year}${month}${day}_F_0000.CSV`, + headers, }; } } diff --git a/src/data-sync/data-sync.module.ts b/src/data-sync/data-sync.module.ts index e1ab972..bbf81af 100644 --- a/src/data-sync/data-sync.module.ts +++ b/src/data-sync/data-sync.module.ts @@ -4,9 +4,16 @@ import { DataSyncService } from './data-sync.service'; import { BseService } from './bse.service'; import { AssetManagement } from './asset-management.service'; import { NseService } from './nse.service'; +import { HttpClient } from './utils/httpClient'; @Module({ imports: [AssetManagementModule], - providers: [AssetManagement, DataSyncService, BseService, NseService], + providers: [ + HttpClient, + AssetManagement, + DataSyncService, + BseService, + NseService, + ], }) export class DataSyncModule {} diff --git a/src/data-sync/data-sync.service.ts b/src/data-sync/data-sync.service.ts index 4782040..40cbcab 100644 --- a/src/data-sync/data-sync.service.ts +++ b/src/data-sync/data-sync.service.ts @@ -1,43 +1,56 @@ import { Injectable, Logger } from '@nestjs/common'; -import { Timeout } from '@nestjs/schedule'; -import axios from 'axios'; +import { AxiosHeaders } from 'axios'; import { NseService } from './nse.service'; import { BseService } from './bse.service'; import * as unzipper from 'unzipper'; -import { PassThrough, Readable } from 'stream'; +import { PassThrough } from 'stream'; import { Cron, CronExpression } from '@nestjs/schedule'; +import { getCurrentDate, getUtcTradeDays } from './utils/trade-days'; +import { HttpClient } from './utils/httpClient'; @Injectable() export class DataSyncService { constructor( + private httpClient: HttpClient, private nseService: NseService, private bseService: BseService, ) {} private readonly logger = new Logger(DataSyncService.name); - async handleAssetDataBSE(url: string, userAgent: string, referer: string) { - const response = await axios.get(url, { - headers: { - 'User-Agent': userAgent, - Referer: referer, - encoding: null, - }, - responseType: 'stream', - }); - await this.bseService.handleAssetData(response.data); + @Cron(CronExpression.EVERY_DAY_AT_6PM, { timeZone: 'Asia/Kolkata' }) + async execute() { + const currentDate = getCurrentDate(); + this.logger.log( + `Data Sync Started for ${currentDate.toISOString().slice(0, 10)}`, + ); + + const workdays = getUtcTradeDays(currentDate, currentDate); + const workdaysStr = workdays.map((date) => date.toISOString().slice(0, 10)); + for (const dateStr of workdaysStr) { + await this.handleBSEDataSync(new Date(dateStr)); + await this.handleNSEDataSync(new Date(dateStr)); + } } - @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) - async handleDeliveryDataBSE(url: string, userAgent: string, referer: string) { + async handleBSEDataSync(date: Date) { + const start = performance.now(); + this.logger.log('BSE Data Sync Started'); - const response = await axios.get(url, { - headers: { - 'User-Agent': userAgent, - Referer: referer, - encoding: null, - }, - responseType: 'stream', - }); + + const { assetUrl, tradingURL, deliveryURL, headers } = + this.bseService.generateFileUrls(date); + const assetResponse = await this.httpClient.get(assetUrl, headers); + await this.bseService.handleAssetData(assetResponse.data); + const tradeResponse = await this.httpClient.get(tradingURL, headers); + await this.bseService.handleTradingData(tradeResponse.data); + await this.handleDeliveryDataBSE(deliveryURL, headers); + const end = performance.now(); + const timeTaken = end - start; + this.logger.log(`BSE Data Sync Finished. Time taken: ${timeTaken} ms`); + } + + async handleDeliveryDataBSE(url: string, headers: AxiosHeaders) { + const response = await this.httpClient.get(url, headers); response.data.pipe(unzipper.Parse()).on('entry', async (entry) => { const fileName = entry.path; if (fileName.includes('SCBSEALL')) { @@ -54,59 +67,19 @@ export class DataSyncService { }); } - async handleTradingDataBSE(url: string, userAgent: string, referer: string) { - const response = await axios.get(url, { - headers: { - 'User-Agent': userAgent, - Referer: referer, - encoding: null, - }, - responseType: 'stream', - }); - await this.bseService.handleTradingData(response.data); - } - - @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) - async handleBSEDataSync() { - const { assetUrl, tradingURL, deliveryURL, userAgent, referer } = - this.bseService.generateFileUrls(new Date(2024, 3, 19)); - await this.handleAssetDataBSE(assetUrl, userAgent, referer); - await this.handleDeliveryDataBSE(deliveryURL, userAgent, referer); - await this.handleTradingDataBSE(tradingURL, userAgent, referer); - } - - @Timeout(5000) - async handleNSEDataSync() { + async handleNSEDataSync(date: Date) { + const start = performance.now(); this.logger.log('NSE Data Sync Started'); - const { assetUrl, tradingURL, userAgent, referer } = - this.nseService.generateFileUrls(new Date(2024, 3, 19)); - - const assetResponse = await axios.get(assetUrl, { - headers: { - Accept: '*/*"', - Connection: 'keep-alive', - 'User-Agent': userAgent, - Referer: referer, - }, - }); + const { assetUrl, tradingURL, headers } = + this.nseService.generateFileUrls(date); - const assetResponseStream = Readable.from( - JSON.stringify(assetResponse.data), - ); - - await this.nseService.handleAssetData(assetResponseStream); - const tradeResponse = await axios.get(tradingURL, { - headers: { - Accept: '*/*"', - Connection: 'keep-alive', - 'User-Agent': userAgent, - Referer: referer, - }, - }); - const tradeResponseStream = Readable.from( - JSON.stringify(tradeResponse.data), - ); + const assetResponse = await this.httpClient.get(assetUrl, headers); - await this.nseService.handleTradingData(tradeResponseStream); + await this.nseService.handleAssetData(assetResponse.data); + const tradeResponse = await this.httpClient.get(tradingURL, headers); + await this.nseService.handleTradingData(tradeResponse.data); + const end = performance.now(); + const timeTaken = end - start; + this.logger.log(`NSE Data Sync Finished. Time taken: ${timeTaken} ms`); } } diff --git a/src/data-sync/nse.service.ts b/src/data-sync/nse.service.ts index 445466c..f2d7501 100644 --- a/src/data-sync/nse.service.ts +++ b/src/data-sync/nse.service.ts @@ -5,6 +5,7 @@ import { AssetDto, DeliveryDataDTO, TradingDataDTO } from './dto'; import { CSV_SEPARATOR } from './types/enums/csv'; import { Exchange } from 'src/asset-management/types/enums'; import { Stream } from 'stream'; +import { AxiosHeaders } from 'axios'; enum STOCK_DATA_CSV_HEADERS { SYMBOL = 'SYMBOL', @@ -38,6 +39,7 @@ enum DELIVERY_DATA_CSV_HEADERS { @Injectable() export class NseService { constructor(private readonly AM: AssetManagement) {} + private logger = new Logger(NseService.name); async handleAssetData(csvData: Stream) { const nseExchange = await this.AM.exchangeService.findOrCreateExchange( @@ -72,7 +74,7 @@ export class NseService { ); if (!assetExchange) { if (['EQ', 'BE', 'SM'].includes(series)) { - Logger.warn( + this.logger.warn( `Asset Details not found for symbol: ${symbol} and series: ${series}`, ); } @@ -128,13 +130,19 @@ export class NseService { const month = ('0' + (date.getMonth() + 1)).slice(-2); // Months are 0-based in JavaScript const day = ('0' + date.getDate()).slice(-2); - return { - userAgent: + const headers = new AxiosHeaders({ + Accept: '*/*"', + Connection: 'keep-alive', + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', - referer: + Referer: 'https://www1.nseindia.com/products/content/equities/equities/archieve_eq.htm', + }); + + return { assetUrl: `https://nsearchives.nseindia.com/content/equities/EQUITY_L.csv`, tradingURL: `https://nsearchives.nseindia.com/products/content/sec_bhavdata_full_${day}${month}${year}.csv`, + headers, }; } } diff --git a/src/data-sync/utils/httpClient.ts b/src/data-sync/utils/httpClient.ts new file mode 100644 index 0000000..c85522a --- /dev/null +++ b/src/data-sync/utils/httpClient.ts @@ -0,0 +1,18 @@ +import { Logger } from '@nestjs/common'; +import axios, { AxiosHeaders } from 'axios'; + +export class HttpClient { + private readonly logger = new Logger(HttpClient.name); + + async get(url: string, headers: AxiosHeaders) { + try { + return await axios.get(url, { + headers, + responseType: 'stream', + }); + } catch (error) { + this.logger.error(`Failed to get data from ${url}: ${error.message}`); + throw error; + } + } +} diff --git a/src/data-sync/utils/trade-days.test.ts b/src/data-sync/utils/trade-days.test.ts new file mode 100644 index 0000000..8bfd175 --- /dev/null +++ b/src/data-sync/utils/trade-days.test.ts @@ -0,0 +1,58 @@ +import { getCurrentDate, getUtcTradeDays, tradingHolidays } from './trade-days'; + +describe('getUtcTradeDays', () => { + it('should return an array of trade days between the start and end dates', () => { + const startDate = new Date('2024-01-01'); + const endDate = new Date('2024-12-31'); + const tradeDays = getUtcTradeDays(startDate, endDate); + + expect(tradeDays).toHaveLength(246); // Expected number of trade days in 2024 + + // Ensure all trade days fall within the specified range + tradeDays.forEach((tradeDay) => { + expect(tradeDay >= startDate && tradeDay <= endDate).toBe(true); + }); + }); + + it('should exclude trading holidays from the result', () => { + const startDate = new Date('2024-01-01'); + const endDate = new Date('2024-12-31'); + const tradeDays = getUtcTradeDays(startDate, endDate); + + // Ensure no trading holidays are present in the trade days array + tradingHolidays.forEach((holiday) => { + const holidayDate = new Date(holiday); + expect(tradeDays.includes(holidayDate)).toBe(false); + }); + }); + + it('should return single trade day if start and end dates are the same', () => { + const date = getCurrentDate(); + const tradeDays = getUtcTradeDays(date, date); + + expect(tradeDays).toHaveLength(1); + expect(tradeDays[0]).toEqual(date); + }); +}); + +describe('getCurrentDate', () => { + it('should return the current date when no argument is passed', () => { + const result = getCurrentDate(); + const expected = new Date(new Date().toISOString().split('T')[0]); + expect(result).toEqual(expected); + }); + + it('should return the date 5 days in the future when 5 is passed', () => { + const result = getCurrentDate(5); + const expected = new Date(new Date().toISOString().split('T')[0]); + expected.setDate(expected.getDate() + 5); + expect(result).toEqual(expected); + }); + + it('should return the date 3 days in the past when -3 is passed', () => { + const result = getCurrentDate(-3); + const expected = new Date(new Date().toISOString().split('T')[0]); + expected.setDate(expected.getDate() - 3); + expect(result).toEqual(expected); + }); +}); diff --git a/src/data-sync/utils/trade-days.ts b/src/data-sync/utils/trade-days.ts new file mode 100644 index 0000000..1484572 --- /dev/null +++ b/src/data-sync/utils/trade-days.ts @@ -0,0 +1,71 @@ +export const tradingHolidays = [ + '2024-01-22', // Special Holiday + '2024-01-26', // Republic Day + '2024-03-08', // Mahashivratri + '2024-03-25', // Holi + '2024-03-29', // Good Friday + '2024-04-11', // Id-Ul-Fitr (Ramadan Eid) + '2024-04-17', // Shri Ram Navmi + '2024-05-01', // Maharashtra Day + '2024-05-20', // General Parliamentary Elections + '2024-06-17', // Bakri Id + '2024-07-17', // Moharram + '2024-08-15', // Independence Day + '2024-10-02', // Mahatma Gandhi Jayanti + '2024-11-01', // Diwali Laxmi Pujan* + '2024-11-15', // Gurunanak Jayanti + '2024-12-25', // Christmas +]; +export const getUtcTradeDays = (startDate: Date, endDate: Date): Date[] => { + const workdays: Date[] = []; + const currentDate = new Date( + Date.UTC( + startDate.getUTCFullYear(), + startDate.getUTCMonth(), + startDate.getUTCDate(), + ), + ); // clone the date and ignore time + const excludeDates = tradingHolidays; + + // convert the excludeDates strings to Date objects and ignore the time part + const excludeDateObjects = excludeDates.map((dateStr) => { + const d = new Date(dateStr); + return new Date( + Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()), + ); + }); + + while (currentDate <= endDate) { + const dayOfWeek = currentDate.getUTCDay(); + const isWeekend = dayOfWeek === 0 || dayOfWeek === 6; // 0 (Sunday) or 6 (Saturday) + + // check if the current date is in the excludeDates array + const isExcluded = excludeDateObjects.some( + (d) => d.getTime() === currentDate.getTime(), + ); + + if (!isWeekend && !isExcluded) { + // it's a workday and it's not excluded + workdays.push(new Date(currentDate.getTime())); // push a clone of the date + } + + // go to the next day + currentDate.setUTCDate(currentDate.getUTCDate() + 1); + } + + return workdays; +}; + +export const getCurrentDate = (days?: number): Date => { + const date = new Date(); + + if (days) { + date.setDate(date.getDate() + days); + } + + // Convert the date to 'YYYY-MM-DD' format in UTC + const dateStr = date.toISOString().split('T')[0]; + + // Create a new Date object from the date string + return new Date(dateStr); +}; diff --git a/src/main.ts b/src/main.ts index 13cad38..fa6b58d 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,7 +2,9 @@ import { NestFactory } from '@nestjs/core'; import { AppModule } from './app.module'; async function bootstrap() { - const app = await NestFactory.create(AppModule); + const app = await NestFactory.create(AppModule, { + logger: ['error', 'log'], + }); await app.listen(3000); } bootstrap();