From 5025b98ce2c37f5f7fe0397c0ca9b1bed652f704 Mon Sep 17 00:00:00 2001 From: Danish Joseph Date: Mon, 22 Apr 2024 02:17:24 +0200 Subject: [PATCH] Setup data sync module (#12) * Add DataSyncModule to app.module.ts * Update package.json and yarn.lock to add @nestjs/schedule dependency * Add @nestjs/schedule and AssetManagementModule to DataSyncModule imports * Add new files and update existing files in asset-management module * Add new files and update existing files in data-sync module * Remove unused files and configurations * Add Docker configuration files * Add CI workflow for building and publishing Docker image * Add pull_request trigger to CI workflow * Update NodeJS version in CI workflow to use actions/setup-node@v4 * Refactor logger instantiation in asset.service.ts and asset-exchange.service.ts * Update CI workflow to remove creation of .env file * Update CI workflow to include Docker image build and push * Update CI workflow to include Docker image save and load * Update CI workflow to include Docker image save and load * Refactor Docker image naming in CI workflow * Add GitHub Actions step to comment on successful pull requests with Docker image artifact link * Update GitHub Actions step to use actions/github-script@v7 in ci.yml * Update permissions in ci.yml --- .dockerignore | 7 + .github/workflows/ci.yml | 91 +++++++++ .idx/.gitignore | 2 - .idx/dev.nix | 12 -- Dockerfile | 17 ++ docker-compose.yml | 31 ++-- package.json | 7 +- src/app.module.ts | 4 + .../asset-management.module.ts | 21 ++- src/asset-management/dto/asset.dto.ts | 35 ++++ src/asset-management/dto/delivery-data.dto.ts | 18 ++ src/asset-management/dto/index.ts | 3 + src/asset-management/dto/trading-data.dto.ts | 36 ++++ .../repositories/base.repository.ts | 21 ++- .../services/asset-exchange.service.test.ts | 66 +++++++ .../services/asset-exchange.service.ts | 24 +++ .../services/asset.service.test.ts | 68 ++++++- .../services/asset.service.ts | 37 +++- .../services/delivery-data.service.ts | 15 +- .../services/exchange.service.test.ts | 21 +-- .../services/exchange.service.ts | 18 +- src/asset-management/services/index.ts | 1 + .../services/trading-data.service.ts | 15 +- src/asset-management/types/enums/exchange.ts | 4 + src/asset-management/types/enums/index.ts | 1 + .../types/interface/asset-exchange.ts | 8 + src/asset-management/types/interface/asset.ts | 13 ++ .../types/interface/exchange.ts | 4 + src/asset-management/types/interface/index.ts | 3 + .../utils/validate-dto-error.ts | 10 + src/data-sync/asset-management.service.ts | 19 ++ src/data-sync/bse.service.ts | 174 ++++++++++++++++++ src/data-sync/data-sync.module.ts | 12 ++ src/data-sync/data-sync.service.ts | 108 +++++++++++ src/data-sync/dto/asset.dto.ts | 35 ++++ src/data-sync/dto/delivery-data.dto.ts | 18 ++ src/data-sync/dto/index.ts | 3 + src/data-sync/dto/trading-data.dto.ts | 36 ++++ src/data-sync/nse.service.ts | 140 ++++++++++++++ src/data-sync/types/enums/csv.ts | 4 + src/data-sync/utils/csv-parser.ts | 24 +++ src/data-sync/utils/validate-dto-error.ts | 10 + yarn.lock | 131 ++++++++++++- 43 files changed, 1263 insertions(+), 64 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/ci.yml delete mode 100644 .idx/.gitignore delete mode 100644 .idx/dev.nix create mode 100644 Dockerfile create mode 100644 src/asset-management/dto/asset.dto.ts create mode 100644 src/asset-management/dto/delivery-data.dto.ts create mode 100644 src/asset-management/dto/index.ts create mode 100644 src/asset-management/dto/trading-data.dto.ts create mode 100644 src/asset-management/services/asset-exchange.service.test.ts create mode 100644 src/asset-management/services/asset-exchange.service.ts create mode 100644 src/asset-management/types/enums/exchange.ts create mode 100644 src/asset-management/types/enums/index.ts create mode 100644 src/asset-management/types/interface/asset-exchange.ts create mode 100644 src/asset-management/types/interface/asset.ts create mode 100644 src/asset-management/types/interface/exchange.ts create mode 100644 src/asset-management/types/interface/index.ts create mode 100644 src/asset-management/utils/validate-dto-error.ts create mode 100644 src/data-sync/asset-management.service.ts create mode 100644 src/data-sync/bse.service.ts create mode 100644 src/data-sync/data-sync.module.ts create mode 100644 src/data-sync/data-sync.service.ts create mode 100644 src/data-sync/dto/asset.dto.ts create mode 100644 src/data-sync/dto/delivery-data.dto.ts create mode 100644 src/data-sync/dto/index.ts create mode 100644 src/data-sync/dto/trading-data.dto.ts create mode 100644 src/data-sync/nse.service.ts create mode 100644 src/data-sync/types/enums/csv.ts create mode 100644 src/data-sync/utils/csv-parser.ts create mode 100644 src/data-sync/utils/validate-dto-error.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..96a630e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +node_modules +dist +test +.dockerignore +Dockerfile +README.md +.env \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5d15be5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,91 @@ +name: CI +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + issues: write + pull-requests: write + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up NodeJS + uses: actions/setup-node@v4 + with: + node-version: 18 + + - name: Install dependencies + run: yarn install + + - name: Run tests + run: yarn test --colors + + - name: Build Docker image + run: | + docker build -t stockdog:pr-${{ github.event.number }} . + docker save stockdog:pr-${{ github.event.number }} > stockdog-pr-${{ github.event.number }}.image.tar + + - name: Save Docker image as artifact + uses: actions/upload-artifact@v4 + with: + name: docker-image + path: stockdog-pr-${{ github.event.number }}.image.tar + + - name: Comment PR + if: success() && github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const artifactUrl = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; + const message = `:sparkles: **Docker Image Artifact** :sparkles:\n\nYour Docker image artifact can be found at the following location:\n\n:point_right: [Click Here](${artifactUrl}) :point_left:\n\nHappy coding! :rocket:`; + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: message, + }); + + push: + needs: build + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download Docker image from artifact + uses: actions/download-artifact@v4 + with: + name: docker-image + path: stockdog-pr-${{ github.event.number }}.image.tar + + - name: Load Docker image + run: docker load < stockdog-pr-${{ github.event.number }}.image.tar + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - name: Push Docker image + run: docker push ${{ secrets.DOCKER_HUB_USERNAME }}/stockdog:pr-${{ github.event.number }} + + - name: Update image metadata + uses: docker/metadata-action@v5 + with: + images: ${{ secrets.DOCKER_HUB_USERNAME }}/stockdog:pr-${{ github.event.number }} + tags: | + type=sha + labels: | + org.opencontainers.image.title=StockDog App + org.opencontainers.image.description=Stock market analysis app + org.opencontainers.image.url=https://github.com/${{github.repository}} + org.opencontainers.image.revision=${{github.sha}} + org.opencontainers.image.licenses=MIT diff --git a/.idx/.gitignore b/.idx/.gitignore deleted file mode 100644 index 96be05f..0000000 --- a/.idx/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ - -gc/ diff --git a/.idx/dev.nix b/.idx/dev.nix deleted file mode 100644 index be7f2c0..0000000 --- a/.idx/dev.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ pkgs, ... }: { -packages = [ - pkgs.yarn - pkgs.nodejs_18 - pkgs.docker-compose - pkgs.bash-completion - pkgs.git - ]; - -services.docker.enable = true; - -} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4c7934e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM node:21-alpine + +WORKDIR /app + +COPY package.json ./ + +COPY yarn.lock ./ + +RUN yarn install + +COPY . . + +RUN yarn build + +EXPOSE 3000 + +CMD [ "yarn", "start:prod" ] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 720b31f..c68a96b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,21 @@ -version: '3.8' - -services: - db: - image: postgres:13 - environment: +version: '3.8' + +services: + db: + image: postgres:13 + environment: POSTGRES_USER: ${DB_USERNAME} POSTGRES_PASSWORD: ${DB_PASSWORD} POSTGRES_DB: ${DB_NAME} - ports: - - "5432:5432" - volumes: - - db-data:/var/lib/postgresql/data - -volumes: - db-data: + ports: + - '5432:5432' + volumes: + - db-data:/var/lib/postgresql/data + networks: + - stockdog + +volumes: + db-data: + +networks: + stockdog: diff --git a/package.json b/package.json index e16bd3d..d53db50 100644 --- a/package.json +++ b/package.json @@ -24,11 +24,16 @@ "@nestjs/config": "^3.2.2", "@nestjs/core": "^10.0.0", "@nestjs/platform-express": "^10.0.0", + "@nestjs/schedule": "^4.0.2", "@nestjs/typeorm": "^10.0.2", + "axios": "^1.6.8", + "class-validator": "^0.14.1", + "csv-parser": "^3.0.0", "pg": "^8.11.5", "reflect-metadata": "^0.2.0", "rxjs": "^7.8.1", - "typeorm": "^0.3.20" + "typeorm": "^0.3.20", + "unzipper": "^0.11.3" }, "devDependencies": { "@nestjs/cli": "^10.0.0", diff --git a/src/app.module.ts b/src/app.module.ts index 57faa0d..a6954ba 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -4,15 +4,19 @@ import { AppService } from './app.service'; import { ConfigModule } from '@nestjs/config'; import { AssetManagementModule } from './asset-management/asset-management.module'; import { DatabaseModule } from './database.module'; +import { DataSyncModule } from './data-sync/data-sync.module'; +import { ScheduleModule } from '@nestjs/schedule'; @Module({ imports: [ + ScheduleModule.forRoot(), ConfigModule.forRoot({ isGlobal: true, envFilePath: ['.env'], }), DatabaseModule, AssetManagementModule, + DataSyncModule, ], controllers: [AppController], providers: [AppService], diff --git a/src/asset-management/asset-management.module.ts b/src/asset-management/asset-management.module.ts index fcecd95..2981ad8 100644 --- a/src/asset-management/asset-management.module.ts +++ b/src/asset-management/asset-management.module.ts @@ -1,4 +1,4 @@ -import { Module } from '@nestjs/common'; +import { Logger, Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { AssetExchange } from './entities/asset-exchange.entity'; import { Asset } from './entities/asset.entity'; @@ -13,6 +13,12 @@ import { TradingDataRepository, } from './repositories'; import { ExchangeService } from './services/exchange.service'; +import { + AssetService, + DeliveryDataService, + TradingDataService, + AssetExchangeService, +} from './services'; @Module({ imports: [ @@ -25,13 +31,24 @@ import { ExchangeService } from './services/exchange.service'; ]), ], providers: [ + Logger, AssetRepository, ExchangeRepository, AssetExchangeRepository, TradingDataRepository, DeliveryDataRepository, ExchangeService, + AssetService, + DeliveryDataService, + TradingDataService, + AssetExchangeService, + ], + exports: [ + ExchangeService, + AssetService, + DeliveryDataService, + TradingDataService, + AssetExchangeService, ], - exports: [ExchangeService], }) export class AssetManagementModule {} diff --git a/src/asset-management/dto/asset.dto.ts b/src/asset-management/dto/asset.dto.ts new file mode 100644 index 0000000..7703acd --- /dev/null +++ b/src/asset-management/dto/asset.dto.ts @@ -0,0 +1,35 @@ +import { IsString, IsOptional, IsNotEmpty, IsNumber } from 'class-validator'; +import { AssetExchange } from '../entities'; + +export class AssetDto { + @IsNotEmpty() + @IsString() + isin: string; + + @IsNotEmpty() + @IsString() + symbol: string; + + @IsNotEmpty() + @IsString() + name: string; + + @IsNotEmpty() + @IsString() + assetExchangeCode: string; + + @IsOptional() + @IsNumber() + faceValue: number; + + @IsOptional() + @IsString() + industry: string; + + @IsOptional() + @IsString() + sector: string; + + @IsOptional() + assetExchanges: AssetExchange[]; +} diff --git a/src/asset-management/dto/delivery-data.dto.ts b/src/asset-management/dto/delivery-data.dto.ts new file mode 100644 index 0000000..437073a --- /dev/null +++ b/src/asset-management/dto/delivery-data.dto.ts @@ -0,0 +1,18 @@ +import { IsDate, IsNumber, IsOptional } from 'class-validator'; +import { AssetExchange } from '../entities'; + +export class DeliveryDataDTO { + @IsDate() + date: Date; + + @IsNumber() + @IsOptional() + deliveryQuantity: number; + + @IsNumber() + @IsOptional() + deliveryPercentage: number; + + @IsOptional() + assetExchange: AssetExchange | null; +} diff --git a/src/asset-management/dto/index.ts b/src/asset-management/dto/index.ts new file mode 100644 index 0000000..c6e6a2b --- /dev/null +++ b/src/asset-management/dto/index.ts @@ -0,0 +1,3 @@ +export * from './asset.dto'; +export * from './trading-data.dto'; +export * from './delivery-data.dto'; diff --git a/src/asset-management/dto/trading-data.dto.ts b/src/asset-management/dto/trading-data.dto.ts new file mode 100644 index 0000000..0cc559a --- /dev/null +++ b/src/asset-management/dto/trading-data.dto.ts @@ -0,0 +1,36 @@ +import { IsNumber, IsDate } from 'class-validator'; +import { AssetExchange } from '../entities'; + +export class TradingDataDTO { + @IsDate() + date: Date; + + @IsNumber() + open: number; + + @IsNumber() + high: number; + + @IsNumber() + low: number; + + @IsNumber() + close: number; + + @IsNumber() + lastPrice: number; + + @IsNumber() + previousClose: number; + + @IsNumber() + volume: number; + + @IsNumber() + turnover: number; + + @IsNumber() + totalTrades: number; + + assetExchange: AssetExchange; +} diff --git a/src/asset-management/repositories/base.repository.ts b/src/asset-management/repositories/base.repository.ts index 4c326a4..c7440da 100644 --- a/src/asset-management/repositories/base.repository.ts +++ b/src/asset-management/repositories/base.repository.ts @@ -3,7 +3,10 @@ import { FindOneOptions, SelectQueryBuilder, FindOptionsWhere, + InsertResult, } from 'typeorm'; +import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'; +import { UpsertOptions } from 'typeorm/repository/UpsertOptions'; export interface DatabaseRepository { findAll(): Promise; @@ -11,6 +14,10 @@ export interface DatabaseRepository { create(item: T): Promise; update(id: string, item: T): Promise; delete(id: string): Promise; + upsert( + item: QueryDeepPartialEntity, + options: UpsertOptions, + ): Promise; createQueryBuilder(alias: string): SelectQueryBuilder; } @@ -25,8 +32,11 @@ export abstract class BaseRepository implements DatabaseRepository { return this.repository.findOne(id); } - async findOneBy(props: FindOptionsWhere): Promise { - return this.repository.findOneBy(props); + async findOneBy( + props: FindOptionsWhere, + relations?: string[], + ): Promise { + return this.repository.findOne({ where: props, relations: relations }); } async create(item: T): Promise { @@ -47,6 +57,13 @@ export abstract class BaseRepository implements DatabaseRepository { await this.repository.delete(id); } + async upsert( + item: QueryDeepPartialEntity, + options: UpsertOptions, + ): Promise { + return this.repository.upsert(item, options); + } + createQueryBuilder(alias: string): SelectQueryBuilder { return this.repository.createQueryBuilder(alias); } diff --git a/src/asset-management/services/asset-exchange.service.test.ts b/src/asset-management/services/asset-exchange.service.test.ts new file mode 100644 index 0000000..fe308fd --- /dev/null +++ b/src/asset-management/services/asset-exchange.service.test.ts @@ -0,0 +1,66 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { AssetExchangeRepository } from '../repositories'; +import { AssetExchangeService } from './asset-exchange.service'; +import { Exchange } from '../types/enums'; +import { AssetExchange } from '../entities'; + +describe('AssetExchangeService', () => { + let assetExchangeService: AssetExchangeService; + let assetExchangeRepository: jest.Mocked; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + AssetExchangeService, + { + provide: AssetExchangeRepository, + useValue: { + findOneBy: jest.fn(), + }, + }, + ], + }).compile(); + + assetExchangeService = + module.get(AssetExchangeService); + assetExchangeRepository = module.get(AssetExchangeRepository); + expect(assetExchangeService).toBeDefined(); + expect(assetExchangeRepository).toBeDefined(); + }); + + it('should find by symbol with NSE exchange', async () => { + const symbol = 'test-symbol'; + const exchange = { + abbreviation: Exchange.NSE, + name: 'National Stock Exchange', + }; + const relations = ['relation1', 'relation2']; + const expectedData = { + id: 1, + symbol, + exchange, + asset: jest.fn(), + tradingData: jest.fn(), + deliveryData: jest.fn(), + }; + + assetExchangeRepository.findOneBy.mockResolvedValue( + expectedData as unknown as AssetExchange, + ); + + const result = await assetExchangeService.findBySymbol( + symbol, + exchange, + relations, + ); + + expect(result).toEqual(expectedData); + expect(assetExchangeRepository.findOneBy).toHaveBeenCalledWith( + { + asset: { symbol }, + exchange, + }, + relations, + ); + }); +}); diff --git a/src/asset-management/services/asset-exchange.service.ts b/src/asset-management/services/asset-exchange.service.ts new file mode 100644 index 0000000..66fa539 --- /dev/null +++ b/src/asset-management/services/asset-exchange.service.ts @@ -0,0 +1,24 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { AssetExchangeRepository } from '../repositories'; +import { Exchange } from '../types/interface'; +import { Exchange as Ex } from '../types/enums'; + +@Injectable() +export class AssetExchangeService { + constructor( + private readonly assetExchangeRepository: AssetExchangeRepository, + ) {} + + async findBySymbol(symbol: string, exchange: Exchange, relations: string[]) { + const identifier = + exchange.abbreviation === Ex.NSE ? 'symbol' : 'assetExchangeCode'; + + return this.assetExchangeRepository.findOneBy( + { + asset: { [identifier]: symbol }, + exchange, + }, + relations, + ); + } +} diff --git a/src/asset-management/services/asset.service.test.ts b/src/asset-management/services/asset.service.test.ts index 166339f..51b5c50 100644 --- a/src/asset-management/services/asset.service.test.ts +++ b/src/asset-management/services/asset.service.test.ts @@ -1,10 +1,13 @@ import { Test, TestingModule } from '@nestjs/testing'; import { AssetService } from './asset.service'; -import { AssetRepository } from '../repositories'; +import { AssetRepository, AssetExchangeRepository } from '../repositories'; +import { AssetDto } from '../dto'; +import { Exchange, Asset, AssetExchange } from '../entities'; describe('AssetService', () => { let assetService: AssetService; let assetRepository: AssetRepository; + let assetExchangeRepository: AssetExchangeRepository; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ @@ -13,7 +16,15 @@ describe('AssetService', () => { { provide: AssetRepository, useValue: { - // Mock any methods you want to test + findOneBy: jest.fn(), + create: jest.fn(), + }, + }, + { + provide: AssetExchangeRepository, + useValue: { + findOneBy: jest.fn(), + create: jest.fn(), }, }, ], @@ -21,9 +32,60 @@ describe('AssetService', () => { assetService = module.get(AssetService); assetRepository = module.get(AssetRepository); + assetExchangeRepository = module.get( + AssetExchangeRepository, + ); expect(assetService).toBeDefined(); expect(assetRepository).toBeDefined(); }); - // Add more test cases for other methods in the AssetService class + it('should create asset and asset exchange', async () => { + const assetData: AssetDto = { + isin: 'test-isin', + name: 'Test Asset', + symbol: 'TST', + assetExchangeCode: 'TST123', + faceValue: 0, + industry: '', + sector: '', + assetExchanges: [], + }; + const exchange = { + id: 1, + name: 'Test Exchange', + abbreviation: 'TST', + } as Exchange; + const asset: Asset = { id: 1, ...assetData }; + const assetExchange: AssetExchange = { + id: 1, + asset, + exchange, + tradingData: [], + deliveryData: [], + }; + + assetRepository.findOneBy = jest.fn().mockResolvedValueOnce(null); + assetExchangeRepository.findOneBy = jest.fn().mockResolvedValueOnce(null); + assetRepository.create = jest + .fn() + .mockResolvedValueOnce({ id: 1, ...assetData }); + assetExchangeRepository.create = jest + .fn() + .mockResolvedValueOnce(assetExchange); + + await assetService.createAsset(assetData, exchange); + + expect(assetRepository.findOneBy).toHaveBeenCalledWith({ + isin: assetData.isin, + }); + expect(assetExchangeRepository.findOneBy).toHaveBeenCalledWith({ + asset: { isin: assetData.isin }, + exchange, + }); + expect(assetRepository.create).toHaveBeenCalledWith(assetData); + expect(assetExchangeRepository.create).toHaveBeenCalledWith({ + asset, + exchange, + }); + }); }); diff --git a/src/asset-management/services/asset.service.ts b/src/asset-management/services/asset.service.ts index c769adf..3de69fa 100644 --- a/src/asset-management/services/asset.service.ts +++ b/src/asset-management/services/asset.service.ts @@ -1,7 +1,38 @@ -import { Injectable } from '@nestjs/common'; -import { AssetRepository } from '../repositories'; +import { Injectable, Logger } from '@nestjs/common'; +import { AssetExchangeRepository, AssetRepository } from '../repositories'; +import { validateAndThrowError } from '../utils/validate-dto-error'; +import { Asset, AssetExchange, Exchange } from '../entities'; +import { AssetDto } from '../dto'; @Injectable() export class AssetService { - constructor(private readonly assetRepository: AssetRepository) {} + constructor( + private readonly assetRepository: AssetRepository, + private assetExchangeRepository: AssetExchangeRepository, + ) {} + + async createAsset(assetData: AssetDto, exchange: Exchange) { + await validateAndThrowError(assetData, 'AssetDto'); + + const [existingStock, existingAssetExchange] = await Promise.all([ + this.assetRepository.findOneBy({ isin: assetData.isin }), + this.assetExchangeRepository.findOneBy({ + asset: { isin: assetData.isin }, + exchange, + }), + ]); + let savedStock; + if (!existingStock) { + savedStock = await this.assetRepository.create(assetData as Asset); + } else { + savedStock = existingStock; + } + + if (!existingAssetExchange) { + await this.assetExchangeRepository.create({ + asset: savedStock, + exchange, + } as AssetExchange); + } + } } diff --git a/src/asset-management/services/delivery-data.service.ts b/src/asset-management/services/delivery-data.service.ts index 099e78d..7398c58 100644 --- a/src/asset-management/services/delivery-data.service.ts +++ b/src/asset-management/services/delivery-data.service.ts @@ -1,12 +1,25 @@ +import { Injectable } from '@nestjs/common'; +import { DeliveryDataDTO } from '../dto'; import { DeliveryData } from '../entities'; import { DeliveryDataRepository } from '../repositories'; +import { validateAndThrowError } from '../utils/validate-dto-error'; +@Injectable() export class DeliveryDataService { constructor( private readonly deliveryDataRepository: DeliveryDataRepository, ) {} async findDeliveryDataById(id: string): Promise { - return this.deliveryDataRepository.findById(id); + const deliveryData = await this.deliveryDataRepository.findById(id); + if (!deliveryData) { + throw new Error(`Delivery data with id ${id} not found`); + } + return deliveryData; + } + + async saveDeliveryData(deliveryData: DeliveryDataDTO): Promise { + await validateAndThrowError(deliveryData, 'deliveryDataDTO'); + return this.deliveryDataRepository.create(deliveryData as DeliveryData); } } diff --git a/src/asset-management/services/exchange.service.test.ts b/src/asset-management/services/exchange.service.test.ts index e268a75..7e623fa 100644 --- a/src/asset-management/services/exchange.service.test.ts +++ b/src/asset-management/services/exchange.service.test.ts @@ -1,6 +1,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ExchangeService } from './exchange.service'; import { ExchangeRepository } from '../repositories/exchange.repository'; +import { Exchange } from '../types/enums'; describe('ExchangeService', () => { let exchangeService: ExchangeService; @@ -26,11 +27,11 @@ describe('ExchangeService', () => { describe('findOrCreateExchange', () => { it('should return an existing exchange if found', async () => { - const abbreviation = 'ABC'; + const abbreviation = Exchange.NSE; const existingExchange = { id: 1, abbreviation, - name: 'Existing Exchange', + name: 'National Stock Exchange', assetExchanges: [], }; @@ -38,10 +39,7 @@ describe('ExchangeService', () => { .spyOn(exchangeRepository, 'findOneBy') .mockResolvedValue(existingExchange); - const result = await exchangeService.findOrCreateExchange( - abbreviation, - 'New Exchange', - ); + const result = await exchangeService.findOrCreateExchange(abbreviation); expect(result).toEqual(existingExchange); expect(exchangeRepository.findOneBy).toHaveBeenCalledWith({ @@ -51,21 +49,18 @@ describe('ExchangeService', () => { }); it('should create a new exchange if not found', async () => { - const abbreviation = 'XYZ'; + const abbreviation = Exchange.NSE; const newExchange = { id: 1, abbreviation, - name: 'New Exchange', + name: 'National Stock Exchange', assetExchanges: [], }; jest.spyOn(exchangeRepository, 'findOneBy').mockResolvedValue(null); jest.spyOn(exchangeRepository, 'create').mockResolvedValue(newExchange); - const result = await exchangeService.findOrCreateExchange( - abbreviation, - 'New Exchange', - ); + const result = await exchangeService.findOrCreateExchange(abbreviation); expect(result).toEqual(newExchange); expect(exchangeRepository.findOneBy).toHaveBeenCalledWith({ @@ -73,7 +68,7 @@ describe('ExchangeService', () => { }); expect(exchangeRepository.create).toHaveBeenCalledWith({ abbreviation, - name: 'New Exchange', + name: 'National Stock Exchange', }); }); }); diff --git a/src/asset-management/services/exchange.service.ts b/src/asset-management/services/exchange.service.ts index 10dd188..b6af3e9 100644 --- a/src/asset-management/services/exchange.service.ts +++ b/src/asset-management/services/exchange.service.ts @@ -1,19 +1,25 @@ import { Injectable } from '@nestjs/common'; import { ExchangeRepository } from '../repositories'; import { Exchange } from '../entities'; +import { Exchange as Ex } from '../types/enums'; @Injectable() export class ExchangeService { constructor(private readonly exchangeRepository: ExchangeRepository) {} - async findOrCreateExchange( - abbreviation: string, - name: string, - ): Promise { - const exchange = await this.exchangeRepository.findOneBy({ abbreviation }); + async findOrCreateExchange(exchange: Ex): Promise { + let name = 'National Stock Exchange'; + let abbreviation = 'NSE'; + if (exchange === Ex.BSE) { + name = 'Bombay Stock Exchange'; + abbreviation = 'BSE'; + } + const existingExchange = await this.exchangeRepository.findOneBy({ + abbreviation, + }); return ( - exchange || + existingExchange || this.exchangeRepository.create({ name, abbreviation } as Exchange) ); } diff --git a/src/asset-management/services/index.ts b/src/asset-management/services/index.ts index be10ebf..f384503 100644 --- a/src/asset-management/services/index.ts +++ b/src/asset-management/services/index.ts @@ -2,3 +2,4 @@ export * from './exchange.service'; export * from './asset.service'; export * from './delivery-data.service'; export * from './trading-data.service'; +export * from './asset-exchange.service'; diff --git a/src/asset-management/services/trading-data.service.ts b/src/asset-management/services/trading-data.service.ts index 512ed4a..46665d9 100644 --- a/src/asset-management/services/trading-data.service.ts +++ b/src/asset-management/services/trading-data.service.ts @@ -1,10 +1,23 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { TradingDataDTO } from '../dto'; import { TradingData } from '../entities'; import { TradingDataRepository } from '../repositories'; +import { validateAndThrowError } from '../utils/validate-dto-error'; +@Injectable() export class TradingDataService { constructor(private readonly tradingDataRepository: TradingDataRepository) {} async findTradingDataById(id: string): Promise { - return this.tradingDataRepository.findById(id); + const tradingData = await this.tradingDataRepository.findById(id); + if (!tradingData) { + throw new Error(`Trading data with id ${id} not found`); + } + return tradingData; + } + + async saveTradingData(tradingData: TradingDataDTO): Promise { + await validateAndThrowError(tradingData, 'TradingDataDTO'); + return this.tradingDataRepository.create(tradingData as TradingData); } } diff --git a/src/asset-management/types/enums/exchange.ts b/src/asset-management/types/enums/exchange.ts new file mode 100644 index 0000000..7df3f6e --- /dev/null +++ b/src/asset-management/types/enums/exchange.ts @@ -0,0 +1,4 @@ +export enum Exchange { + NSE = 'NSE', + BSE = 'BSE', +} diff --git a/src/asset-management/types/enums/index.ts b/src/asset-management/types/enums/index.ts new file mode 100644 index 0000000..40165d3 --- /dev/null +++ b/src/asset-management/types/enums/index.ts @@ -0,0 +1 @@ +export * from './exchange'; diff --git a/src/asset-management/types/interface/asset-exchange.ts b/src/asset-management/types/interface/asset-exchange.ts new file mode 100644 index 0000000..52a7708 --- /dev/null +++ b/src/asset-management/types/interface/asset-exchange.ts @@ -0,0 +1,8 @@ +import { Asset } from './asset'; +import { Exchange } from './exchange'; + +export interface AssetExchange { + id: number; + asset: Asset; + exchange: Exchange; +} diff --git a/src/asset-management/types/interface/asset.ts b/src/asset-management/types/interface/asset.ts new file mode 100644 index 0000000..7f86bf0 --- /dev/null +++ b/src/asset-management/types/interface/asset.ts @@ -0,0 +1,13 @@ +import { AssetExchange } from 'src/asset-management/entities'; + +export interface Asset { + id: number; + isin: string; + symbol: string; + name: string; + assetExchangeCode: string; + faceValue: number; + industry: string; + sector: string; + assetExchanges: AssetExchange[]; +} diff --git a/src/asset-management/types/interface/exchange.ts b/src/asset-management/types/interface/exchange.ts new file mode 100644 index 0000000..70b9f00 --- /dev/null +++ b/src/asset-management/types/interface/exchange.ts @@ -0,0 +1,4 @@ +export interface Exchange { + abbreviation: string; + name: string; +} diff --git a/src/asset-management/types/interface/index.ts b/src/asset-management/types/interface/index.ts new file mode 100644 index 0000000..f36888e --- /dev/null +++ b/src/asset-management/types/interface/index.ts @@ -0,0 +1,3 @@ +export * from './asset'; +export * from './exchange'; +export * from './asset-exchange'; diff --git a/src/asset-management/utils/validate-dto-error.ts b/src/asset-management/utils/validate-dto-error.ts new file mode 100644 index 0000000..9f9a7a5 --- /dev/null +++ b/src/asset-management/utils/validate-dto-error.ts @@ -0,0 +1,10 @@ +import { validate } from 'class-validator'; + +export const validateAndThrowError = async (dto, dtoName) => { + const errors = await validate(dto); + if (errors.length > 0) { + throw new Error( + `Validation failed for ${dtoName} : ${JSON.stringify(errors)}` + ); + } +}; diff --git a/src/data-sync/asset-management.service.ts b/src/data-sync/asset-management.service.ts new file mode 100644 index 0000000..09c1130 --- /dev/null +++ b/src/data-sync/asset-management.service.ts @@ -0,0 +1,19 @@ +import { Injectable } from '@nestjs/common'; +import { + AssetService, + DeliveryDataService, + ExchangeService, + TradingDataService, + AssetExchangeService, +} from 'src/asset-management/services'; + +@Injectable() +export class AssetManagement { + constructor( + public exchangeService: ExchangeService, + public assetService: AssetService, + public deliveryDataService: DeliveryDataService, + public tradingDataService: TradingDataService, + public assetExchangeService: AssetExchangeService, + ) {} +} diff --git a/src/data-sync/bse.service.ts b/src/data-sync/bse.service.ts new file mode 100644 index 0000000..43d859c --- /dev/null +++ b/src/data-sync/bse.service.ts @@ -0,0 +1,174 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { AssetManagement } from './asset-management.service'; +import { CSV_SEPARATOR } from './types/enums/csv'; +import { AssetDto, DeliveryDataDTO, TradingDataDTO } from './dto'; +import parseCSV from './utils/csv-parser'; +import { Stream } from 'stream'; +import { Exchange } from 'src/asset-management/types/enums'; +enum STOCK_DATA_CSV_HEADERS { + SYMBOL = 'Security Id', + NAME_OF_COMPANY = 'Issuer Name', + ISIN_NUMBER = 'ISIN No', + FACE_VALUE = 'Face Value', + INDUSTRY = 'Industry New Name', + SECTOR = 'Sector Name', + ASSET_EXCHANGE_CODE = 'Security Code', +} + +enum TRADE_DATA_CSV_HEADERS { + SYMBOL = 'FinInstrmId', + NAME = 'FinInstrmNm', + DATE = 'TradDt', + OPEN_PRICE = 'OpnPric', + HIGH_PRICE = 'HghPric', + LOW_PRICE = 'LwPric', + CLOSE_PRICE = 'ClsPric', + LAST_PRICE = 'LastPric', + PREV_CLOSE = 'PrvsClsgPric', + VOLUME = 'TtlTradgVol', + TURNOVER = 'TtlTrfVal', + TOTAL_TRADES = 'TtlNbOfTxsExctd', +} + +enum DELIVERY_DATA_CSV_HEADERS { + SYMBOL = 'SCRIP CODE', + DATE = 'DATE', + DELIV_QTY = 'DELIVERY QTY', + DELIV_PER = 'DELV. PER.', +} + +@Injectable() +export class BseService { + constructor(private readonly AM: AssetManagement) {} + + async handleAssetData(csvData: Stream) { + const bseExchange = await this.AM.exchangeService.findOrCreateExchange( + Exchange.BSE, + ); + const parser = await parseCSV(csvData, CSV_SEPARATOR.COMMA); + for await (const record of parser) { + const assetData = new AssetDto(); + assetData.name = record[STOCK_DATA_CSV_HEADERS.NAME_OF_COMPANY]; + assetData.isin = record[STOCK_DATA_CSV_HEADERS.ISIN_NUMBER]; + assetData.faceValue = parseFloat( + record[STOCK_DATA_CSV_HEADERS.FACE_VALUE], + ); + assetData.symbol = record[STOCK_DATA_CSV_HEADERS.SYMBOL]; + assetData.industry = record[STOCK_DATA_CSV_HEADERS.INDUSTRY]; + assetData.sector = record[STOCK_DATA_CSV_HEADERS.SECTOR]; + assetData.assetExchangeCode = + record[STOCK_DATA_CSV_HEADERS.ASSET_EXCHANGE_CODE]; + await this.AM.assetService.createAsset(assetData, bseExchange); + } + } + + async handleDeliveryData(streamData) { + const bseExchange = await this.AM.exchangeService.findOrCreateExchange( + Exchange.BSE, + ); + const parser = await parseCSV(streamData, CSV_SEPARATOR.PIPE); + for await (const record of parser) { + const assetExchangeCode = record[TRADE_DATA_CSV_HEADERS.SYMBOL]; + const assetExchange = await this.AM.assetExchangeService.findBySymbol( + assetExchangeCode, + bseExchange, + ['asset'], + ); + if (!assetExchange) { + Logger.warn( + `Asset Details not found for stock: ${ + record[TRADE_DATA_CSV_HEADERS.NAME] + }`, + ); + continue; + } + const deliveryDataDto = new DeliveryDataDTO(); + const date = record[DELIVERY_DATA_CSV_HEADERS.DATE]; + const parsedDate = new Date( + `${date.slice(4)}-${date.slice(2, 4)}-${date.slice(0, 2)}`, + ); + deliveryDataDto.date = parsedDate; + const qty = parseInt(record[DELIVERY_DATA_CSV_HEADERS.DELIV_QTY]) ?? null; + deliveryDataDto.deliveryQuantity = Number.isNaN(qty) ? null : qty; + const percent = + parseInt(record[DELIVERY_DATA_CSV_HEADERS.DELIV_PER]) ?? null; + deliveryDataDto.deliveryPercentage = Number.isNaN(percent) + ? null + : percent; + deliveryDataDto.assetExchange = assetExchange; + + await this.AM.deliveryDataService.saveDeliveryData(deliveryDataDto); + } + } + + async handleTradingData(csvData: Stream) { + const bseExchange = await this.AM.exchangeService.findOrCreateExchange( + Exchange.BSE, + ); + const parser = await parseCSV(csvData, CSV_SEPARATOR.COMMA); + for await (const record of parser) { + const assetExchangeCode = record[TRADE_DATA_CSV_HEADERS.SYMBOL]; + const assetExchange = await this.AM.assetExchangeService.findBySymbol( + assetExchangeCode, + bseExchange, + ['asset'], + ); + if (!assetExchange) { + Logger.warn( + `Asset Details not found for stock: ${ + record[TRADE_DATA_CSV_HEADERS.NAME] + } AssetExchangeCode: ${assetExchangeCode}`, + ); + continue; + } + const tradingDataDto = new TradingDataDTO(); + tradingDataDto.date = new Date(record[TRADE_DATA_CSV_HEADERS.DATE]); + tradingDataDto.open = parseFloat( + record[TRADE_DATA_CSV_HEADERS.OPEN_PRICE], + ); + tradingDataDto.high = parseFloat( + record[TRADE_DATA_CSV_HEADERS.HIGH_PRICE], + ); + tradingDataDto.low = parseFloat(record[TRADE_DATA_CSV_HEADERS.LOW_PRICE]); + tradingDataDto.close = parseFloat( + record[TRADE_DATA_CSV_HEADERS.CLOSE_PRICE], + ); + const lastPrice = parseFloat(record[TRADE_DATA_CSV_HEADERS.LAST_PRICE]); + + tradingDataDto.lastPrice = Number.isNaN(lastPrice) + ? 0 + : tradingDataDto.close; + tradingDataDto.previousClose = parseFloat( + record[TRADE_DATA_CSV_HEADERS.PREV_CLOSE], + ); + tradingDataDto.previousClose = parseFloat( + record[TRADE_DATA_CSV_HEADERS.PREV_CLOSE], + ); + tradingDataDto.volume = parseInt(record[TRADE_DATA_CSV_HEADERS.VOLUME]); + tradingDataDto.turnover = parseFloat( + record[TRADE_DATA_CSV_HEADERS.TURNOVER], + ); + tradingDataDto.totalTrades = parseFloat( + record[TRADE_DATA_CSV_HEADERS.TOTAL_TRADES], + ); + + tradingDataDto.assetExchange = assetExchange; + await this.AM.tradingDataService.saveTradingData(tradingDataDto); + } + } + + generateFileUrls(date: Date) { + const year = date.getFullYear(); + const month = ('0' + (date.getMonth() + 1)).slice(-2); // Months are 0-based in JavaScript + const day = ('0' + date.getDate()).slice(-2); + + return { + userAgent: + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11', + referer: 'https://www.bseindia.com/markets/marketinfo/BhavCopy.aspx', + assetUrl: `https://api.bseindia.com/BseIndiaAPI/api/LitsOfScripCSVDownload/w?segment=Equity&status=Active&industry=&Group=&Scripcode=`, + deliveryURL: `https://www.bseindia.com/BSEDATA/gross/${year}/SCBSEALL${day}${month}.zip`, + tradingURL: `https://www.bseindia.com/download/BhavCopy/Equity/BhavCopy_BSE_CM_0_0_0_${year}${month}${day}_F_0000.CSV`, + }; + } +} diff --git a/src/data-sync/data-sync.module.ts b/src/data-sync/data-sync.module.ts new file mode 100644 index 0000000..fb936d3 --- /dev/null +++ b/src/data-sync/data-sync.module.ts @@ -0,0 +1,12 @@ +import { Logger, Module } from '@nestjs/common'; +import { AssetManagementModule } from 'src/asset-management/asset-management.module'; +import { DataSyncService } from './data-sync.service'; +import { BseService } from './bse.service'; +import { AssetManagement } from './asset-management.service'; +import { NseService } from './nse.service'; + +@Module({ + imports: [AssetManagementModule], + providers: [AssetManagement, DataSyncService, BseService, NseService], +}) +export class DataSyncModule {} diff --git a/src/data-sync/data-sync.service.ts b/src/data-sync/data-sync.service.ts new file mode 100644 index 0000000..f3d63d9 --- /dev/null +++ b/src/data-sync/data-sync.service.ts @@ -0,0 +1,108 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Timeout } from '@nestjs/schedule'; +import axios from 'axios'; +import { NseService } from './nse.service'; +import { BseService } from './bse.service'; +import * as unzipper from 'unzipper'; +import { PassThrough, Readable } from 'stream'; +import { Cron, CronExpression } from '@nestjs/schedule'; + +@Injectable() +export class DataSyncService { + constructor( + private nseService: NseService, + private bseService: BseService, + ) {} + private readonly logger = new Logger(DataSyncService.name); + + async handleAssetDataBSE(url: string, userAgent: string, referer: string) { + const response = await axios.get(url, { + headers: { + 'User-Agent': userAgent, + Referer: referer, + encoding: null, + }, + responseType: 'stream', + }); + await this.bseService.handleAssetData(response.data); + } + + @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) + async handleDeliveryDataBSE(url: string, userAgent: string, referer: string) { + this.logger.log('BSE Data Sync Started'); + const response = await axios.get(url, { + headers: { + 'User-Agent': userAgent, + Referer: referer, + encoding: null, + }, + responseType: 'stream', + }); + response.data.pipe(unzipper.Parse()).on('entry', async (entry) => { + const fileName = entry.path; + if (fileName.includes('SCBSEALL')) { + const dataStream = new PassThrough(); + entry + .on('data', (chunk) => dataStream.write(chunk)) + .on('end', async () => { + dataStream.end(); + await this.bseService.handleDeliveryData(dataStream); + }); + } else { + entry.autodrain(); + } + }); + } + + async handleTradingDataBSE(url: string, userAgent: string, referer: string) { + const response = await axios.get(url, { + headers: { + 'User-Agent': userAgent, + Referer: referer, + encoding: null, + }, + responseType: 'stream', + }); + await this.bseService.handleTradingData(response.data); + } + + @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) + async handleBSEDataSync() { + const { assetUrl, tradingURL, deliveryURL, userAgent, referer } = + this.bseService.generateFileUrls(new Date(2024, 3, 19)); + await this.handleAssetDataBSE(assetUrl, userAgent, referer); + await this.handleDeliveryDataBSE(deliveryURL, userAgent, referer); + await this.handleTradingDataBSE(tradingURL, userAgent, referer); + } + + @Timeout(5000) + async handleNSEDataSync() { + this.logger.log('NSE Data Sync Started'); + const { assetUrl, tradingURL, userAgent, referer } = + this.nseService.generateFileUrls(new Date(2024, 3, 19)); + + const assetResponse = await axios.get(assetUrl, { + headers: { + Accept: '*/*"', + Connection: 'keep-alive', + 'User-Agent': userAgent, + Referer: referer, + }, + }); + + let assetResponseStream = Readable.from(JSON.stringify(assetResponse.data)); + + await this.nseService.handleAssetData(assetResponseStream); + const tradeResponse = await axios.get(tradingURL, { + headers: { + Accept: '*/*"', + Connection: 'keep-alive', + 'User-Agent': userAgent, + Referer: referer, + }, + }); + let tradeResponseStream = Readable.from(JSON.stringify(tradeResponse.data)); + + await this.nseService.handleTradingData(tradeResponseStream); + } +} diff --git a/src/data-sync/dto/asset.dto.ts b/src/data-sync/dto/asset.dto.ts new file mode 100644 index 0000000..8657ac3 --- /dev/null +++ b/src/data-sync/dto/asset.dto.ts @@ -0,0 +1,35 @@ +import { IsString, IsOptional, IsNotEmpty, IsNumber } from 'class-validator'; +import { AssetExchange } from 'src/asset-management/entities'; + +export class AssetDto { + @IsNotEmpty() + @IsString() + isin: string; + + @IsNotEmpty() + @IsString() + symbol: string; + + @IsNotEmpty() + @IsString() + name: string; + + @IsNotEmpty() + @IsString() + assetExchangeCode: string; + + @IsOptional() + @IsNumber() + faceValue: number; + + @IsOptional() + @IsString() + industry: string; + + @IsOptional() + @IsString() + sector: string; + + @IsOptional() + assetExchanges: AssetExchange[]; +} diff --git a/src/data-sync/dto/delivery-data.dto.ts b/src/data-sync/dto/delivery-data.dto.ts new file mode 100644 index 0000000..1c83e5c --- /dev/null +++ b/src/data-sync/dto/delivery-data.dto.ts @@ -0,0 +1,18 @@ +import { IsDate, IsNumber, IsOptional } from 'class-validator'; +import { AssetExchange } from 'src/asset-management/entities'; + +export class DeliveryDataDTO { + @IsDate() + date: Date; + + @IsNumber() + @IsOptional() + deliveryQuantity: number; + + @IsNumber() + @IsOptional() + deliveryPercentage: number; + + @IsOptional() + assetExchange: AssetExchange | null; +} diff --git a/src/data-sync/dto/index.ts b/src/data-sync/dto/index.ts new file mode 100644 index 0000000..c6e6a2b --- /dev/null +++ b/src/data-sync/dto/index.ts @@ -0,0 +1,3 @@ +export * from './asset.dto'; +export * from './trading-data.dto'; +export * from './delivery-data.dto'; diff --git a/src/data-sync/dto/trading-data.dto.ts b/src/data-sync/dto/trading-data.dto.ts new file mode 100644 index 0000000..1982d71 --- /dev/null +++ b/src/data-sync/dto/trading-data.dto.ts @@ -0,0 +1,36 @@ +import { IsNumber, IsDate } from 'class-validator'; +import { AssetExchange } from 'src/asset-management/entities'; + +export class TradingDataDTO { + @IsDate() + date: Date; + + @IsNumber() + open: number; + + @IsNumber() + high: number; + + @IsNumber() + low: number; + + @IsNumber() + close: number; + + @IsNumber() + lastPrice: number; + + @IsNumber() + previousClose: number; + + @IsNumber() + volume: number; + + @IsNumber() + turnover: number; + + @IsNumber() + totalTrades: number; + + assetExchange: AssetExchange; +} diff --git a/src/data-sync/nse.service.ts b/src/data-sync/nse.service.ts new file mode 100644 index 0000000..445466c --- /dev/null +++ b/src/data-sync/nse.service.ts @@ -0,0 +1,140 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { AssetManagement } from './asset-management.service'; +import parseCSV from './utils/csv-parser'; +import { AssetDto, DeliveryDataDTO, TradingDataDTO } from './dto'; +import { CSV_SEPARATOR } from './types/enums/csv'; +import { Exchange } from 'src/asset-management/types/enums'; +import { Stream } from 'stream'; + +enum STOCK_DATA_CSV_HEADERS { + SYMBOL = 'SYMBOL', + NAME_OF_COMPANY = 'NAME OF COMPANY', + ISIN_NUMBER = 'ISIN NUMBER', + FACE_VALUE = 'FACE VALUE', +} + +enum TRADE_DATA_CSV_HEADERS { + SYMBOL = 'SYMBOL', + SERIES = 'SERIES', + DATE = 'DATE1', + OPEN_PRICE = 'OPEN_PRICE', + HIGH_PRICE = 'HIGH_PRICE', + LOW_PRICE = 'LOW_PRICE', + CLOSE_PRICE = 'CLOSE_PRICE', + LAST_PRICE = 'LAST_PRICE', + PREV_CLOSE = 'PREV_CLOSE', + VOLUME = 'TTL_TRD_QNTY', + TURNOVER = 'TURNOVER_LACS', + TOTAL_TRADES = 'NO_OF_TRADES', +} + +enum DELIVERY_DATA_CSV_HEADERS { + SYMBOL = 'SYMBOL', + DATE = 'DATE1', + DELIV_QTY = 'DELIV_QTY', + DELIV_PER = 'DELIV_PER', +} + +@Injectable() +export class NseService { + constructor(private readonly AM: AssetManagement) {} + + async handleAssetData(csvData: Stream) { + const nseExchange = await this.AM.exchangeService.findOrCreateExchange( + Exchange.NSE, + ); + const parser = await parseCSV(csvData, CSV_SEPARATOR.COMMA); + for await (const record of parser) { + const assetData = new AssetDto(); + assetData.name = record[STOCK_DATA_CSV_HEADERS.NAME_OF_COMPANY]; + assetData.isin = record[STOCK_DATA_CSV_HEADERS.ISIN_NUMBER]; + assetData.faceValue = parseFloat( + record[STOCK_DATA_CSV_HEADERS.FACE_VALUE], + ); + assetData.symbol = record[STOCK_DATA_CSV_HEADERS.SYMBOL]; + assetData.assetExchangeCode = record[STOCK_DATA_CSV_HEADERS.SYMBOL]; + await this.AM.assetService.createAsset(assetData, nseExchange); + } + } + + async handleTradingData(csvData: Stream) { + const nseExchange = await this.AM.exchangeService.findOrCreateExchange( + Exchange.NSE, + ); + const parser = await parseCSV(csvData, CSV_SEPARATOR.COMMA); + for await (const record of parser) { + const symbol = record[TRADE_DATA_CSV_HEADERS.SYMBOL]; + const series = record[TRADE_DATA_CSV_HEADERS.SERIES]; + const assetExchange = await this.AM.assetExchangeService.findBySymbol( + symbol, + nseExchange, + ['asset'], + ); + if (!assetExchange) { + if (['EQ', 'BE', 'SM'].includes(series)) { + Logger.warn( + `Asset Details not found for symbol: ${symbol} and series: ${series}`, + ); + } + continue; + } + + const tradingDataDto = new TradingDataDTO(); + tradingDataDto.date = new Date(record[TRADE_DATA_CSV_HEADERS.DATE]); + tradingDataDto.open = parseFloat( + record[TRADE_DATA_CSV_HEADERS.OPEN_PRICE], + ); + tradingDataDto.high = parseFloat( + record[TRADE_DATA_CSV_HEADERS.HIGH_PRICE], + ); + tradingDataDto.low = parseFloat(record[TRADE_DATA_CSV_HEADERS.LOW_PRICE]); + tradingDataDto.close = parseFloat( + record[TRADE_DATA_CSV_HEADERS.CLOSE_PRICE], + ); + const lastPrice = parseFloat(record[TRADE_DATA_CSV_HEADERS.LAST_PRICE]); + tradingDataDto.lastPrice = Number.isNaN(lastPrice) + ? 0 + : tradingDataDto.close; + tradingDataDto.previousClose = parseFloat( + record[TRADE_DATA_CSV_HEADERS.PREV_CLOSE], + ); + tradingDataDto.volume = parseInt(record[TRADE_DATA_CSV_HEADERS.VOLUME]); + tradingDataDto.turnover = parseFloat( + record[TRADE_DATA_CSV_HEADERS.TURNOVER], + ); + tradingDataDto.totalTrades = parseFloat( + record[TRADE_DATA_CSV_HEADERS.TOTAL_TRADES], + ); + tradingDataDto.assetExchange = assetExchange; + + const deliveryDataDto = new DeliveryDataDTO(); + deliveryDataDto.date = new Date(record[DELIVERY_DATA_CSV_HEADERS.DATE]); + const qty = parseInt(record[DELIVERY_DATA_CSV_HEADERS.DELIV_QTY]) ?? null; + deliveryDataDto.deliveryQuantity = Number.isNaN(qty) ? null : qty; + const percent = + parseInt(record[DELIVERY_DATA_CSV_HEADERS.DELIV_PER]) ?? null; + deliveryDataDto.deliveryPercentage = Number.isNaN(percent) + ? null + : percent; + deliveryDataDto.assetExchange = assetExchange; + + await this.AM.tradingDataService.saveTradingData(tradingDataDto); + await this.AM.deliveryDataService.saveDeliveryData(deliveryDataDto); + } + } + + generateFileUrls(date: Date) { + const year = date.getFullYear(); + const month = ('0' + (date.getMonth() + 1)).slice(-2); // Months are 0-based in JavaScript + const day = ('0' + date.getDate()).slice(-2); + + return { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36', + referer: + 'https://www1.nseindia.com/products/content/equities/equities/archieve_eq.htm', + assetUrl: `https://nsearchives.nseindia.com/content/equities/EQUITY_L.csv`, + tradingURL: `https://nsearchives.nseindia.com/products/content/sec_bhavdata_full_${day}${month}${year}.csv`, + }; + } +} diff --git a/src/data-sync/types/enums/csv.ts b/src/data-sync/types/enums/csv.ts new file mode 100644 index 0000000..733df5b --- /dev/null +++ b/src/data-sync/types/enums/csv.ts @@ -0,0 +1,4 @@ +export const enum CSV_SEPARATOR { + COMMA = ',', + PIPE = '|', +} diff --git a/src/data-sync/utils/csv-parser.ts b/src/data-sync/utils/csv-parser.ts new file mode 100644 index 0000000..43358d1 --- /dev/null +++ b/src/data-sync/utils/csv-parser.ts @@ -0,0 +1,24 @@ +import * as csvParser from 'csv-parser'; + +const parseCSV = async (csvData, separator: string): Promise => { + return new Promise((resolve, reject) => { + const results = []; + csvData + .pipe(csvParser({ separator })) + .on('data', (data) => { + // Trim the keys of each row + const trimmedData = Object.fromEntries( + Object.entries(data).map(([key, value]) => [ + key.trim(), + (value as string).trim(), + ]), + ); + + results.push(trimmedData); + }) + .on('end', () => resolve(results)) + .on('error', (error) => reject(error)); + }); +}; + +export default parseCSV; diff --git a/src/data-sync/utils/validate-dto-error.ts b/src/data-sync/utils/validate-dto-error.ts new file mode 100644 index 0000000..9f9a7a5 --- /dev/null +++ b/src/data-sync/utils/validate-dto-error.ts @@ -0,0 +1,10 @@ +import { validate } from 'class-validator'; + +export const validateAndThrowError = async (dto, dtoName) => { + const errors = await validate(dto); + if (errors.length > 0) { + throw new Error( + `Validation failed for ${dtoName} : ${JSON.stringify(errors)}` + ); + } +}; diff --git a/yarn.lock b/yarn.lock index f493245..b23e56f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -751,6 +751,14 @@ multer "1.4.4-lts.1" tslib "2.6.2" +"@nestjs/schedule@^4.0.2": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@nestjs/schedule/-/schedule-4.0.2.tgz#573061b152e174f1bf590f5ec00a2c400e8bbee8" + integrity sha512-po9oauE7fO0CjhDKvVC2tzEgjOUwhxYoIsXIVkgfu+xaDMmzzpmXY2s1LT4oP90Z+PaTtPoAHmhslnYmo4mSZg== + dependencies: + cron "3.1.7" + uuid "9.0.1" + "@nestjs/schematics@^10.0.0", "@nestjs/schematics@^10.0.1": version "10.1.1" resolved "https://registry.yarnpkg.com/@nestjs/schematics/-/schematics-10.1.1.tgz#a67fb178a7ad6025ccc3314910b077ac454fcdf3" @@ -998,6 +1006,11 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== +"@types/luxon@~3.4.0": + version "3.4.2" + resolved "https://registry.yarnpkg.com/@types/luxon/-/luxon-3.4.2.tgz#e4fc7214a420173cea47739c33cdf10874694db7" + integrity sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA== + "@types/methods@^1.1.4": version "1.1.4" resolved "https://registry.yarnpkg.com/@types/methods/-/methods-1.1.4.tgz#d3b7ac30ac47c91054ea951ce9eed07b1051e547" @@ -1069,6 +1082,11 @@ "@types/methods" "^1.1.4" "@types/superagent" "^8.1.0" +"@types/validator@^13.11.8": + version "13.11.9" + resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.11.9.tgz#adfe96520b437a0eaa798a475877bf2f75ee402d" + integrity sha512-FCTsikRozryfayPuiI46QzH3fnrOoctTjvOYZkho9BTFLCOZ2rgZJHMOVgCOfttjPJcgOx52EpkY0CMfy87MIw== + "@types/yargs-parser@*": version "21.0.3" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.3.tgz#815e30b786d2e8f0dcd85fd5bcf5e1a04d008f15" @@ -1474,6 +1492,15 @@ asynckit@^0.4.0: resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== +axios@^1.6.8: + version "1.6.8" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.8.tgz#66d294951f5d988a00e87a0ffb955316a619ea66" + integrity sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ== + dependencies: + follow-redirects "^1.15.6" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + babel-jest@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-29.7.0.tgz#f4369919225b684c56085998ac63dbd05be020d5" @@ -1544,6 +1571,11 @@ base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== +big-integer@^1.6.17: + version "1.6.52" + resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.52.tgz#60a887f3047614a8e1bffe5d7173490a97dc8c85" + integrity sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg== + binary-extensions@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" @@ -1558,6 +1590,11 @@ bl@^4.1.0: inherits "^2.0.4" readable-stream "^3.4.0" +bluebird@~3.4.1: + version "3.4.7" + resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3" + integrity sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA== + body-parser@1.20.2: version "1.20.2" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" @@ -1748,6 +1785,15 @@ cjs-module-lexer@^1.0.0: resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz#6c370ab19f8a3394e318fe682686ec0ac684d107" integrity sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ== +class-validator@^0.14.1: + version "0.14.1" + resolved "https://registry.yarnpkg.com/class-validator/-/class-validator-0.14.1.tgz#ff2411ed8134e9d76acfeb14872884448be98110" + integrity sha512-2VEG9JICxIqTpoK1eMzZqaV+u/EiwEJkMGzTrZf6sU/fwsnOITVgYJ8yojSy6CaXtO9V0Cc6ZQZ8h8m4UBuLwQ== + dependencies: + "@types/validator" "^13.11.8" + libphonenumber-js "^1.10.53" + validator "^13.9.0" + cli-cursor@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" @@ -1974,6 +2020,14 @@ create-require@^1.1.0: resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== +cron@3.1.7: + version "3.1.7" + resolved "https://registry.yarnpkg.com/cron/-/cron-3.1.7.tgz#3423d618ba625e78458fff8cb67001672d49ba0d" + integrity sha512-tlBg7ARsAMQLzgwqVxy8AZl/qlTc5nibqYwtNGoCrd+cV+ugI+tvZC1oT/8dFH8W455YrywGykx/KMmAqOr7Jw== + dependencies: + "@types/luxon" "~3.4.0" + luxon "~3.4.0" + cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" @@ -1983,6 +2037,13 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" +csv-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/csv-parser/-/csv-parser-3.0.0.tgz#b88a6256d79e090a97a1b56451f9327b01d710e7" + integrity sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ== + dependencies: + minimist "^1.2.0" + dayjs@^1.11.9: version "1.11.10" resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.10.tgz#68acea85317a6e164457d6d6947564029a6a16a0" @@ -2095,6 +2156,13 @@ dotenv@16.4.5, dotenv@^16.0.3: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f" integrity sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg== +duplexer2@~0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1" + integrity sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA== + dependencies: + readable-stream "^2.0.2" + eastasianwidth@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb" @@ -2522,6 +2590,11 @@ flatted@^3.2.9: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a" integrity sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw== +follow-redirects@^1.15.6: + version "1.15.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== + foreground-child@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.1.1.tgz#1d173e776d75d2772fed08efe4a0de1ea1b12d0d" @@ -2601,6 +2674,16 @@ fsevents@^2.3.2, fsevents@~2.3.2: resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== +fstream@^1.0.12: + version "1.0.12" + resolved "https://registry.yarnpkg.com/fstream/-/fstream-1.0.12.tgz#4e8ba8ee2d48be4f7d0de505455548eae5932045" + integrity sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg== + dependencies: + graceful-fs "^4.1.2" + inherits "~2.0.0" + mkdirp ">=0.5 0" + rimraf "2" + function-bind@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" @@ -2731,7 +2814,7 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" -graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.9: +graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.2, graceful-fs@^4.2.4, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== @@ -2857,7 +2940,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -3528,6 +3611,11 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" +libphonenumber-js@^1.10.53: + version "1.10.60" + resolved "https://registry.yarnpkg.com/libphonenumber-js/-/libphonenumber-js-1.10.60.tgz#1160ec5b390d46345032aa52be7ffa7a1950214b" + integrity sha512-Ctgq2lXUpEJo5j1762NOzl2xo7z7pqmVWYai0p07LvAkQ32tbPv3wb+tcUeHEiXhKU5buM4H9MXsXo6OlM6C2g== + lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" @@ -3594,6 +3682,11 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" +luxon@~3.4.0: + version "3.4.4" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.4.4.tgz#cf20dc27dc532ba41a169c43fdcc0063601577af" + integrity sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA== + magic-string@0.30.5: version "0.30.5" resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.30.5.tgz#1994d980bd1c8835dc6e78db7cbd4ae4f24746f9" @@ -3715,7 +3808,7 @@ minimatch@^9.0.1: dependencies: brace-expansion "^2.0.1" -minimist@^1.2.6: +minimist@^1.2.0, minimist@^1.2.6: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== @@ -3730,7 +3823,7 @@ minipass@^4.2.4: resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.4.tgz#dbce03740f50a4786ba994c1fb908844d27b038c" integrity sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ== -mkdirp@^0.5.4: +"mkdirp@>=0.5 0", mkdirp@^0.5.4: version "0.5.6" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== @@ -4180,6 +4273,11 @@ proxy-addr@~2.0.7: forwarded "0.2.0" ipaddr.js "1.9.1" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + punycode@^2.1.0: version "2.3.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" @@ -4236,7 +4334,7 @@ react-is@^18.0.0: resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b" integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w== -readable-stream@^2.2.2: +readable-stream@^2.0.2, readable-stream@^2.2.2: version "2.3.8" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b" integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== @@ -4336,6 +4434,13 @@ reusify@^1.0.4: resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== +rimraf@2: + version "2.7.1" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" + integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== + dependencies: + glob "^7.1.3" + rimraf@4.4.1: version "4.4.1" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-4.4.1.tgz#bd33364f67021c5b79e93d7f4fa0568c7c21b755" @@ -4970,6 +5075,17 @@ unpipe@1.0.0, unpipe@~1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +unzipper@^0.11.3: + version "0.11.3" + resolved "https://registry.yarnpkg.com/unzipper/-/unzipper-0.11.3.tgz#7c7cfb19b368192721c0258b27ee0b33ceabe5ec" + integrity sha512-Xs72T2f5DkRU1ULU/5kywmMXHI3ZJIL6a1Eb/7eTwmqbOS35ed10PYGDeKMXOuJ5zZrp/HYNeRj80HZsBtOyrA== + dependencies: + big-integer "^1.6.17" + bluebird "~3.4.1" + duplexer2 "~0.1.4" + fstream "^1.0.12" + graceful-fs "^4.2.2" + update-browserslist-db@^1.0.13: version "1.0.13" resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz#3c5e4f5c083661bd38ef64b6328c26ed6c8248c4" @@ -5014,6 +5130,11 @@ v8-to-istanbul@^9.0.1: "@types/istanbul-lib-coverage" "^2.0.1" convert-source-map "^2.0.0" +validator@^13.9.0: + version "13.11.0" + resolved "https://registry.yarnpkg.com/validator/-/validator-13.11.0.tgz#23ab3fd59290c61248364eabf4067f04955fbb1b" + integrity sha512-Ii+sehpSfZy+At5nPdnyMhx78fEoPDkR2XW/zimHEL3MyGJQOCQ7WeP20jPYRz7ZCpcKLB21NxuXHF3bxjStBQ== + vary@^1, vary@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"