Setup data sync module (#12)

* Add DataSyncModule to app.module.ts

* Update package.json and yarn.lock to add @nestjs/schedule dependency

* Add @nestjs/schedule and AssetManagementModule to DataSyncModule imports

* Add new files and update existing files in asset-management module

* Add new files and update existing files in data-sync module

* Remove unused files and configurations

* Add Docker configuration files

* Add CI workflow for building and publishing Docker image

* Add pull_request trigger to CI workflow

* Update NodeJS version in CI workflow to use actions/setup-node@v4

* Refactor logger instantiation in asset.service.ts and asset-exchange.service.ts

* Update CI workflow to remove creation of .env file

* Update CI workflow to include Docker image build and push

* Update CI workflow to include Docker image save and load

* Update CI workflow to include Docker image save and load

* Refactor Docker image naming in CI workflow

* Add GitHub Actions step to comment on successful pull requests with Docker image artifact link

* Update GitHub Actions step to use actions/github-script@v7 in ci.yml

* Update permissions in ci.yml

.dockerignore

@@ -0,0 +1,7 @@
+node_modules
+dist
+test
+.dockerignore
+Dockerfile
+README.md
+.env

.github/workflows/ci.yml

@@ -0,0 +1,91 @@
+name: CI
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - name: Install dependencies
+        run: yarn install
+
+      - name: Run tests
+        run: yarn test --colors
+
+      - name: Build Docker image
+        run: |
+          docker build -t stockdog:pr-${{ github.event.number }} .
+          docker save stockdog:pr-${{ github.event.number }} > stockdog-pr-${{ github.event.number }}.image.tar
+
+      - name: Save Docker image as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-image
+          path: stockdog-pr-${{ github.event.number }}.image.tar
+
+      - name: Comment PR
+        if: success() && github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const artifactUrl = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;
+            const message = `:sparkles: **Docker Image Artifact** :sparkles:\n\nYour Docker image artifact can be found at the following location:\n\n:point_right: [Click Here](${artifactUrl}) :point_left:\n\nHappy coding! :rocket:`;
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: message,
+            });
+
+  push:
+    needs: build
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download Docker image from artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+          path: stockdog-pr-${{ github.event.number }}.image.tar
+
+      - name: Load Docker image
+        run: docker load < stockdog-pr-${{ github.event.number }}.image.tar
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Push Docker image
+        run: docker push ${{ secrets.DOCKER_HUB_USERNAME }}/stockdog:pr-${{ github.event.number }}
+
+      - name: Update image metadata
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ secrets.DOCKER_HUB_USERNAME }}/stockdog:pr-${{ github.event.number }}
+          tags: |
+            type=sha
+          labels: |
+            org.opencontainers.image.title=StockDog App
+            org.opencontainers.image.description=Stock market analysis app
+            org.opencontainers.image.url=https://github.com/${{github.repository}}
+            org.opencontainers.image.revision=${{github.sha}}
+            org.opencontainers.image.licenses=MIT

Dockerfile

@@ -0,0 +1,17 @@
+FROM node:21-alpine
+
+WORKDIR /app
+
+COPY package.json ./
+
+COPY yarn.lock ./
+
+RUN yarn install
+
+COPY . .
+
+RUN yarn build
+
+EXPOSE 3000
+
+CMD [ "yarn", "start:prod" ]

docker-compose.yml

@@ -1,16 +1,21 @@
-version: '3.8'  
-  
-services:  
-  db:  
-    image: postgres:13  
-    environment:  
+version: '3.8'
+
+services:
+  db:
+    image: postgres:13
+    environment:
       POSTGRES_USER: ${DB_USERNAME}
       POSTGRES_PASSWORD: ${DB_PASSWORD}
       POSTGRES_DB: ${DB_NAME}
-    ports:  
-      - "5432:5432"  
-    volumes:  
-      - db-data:/var/lib/postgresql/data  
-
-volumes:  
-  db-data:  
+    ports:
+      - '5432:5432'
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    networks:
+      - stockdog
+
+volumes:
+  db-data:
+
+networks:
+  stockdog:

package.json

@@ -24,11 +24,16 @@
     "@nestjs/config": "^3.2.2",
     "@nestjs/core": "^10.0.0",
     "@nestjs/platform-express": "^10.0.0",
+    "@nestjs/schedule": "^4.0.2",
     "@nestjs/typeorm": "^10.0.2",
+    "axios": "^1.6.8",
+    "class-validator": "^0.14.1",
+    "csv-parser": "^3.0.0",
     "pg": "^8.11.5",
     "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1",
-    "typeorm": "^0.3.20"
+    "typeorm": "^0.3.20",
+    "unzipper": "^0.11.3"
   },
   "devDependencies": {
     "@nestjs/cli": "^10.0.0",

src/app.module.ts

@@ -4,15 +4,19 @@ import { AppService } from './app.service';
 import { ConfigModule } from '@nestjs/config';
 import { AssetManagementModule } from './asset-management/asset-management.module';
 import { DatabaseModule } from './database.module';
+import { DataSyncModule } from './data-sync/data-sync.module';
+import { ScheduleModule } from '@nestjs/schedule';
 
 @Module({
   imports: [
+    ScheduleModule.forRoot(),
     ConfigModule.forRoot({
       isGlobal: true,
       envFilePath: ['.env'],
     }),
     DatabaseModule,
     AssetManagementModule,
+    DataSyncModule,
   ],
   controllers: [AppController],
   providers: [AppService],

src/asset-management/asset-management.module.ts

@@ -1,4 +1,4 @@
-import { Module } from '@nestjs/common';
+import { Logger, Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { AssetExchange } from './entities/asset-exchange.entity';
 import { Asset } from './entities/asset.entity';
@@ -13,6 +13,12 @@ import {
   TradingDataRepository,
 } from './repositories';
 import { ExchangeService } from './services/exchange.service';
+import {
+  AssetService,
+  DeliveryDataService,
+  TradingDataService,
+  AssetExchangeService,
+} from './services';
 
 @Module({
   imports: [
@@ -25,13 +31,24 @@ import { ExchangeService } from './services/exchange.service';
     ]),
   ],
   providers: [
+    Logger,
     AssetRepository,
     ExchangeRepository,
     AssetExchangeRepository,
     TradingDataRepository,
     DeliveryDataRepository,
     ExchangeService,
+    AssetService,
+    DeliveryDataService,
+    TradingDataService,
+    AssetExchangeService,
+  ],
+  exports: [
+    ExchangeService,
+    AssetService,
+    DeliveryDataService,
+    TradingDataService,
+    AssetExchangeService,
   ],
-  exports: [ExchangeService],
 })
 export class AssetManagementModule {}

src/asset-management/dto/asset.dto.ts

@@ -0,0 +1,35 @@
+import { IsString, IsOptional, IsNotEmpty, IsNumber } from 'class-validator';
+import { AssetExchange } from '../entities';
+
+export class AssetDto {
+  @IsNotEmpty()
+  @IsString()
+  isin: string;
+
+  @IsNotEmpty()
+  @IsString()
+  symbol: string;
+
+  @IsNotEmpty()
+  @IsString()
+  name: string;
+
+  @IsNotEmpty()
+  @IsString()
+  assetExchangeCode: string;
+
+  @IsOptional()
+  @IsNumber()
+  faceValue: number;
+
+  @IsOptional()
+  @IsString()
+  industry: string;
+
+  @IsOptional()
+  @IsString()
+  sector: string;
+
+  @IsOptional()
+  assetExchanges: AssetExchange[];
+}

src/asset-management/dto/delivery-data.dto.ts

@@ -0,0 +1,18 @@
+import { IsDate, IsNumber, IsOptional } from 'class-validator';
+import { AssetExchange } from '../entities';
+
+export class DeliveryDataDTO {
+  @IsDate()
+  date: Date;
+
+  @IsNumber()
+  @IsOptional()
+  deliveryQuantity: number;
+
+  @IsNumber()
+  @IsOptional()
+  deliveryPercentage: number;
+
+  @IsOptional()
+  assetExchange: AssetExchange | null;
+}

src/asset-management/dto/index.ts

@@ -0,0 +1,3 @@
+export * from './asset.dto';
+export * from './trading-data.dto';
+export * from './delivery-data.dto';

src/asset-management/dto/trading-data.dto.ts

@@ -0,0 +1,36 @@
+import { IsNumber, IsDate } from 'class-validator';
+import { AssetExchange } from '../entities';
+
+export class TradingDataDTO {
+  @IsDate()
+  date: Date;
+
+  @IsNumber()
+  open: number;
+
+  @IsNumber()
+  high: number;
+
+  @IsNumber()
+  low: number;
+
+  @IsNumber()
+  close: number;
+
+  @IsNumber()
+  lastPrice: number;
+
+  @IsNumber()
+  previousClose: number;
+
+  @IsNumber()
+  volume: number;
+
+  @IsNumber()
+  turnover: number;
+
+  @IsNumber()
+  totalTrades: number;
+
+  assetExchange: AssetExchange;
+}

src/asset-management/repositories/base.repository.ts

@@ -3,14 +3,21 @@ import {
   FindOneOptions,
   SelectQueryBuilder,
   FindOptionsWhere,
+  InsertResult,
 } from 'typeorm';
+import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity';
+import { UpsertOptions } from 'typeorm/repository/UpsertOptions';
 
 export interface DatabaseRepository<T> {
   findAll(): Promise<T[]>;
   findById(id: any): Promise<T | null>;
   create(item: T): Promise<T>;
   update(id: string, item: T): Promise<T | null>;
   delete(id: string): Promise<void>;
+  upsert(
+    item: QueryDeepPartialEntity<T>,
+    options: UpsertOptions<T>,
+  ): Promise<InsertResult>;
   createQueryBuilder(alias: string): SelectQueryBuilder<T>;
 }
 
@@ -25,8 +32,11 @@ export abstract class BaseRepository<T> implements DatabaseRepository<T> {
     return this.repository.findOne(id);
   }
 
-  async findOneBy(props: FindOptionsWhere<T>): Promise<T | null> {
-    return this.repository.findOneBy(props);
+  async findOneBy(
+    props: FindOptionsWhere<T>,
+    relations?: string[],
+  ): Promise<T | null> {
+    return this.repository.findOne({ where: props, relations: relations });
   }
 
   async create(item: T): Promise<T> {
@@ -47,6 +57,13 @@ export abstract class BaseRepository<T> implements DatabaseRepository<T> {
     await this.repository.delete(id);
   }
 
+  async upsert(
+    item: QueryDeepPartialEntity<T>,
+    options: UpsertOptions<T>,
+  ): Promise<InsertResult> {
+    return this.repository.upsert(item, options);
+  }
+
   createQueryBuilder(alias: string): SelectQueryBuilder<T> {
     return this.repository.createQueryBuilder(alias);
   }