Merge pull request #46 from boostcampwm-2024/task-#45-newsAI-cicd

Task #45 news ai cicd

packages/backend/src/main.ts

@@ -35,12 +35,12 @@ async function bootstrap() {
 
   // 4초 후 customQueryLogger 작동
   // 서버 초기화 쿼리는 로그 찍을 필요 없음
-  setTimeout(() => {
-    const dataSource = app.get(DataSource);
-    dataSource.setOptions({
-      logger: new CustomQueryLogger()
-    });
-  }, 4000);
+  // setTimeout(() => {
+  //   const dataSource = app.get(DataSource);
+  //   dataSource.setOptions({
+  //     logger: new CustomQueryLogger()
+  //   });
+  // }, 4000);
 
   app.setGlobalPrefix('api');
   app.use(session({ ...sessionConfig, store }));

packages/backend/src/news/StockNewsOrchestrationService.ts

@@ -0,0 +1,58 @@
+import { Inject, Injectable } from '@nestjs/common';
+import { Logger } from 'winston';
+import { NewsCrawlingService } from '@/news/newsCrawling.service';
+import { NewsSummaryService } from '@/news/newsSummary.service';
+import { StockNewsRepository } from '@/news/stockNews.repository';
+import { CrawlingDataDto } from '@/news/dto/crawlingData.dto';
+import { Cron } from '@nestjs/schedule';
+import { CreateStockNewsDto } from '@/news/dto/stockNews.dto';
+
+@Injectable()
+export class StockNewsOrchestrationService {
+  constructor(
+    @Inject('winston') private readonly logger: Logger,
+    private readonly newsCrawlingService: NewsCrawlingService,
+    private readonly newsSummaryService: NewsSummaryService,
+    private readonly stockNewsRepository: StockNewsRepository,
+  ) {}
+
+  @Cron('19 15 0 * * *') //오후 3시 19분
+  public async orchestrateStockProcessing() {
+    const stockNameList = [
+      '삼성전자',
+      'SK하이닉스',
+      'LG에너지솔루션',
+      '삼성바이오로직스',
+      '현대차',
+      '기아',
+      '셀트리온',
+      'NAVER',
+      'KB금융',
+      'HD현대중공업',
+    ];
+    // 주식 데이터 크롤링
+    for (const stockName of stockNameList) {
+      const stockDataList =
+        await this.newsCrawlingService.getNewsLinks(stockName);
+
+      const stockNewsData: CrawlingDataDto =
+        await this.newsCrawlingService.crawling(
+          stockDataList!.stock,
+          stockDataList!.response,
+        );
+      // 데이터 요약
+      const summarizedData =
+        await this.newsSummaryService.summarizeNews(stockNewsData);
+
+      console.log('summarizedData');
+      console.log(summarizedData);
+
+      // DB에 저장
+      if (summarizedData) {
+        await this.stockNewsRepository.create(summarizedData);
+      } else {
+        this.logger.error('Failed to summarize news');
+      }
+    }
+  }
+}

packages/backend/src/news/dto/crawlNewsItem.dto.ts

@@ -0,0 +1,6 @@
+export class CrawlNewsItemDto {
+  date: string;
+  title: string;
+  content: string;
+  url: string;
+}

packages/backend/src/news/dto/crawlingData.dto.ts

@@ -0,0 +1,6 @@
+import { CrawlNewsItemDto } from './crawlNewsItem.dto';
+
+export class CrawlingDataDto {
+  stockName: string;
+  news: CrawlNewsItemDto[];
+}

packages/backend/src/news/dto/newsInfoDto.ts

@@ -0,0 +1,9 @@
+import { NewsItemDto } from './newsItemDto';
+
+export class NewsInfoDto {
+  lastBuildDate: string;
+  total: number;
+  start: number;
+  display: number;
+  items: NewsItemDto[];
+}

packages/backend/src/news/dto/newsItemDto.ts

@@ -0,0 +1,7 @@
+export class NewsItemDto {
+  title: string;
+  original_link: string;
+  link: string;
+  description: string;
+  pubDate: Date;
+}

packages/backend/src/news/newsCrawling.service.ts

@@ -0,0 +1,71 @@
+import { Inject, Injectable } from '@nestjs/common';
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+import { Logger } from 'winston';
+import { NewsInfoDto } from './dto/newsInfoDto';
+import { NewsItemDto } from './dto/newsItemDto';
+import { CrawlingDataDto } from '@/news/dto/crawlingData.dto';
+
+@Injectable()
+export class NewsCrawlingService {
+  constructor(@Inject('winston') private readonly logger: Logger) {
+  }
+
+  // naver news API 이용해 뉴스 정보 얻어오기
+  async getNewsLinks(stockName: string) {
+    const encodedStockName = encodeURI(stockName);
+    const newsUrl = `${process.env.NAVER_NEWS_URL}?query=${encodedStockName}&display=10&sort=sim`;
+    try {
+      const res: NewsInfoDto = await axios(newsUrl, {
+        method: 'GET',
+        headers: {
+          'X-Naver-Client-Id': process.env.NAVER_CLIENT_ID,
+          'X-Naver-Client-Secret': process.env.NAVER_CLIENT_SECRET,
+        },
+      }).then((r) => r.data);
+      return {
+        stock: stockName,
+        response: await this.extractNaverNews(res),
+      };
+    } catch (err) {
+      this.logger.error(err);
+    }
+  }
+
+  async extractNaverNews(newsData: NewsInfoDto) {
+    return newsData.items.filter((e) => e.link.includes('n.news.naver.com'));
+  }
+
+  // 얻어온 뉴스 정보들 중 naver news에 기사가 있는 사이트에서 제목, 본문, 생성 날짜등을 크롤링해오기
+  async crawling(stock: string, news: NewsItemDto[]) {
+    return {
+      stockName: stock,
+      news: await Promise.all(
+        news.map(async (n) => {
+          const url = decodeURI(n.link);
+          return await axios(url, {
+            method: 'GET',
+            headers: {
+              'User-Agent':
+                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
+              'Accept-Language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
+            },
+          }).then(async (r) => {
+            const htmlString = await r.data;
+            const $ = cheerio.load(htmlString);
+
+            const date = $('span._ARTICLE_DATE_TIME').attr('data-date-time');
+            const title = $('#title_area').text();
+            const content = $('#dic_area').text();
+            return {
+              date: date,
+              title: title,
+              content: content,
+              url: url,
+            };
+          });
+        }),
+      ),
+    } as CrawlingDataDto;
+  }
+}

packages/backend/src/news/newsClova.service.ts → packages/backend/src/news/newsSummary.service.ts

@@ -1,19 +1,20 @@
 import { Inject, Injectable } from '@nestjs/common';
 import axios from 'axios';
 import { Logger } from 'winston';
-import { SAMPLE_NEWS_SCRAP } from './sample';
 import { CreateStockNewsDto } from './dto/stockNews.dto';
+import { SAMPLE_NEWS_SCRAP } from './sample';
+import { CrawlingDataDto } from '@/news/dto/crawlingData.dto';
 
 @Injectable()
-export class NewsClovaService {
+export class NewsSummaryService {
   private readonly CLOVA_API_URL =
     'https://clovastudio.stream.ntruss.com/testapp/v1/chat-completions/HCX-003';
   private readonly CLOVA_API_KEY = process.env.CLOVA_API_KEY;
 
   constructor(@Inject('winston') private readonly logger: Logger) {}
 
   // TODO: 뉴스 데이터를 넣어주는 파라미터 추가
-  async summarizeNews() {
+  async summarizeNews(stockNewsData: CrawlingDataDto) {
     try {
       const clovaResponse = await axios.post(
         this.CLOVA_API_URL,
@@ -25,7 +26,7 @@ export class NewsClovaService {
             },
             {
               role: 'user',
-              content: JSON.stringify(SAMPLE_NEWS_SCRAP), // TODO: 파라미터값으로 변경
+              content: JSON.stringify(stockNewsData),
             },
           ],
           ...this.getParameters(),
@@ -41,9 +42,9 @@ export class NewsClovaService {
       }
 
       const summarizedNews = new CreateStockNewsDto();
-      summarizedNews.stock_id = content.stock_id;
-      summarizedNews.stock_name = content.stock_name;
-      summarizedNews.link = content.link;
+      summarizedNews.stock_id = content.stockId;
+      summarizedNews.stock_name = content.stockName;
+      summarizedNews.link = this.formatLinks(content?.link);
       summarizedNews.title = content.title;
       summarizedNews.summary = content.summary;
       summarizedNews.positive_content = content.positive_content;
@@ -61,6 +62,16 @@ export class NewsClovaService {
     }
   }
 
+  private formatLinks(links: unknown): string {
+    if (Array.isArray(links)) {
+      return links.join(",");
+    }
+    if (typeof links === 'string') {
+      return links;
+    }
+    return "";
+  }
+
   private getSystemPrompt() {
     return '당신은 AI 기반 주식 분석 전문가입니다. 입력으로 주어지는 JSON 형식의 뉴스 데이터를 분석하여, JSON 형식으로 종합적인 분석 결과를 도출해 주세요.\r\n\r\n[입력 형식]\r\n{\r\n  "stock_name": "종목 이름",\r\n  "news": [\r\n    {\r\n      "date": "기사 날짜",\r\n      "title": "기사 제목",\r\n      "content": "기사 내용",\r\n      "url": "기사 링크"\r\n    },\r\n    ...\r\n  ]\r\n}\r\n\r\n[출력 형식]\r\n{\r\n  "stock_id": "종목 번호",\r\n  "stock_name": "종목 이름",\r\n  "link": "기사 링크들",\r\n  "title": "요약 타이틀",\r\n  "summary": "요약 내용",\r\n  "positive_content": "긍정적 측면",\r\n  "negative_content": "부정적 측면"\r\n}\r\n\r\n[분석 지침]\r\n분석해야 할 항목은 다음과 같습니다:\r\n\r\n1. **종목 정보:**\r\n   - stock_name을 기반으로 해당 종목의 stock_id를 찾아 포함\r\n   - 제공된 모든 뉴스의 url을 쉼표로 구분하여 link 필드에 포함\r\n\r\n2. **종합 분석:**\r\n   - title: 전체 뉴스 내용을 관통하는 핵심 주제나 이슈를 간단한 제목으로 작성\r\n   - summary: 모든 뉴스 기사의 주요 내용을 종합적으로 요약하여 작성\r\n\r\n3. **영향 분석:**\r\n   - positive_content: 기업, 산업, 경제에 긍정적 영향을 줄 수 있는 요소들을 분석하여 작성\r\n   - negative_content: 위험 요소나 부정적 영향을 줄 수 있는 요소들을 분석하여 작성\r\n\r\n[제약 사항]\r\n1. 모든 뉴스 기사의 내용을 종합적으로 고려하여 분석합니다.\r\n2. positive_content 와 negative_content 내용이 없는 경우 "해당사항 없음"으로 작성합니다.\r\n3. 요약과 분석은 객관적이고 사실에 기반하여 작성합니다.\r\n4. 특정 종목의 stock_id를 모르는 경우 빈 문자열("")을 반환합니다.\r\n5. 반드시 JSON 형식으로 응답합니다.';
   }

packages/backend/src/news/stockNews.controller.ts

@@ -1,34 +1,42 @@
 import { Controller, Post, Body, Get, Param } from '@nestjs/common';
 import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger';
-import { StockNewsService } from '@/news/stockNews.service';
 import { CreateStockNewsDto, StockNewsResponse } from '@/news/dto/stockNews.dto';
+import { StockNewsRepository } from '@/news/stockNews.repository';
+import { StockNewsOrchestrationService } from '@/news/StockNewsOrchestrationService';
 
 @ApiTags('Stock News')
 @Controller('stock/news')
 export class StockNewsController {
-  constructor(private readonly stockNewsService: StockNewsService) {}
+  constructor(
+    private readonly stockNewsRepository: StockNewsRepository,
+    private readonly stockNewsOrchestrationService: StockNewsOrchestrationService) {}
 
   @Post()
   @ApiOperation({ summary: '주식 뉴스 정보 저장' })
   @ApiResponse({ status: 201, type: StockNewsResponse })
   async create(@Body() createStockNewsDto: CreateStockNewsDto) {
-    const stockNews = await this.stockNewsService.create(createStockNewsDto);
+    const stockNews = await this.stockNewsRepository.create(createStockNewsDto);
     return new StockNewsResponse(stockNews);
   }
 
   @Get(':stockId')
   @ApiOperation({ summary: '종목별 뉴스 조회' })
   @ApiResponse({ status: 200, type: [StockNewsResponse] })
   async findByStockId(@Param('stockId') stockId: string) {
-    const newsList = await this.stockNewsService.findByStockId(stockId);
-    return newsList.map(news => new StockNewsResponse(news));
+    const newsList = await this.stockNewsRepository.findByStockId(stockId);
+    return newsList.map((news) => new StockNewsResponse(news));
   }
 
   @Get(':stockId/latest')
   @ApiOperation({ summary: '종목별 최신 뉴스 조회' })
   @ApiResponse({ status: 200, type: StockNewsResponse })
   async findLatestByStockId(@Param('stockId') stockId: string) {
-    const news = await this.stockNewsService.findLatestByStockId(stockId);
+    const news = await this.stockNewsRepository.findLatestByStockId(stockId);
     return news ? new StockNewsResponse(news) : null;
   }
+
+  @Post('test')
+  async testAI(){
+    await this.stockNewsOrchestrationService.orchestrateStockProcessing();
+  }
 }

packages/backend/src/news/stockNews.module.ts

@@ -1,14 +1,16 @@
+import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { StockNews } from '@/news/domain/stockNews.entity';
 import { StockNewsController } from '@/news/stockNews.controller';
-import { Module } from '@nestjs/common';
-import { StockNewsService } from '@/news/stockNews.service';
+import { StockNewsRepository } from '@/news/stockNews.repository';
+import { NewsCrawlingService } from '@/news/newsCrawling.service';
+import { StockNewsOrchestrationService } from '@/news/StockNewsOrchestrationService';
+import { NewsSummaryService } from '@/news/newsSummary.service';
 
 @Module({
   imports: [TypeOrmModule.forFeature([StockNews])],
   controllers: [StockNewsController],
-  providers: [StockNewsService],
-  exports: [StockNewsService],
+  providers: [StockNewsRepository, NewsCrawlingService, StockNewsOrchestrationService, NewsSummaryService],
+  exports: [],
 })
-
-export class StockNewsModule {}
+export class StockNewsModule {}

packages/backend/src/news/stockNews.service.ts → packages/backend/src/news/stockNews.repository.ts

@@ -5,13 +5,14 @@ import { StockNews } from '@/news/domain/stockNews.entity';
 import { CreateStockNewsDto } from '@/news/dto/stockNews.dto';
 
 @Injectable()
-export class StockNewsService {
+export class StockNewsRepository {
   constructor(
     @InjectRepository(StockNews)
     private readonly stockNewsRepository: Repository<StockNews>,
   ) {}
 
   async create(dto: CreateStockNewsDto): Promise<StockNews> {
+    console.log(dto);
     const stockNews = new StockNews();
     stockNews.stockId = dto.stock_id;
     stockNews.stockName = dto.stock_name;