[Fix] 데이터 연동 오류 현상 수정

- 전시 데이터 연동 오류 수정 - 축제 데이터 연동 오류 수정
art-friendly-project · Aug 7, 2024 · ea3ef4e · ea3ef4e
1 parent 6e959da
commit ea3ef4e
Show file tree

Hide file tree

Showing 13 changed files with 60 additions and 21 deletions.
diff --git a/src/main/java/org/com/artfriendlybatch/domain/exhibition/chunk/ExhibitionInfoChunk.java b/src/main/java/org/com/artfriendlybatch/domain/exhibition/chunk/ExhibitionInfoChunk.java
@@ -7,6 +7,7 @@
 import org.com.artfriendlybatch.domain.exhibition.mapper.ExhibitionInfoMapper;
 import org.com.artfriendlybatch.domain.exhibition.repository.ExhibitionInfoRepository;
 import org.com.artfriendlybatch.domain.exhibition.service.ExhibitionInfoService;
+import org.springframework.batch.core.configuration.annotation.StepScope;
 import org.springframework.batch.item.ItemProcessor;
 import org.springframework.batch.item.ItemReader;
 import org.springframework.batch.item.ItemWriter;
@@ -26,11 +27,13 @@ public ExhibitionInfoChunk(ExhibitionInfoService exhibitionInfoService, Exhibiti
     }
 
     @Bean
+    @StepScope
     public ItemReader<PerformList> exhibitionInfoIntegrateReader() {
         return new PerformReader(exhibitionInfoService); // API를 호출하여 데이터 읽기
     }
 
     @Bean
+    @StepScope
     public ItemProcessor<PerformList, ExhibitionInfoIntegrateDto> exhibitionInfoIntegrateProcessor(ExhibitionInfoMapper exhibitionInfoMapper) {
         return item -> {
             // API 호출 및 ExhibitionInfo 생성 로직(이미 있는 값이면 업데이트로 수정)
@@ -60,6 +63,7 @@ public ItemProcessor<PerformList, ExhibitionInfoIntegrateDto> exhibitionInfoInte
     }
 
     @Bean
+    @StepScope
     public ItemWriter<ExhibitionInfoIntegrateDto> exhibitionInfoIntegrateWriter() {
         return items -> {
             for (ExhibitionInfoIntegrateDto dto : items) {
@@ -75,16 +79,19 @@ public ItemWriter<ExhibitionInfoIntegrateDto> exhibitionInfoIntegrateWriter() {
     }
 
     @Bean
+    @StepScope
     public ItemReader<ExhibitionInfo> exhibitionInfoReader() {
         return new ExhibitionInfoReader(exhibitionInfoService);
     }
 
     @Bean
+    @StepScope
     public ItemProcessor<ExhibitionInfo, ExhibitionInfo> exhibitionInfoProcessor() {
         return ExhibitionInfo::updateProgressStatus;
     }
 
     @Bean
+    @StepScope
     public ItemWriter<ExhibitionInfo> exhibitionInfoWriter() {
         return exhibitionInfoRepository::saveAll;
     }

diff --git a/src/main/java/org/com/artfriendlybatch/domain/exhibition/chunk/ExhibitionInfoReader.java b/src/main/java/org/com/artfriendlybatch/domain/exhibition/chunk/ExhibitionInfoReader.java
@@ -16,13 +16,12 @@ public class ExhibitionInfoReader implements ItemReader<ExhibitionInfo> {
 
     public ExhibitionInfoReader(ExhibitionInfoService exhibitionInfoService) {
         this.exhibitionInfoService = exhibitionInfoService;
+        this.details = fetchDataFromDb();
     }
 
 
     @Override
     public ExhibitionInfo read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
-        this.details = fetchDataFromDb();
-
         if (currentIndex < details.size()) {
             return details.get(currentIndex++); // 현재 인덱스의 항목 반환 후 인덱스 증가
         }

diff --git a/src/main/java/org/com/artfriendlybatch/domain/exhibition/chunk/PerformReader.java b/src/main/java/org/com/artfriendlybatch/domain/exhibition/chunk/PerformReader.java
@@ -17,12 +17,11 @@ public class PerformReader implements ItemReader<PerformList> {
 
     public PerformReader(ExhibitionInfoService exhibitionInfoService) {
         this.exhibitionInfoService = exhibitionInfoService;
+        this.details = fetchDataFromApi();
     }
 
     @Override
     public PerformList read() throws Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
-        this.details = fetchDataFromApi();
-
         if (currentIndex < details.size()) {
             return details.get(currentIndex++); // 현재 인덱스의 항목 반환 후 인덱스 증가
         }

diff --git a/...riendlybatch/domain/exhibition/repository/querydsl/ExhibitionInfoRepositoryCustomImp.java b/...riendlybatch/domain/exhibition/repository/querydsl/ExhibitionInfoRepositoryCustomImp.java
@@ -25,15 +25,13 @@ public ExhibitionInfo findExhibitionInfoBySeq(int seq) {
     @Override
     public List<ExhibitionInfo> findExhibitionInfoByProgressStatusIn(List<String> progressStatus) {
         return queryFactory.selectFrom(exhibitionInfo)
-                .from(exhibitionInfo)
                 .where(exhibitionInfo.progressStatus.in(progressStatus))
                 .fetch();
     }
 
     @Override
     public List<ExhibitionInfo> findExhibitionInfoByProgressStatus(String progressStatus) {
         return queryFactory.selectFrom(exhibitionInfo)
-                .from(exhibitionInfo)
                 .where(exhibitionInfo.progressStatus.eq(progressStatus))
                 .fetch();
     }

diff --git a/src/main/java/org/com/artfriendlybatch/domain/exhibition/service/ExhibitionInfoService.java b/src/main/java/org/com/artfriendlybatch/domain/exhibition/service/ExhibitionInfoService.java
@@ -139,7 +139,7 @@ private ExhibitionApiRspDto callExhibitionDetailsApi(int seq) {
         ExhibitionApiRspDto exhibitionApiRspDto = exhibitionApiClient.getDetail(String.valueOf(seq), serviceKey);
 
         if(exhibitionApiRspDto.getComMsgHeader().getSuccessYN().equals("N"))
-            throw new RuntimeException("api 연동 에러");
+            throw new RuntimeException("전시 정보 api 호출 오류");
 
         return exhibitionApiRspDto;
     }
@@ -163,8 +163,9 @@ private ExhibitionsApiRspData callExhibitionApi(ApiRequestBuilder apiRequestBuil
                 serviceKey
         );
 
-        if(responseData.getComMsgHeader().getSuccessYN().equals("N"))
-            throw new RuntimeException("api 연동 에러");
+        if(responseData.getComMsgHeader().getSuccessYN().equals("N")) {
+            throw new RuntimeException("전시 정보 api 호출 오류");
+        }
 
         return responseData;
     }

diff --git a/src/main/java/org/com/artfriendlybatch/domain/festival/callable/FestivalInfoCallable.java b/src/main/java/org/com/artfriendlybatch/domain/festival/callable/FestivalInfoCallable.java
@@ -5,6 +5,8 @@
 import org.com.artfriendlybatch.domain.festival.entity.FestivalInfo;
 import org.com.artfriendlybatch.domain.festival.mapper.FestivalMapper;
 import org.com.artfriendlybatch.domain.festival.service.FestivalService;
+import org.springframework.transaction.annotation.Propagation;
+import org.springframework.transaction.annotation.Transactional;
 
 import java.util.concurrent.Callable;
 
@@ -24,7 +26,8 @@ public FestivalInfo call() throws Exception {
         return getFestivalInfo();
     }
 
-    private FestivalInfo getFestivalInfo() throws Exception {
+    @Transactional(propagation = Propagation.REQUIRES_NEW)
+    public FestivalInfo getFestivalInfo() {
         CommonInfoRspDto commonInfoRspDto = festivalService.getCommonInfo(contentId);
         DetailInfoRspDto detailInfoRspDto = festivalService.getDetailInfo(contentId);
 

diff --git a/src/main/java/org/com/artfriendlybatch/domain/festival/entity/FestivalInfo.java b/src/main/java/org/com/artfriendlybatch/domain/festival/entity/FestivalInfo.java
@@ -7,13 +7,14 @@
 import lombok.Builder;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
+import org.com.artfriendlybatch.domain.common.BaseTimeEntity;
 
 import java.time.LocalDate;
 
 @Entity(name = "festival_info")
 @Getter
 @NoArgsConstructor(access = AccessLevel.PROTECTED)
-public class FestivalInfo {
+public class FestivalInfo extends BaseTimeEntity {
     @Id
     @GeneratedValue(strategy = GenerationType.IDENTITY)
     private Long id;

diff --git a/src/main/java/org/com/artfriendlybatch/domain/festival/mapper/FestivalMapper.java b/src/main/java/org/com/artfriendlybatch/domain/festival/mapper/FestivalMapper.java
@@ -56,17 +56,17 @@ else if (now.isBefore(startDate)) {
                 .title(commonItem.getTitle())
                 .seq(Integer.parseInt(commonItem.getContentid()))
                 .phone(commonItem.getTel())
-                .homepageUrl(commonItem.getHomepage())
+                .homepageUrl(commonItem.getHomepage().length() > 1000 ? "" : commonItem.getHomepage())
                 .imageUrl(commonItem.getFirstimage())
                 .placeAddr(commonItem.getAddr1())
                 .gpsX(Double.parseDouble(commonItem.getMapx()))
                 .gpsY(Double.parseDouble(commonItem.getMapy()))
-                .description(commonItem.getOverview().length() >= 1000 ? "" : commonItem.getOverview())
+                .description(commonItem.getOverview().length() > 1000 ? "" : commonItem.getOverview())
                 .place(detailItem.getEventplace())
                 .startDate(startDate)
                 .endDate(endDate)
                 .organizer(detailItem.getSponsor1())
-                .price(detailItem.getUsetimefestival())
+                .price(detailItem.getUsetimefestival().length() > 1000 ? "" : detailItem.getUsetimefestival())
                 .progressStatus(progressStatus)
                 .area(area)
                 .build();

diff --git a/src/main/java/org/com/artfriendlybatch/domain/festival/service/FestivalService.java b/src/main/java/org/com/artfriendlybatch/domain/festival/service/FestivalService.java
@@ -1,6 +1,5 @@
 package org.com.artfriendlybatch.domain.festival.service;
 
-import jakarta.transaction.Transactional;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -17,6 +16,7 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -25,7 +25,7 @@
 
 @Slf4j
 @Service
-@Transactional
+@Transactional(readOnly = true)
 @RequiredArgsConstructor
 public class FestivalService {
     private final FestivalApiClient festivalApiClient;
@@ -40,7 +40,7 @@ public class FestivalService {
     private String serviceKey;
     private final String contentTypeId = "15"; // 축제 행사
 
-    public AreaBaseInfoRspDto getAreaBaseList() throws Exception {
+    public AreaBaseInfoRspDto getAreaBaseList() {
         final int numOfRows = 100;
         final int pageNo = 1;
         final String arrange = "R"; // 생성일 순, 이미지 있는 것 부터 정렬
@@ -57,7 +57,7 @@ public AreaBaseInfoRspDto getAreaBaseList() throws Exception {
 
     }
 
-    public CommonInfoRspDto getCommonInfo(String contentId) throws Exception {
+    public CommonInfoRspDto getCommonInfo(String contentId) {
         final String defaultYN = "Y";
         final String firstImageYN = "Y";
         final String addrinfoYN = "Y";
@@ -73,18 +73,18 @@ public CommonInfoRspDto getCommonInfo(String contentId) throws Exception {
             throw new RuntimeException("공통 정보 api 호출 오류");
     }
 
-    public DetailInfoRspDto getDetailInfo(String contentId) throws Exception {
+    public DetailInfoRspDto getDetailInfo(String contentId) {
         ResponseEntity<DetailInfoRspDto> detailInfoRspDtoResponseEntity = festivalApiClient.getDetailInfo(mobileOS, mobileApp, serviceKey, type, contentId, contentTypeId);
 
         if(detailInfoRspDtoResponseEntity.getStatusCode().is2xxSuccessful()) {
             return detailInfoRspDtoResponseEntity.getBody();
         }
         else
-            throw new RuntimeException("공통 정보 api 호출 오류");
+            throw new RuntimeException("상세 정보 api 호출 오류");
     }
 
     @Transactional
-    public void integrateFestivalInfo() throws Exception {
+    public void integrateFestivalInfo() {
         List<FestivalInfo> festivalInfoList = new ArrayList<>();
         ExecutorService excutor = Executors.newFixedThreadPool(10);
 

diff --git a/src/main/java/org/com/artfriendlybatch/global/batch/policy/CustomSkipPolicy.java b/src/main/java/org/com/artfriendlybatch/global/batch/policy/CustomSkipPolicy.java
@@ -0,0 +1,11 @@
+package org.com.artfriendlybatch.global.batch.policy;
+
+import org.springframework.batch.core.step.skip.SkipLimitExceededException;
+import org.springframework.batch.core.step.skip.SkipPolicy;
+
+public class CustomSkipPolicy implements SkipPolicy {
+    @Override
+    public boolean shouldSkip(Throwable t, long skipCount) throws SkipLimitExceededException {
+        return t instanceof RuntimeException; // 예외 유형에 따라 조정
+    }
+}
diff --git a/src/main/java/org/com/artfriendlybatch/global/config/batch/ExhibitionBatchConfiguration.java b/src/main/java/org/com/artfriendlybatch/global/config/batch/ExhibitionBatchConfiguration.java
@@ -5,10 +5,12 @@
 import org.com.artfriendlybatch.domain.exhibition.entity.ExhibitionInfo;
 import org.com.artfriendlybatch.domain.exhibition.mapper.ExhibitionInfoMapper;
 import org.com.artfriendlybatch.domain.exhibition.chunk.ExhibitionInfoChunk;
+import org.com.artfriendlybatch.global.batch.policy.CustomSkipPolicy;
 import org.com.artfriendlybatch.global.batch.task.ExhibitionCrawlingTask;
 import org.springframework.batch.core.Job;
 import org.springframework.batch.core.Step;
 import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
+import org.springframework.batch.core.configuration.annotation.JobScope;
 import org.springframework.batch.core.job.builder.JobBuilder;
 import org.springframework.batch.core.launch.support.RunIdIncrementer;
 import org.springframework.batch.core.repository.JobRepository;
@@ -60,12 +62,16 @@ public Step crawlingStep(JobRepository jobRepository, PlatformTransactionManager
     }
 
     @Bean(name = "integrateStep")
+    @JobScope
     Step exhibitionIntegrateStep(JobRepository jobRepository, PlatformTransactionManager tx, ExhibitionInfoMapper exhibitionInfoMapper) {
         return new StepBuilder("exhibitionInfoIntegrateStep", jobRepository)
                 .<PerformList, ExhibitionInfoIntegrateDto>chunk(10, tx)
                 .reader(exhibitionInfoChunk.exhibitionInfoIntegrateReader())
                 .processor(exhibitionInfoChunk.exhibitionInfoIntegrateProcessor(exhibitionInfoMapper))
                 .writer(exhibitionInfoChunk.exhibitionInfoIntegrateWriter())
+                .faultTolerant()
+                .skipLimit(10)
+                .skipPolicy(new CustomSkipPolicy())
                 .build();
     }
 

diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml
@@ -30,6 +30,13 @@ spring:
 
   application:
     name: Crawler
+  cloud:
+    openfeign:
+      client:
+        config:
+          default:
+            connect-timeout: 5000
+            read-timeout: 5000
 
 server:
   port: 8090

diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
@@ -30,6 +30,13 @@ spring:
 
   application:
     name: Crawler
+  cloud:
+    openfeign:
+      client:
+        config:
+          default:
+            connect-timeout: 5000
+            read-timeout: 5000
 
 logging:
   level: