From 3caedb3b9179c1ef9c06808484d48fe7a370b3e5 Mon Sep 17 00:00:00 2001 From: eojin0814 Date: Mon, 13 Nov 2023 22:27:55 +0900 Subject: [PATCH] =?UTF-8?q?feat(#1)=20:=20Spring=20Batch=20=EA=B5=AC?= =?UTF-8?q?=ED=98=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 15 +++ .../server/KeywordServiceApplication.java | 2 + .../domain/morpheme/domain/entity/Issue.java | 21 ++++ .../morpheme/domain/entity/Morpheme.java | 28 +++++ .../domain/morpheme/domain/entity/Topic.java | 21 ++++ .../domain/repository/MorphemeRepository.java | 10 ++ .../domain/service/MorphemeService.java | 40 +++++++ .../domain/service/NewsAPIService.java | 108 ++++++++++++++++++ .../presentation/MorphemeController.java | 36 ++++++ .../global/config/RestTemplateConfig.java | 22 ++++ .../global/config/batch/BatchJobConfig.java | 67 +++++++++++ .../config/batch/GetAnalysisTasklet.java | 34 ++++++ .../config/batch/GetMorphemeTasklet.java | 30 +++++ .../config/batch/GetNewsApiTasklet.java | 26 +++++ .../global/security/SecurityConfig.java | 3 +- 15 files changed, 462 insertions(+), 1 deletion(-) create mode 100644 src/main/java/gwangjang/server/domain/morpheme/domain/entity/Issue.java create mode 100644 src/main/java/gwangjang/server/domain/morpheme/domain/entity/Morpheme.java create mode 100644 src/main/java/gwangjang/server/domain/morpheme/domain/entity/Topic.java create mode 100644 src/main/java/gwangjang/server/domain/morpheme/domain/repository/MorphemeRepository.java create mode 100644 src/main/java/gwangjang/server/domain/morpheme/domain/service/MorphemeService.java create mode 100644 src/main/java/gwangjang/server/domain/morpheme/domain/service/NewsAPIService.java create mode 100644 src/main/java/gwangjang/server/domain/morpheme/presentation/MorphemeController.java create mode 100644 src/main/java/gwangjang/server/global/config/RestTemplateConfig.java create mode 100644 src/main/java/gwangjang/server/global/config/batch/BatchJobConfig.java create mode 100644 src/main/java/gwangjang/server/global/config/batch/GetAnalysisTasklet.java create mode 100644 src/main/java/gwangjang/server/global/config/batch/GetMorphemeTasklet.java create mode 100644 src/main/java/gwangjang/server/global/config/batch/GetNewsApiTasklet.java diff --git a/build.gradle b/build.gradle index e3bbda2..31a9feb 100644 --- a/build.gradle +++ b/build.gradle @@ -19,6 +19,7 @@ configurations { repositories { mavenCentral() + maven { url 'https://jitpack.io' } } ext { @@ -69,6 +70,20 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-data-redis' + // 형태소 + implementation 'com.github.shin285:KOMORAN:3.3.4' + + implementation 'org.json:json:20210307' + implementation 'com.googlecode.json-simple:json-simple:1.1' + + + + implementation group: 'io.springfox', name: 'springfox-boot-starter', version: '3.0.0' // 사용 가능한 최신 버전으로 업데이트하세요 + + //spring batch + implementation 'org.springframework.boot:spring-boot-starter-batch' + + implementation "org.springframework.boot:spring-boot-starter-quartz" } diff --git a/src/main/java/gwangjang/server/KeywordServiceApplication.java b/src/main/java/gwangjang/server/KeywordServiceApplication.java index c4d2fea..c4e7677 100644 --- a/src/main/java/gwangjang/server/KeywordServiceApplication.java +++ b/src/main/java/gwangjang/server/KeywordServiceApplication.java @@ -1,9 +1,11 @@ package gwangjang.server; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @SpringBootApplication +@EnableBatchProcessing public class KeywordServiceApplication { public static void main(String[] args) { diff --git a/src/main/java/gwangjang/server/domain/morpheme/domain/entity/Issue.java b/src/main/java/gwangjang/server/domain/morpheme/domain/entity/Issue.java new file mode 100644 index 0000000..7dc80af --- /dev/null +++ b/src/main/java/gwangjang/server/domain/morpheme/domain/entity/Issue.java @@ -0,0 +1,21 @@ +package gwangjang.server.domain.morpheme.domain.entity; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@NoArgsConstructor +@AllArgsConstructor +@Getter +public class Issue { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "issue_id") + private Long id; + private String issueTitle; + @ManyToOne + @JoinColumn(name = "topic_id") + private Topic topic; +} diff --git a/src/main/java/gwangjang/server/domain/morpheme/domain/entity/Morpheme.java b/src/main/java/gwangjang/server/domain/morpheme/domain/entity/Morpheme.java new file mode 100644 index 0000000..409bec6 --- /dev/null +++ b/src/main/java/gwangjang/server/domain/morpheme/domain/entity/Morpheme.java @@ -0,0 +1,28 @@ +package gwangjang.server.domain.morpheme.domain.entity; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + + +@Entity +@NoArgsConstructor +@AllArgsConstructor +@Getter +@Setter +public class Morpheme { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "morpheme_id") + private Long id; + + private String word; + + private int count; + + private int issueId; + + +} diff --git a/src/main/java/gwangjang/server/domain/morpheme/domain/entity/Topic.java b/src/main/java/gwangjang/server/domain/morpheme/domain/entity/Topic.java new file mode 100644 index 0000000..2777197 --- /dev/null +++ b/src/main/java/gwangjang/server/domain/morpheme/domain/entity/Topic.java @@ -0,0 +1,21 @@ +package gwangjang.server.domain.morpheme.domain.entity; + +import jakarta.persistence.*; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; + +@Entity +@NoArgsConstructor +@AllArgsConstructor +@Getter +public class Topic { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "topic_id") + private Long id; + + private String topicTitle; + + +} diff --git a/src/main/java/gwangjang/server/domain/morpheme/domain/repository/MorphemeRepository.java b/src/main/java/gwangjang/server/domain/morpheme/domain/repository/MorphemeRepository.java new file mode 100644 index 0000000..6d74f48 --- /dev/null +++ b/src/main/java/gwangjang/server/domain/morpheme/domain/repository/MorphemeRepository.java @@ -0,0 +1,10 @@ +package gwangjang.server.domain.morpheme.domain.repository; + +import gwangjang.server.domain.morpheme.domain.entity.Morpheme; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface MorphemeRepository extends JpaRepository { + Morpheme findByWordAndIssueId(String word, int issueId); +} diff --git a/src/main/java/gwangjang/server/domain/morpheme/domain/service/MorphemeService.java b/src/main/java/gwangjang/server/domain/morpheme/domain/service/MorphemeService.java new file mode 100644 index 0000000..4cb37b0 --- /dev/null +++ b/src/main/java/gwangjang/server/domain/morpheme/domain/service/MorphemeService.java @@ -0,0 +1,40 @@ +package gwangjang.server.domain.morpheme.domain.service; + +import gwangjang.server.domain.morpheme.domain.entity.Morpheme; +import gwangjang.server.domain.morpheme.domain.repository.MorphemeRepository; +import gwangjang.server.global.annotation.DomainService; +import jakarta.transaction.Transactional; +import kr.co.shineware.nlp.komoran.model.Token; +import lombok.RequiredArgsConstructor; + +import java.util.List; + +@DomainService +@Transactional +@RequiredArgsConstructor +public class MorphemeService { + private final MorphemeRepository morphemeRepository; + + @Transactional + public void saveOrUpdateWord(List tokens , int id) { + for (Token token : tokens) { + String word = token.getMorph(); + System.out.println("save " + word); + Morpheme existingWord = morphemeRepository.findByWordAndIssueId(word,3); + if (existingWord != null) { + // 단어가 이미 존재하면 count를 업데이트 + existingWord.setCount(existingWord.getCount() + 1); + System.out.println(existingWord.getWord()); + morphemeRepository.save(existingWord); + } else { + // 단어가 존재하지 않으면 새로운 레코드를 생성 + Morpheme newWord = new Morpheme(); + newWord.setWord(word); + System.out.println("else save " +word); + newWord.setCount(1); + newWord.setIssueId(id); + morphemeRepository.save(newWord); + } + } + } +} diff --git a/src/main/java/gwangjang/server/domain/morpheme/domain/service/NewsAPIService.java b/src/main/java/gwangjang/server/domain/morpheme/domain/service/NewsAPIService.java new file mode 100644 index 0000000..00e06fa --- /dev/null +++ b/src/main/java/gwangjang/server/domain/morpheme/domain/service/NewsAPIService.java @@ -0,0 +1,108 @@ +package gwangjang.server.domain.morpheme.domain.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import gwangjang.server.global.annotation.DomainService; +import jakarta.transaction.Transactional; +import kr.co.shineware.nlp.komoran.constant.DEFAULT_MODEL; +import kr.co.shineware.nlp.komoran.core.Komoran; +import kr.co.shineware.nlp.komoran.model.KomoranResult; +import kr.co.shineware.nlp.komoran.model.Token; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.List; + + +@DomainService +@Transactional +public class NewsAPIService { + @Value("${naver.client-id}") + private String NAVER_API_ID; + + @Value("${naver.secret}") + private String NAVER_API_SECRET; + private final RestTemplate restTemplate; + ObjectMapper objectMapper = new ObjectMapper(); + + + + @Autowired + public NewsAPIService(RestTemplate restTemplate) { + this.restTemplate = restTemplate; + } + + + public String naverAPI(String name) throws JsonProcessingException { + + StringBuilder rslt = new StringBuilder(); + + for (int start = 1; start <= 1000; start += 100) { + URI uri = UriComponentsBuilder + .fromUriString("https://openapi.naver.com/") + .path("/v1/search/news.json") + .queryParam("query", name) + .queryParam("display", 100) + .queryParam("start", start) + .queryParam("sort", "sim") + .encode(StandardCharsets.UTF_8) + .build() + .toUri(); + + RequestEntity req = RequestEntity + .get(uri) + .header("X-Naver-Client-Id", NAVER_API_ID) + .header("X-Naver-Client-Secret", NAVER_API_SECRET) + .build(); + + ResponseEntity result = restTemplate.exchange(req, String.class); + String json = result.getBody(); + System.out.println(json); + + try { + JSONParser parser = new JSONParser(); + JSONObject jsonData = (JSONObject) parser.parse(json); + JSONArray items = (JSONArray) jsonData.get("items"); + + for (Object obj : items) { + JSONObject item = (JSONObject) obj; + + String title = (String) item.get("title"); + String description = (String) item.get("description"); + rslt.append(title); + rslt.append(description); + } + + } catch (Exception e) { + e.printStackTrace(); + } + } + + return rslt.toString(); + } + + public List analysis(String msg) { + + Komoran komoran = new Komoran(DEFAULT_MODEL.FULL); + KomoranResult analyzeResultList = komoran.analyze(msg); + + System.out.println(analyzeResultList.getPlainText()); + + List tokenList = analyzeResultList.getTokenList(); + + for (Token token : tokenList) { + System.out.format("%s\n", token.getMorph()); + } + return tokenList; + } + +} diff --git a/src/main/java/gwangjang/server/domain/morpheme/presentation/MorphemeController.java b/src/main/java/gwangjang/server/domain/morpheme/presentation/MorphemeController.java new file mode 100644 index 0000000..89ba667 --- /dev/null +++ b/src/main/java/gwangjang/server/domain/morpheme/presentation/MorphemeController.java @@ -0,0 +1,36 @@ +package gwangjang.server.domain.morpheme.presentation; + +import com.fasterxml.jackson.core.JsonProcessingException; +import gwangjang.server.domain.morpheme.domain.service.MorphemeService; +import gwangjang.server.domain.morpheme.domain.service.NewsAPIService; +import io.swagger.annotations.ApiOperation; +import kr.co.shineware.nlp.komoran.model.Token; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@RequestMapping("/keyword") +@RequiredArgsConstructor +public class MorphemeController { + + private final NewsAPIService newsAPIService; + private final MorphemeService morphemeService; + @GetMapping("/analysis/{msg}") + public String analysis(@PathVariable String msg) throws JsonProcessingException { + String newsList1 = newsAPIService.naverAPI("주 69시간 근로시간 제도 개편"); + String newsList2 = newsAPIService.naverAPI("이태원 참사"); + String newsList3 = newsAPIService.naverAPI("국민연금 개혁"); + List newsAnalysis1 =newsAPIService.analysis(newsList1); + List newsAnalysis2 =newsAPIService.analysis(newsList2); + List newsAnalysis3 =newsAPIService.analysis(newsList3); + morphemeService.saveOrUpdateWord(newsAnalysis1, 100 ); + morphemeService.saveOrUpdateWord(newsAnalysis2, 200); + morphemeService.saveOrUpdateWord(newsAnalysis3, 300); + return "success"; + } +} diff --git a/src/main/java/gwangjang/server/global/config/RestTemplateConfig.java b/src/main/java/gwangjang/server/global/config/RestTemplateConfig.java new file mode 100644 index 0000000..fafdd77 --- /dev/null +++ b/src/main/java/gwangjang/server/global/config/RestTemplateConfig.java @@ -0,0 +1,22 @@ +package gwangjang.server.global.config; + + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.web.client.RestTemplate; + +@Configuration +public class RestTemplateConfig { + @Bean + public RestTemplate restTemplate() { + RestTemplate restTemplate = new RestTemplate(); + SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); + requestFactory.setConnectTimeout(5000); + requestFactory.setReadTimeout(5000); + + restTemplate.setRequestFactory(requestFactory); + + return restTemplate; + } +} \ No newline at end of file diff --git a/src/main/java/gwangjang/server/global/config/batch/BatchJobConfig.java b/src/main/java/gwangjang/server/global/config/batch/BatchJobConfig.java new file mode 100644 index 0000000..986a79c --- /dev/null +++ b/src/main/java/gwangjang/server/global/config/batch/BatchJobConfig.java @@ -0,0 +1,67 @@ +package gwangjang.server.global.config.batch; + +import gwangjang.server.domain.morpheme.domain.service.MorphemeService; +import gwangjang.server.domain.morpheme.domain.service.NewsAPIService; +import lombok.RequiredArgsConstructor; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; + +@RequiredArgsConstructor +@Configuration +public class BatchJobConfig { + + private final NewsAPIService newsApiService; + private final MorphemeService morphemeService; + + + @Bean(name="apiJob") + public Job apiJob(JobRepository jobRepository, PlatformTransactionManager platformTransactionManager){ + return new JobBuilder("apiJob", jobRepository) + .start(getNewsApiTasklet(jobRepository, platformTransactionManager)) + .next(getAnalysisTasklet(jobRepository,platformTransactionManager)) + .next(getMorphemeApiStep(jobRepository,platformTransactionManager)) + .build(); + } + @Bean + public Step getNewsApiTasklet(JobRepository jobRepository, PlatformTransactionManager platformTransactionManager){ + return new StepBuilder("newsApiService", jobRepository) + .tasklet(new GetNewsApiTasklet(newsApiService), platformTransactionManager) + .build(); + } + @Bean + public Step getAnalysisTasklet(JobRepository jobRepository, PlatformTransactionManager platformTransactionManager){ + return new StepBuilder("newsApiService", jobRepository) + .tasklet(new GetAnalysisTasklet(newsApiService), platformTransactionManager) + .build(); + } + @Bean + public Step getMorphemeApiStep(JobRepository jobRepository, PlatformTransactionManager platformTransactionManager){ + return new StepBuilder("morphemeService", jobRepository) + .tasklet(new GetMorphemeTasklet(morphemeService), platformTransactionManager) + .build(); + } + @Bean + public Job stepNextConditionalJob(JobRepository jobRepository, PlatformTransactionManager transactionManager) { + return new JobBuilder("stepNextConditionalJob" ,jobRepository) + .start(getNewsApiTasklet(jobRepository, transactionManager)) + .on("*") // Any exit status + .to(getAnalysisTasklet(jobRepository, transactionManager)) + .from(getAnalysisTasklet(jobRepository, transactionManager)) + .on("*") // Any exit status + .to(getMorphemeApiStep(jobRepository, transactionManager)) + .from(getMorphemeApiStep(jobRepository, transactionManager)) + .on("*") // Any exit status + .to(getNewsApiTasklet(jobRepository, transactionManager)) + .end() + .build(); + } + + +} \ No newline at end of file diff --git a/src/main/java/gwangjang/server/global/config/batch/GetAnalysisTasklet.java b/src/main/java/gwangjang/server/global/config/batch/GetAnalysisTasklet.java new file mode 100644 index 0000000..8c06b11 --- /dev/null +++ b/src/main/java/gwangjang/server/global/config/batch/GetAnalysisTasklet.java @@ -0,0 +1,34 @@ +package gwangjang.server.global.config.batch; + +import gwangjang.server.domain.morpheme.domain.service.NewsAPIService; +import kr.co.shineware.nlp.komoran.model.Token; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.stereotype.Component; + +import java.util.List; + +@RequiredArgsConstructor +@Component +public class GetAnalysisTasklet implements Tasklet { + private final NewsAPIService newsAPIService; + + @Override + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { + // Retrieve the result from JobExecutionContext + String userApiResult = (String) chunkContext.getStepContext().getStepExecution().getJobExecution() + .getExecutionContext().get("userApiResult"); + + // Use the result as needed + List analysisResult = newsAPIService.analysis(userApiResult); + + // Store the analysis result in JobExecutionContext + chunkContext.getStepContext().getStepExecution().getJobExecution().getExecutionContext() + .put("analysisResult", analysisResult); + + return RepeatStatus.FINISHED; + } +} diff --git a/src/main/java/gwangjang/server/global/config/batch/GetMorphemeTasklet.java b/src/main/java/gwangjang/server/global/config/batch/GetMorphemeTasklet.java new file mode 100644 index 0000000..54f0167 --- /dev/null +++ b/src/main/java/gwangjang/server/global/config/batch/GetMorphemeTasklet.java @@ -0,0 +1,30 @@ +package gwangjang.server.global.config.batch; + +import gwangjang.server.domain.morpheme.domain.service.MorphemeService; +import kr.co.shineware.nlp.komoran.model.Token; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.stereotype.Component; + +import java.util.List; + +@RequiredArgsConstructor +@Component +public class GetMorphemeTasklet implements Tasklet { + private final MorphemeService morphemeService; + + @Override + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { + // Retrieve the result from JobExecutionContext + List analysisResult = (List) chunkContext.getStepContext().getStepExecution().getJobExecution() + .getExecutionContext().get("analysisResult"); + + // Use the result as needed + morphemeService.saveOrUpdateWord(analysisResult, 3); // Assuming 3 is the issueId, replace with the correct value + + return RepeatStatus.FINISHED; + } +} diff --git a/src/main/java/gwangjang/server/global/config/batch/GetNewsApiTasklet.java b/src/main/java/gwangjang/server/global/config/batch/GetNewsApiTasklet.java new file mode 100644 index 0000000..50df972 --- /dev/null +++ b/src/main/java/gwangjang/server/global/config/batch/GetNewsApiTasklet.java @@ -0,0 +1,26 @@ +package gwangjang.server.global.config.batch; + +import gwangjang.server.domain.morpheme.domain.service.NewsAPIService; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.stereotype.Component; + +@RequiredArgsConstructor +@Component +public class GetNewsApiTasklet implements Tasklet { + private final NewsAPIService newsAPIService; + + @Override + public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { + String userApiResult = newsAPIService.naverAPI("test"); + + // Store the result in JobExecutionContext + chunkContext.getStepContext().getStepExecution().getJobExecution().getExecutionContext() + .put("userApiResult", userApiResult); + + return RepeatStatus.FINISHED; + } +} diff --git a/src/main/java/gwangjang/server/global/security/SecurityConfig.java b/src/main/java/gwangjang/server/global/security/SecurityConfig.java index 390df4d..500df48 100644 --- a/src/main/java/gwangjang/server/global/security/SecurityConfig.java +++ b/src/main/java/gwangjang/server/global/security/SecurityConfig.java @@ -20,7 +20,7 @@ public class SecurityConfig { @Bean public WebSecurityCustomizer webSecurityCustomizer() { return (web) -> web.ignoring() - .requestMatchers("/resource/**", "/css/**", "/js/**", "/img/**", "/lib/**"); + .requestMatchers("/resource/**", "/css/**", "/js/**", "/img/**", "/lib/**","/**"); }; // .requestMatchers(new AntPathRequestMatcher( "/**/*.html")); @@ -47,6 +47,7 @@ public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { http .authorizeHttpRequests( authorize -> authorize + .requestMatchers("/**").permitAll() .requestMatchers("/auth/**").permitAll() .anyRequest().authenticated() );