Req_tracing (#149)

* harvester will stop for fatal errors

* adds accepts header for url check

* adding mock stats

* adding mock stats

* added committed at to the HarvesterRun

* added committed while harvesting

* collecting stats during harvester

build.gradle

@@ -175,6 +175,7 @@ jacocoTestCoverageVerification {
                     'it.gov.innovazione.ndc.eventhandler.*',
                     'it.gov.innovazione.ndc.harvester.service.RepositoryService',
                     'it.gov.innovazione.ndc.harvester.service.HarvesterRunService',
+                    'it.gov.innovazione.ndc.harvester.service.SemanticContentStatsService',
                     'it.gov.innovazione.ndc.harvester.SecurityUtils',
                     'it.gov.innovazione.ndc.*Exception',
                     'it.gov.innovazione.ndc.harvester.context.*',
@@ -195,7 +196,7 @@ jacocoTestCoverageVerification {
                     'it.gov.innovazione.ndc.repository.TripleStoreRepository',
                     'it.gov.innovazione.ndc.service.EventCleaner',
                     'it.gov.innovazione.ndc.service.TemplateService',
-                    'it.gov.innovazione.ndc.harvester.service.startupjob.TempEraserStartupJob',
+                    'it.gov.innovazione.ndc.harvester.service.startupjob.*',
                     'it.gov.innovazione.ndc.alerter.*'
             ]
         }

src/integration/java/it/gov/innovazione/ndc/integration/BaseIntegrationTest.java

@@ -5,6 +5,7 @@
 import it.gov.innovazione.ndc.harvester.HarvesterService;
 import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata;
 import it.gov.innovazione.ndc.harvester.service.RepositoryService;
+import it.gov.innovazione.ndc.harvester.service.SemanticContentStatsService;
 import it.gov.innovazione.ndc.harvester.service.startupjob.StartupJobsRunner;
 import it.gov.innovazione.ndc.repository.TripleStoreProperties;
 import org.elasticsearch.client.RestClient;
@@ -66,6 +67,9 @@ public class BaseIntegrationTest {
     @SpyBean
     RepositoryService repositoryService;
 
+    @SpyBean
+    SemanticContentStatsService semanticContentStatsService;
+
     @Autowired
     TripleStoreProperties virtuosoProps;
 
@@ -96,6 +100,7 @@ private void dataIsHarvested() throws IOException {
         doReturn(cloneDir).when(agencyRepositoryService).cloneRepo(REPO_URL, null);
         doNothing().when(agencyRepositoryService).removeClonedRepo(cloneDir);
         doNothing().when(repositoryService).storeRightsHolders(any(), any());
+        doNothing().when(semanticContentStatsService).saveStats();
 
         harvesterService.harvest(asRepo(REPO_URL));
 

src/main/java/it/gov/innovazione/ndc/controller/SemanticAssetStats.java

@@ -0,0 +1,35 @@
+package it.gov.innovazione.ndc.controller;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+
+@Data
+@Builder
+public class SemanticAssetStats {
+    private final SemanticAssetTypeStats totalStats;
+    private final SemanticAssetTypeStats controlledVocabularyStats;
+    private final SemanticAssetTypeStats ontologyStats;
+    private final SemanticAssetTypeStats schemaStats;
+
+    @Data
+    @Builder
+    public static class SemanticAssetTypeStats {
+        private final long current;
+        private final long lastYear;
+
+        public long getIncrementOverLastYear() {
+            return current - lastYear;
+        }
+
+        public double getIncrementPercentageOverLastYear() {
+            BigDecimal bigDecimal = BigDecimal.valueOf(
+                    lastYear == 0
+                            ? 0
+                            : ((double) current - lastYear) / lastYear * 100);
+            return bigDecimal.setScale(1, RoundingMode.HALF_UP).doubleValue();
+        }
+    }
+}

src/main/java/it/gov/innovazione/ndc/controller/SemanticAssetsController.java

@@ -56,6 +56,35 @@ List<RightsHolder> getRightsHolders() {
 
     }
 
+    @Operation(tags = {"semantic-assets"},
+            summary = "Retrieves the statistics",
+            description = "Retrieves the statistics of the semantic assets.",
+            operationId = "getStats",
+            responses = {@ApiResponse(responseCode = "200", description = "OK", content = {
+                @Content(mediaType = "application/json", schema = @Schema(implementation = SemanticAssetStats.class))
+            })})
+    @GetMapping(value = "/semantic-assets/stats", produces = {"application/json"})
+    SemanticAssetStats getStats() {
+        return SemanticAssetStats.builder()
+                .totalStats(SemanticAssetStats.SemanticAssetTypeStats.builder()
+                        .current(118)
+                        .lastYear(115)
+                        .build())
+                .controlledVocabularyStats(SemanticAssetStats.SemanticAssetTypeStats.builder()
+                        .current(120)
+                        .lastYear(118)
+                        .build())
+                .ontologyStats(SemanticAssetStats.SemanticAssetTypeStats.builder()
+                        .current(100)
+                        .lastYear(98)
+                        .build())
+                .schemaStats(SemanticAssetStats.SemanticAssetTypeStats.builder()
+                        .current(80)
+                        .lastYear(80)
+                        .build())
+                .build();
+    }
+
     @Override
     public ResponseEntity<SearchResult> search(
             String q,

src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterUpdateCommitDateEvent.java

@@ -0,0 +1,13 @@
+package it.gov.innovazione.ndc.eventhandler.event;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.time.Instant;
+
+@Builder
+@Data
+public class HarvesterUpdateCommitDateEvent {
+    private final String runId;
+    private final Instant commitDate;
+}

src/main/java/it/gov/innovazione/ndc/eventhandler/handler/HarvesterCommitDateUpdateEventListener.java

@@ -0,0 +1,42 @@
+package it.gov.innovazione.ndc.eventhandler.handler;
+
+import it.gov.innovazione.ndc.eventhandler.NdcEventHandler;
+import it.gov.innovazione.ndc.eventhandler.NdcEventWrapper;
+import it.gov.innovazione.ndc.eventhandler.event.HarvesterUpdateCommitDateEvent;
+import it.gov.innovazione.ndc.harvester.service.HarvesterRunService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class HarvesterCommitDateUpdateEventListener implements NdcEventHandler {
+
+    private static final Collection<Class<?>> SUPPORTED_EVENTS = List.of(HarvesterUpdateCommitDateEvent.class);
+
+    private final HarvesterRunService harvesterRunService;
+
+    @Override
+    public boolean canHandle(NdcEventWrapper<?> event) {
+        return SUPPORTED_EVENTS.contains(event.getPayload().getClass());
+    }
+
+    @Override
+    public void handle(NdcEventWrapper<?> event) {
+        HarvesterUpdateCommitDateEvent payload = (HarvesterUpdateCommitDateEvent) event.getPayload();
+        if (Objects.isNull(payload) || Objects.isNull(payload.getCommitDate())) {
+            log.warn("Received invalid HarvesterUpdateCommitDateEvent: {}", payload);
+            return;
+        }
+        harvesterRunService.getAllRuns().stream()
+                .filter(harvesterRun -> payload.getRunId().equals(harvesterRun.getId()))
+                .findAny()
+                .map(harvesterRun -> harvesterRun.withRevisionCommittedAt(payload.getCommitDate()))
+                .ifPresent(harvesterRunService::updateHarvesterRunCommittedAt);
+    }
+}

src/main/java/it/gov/innovazione/ndc/harvester/AgencyRepositoryService.java

@@ -1,5 +1,7 @@
 package it.gov.innovazione.ndc.harvester;
 
+import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher;
+import it.gov.innovazione.ndc.eventhandler.event.HarvesterUpdateCommitDateEvent;
 import it.gov.innovazione.ndc.harvester.exception.InvalidAssetFolderException;
 import it.gov.innovazione.ndc.harvester.model.CvPath;
 import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath;
@@ -11,6 +13,7 @@
 import it.gov.innovazione.ndc.harvester.util.GitUtils;
 import it.gov.innovazione.ndc.harvester.util.PropertiesUtils;
 import it.gov.innovazione.ndc.harvester.util.Version;
+import it.gov.innovazione.ndc.model.harvester.HarvesterRun;
 import it.gov.innovazione.ndc.service.logging.HarvesterStage;
 import it.gov.innovazione.ndc.service.logging.LoggingContext;
 import lombok.SneakyThrows;
@@ -19,6 +22,7 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.time.Instant;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
@@ -40,19 +44,22 @@ public class AgencyRepositoryService {
     private final ControlledVocabularyFolderScanner controlledVocabularyFolderScanner;
     private final SchemaFolderScanner schemaFolderScanner;
     private final List<String> lowerSkipWords;
+    private final NdcEventPublisher eventPublisher;
 
     public AgencyRepositoryService(FileUtils fileUtils,
                                    GitUtils gitUtils,
                                    OntologyFolderScanner ontologyFolderScanner,
                                    ControlledVocabularyFolderScanner controlledVocabularyFolderScanner,
                                    SchemaFolderScanner schemaFolderScanner,
-                                   AgencyRepositoryServiceProperties agencyRepositoryServiceProperties) {
+                                   AgencyRepositoryServiceProperties agencyRepositoryServiceProperties,
+                                   NdcEventPublisher eventPublisher) {
         this.fileUtils = fileUtils;
         this.gitUtils = gitUtils;
         this.ontologyFolderScanner = ontologyFolderScanner;
         this.controlledVocabularyFolderScanner = controlledVocabularyFolderScanner;
         this.schemaFolderScanner = schemaFolderScanner;
         this.lowerSkipWords = PropertiesUtils.lowerSkipWords(agencyRepositoryServiceProperties.getSkipWords(), MIN_SKIP_WORD_LENGTH);
+        this.eventPublisher = eventPublisher;
     }
 
     public Path cloneRepo(String repoUrl) throws IOException {
@@ -62,7 +69,12 @@ public Path cloneRepo(String repoUrl) throws IOException {
     public Path cloneRepo(String repoUrl, String revision) throws IOException {
         Path cloneDir = fileUtils.createTempDirectory(TEMP_DIR_PREFIX);
         log.info("Cloning repo {} @ revision {}, at location {}", repoUrl, revision, cloneDir);
-        gitUtils.cloneRepo(repoUrl, cloneDir.toFile(), revision);
+        Instant instant = gitUtils.cloneRepoAndGetLastCommitDate(repoUrl, cloneDir.toFile(), revision);
+        eventPublisher.publishEvent("harvester", "harvester.get.commit.date", null, "harvester",
+                HarvesterUpdateCommitDateEvent.builder()
+                        .runId(HarvesterRun.getCurrentRunId())
+                        .commitDate(instant)
+                        .build());
         return cloneDir;
     }
 

src/main/java/it/gov/innovazione/ndc/harvester/context/HarvestExecutionContextUtils.java

@@ -1,11 +1,16 @@
 package it.gov.innovazione.ndc.harvester.context;
 
+import it.gov.innovazione.ndc.model.harvester.SemanticContentStats;
 import lombok.NoArgsConstructor;
 
+import java.util.ArrayList;
+import java.util.List;
+
 @NoArgsConstructor(access = lombok.AccessLevel.PRIVATE)
 public class HarvestExecutionContextUtils {
 
     private static final ThreadLocal<HarvestExecutionContext> CONTEXT_HOLDER = new ThreadLocal<>();
+    private static final ThreadLocal<List<SemanticContentStats>> SEMANTIC_CONTENT_STATS_HOLDER = ThreadLocal.withInitial(ArrayList::new);
 
     public static HarvestExecutionContext getContext() {
         return CONTEXT_HOLDER.get();
@@ -17,5 +22,18 @@ public static void setContext(HarvestExecutionContext context) {
 
     public static void clearContext() {
         CONTEXT_HOLDER.remove();
+        clearSemanticContentStats();
+    }
+
+    public static List<SemanticContentStats> getSemanticContentStats() {
+        return SEMANTIC_CONTENT_STATS_HOLDER.get();
+    }
+
+    public static void addSemanticContentStat(SemanticContentStats stats) {
+        SEMANTIC_CONTENT_STATS_HOLDER.get().add(stats);
+    }
+
+    public static void clearSemanticContentStats() {
+        SEMANTIC_CONTENT_STATS_HOLDER.get().clear();
     }
 }

src/main/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvester.java

@@ -13,9 +13,12 @@
 import it.gov.innovazione.ndc.harvester.context.HarvestExecutionContextUtils;
 import it.gov.innovazione.ndc.harvester.exception.SinglePathProcessingException;
 import it.gov.innovazione.ndc.harvester.harvesters.utils.PathUtils;
+import it.gov.innovazione.ndc.harvester.model.HarvesterStatsHolder;
 import it.gov.innovazione.ndc.harvester.model.Instance;
 import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath;
+import it.gov.innovazione.ndc.harvester.service.SemanticContentStatsService;
 import it.gov.innovazione.ndc.model.harvester.Repository;
+import it.gov.innovazione.ndc.model.harvester.SemanticContentStats;
 import it.gov.innovazione.ndc.service.logging.HarvesterStage;
 import it.gov.innovazione.ndc.service.logging.LoggingContext;
 import lombok.RequiredArgsConstructor;
@@ -45,6 +48,7 @@ public abstract class BaseSemanticAssetHarvester<P extends SemanticAssetPath> im
     private final SemanticAssetType type;
     private final NdcEventPublisher eventPublisher;
     private final ConfigService configService;
+    private final SemanticContentStatsService semanticContentStatsService;
 
     @Override
     public SemanticAssetType getType() {
@@ -70,7 +74,8 @@ public void harvest(Repository repository, Path rootPath) {
 
         for (P path : paths) {
             try {
-                processPath(repository.getUrl(), path);
+                HarvesterStatsHolder harvesterStatsHolder = processPath(repository.getUrl(), path);
+                semanticContentStatsService.updateStats(harvesterStatsHolder);
                 log.debug("Path {} processed correctly for {}", path, type);
             } catch (SinglePathProcessingException e) {
                 boolean isInfrastuctureError = checkInfrastructureError(e);
@@ -112,6 +117,7 @@ public void harvest(Repository repository, Path rootPath) {
                 }
             }
         }
+        semanticContentStatsService.saveStats();
     }
 
     private boolean checkInfrastructureError(SinglePathProcessingException e) {
@@ -190,7 +196,7 @@ public void cleanUpBeforeHarvesting(String repoUrl, Instance instance) {
         // by default nothing specific
     }
 
-    protected abstract void processPath(String repoUrl, P path);
+    protected abstract HarvesterStatsHolder processPath(String repoUrl, P path);
 
     protected abstract List<P> scanForPaths(Path rootPath);
 }

src/main/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvester.java

@@ -5,8 +5,10 @@
 import it.gov.innovazione.ndc.harvester.AgencyRepositoryService;
 import it.gov.innovazione.ndc.harvester.SemanticAssetType;
 import it.gov.innovazione.ndc.harvester.model.CvPath;
+import it.gov.innovazione.ndc.harvester.model.HarvesterStatsHolder;
 import it.gov.innovazione.ndc.harvester.model.Instance;
 import it.gov.innovazione.ndc.harvester.pathprocessors.ControlledVocabularyPathProcessor;
+import it.gov.innovazione.ndc.harvester.service.SemanticContentStatsService;
 import org.springframework.stereotype.Component;
 
 import java.nio.file.Path;
@@ -17,15 +19,20 @@ public class ControlledVocabularyHarvester extends BaseSemanticAssetHarvester<Cv
     private final AgencyRepositoryService agencyRepositoryService;
     private final ControlledVocabularyPathProcessor pathProcessor;
 
-    public ControlledVocabularyHarvester(AgencyRepositoryService agencyRepositoryService, ControlledVocabularyPathProcessor pathProcessor, NdcEventPublisher ndcEventPublisher, ConfigService configService) {
-        super(SemanticAssetType.CONTROLLED_VOCABULARY, ndcEventPublisher, configService);
+    public ControlledVocabularyHarvester(
+            AgencyRepositoryService agencyRepositoryService,
+            ControlledVocabularyPathProcessor pathProcessor,
+            NdcEventPublisher ndcEventPublisher,
+            ConfigService configService,
+            SemanticContentStatsService semanticContentStatsService) {
+        super(SemanticAssetType.CONTROLLED_VOCABULARY, ndcEventPublisher, configService, semanticContentStatsService);
         this.agencyRepositoryService = agencyRepositoryService;
         this.pathProcessor = pathProcessor;
     }
 
     @Override
-    protected void processPath(String repoUrl, CvPath path) {
-        pathProcessor.process(repoUrl, path);
+    protected HarvesterStatsHolder processPath(String repoUrl, CvPath path) {
+        return pathProcessor.process(repoUrl, path);
     }
 
     @Override

src/main/java/it/gov/innovazione/ndc/harvester/harvesters/OntologyHarvester.java

@@ -4,8 +4,10 @@
 import it.gov.innovazione.ndc.eventhandler.event.ConfigService;
 import it.gov.innovazione.ndc.harvester.AgencyRepositoryService;
 import it.gov.innovazione.ndc.harvester.SemanticAssetType;
+import it.gov.innovazione.ndc.harvester.model.HarvesterStatsHolder;
 import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath;
 import it.gov.innovazione.ndc.harvester.pathprocessors.OntologyPathProcessor;
+import it.gov.innovazione.ndc.harvester.service.SemanticContentStatsService;
 import org.springframework.stereotype.Component;
 
 import java.nio.file.Path;
@@ -16,15 +18,19 @@ public class OntologyHarvester extends BaseSemanticAssetHarvester<SemanticAssetP
     private final AgencyRepositoryService agencyRepositoryService;
     private final OntologyPathProcessor pathProcessor;
 
-    public OntologyHarvester(AgencyRepositoryService agencyRepositoryService, OntologyPathProcessor pathProcessor, NdcEventPublisher ndcEventPublisher, ConfigService configService) {
-        super(SemanticAssetType.ONTOLOGY, ndcEventPublisher, configService);
+    public OntologyHarvester(AgencyRepositoryService agencyRepositoryService,
+                             OntologyPathProcessor pathProcessor,
+                             NdcEventPublisher ndcEventPublisher,
+                             ConfigService configService,
+                             SemanticContentStatsService semanticContentStatsService) {
+        super(SemanticAssetType.ONTOLOGY, ndcEventPublisher, configService, semanticContentStatsService);
         this.agencyRepositoryService = agencyRepositoryService;
         this.pathProcessor = pathProcessor;
     }
 
     @Override
-    protected void processPath(String repoUrl, SemanticAssetPath path) {
-        pathProcessor.process(repoUrl, path);
+    protected HarvesterStatsHolder processPath(String repoUrl, SemanticAssetPath path) {
+        return pathProcessor.process(repoUrl, path);
     }
 
     @Override