diff --git a/build.gradle b/build.gradle index e936bb23..a32c7c2e 100644 --- a/build.gradle +++ b/build.gradle @@ -52,6 +52,7 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-data-jpa' implementation 'org.springframework.boot:spring-boot-starter-security' implementation 'org.springframework.boot:spring-boot-starter-validation' + implementation 'org.springframework.boot:spring-boot-starter-mail' implementation 'org.mapstruct:mapstruct:1.5.5.Final' @@ -203,6 +204,8 @@ jacocoTestCoverageVerification { 'it.gov.innovazione.ndc.harvester.service.ConfigService.NdcConfiguration', 'it.gov.innovazione.ndc.harvester.service.ConfigService', 'it.gov.innovazione.ndc.service.GithubService*', + 'it.gov.innovazione.ndc.service.EmailService', + 'it.gov.innovazione.ndc.service.AlerterMailSender', 'it.gov.innovazione.ndc.harvester.harvesters.utils.PathUtils', 'it.gov.innovazione.ndc.harvester.service.OnceLogger', 'it.gov.innovazione.ndc.harvester.service.ConfigReaderService', diff --git a/src/integration/java/it/gov/innovazione/ndc/harvester/csv/NoDeepestLevelExtractorIntTest.java b/src/integration/java/it/gov/innovazione/ndc/harvester/csv/NoDeepestLevelExtractorIntTest.java index 62789738..3147a705 100644 --- a/src/integration/java/it/gov/innovazione/ndc/harvester/csv/NoDeepestLevelExtractorIntTest.java +++ b/src/integration/java/it/gov/innovazione/ndc/harvester/csv/NoDeepestLevelExtractorIntTest.java @@ -1,5 +1,6 @@ package it.gov.innovazione.ndc.harvester.csv; +import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; import it.gov.innovazione.ndc.repository.MarkerElasticSearchRepository; import it.gov.innovazione.ndc.repository.VirtuosoClient; import it.gov.innovazione.ndc.service.GithubService; @@ -28,6 +29,9 @@ class NoDeepestLevelExtractorIntTest { @MockBean private GithubService githubService; + @MockBean + private NdcEventPublisher eventPublisher; + @Test void shouldNotBeActivated() { assertThat(extractors.stream().noneMatch(e -> e.getClass().isAssignableFrom(DeepestLevelExtractor.class))).isTrue(); diff --git a/src/integration/java/it/gov/innovazione/ndc/harvester/csv/UseDeepestLevelExtractorIntTest.java b/src/integration/java/it/gov/innovazione/ndc/harvester/csv/UseDeepestLevelExtractorIntTest.java index 3b077979..62457d37 100644 --- a/src/integration/java/it/gov/innovazione/ndc/harvester/csv/UseDeepestLevelExtractorIntTest.java +++ b/src/integration/java/it/gov/innovazione/ndc/harvester/csv/UseDeepestLevelExtractorIntTest.java @@ -1,5 +1,6 @@ package it.gov.innovazione.ndc.harvester.csv; +import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; import it.gov.innovazione.ndc.repository.MarkerElasticSearchRepository; import it.gov.innovazione.ndc.repository.VirtuosoClient; import it.gov.innovazione.ndc.service.GithubService; @@ -28,6 +29,9 @@ class UseDeepestLevelExtractorIntTest { @MockBean private GithubService githubService; + @MockBean + private NdcEventPublisher eventPublisher; + @Test void shouldBeActivated() { assertThat(extractors.stream().anyMatch(e -> e.getClass().isAssignableFrom(DeepestLevelExtractor.class))).isTrue(); diff --git a/src/integration/java/it/gov/innovazione/ndc/integration/RestApiIntegrationTests.java b/src/integration/java/it/gov/innovazione/ndc/integration/RestApiIntegrationTests.java index 887e525e..e8dbf942 100644 --- a/src/integration/java/it/gov/innovazione/ndc/integration/RestApiIntegrationTests.java +++ b/src/integration/java/it/gov/innovazione/ndc/integration/RestApiIntegrationTests.java @@ -1,11 +1,11 @@ package it.gov.innovazione.ndc.integration; import io.restassured.response.Response; +import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; import it.gov.innovazione.ndc.gen.dto.AssetType; import it.gov.innovazione.ndc.gen.dto.SearchResult; import it.gov.innovazione.ndc.gen.dto.SearchResultItem; import it.gov.innovazione.ndc.harvester.SemanticAssetType; -import it.gov.innovazione.ndc.harvester.util.FileUtils; import it.gov.innovazione.ndc.model.profiles.NDC; import it.gov.innovazione.ndc.service.GithubService; import junit.framework.AssertionFailedError; @@ -39,6 +39,8 @@ public class RestApiIntegrationTests extends BaseIntegrationTest { @MockBean private GithubService githubService; + @MockBean + private NdcEventPublisher ndcEventPublisher; @DynamicPropertySource static void updateDynamicPropertySource(DynamicPropertyRegistry registry) { diff --git a/src/integration/java/it/gov/innovazione/ndc/service/VocabularyDataServiceIntegrationTest.java b/src/integration/java/it/gov/innovazione/ndc/service/VocabularyDataServiceIntegrationTest.java index d9877a44..4cc9a983 100644 --- a/src/integration/java/it/gov/innovazione/ndc/service/VocabularyDataServiceIntegrationTest.java +++ b/src/integration/java/it/gov/innovazione/ndc/service/VocabularyDataServiceIntegrationTest.java @@ -2,6 +2,7 @@ import it.gov.innovazione.ndc.controller.exception.VocabularyDataNotFoundException; import it.gov.innovazione.ndc.controller.exception.VocabularyItemNotFoundException; +import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; import it.gov.innovazione.ndc.gen.dto.VocabularyData; import it.gov.innovazione.ndc.harvester.csv.CsvParser; import it.gov.innovazione.ndc.integration.Containers; @@ -40,6 +41,9 @@ public class VocabularyDataServiceIntegrationTest { @MockBean private GithubService githubService; + @MockBean + private NdcEventPublisher eventPublisher; + @DynamicPropertySource static void updateTestcontainersProperties(DynamicPropertyRegistry registry) { registry.add("spring.elasticsearch.rest.uris", elasticsearchContainer::getHttpHostAddress); diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/AlerterService.java b/src/main/java/it/gov/innovazione/ndc/alerter/AlerterService.java new file mode 100644 index 00000000..c0247541 --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/alerter/AlerterService.java @@ -0,0 +1,37 @@ +package it.gov.innovazione.ndc.alerter; + +import it.gov.innovazione.ndc.alerter.data.EventService; +import it.gov.innovazione.ndc.alerter.dto.EventDto; +import it.gov.innovazione.ndc.alerter.event.AlertableEvent; +import lombok.RequiredArgsConstructor; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Service; + +import java.time.Instant; + +import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; + +@Service +@RequiredArgsConstructor +public class AlerterService { + + private final EventService eventService; + + public void alert(AlertableEvent alertableEvent) { + eventService.create(EventDto.builder() + .name(alertableEvent.getName()) + .description(alertableEvent.getDescription()) + .category(alertableEvent.getCategory()) + .context(alertableEvent.getContext()) + .severity(alertableEvent.getSeverity()) + .createdBy(getUser()) + .occurredAt(defaultIfNull(alertableEvent.getOccurredAt(), Instant.now())) + .build()); + } + + private String getUser() { + return defaultIfNull(SecurityContextHolder.getContext().getAuthentication().getName(), "system"); + + } + +} diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/controller/EventController.java b/src/main/java/it/gov/innovazione/ndc/alerter/controller/EventController.java index a908d336..f0c829cf 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/controller/EventController.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/controller/EventController.java @@ -7,12 +7,9 @@ import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import javax.validation.Valid; - @RestController @RequiredArgsConstructor @RequestMapping("/event") @@ -25,12 +22,6 @@ public class EventController extends AbstractCrudController { @Getter(AccessLevel.PROTECTED) private final EventMapper entityMapper; - @Override - public EventDto create(@Valid @RequestBody EventDto entity) { - // todo: logic to handle the event - return super.create(entity); - } - @Override protected void handlePreUpdate(EventDto entity) { throw IMMUTABLE_EXCEPTION; diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/data/EntityService.java b/src/main/java/it/gov/innovazione/ndc/alerter/data/EntityService.java index 59a49bd9..eda5ccaf 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/data/EntityService.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/data/EntityService.java @@ -7,8 +7,12 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; import org.springframework.http.HttpStatus; +import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.ResponseStatus; +import java.util.List; +import java.util.stream.Collectors; + @RequiredArgsConstructor public abstract class EntityService { @@ -88,6 +92,13 @@ private ConflictingOperationException entityDoesNotExistsException() { return new ConflictingOperationException(getEntityName() + " does not exist"); } + @Transactional(readOnly = true) + public List findAll() { + return getRepository().findAll().stream() + .map(a -> getEntityMapper().toDto(a)) + .collect(Collectors.toList()); + } + @ResponseStatus(HttpStatus.CONFLICT) public static class ConflictingOperationException extends RuntimeException { diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/data/EventRepository.java b/src/main/java/it/gov/innovazione/ndc/alerter/data/EventRepository.java index 4926590c..15d70a66 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/data/EventRepository.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/data/EventRepository.java @@ -4,9 +4,12 @@ import org.springframework.stereotype.Repository; import java.time.Instant; +import java.util.List; @Repository interface EventRepository extends NameableRepository { boolean existsByNameAndOccurredAt(String name, Instant occurredAt); + + List findByCreatedAtAfter(Instant instant); } diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/data/EventService.java b/src/main/java/it/gov/innovazione/ndc/alerter/data/EventService.java index eed76d5e..938dcd11 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/data/EventService.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/data/EventService.java @@ -8,6 +8,11 @@ import org.springframework.data.domain.Sort; import org.springframework.stereotype.Service; +import java.time.Instant; +import java.util.List; + +import static java.util.stream.Collectors.toList; + @Service @RequiredArgsConstructor public class EventService extends EntityService { @@ -30,4 +35,10 @@ protected void assertEntityDoesNotExists(EventDto dto) { throw new ConflictingOperationException("An event with the same name/occurredAt already exists: " + dto.getName() + "/" + dto.getOccurredAt()); } } + + public List getEventsNewerThan(Instant instant) { + return repository.findByCreatedAtAfter(instant).stream() + .map(entityMapper::toDto) + .collect(toList()); + } } diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileInitializer.java b/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileInitializer.java index 46daea00..e12cb7e7 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileInitializer.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileInitializer.java @@ -37,7 +37,7 @@ public void init() { .name(pair.getLeft()) .eventCategories(pair.getRight()) .minSeverity(Severity.INFO) - .aggregationTime(60) + .aggregationTime(60L) .build()) .forEach(p -> { log.info("Creating default profile: {}", p.getName()); diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileMapper.java b/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileMapper.java index e3ed93cc..da078248 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileMapper.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileMapper.java @@ -5,6 +5,7 @@ import it.gov.innovazione.ndc.alerter.entities.Profile; import lombok.SneakyThrows; import org.mapstruct.Mapper; +import org.mapstruct.Mapping; import java.util.ArrayList; import java.util.Arrays; @@ -14,6 +15,10 @@ @Mapper(componentModel = "spring") public abstract class ProfileMapper implements EntityMapper { + @Override + @Mapping(target = "lastAlertedAt", ignore = true) + public abstract Profile toEntity(ProfileDto dto); + @SneakyThrows protected List stringListToEventCategoryList(List list) { if (list == null) { diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileService.java b/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileService.java index 296647f0..af9e4643 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileService.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/data/ProfileService.java @@ -10,6 +10,8 @@ import org.springframework.stereotype.Service; import javax.transaction.Transactional; +import java.time.Duration; +import java.time.Instant; @Service @RequiredArgsConstructor @@ -28,4 +30,18 @@ public class ProfileService extends EntityService { @Getter(AccessLevel.PROTECTED) private final Sort defaultSorting = Sort.by("name").ascending(); + + public void setLastUpdated(String id) { + Profile profile = repository.findById(id) + .orElseThrow(() -> new IllegalStateException("Profile not found: " + id)); + profile.setLastAlertedAt(Instant.now()); + repository.save(profile); + } + + public void setAllLastUpdated(Duration backoff) { + repository.findAll().forEach(profile -> { + profile.setLastAlertedAt(Instant.now().minus(backoff)); + repository.save(profile); + }); + } } diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/data/UserRepository.java b/src/main/java/it/gov/innovazione/ndc/alerter/data/UserRepository.java index 9089ad43..321d7e11 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/data/UserRepository.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/data/UserRepository.java @@ -6,4 +6,5 @@ @Repository interface UserRepository extends NameableRepository { + boolean existsByNameAndSurnameAndEmail(String name, String surname, String email); } diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/data/UserService.java b/src/main/java/it/gov/innovazione/ndc/alerter/data/UserService.java index 71dca534..165485a0 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/data/UserService.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/data/UserService.java @@ -19,4 +19,12 @@ public class UserService extends EntityService { private final String entityName = "User"; @Getter(AccessLevel.PROTECTED) private final Sort defaultSorting = Sort.by("name").ascending(); + + @Override + protected void assertEntityDoesNotExists(UserDto dto) { + if (repository.existsByNameAndSurnameAndEmail( + dto.getName(), dto.getSurname(), dto.getEmail())) { + throw new ConflictingOperationException("An user with the same name/surname/email already exists: " + dto.getName() + "/" + dto.getSurname() + "/" + dto.getEmail()); + } + } } diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/dto/EventDto.java b/src/main/java/it/gov/innovazione/ndc/alerter/dto/EventDto.java index db310c60..420290fe 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/dto/EventDto.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/dto/EventDto.java @@ -3,6 +3,7 @@ import it.gov.innovazione.ndc.alerter.entities.EventCategory; import it.gov.innovazione.ndc.alerter.entities.Nameable; import it.gov.innovazione.ndc.alerter.entities.Severity; +import lombok.Builder; import lombok.Data; import javax.validation.constraints.NotBlank; @@ -11,6 +12,7 @@ import java.util.Map; @Data +@Builder public class EventDto implements Nameable { private String id; @NotBlank(message = "Name is mandatory") @@ -19,9 +21,13 @@ public class EventDto implements Nameable { private String description; @NotNull private EventCategory category; + @Builder.Default private Severity severity = Severity.INFO; + @Builder.Default private Map context = Map.of(); + @Builder.Default private Instant occurredAt = Instant.now(); private Instant createdAt; + @Builder.Default private String createdBy = "system"; } diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/dto/ProfileDto.java b/src/main/java/it/gov/innovazione/ndc/alerter/dto/ProfileDto.java index 1cacce27..35a7375c 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/dto/ProfileDto.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/dto/ProfileDto.java @@ -6,6 +6,7 @@ import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotEmpty; +import java.time.Instant; import java.util.List; @Data @@ -17,4 +18,5 @@ public class ProfileDto implements Nameable { private List eventCategories; private Severity minSeverity = Severity.INFO; private Integer aggregationTime = 60; + private Instant lastAlertedAt; } diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/entities/Profile.java b/src/main/java/it/gov/innovazione/ndc/alerter/entities/Profile.java index 7715b643..e4ff32a4 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/entities/Profile.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/entities/Profile.java @@ -13,6 +13,7 @@ import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.Id; +import java.time.Instant; import java.util.List; @Data @@ -34,5 +35,8 @@ public class Profile implements Nameable { @Column(nullable = false) private Severity minSeverity; @Column(nullable = false) - private Integer aggregationTime; + private Long aggregationTime; + @Column + @Builder.Default + private Instant lastAlertedAt = Instant.now(); } diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/entities/User.java b/src/main/java/it/gov/innovazione/ndc/alerter/entities/User.java index f911370a..dadd6ba5 100644 --- a/src/main/java/it/gov/innovazione/ndc/alerter/entities/User.java +++ b/src/main/java/it/gov/innovazione/ndc/alerter/entities/User.java @@ -11,18 +11,21 @@ import javax.persistence.GeneratedValue; import javax.persistence.Id; import javax.persistence.ManyToOne; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; @Data @Entity @Builder @NoArgsConstructor @AllArgsConstructor +@Table(uniqueConstraints = {@UniqueConstraint(columnNames = {"name", "surname", "email"})}) public class User implements Nameable { @Id @GeneratedValue(generator = "uuid") @GenericGenerator(name = "uuid", strategy = "org.hibernate.id.UUIDGenerator") private String id; - @Column(unique = true, nullable = false) + @Column(nullable = false) private String name; @Column(nullable = false) private String surname; diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/event/AlertableEvent.java b/src/main/java/it/gov/innovazione/ndc/alerter/event/AlertableEvent.java new file mode 100644 index 00000000..cc20c5d7 --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/alerter/event/AlertableEvent.java @@ -0,0 +1,23 @@ +package it.gov.innovazione.ndc.alerter.event; + +import it.gov.innovazione.ndc.alerter.entities.EventCategory; +import it.gov.innovazione.ndc.alerter.entities.Severity; + +import java.time.Instant; +import java.util.Map; + +public interface AlertableEvent { + String getName(); + + String getDescription(); + + default Instant getOccurredAt() { + return Instant.now(); + } + + EventCategory getCategory(); + + Severity getSeverity(); + + Map getContext(); +} diff --git a/src/main/java/it/gov/innovazione/ndc/alerter/event/DefaultAlertableEvent.java b/src/main/java/it/gov/innovazione/ndc/alerter/event/DefaultAlertableEvent.java new file mode 100644 index 00000000..bbde8b9b --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/alerter/event/DefaultAlertableEvent.java @@ -0,0 +1,29 @@ +package it.gov.innovazione.ndc.alerter.event; + +import it.gov.innovazione.ndc.alerter.entities.EventCategory; +import it.gov.innovazione.ndc.alerter.entities.Severity; +import lombok.Builder; +import lombok.Getter; +import org.springframework.security.core.context.SecurityContext; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.security.Principal; +import java.time.Instant; +import java.util.Map; +import java.util.Optional; + +@Getter +@Builder +public class DefaultAlertableEvent implements AlertableEvent { + private final String name; + private final String description; + @Builder.Default + private final String createdBy = Optional.ofNullable(SecurityContextHolder.getContext()) + .map(SecurityContext::getAuthentication) + .map(Principal::getName) + .orElse("system"); + private final Instant occurredAt; + private final EventCategory category; + private final Severity severity; + private final Map context; +} diff --git a/src/main/java/it/gov/innovazione/ndc/config/GitHubConfig.java b/src/main/java/it/gov/innovazione/ndc/config/GitHubConfig.java index 5f673981..d2983566 100644 --- a/src/main/java/it/gov/innovazione/ndc/config/GitHubConfig.java +++ b/src/main/java/it/gov/innovazione/ndc/config/GitHubConfig.java @@ -1,6 +1,11 @@ package it.gov.innovazione.ndc.config; +import it.gov.innovazione.ndc.alerter.entities.EventCategory; +import it.gov.innovazione.ndc.alerter.entities.Severity; +import it.gov.innovazione.ndc.alerter.event.DefaultAlertableEvent; +import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; import it.gov.innovazione.ndc.service.NdcGitHubClient; +import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -9,15 +14,31 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -@Configuration +import java.time.Instant; + @Slf4j +@Configuration +@RequiredArgsConstructor public class GitHubConfig { + private final NdcEventPublisher eventPublisher; + + @Bean @SneakyThrows NdcGitHubClient gitHub(@Value("${github.personal-access-token}") String token) { if (token == null || token.isEmpty() || StringUtils.equalsAnyIgnoreCase("no_token", token)) { log.warn("GitHub personal access token not provided. The GitHub issuer capability will be disabled"); + eventPublisher.publishAlertableEvent( + "GitHubConfig", + DefaultAlertableEvent.builder() + .name("GitHubConfig not provided") + .description("GitHubConfig personal access token not provided. The GitHub issuer capability will be disabled") + .occurredAt(Instant.now()) + .category(EventCategory.APPLICATION) + .severity(Severity.WARNING) + .build()); + return NdcGitHubClient.builder() .enabled(false) .build(); diff --git a/src/main/java/it/gov/innovazione/ndc/controller/ConfigurationController.java b/src/main/java/it/gov/innovazione/ndc/controller/ConfigurationController.java index 15a28528..afe4729f 100644 --- a/src/main/java/it/gov/innovazione/ndc/controller/ConfigurationController.java +++ b/src/main/java/it/gov/innovazione/ndc/controller/ConfigurationController.java @@ -1,7 +1,12 @@ package it.gov.innovazione.ndc.controller; +import it.gov.innovazione.ndc.alerter.entities.EventCategory; +import it.gov.innovazione.ndc.alerter.entities.Severity; +import it.gov.innovazione.ndc.alerter.event.AlertableEvent; +import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.harvester.service.ActualConfigService; -import it.gov.innovazione.ndc.harvester.service.ConfigService; +import lombok.Builder; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.DeleteMapping; @@ -17,6 +22,7 @@ import java.security.Principal; import java.util.Map; +import java.util.stream.Collectors; import static org.springframework.http.HttpStatus.ACCEPTED; import static org.springframework.http.HttpStatus.CREATED; @@ -28,6 +34,7 @@ public class ConfigurationController { private final ActualConfigService configService; + private final NdcEventPublisher eventPublisher; @GetMapping public Map getConfig( @@ -49,6 +56,10 @@ public void setConfig( return; } configService.setConfig(config, principal.getName(), repoId); + eventPublisher.publishAlertableEvent("Configuration", WebConfigAlertableEvent.builder() + .repoId(repoId) + .config(config) + .build()); } @PutMapping("/{configKey}") @@ -63,6 +74,11 @@ public void updateRepository( return; } configService.writeConfigKey(configKey, principal.getName(), value, repoId); + eventPublisher.publishAlertableEvent("Configuration", + WebConfigAlertableEvent.builder() + .repoId(repoId) + .config(Map.of(configKey, value)) + .build()); } @DeleteMapping("/{configKey}") @@ -76,5 +92,43 @@ public void deleteRepository( return; } configService.removeConfigKey(configKey, principal.getName(), repoId); + eventPublisher.publishAlertableEvent("Configuration", + WebConfigAlertableEvent.builder() + .repoId(repoId) + .config(Map.of(configKey, "removed")) + .build()); + } + + @Builder + @RequiredArgsConstructor + private static class WebConfigAlertableEvent implements AlertableEvent { + private final String repoId; + private final Map config; + + @Override + public String getName() { + return "Configuration updated"; + } + + @Override + public String getDescription() { + return "Configuration updated for repository " + repoId; + } + + @Override + public EventCategory getCategory() { + return EventCategory.APPLICATION; + } + + @Override + public Severity getSeverity() { + return Severity.INFO; + } + + @Override + public Map getContext() { + return config.entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey().name(), Map.Entry::getValue)); + } } } diff --git a/src/main/java/it/gov/innovazione/ndc/controller/HarvestJobController.java b/src/main/java/it/gov/innovazione/ndc/controller/HarvestJobController.java index ae44baad..62b74bcb 100644 --- a/src/main/java/it/gov/innovazione/ndc/controller/HarvestJobController.java +++ b/src/main/java/it/gov/innovazione/ndc/controller/HarvestJobController.java @@ -1,10 +1,16 @@ package it.gov.innovazione.ndc.controller; +import it.gov.innovazione.ndc.alerter.entities.EventCategory; +import it.gov.innovazione.ndc.alerter.entities.Severity; +import it.gov.innovazione.ndc.alerter.event.AlertableEvent; +import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; import it.gov.innovazione.ndc.harvester.HarvesterJob; import it.gov.innovazione.ndc.harvester.HarvesterService; import it.gov.innovazione.ndc.harvester.JobExecutionResponse; import it.gov.innovazione.ndc.harvester.service.HarvesterRunService; import it.gov.innovazione.ndc.model.harvester.HarvesterRun; +import lombok.Builder; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -17,6 +23,7 @@ import java.time.LocalDateTime; import java.util.List; +import java.util.Map; @Slf4j @RestController @@ -27,11 +34,19 @@ public class HarvestJobController { private final HarvesterJob harvesterJob; private final HarvesterService harvesterService; private final HarvesterRunService harvesterRunService; + private final NdcEventPublisher eventPublisher; @PostMapping("jobs/harvest") public List startHarvestJob(@RequestParam(required = false, defaultValue = "false") Boolean force) { log.info("Starting Harvest job at " + LocalDateTime.now()); - return harvesterJob.harvest(force); + List harvest = harvesterJob.harvest(force); + eventPublisher.publishAlertableEvent( + "Harvester", + WebHarversterAlertableEvent.builder() + .description("Harvester execution started through REST API") + .context(Map.of("force", force)) + .build()); + return harvest; } @GetMapping("jobs/harvest/run") @@ -47,20 +62,58 @@ public List getAllRunningInstance() { @DeleteMapping("jobs/harvest/run") public void deletePendingRuns() { harvesterRunService.deletePendingRuns(); + eventPublisher.publishAlertableEvent( + "Harvester", + WebHarversterAlertableEvent.builder() + .description("Harvester pending runs deleted") + .build()); } @PostMapping(value = "jobs/harvest", params = "repositoryId") public JobExecutionResponse harvestRepositories( @RequestParam("repositoryId") String repositoryId, - @RequestParam(required = false) String revision, + @RequestParam(required = false, defaultValue = "") String revision, @RequestParam(required = false, defaultValue = "false") Boolean force) { log.info("Starting Harvest job at " + LocalDateTime.now() + "for repository " + repositoryId); - return harvesterJob.harvest(repositoryId, revision, force); + JobExecutionResponse harvest = harvesterJob.harvest(repositoryId, revision, force); + eventPublisher.publishAlertableEvent( + "Harvester", + WebHarversterAlertableEvent.builder() + .description("Harvester execution started through REST API") + .context(Map.of("repositoryId", repositoryId, "revision", revision, "force", force)) + .build()); + return harvest; } @PostMapping("jobs/clear") public void clearRepo(@RequestParam("repo_url") String repoUrl) { harvesterService.clear(repoUrl); + eventPublisher.publishAlertableEvent( + "Harvester", + WebHarversterAlertableEvent.builder() + .description("Harvester repo cleared") + .context(Map.of("repo_url", repoUrl)) + .build()); + } + + @Getter + @Builder + private static class WebHarversterAlertableEvent implements AlertableEvent { + @Builder.Default + private final String name = "Harvester Endpoint"; + private final String description; + @Builder.Default + private final Map context = Map.of(); + + @Override + public EventCategory getCategory() { + return EventCategory.APPLICATION; + } + + @Override + public Severity getSeverity() { + return Severity.INFO; + } } } diff --git a/src/main/java/it/gov/innovazione/ndc/controller/RepositoryController.java b/src/main/java/it/gov/innovazione/ndc/controller/RepositoryController.java index 86a15b5e..1799e138 100644 --- a/src/main/java/it/gov/innovazione/ndc/controller/RepositoryController.java +++ b/src/main/java/it/gov/innovazione/ndc/controller/RepositoryController.java @@ -1,5 +1,9 @@ package it.gov.innovazione.ndc.controller; +import it.gov.innovazione.ndc.alerter.entities.EventCategory; +import it.gov.innovazione.ndc.alerter.entities.Severity; +import it.gov.innovazione.ndc.alerter.event.DefaultAlertableEvent; +import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; import it.gov.innovazione.ndc.harvester.HarvesterService; import it.gov.innovazione.ndc.harvester.service.RepositoryService; import it.gov.innovazione.ndc.model.harvester.Repository; @@ -23,6 +27,7 @@ import java.net.URL; import java.security.Principal; import java.util.List; +import java.util.Map; import java.util.Optional; import static org.springframework.http.HttpStatus.CREATED; @@ -35,6 +40,7 @@ public class RepositoryController { private final RepositoryService repositoryService; private final HarvesterService harvesterService; + private final NdcEventPublisher eventPublisher; @GetMapping public List getAllRepositories() { @@ -54,6 +60,15 @@ public void createRepository( repository.getDescription(), repository.getMaxFileSizeBytes(), principal); + eventPublisher.publishAlertableEvent( + "Configuration", + DefaultAlertableEvent.builder() + .name("Configuration") + .description("Repository " + repository.getName() + " created") + .severity(Severity.INFO) + .category(EventCategory.APPLICATION) + .context(Map.of("repository", repository)) + .build()); } private void assertValidUrl(@RequestBody CreateRepository repository) throws BadRequestException { @@ -79,8 +94,26 @@ public ResponseEntity updateRepository( int updated = repositoryService.updateRepo(id, repository, principal); if (updated == 0) { + eventPublisher.publishAlertableEvent( + "Configuration", + DefaultAlertableEvent.builder() + .name("Configuration") + .description("Repository " + id + " not found") + .severity(Severity.WARNING) + .category(EventCategory.APPLICATION) + .context(Map.of("id", id, "repository", repository)) + .build()); return ResponseEntity.notFound().build(); } + eventPublisher.publishAlertableEvent( + "Configuration", + DefaultAlertableEvent.builder() + .name("Configuration") + .description("Repository " + id + " updated") + .severity(Severity.INFO) + .category(EventCategory.APPLICATION) + .context(Map.of("id", id, "repository", repository)) + .build()); return ResponseEntity.noContent().build(); } @@ -94,6 +127,15 @@ public ResponseEntity deleteRepository( .findFirst(); if (optionalRepository.isEmpty()) { + eventPublisher.publishAlertableEvent( + "Configuration", + DefaultAlertableEvent.builder() + .name("Configuration") + .description("Repository " + id + " not found") + .severity(Severity.WARNING) + .category(EventCategory.APPLICATION) + .context(Map.of("id", id)) + .build()); log.warn("Repository {} not found or not active", id); return ResponseEntity.notFound().build(); } @@ -109,6 +151,15 @@ public ResponseEntity deleteRepository( return ResponseEntity.notFound().build(); } + eventPublisher.publishAlertableEvent( + "Configuration", + DefaultAlertableEvent.builder() + .name("Configuration") + .description("Repository " + id + " deleted") + .severity(Severity.INFO) + .category(EventCategory.APPLICATION) + .context(Map.of("id", id, "repository", repository)) + .build()); log.info("Repository {} deleted", repository); return ResponseEntity.noContent().build(); } diff --git a/src/main/java/it/gov/innovazione/ndc/eventhandler/NdcEventPublisher.java b/src/main/java/it/gov/innovazione/ndc/eventhandler/NdcEventPublisher.java index 1b360b80..79d2c020 100644 --- a/src/main/java/it/gov/innovazione/ndc/eventhandler/NdcEventPublisher.java +++ b/src/main/java/it/gov/innovazione/ndc/eventhandler/NdcEventPublisher.java @@ -1,5 +1,6 @@ package it.gov.innovazione.ndc.eventhandler; +import it.gov.innovazione.ndc.alerter.event.AlertableEvent; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationEventPublisher; @@ -20,6 +21,9 @@ public void publishEvent( String correlationId, String user, T event) { + if (event.getClass().isAssignableFrom(AlertableEvent.class)) { + log.warn("Event is not alertable, publishAlertableEvent should be used instead"); + } try { applicationEventPublisher.publishEvent( NdcEventWrapper.builder() @@ -34,4 +38,20 @@ public void publishEvent( log.error("Error publishing event", e); } } + + public void publishAlertableEvent( + String source, T event) { + try { + applicationEventPublisher.publishEvent( + NdcEventWrapper.builder() + .source(source) + .type(event.getName()) + .correlationId("") + .timestamp(Instant.now()) + .payload(event) + .build()); + } catch (Exception e) { + log.error("Error publishing alertable event", e); + } + } } diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/service/ConfigService.java b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/ConfigService.java similarity index 85% rename from src/main/java/it/gov/innovazione/ndc/harvester/service/ConfigService.java rename to src/main/java/it/gov/innovazione/ndc/eventhandler/event/ConfigService.java index 414bea68..1a67f883 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/service/ConfigService.java +++ b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/ConfigService.java @@ -1,5 +1,6 @@ -package it.gov.innovazione.ndc.harvester.service; +package it.gov.innovazione.ndc.eventhandler.event; +import it.gov.innovazione.ndc.harvester.service.ActualConfigService; import lombok.Builder; import lombok.Data; import lombok.Getter; @@ -8,12 +9,13 @@ import java.time.Instant; import java.util.Map; +import java.util.Objects; import java.util.Optional; @Slf4j public abstract class ConfigService { - private Optional fromGlobal(ActualConfigService.ConfigKey key) { + public Optional fromGlobal(ActualConfigService.ConfigKey key) { try { Class type = null; return Optional.ofNullable(getNdcConfiguration()) @@ -106,13 +108,27 @@ public static class ConfigEntry { } @Builder + @Getter public static class ConfigEvent { private final Map changes; private final String destination; private final Exception error; + + public boolean isChange(ActualConfigService.ConfigKey key) { + return Objects.nonNull(changes) && changes.containsKey(key); + } + + public boolean isChange(ActualConfigService.ConfigKey key, Object newValue) { + try { + return isChange(key) && changes.get(key).getNewValue().getValue().equals(newValue); + } catch (Exception e) { + return false; + } + } } @Builder + @Getter public static class ConfigChange { private final ConfigEntry oldValue; private final ConfigEntry newValue; diff --git a/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvestedFileTooBigEvent.java b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvestedFileTooBigEvent.java index c23267cd..bf887675 100644 --- a/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvestedFileTooBigEvent.java +++ b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvestedFileTooBigEvent.java @@ -1,5 +1,8 @@ package it.gov.innovazione.ndc.eventhandler.event; +import it.gov.innovazione.ndc.alerter.entities.EventCategory; +import it.gov.innovazione.ndc.alerter.entities.Severity; +import it.gov.innovazione.ndc.alerter.event.AlertableEvent; import it.gov.innovazione.ndc.model.harvester.Repository; import lombok.AccessLevel; import lombok.Builder; @@ -8,17 +11,48 @@ import java.io.File; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @Builder @Data -public class HarvestedFileTooBigEvent { +public class HarvestedFileTooBigEvent implements AlertableEvent { private final String runId; private final Repository repository; private final String revision; private final ViolatingSemanticAsset violatingSemanticAsset; private final String relativePathInRepo; + @Override + public String getName() { + return "HarvestedFileTooBig"; + } + + @Override + public String getDescription() { + return "A semantic asset file is too big"; + } + + @Override + public EventCategory getCategory() { + return EventCategory.SEMANTIC; + } + + @Override + public Severity getSeverity() { + return Severity.WARNING; + } + + @Override + public Map getContext() { + return Map.of( + "runId", runId, + "repository", repository, + "revision", revision, + "violatingSemanticAsset", violatingSemanticAsset, + "relativePathInRepo", relativePathInRepo); + } + @Data @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public static class ViolatingSemanticAsset { @@ -40,7 +74,6 @@ public static ViolatingSemanticAsset fromPath( file.length() > maxFileSizeBytes)) .collect(Collectors.toList())); } - } diff --git a/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterFinishedEvent.java b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterFinishedEvent.java index ecc115bf..98addc22 100644 --- a/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterFinishedEvent.java +++ b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterFinishedEvent.java @@ -1,16 +1,59 @@ package it.gov.innovazione.ndc.eventhandler.event; +import it.gov.innovazione.ndc.alerter.entities.EventCategory; +import it.gov.innovazione.ndc.alerter.entities.Severity; +import it.gov.innovazione.ndc.alerter.event.AlertableEvent; import it.gov.innovazione.ndc.model.harvester.HarvesterRun; import it.gov.innovazione.ndc.model.harvester.Repository; import lombok.Builder; import lombok.Data; +import java.util.Map; + @Builder @Data -public class HarvesterFinishedEvent { +public class HarvesterFinishedEvent implements AlertableEvent { private final String runId; private final Repository repository; private final String revision; private final HarvesterRun.Status status; private final Exception exception; + + @Override + public String getName() { + return "Run " + runId + " finished"; + } + + @Override + public String getDescription() { + return "Harvester run " + runId + " finished with status " + status; + } + + @Override + public EventCategory getCategory() { + return EventCategory.APPLICATION; + } + + @Override + public Severity getSeverity() { + if (status == HarvesterRun.Status.ALREADY_RUNNING) { + return Severity.WARNING; + } else if (status == HarvesterRun.Status.SUCCESS || status == HarvesterRun.Status.UNCHANGED) { + return Severity.INFO; + } else if (status == HarvesterRun.Status.FAILURE) { + return Severity.ERROR; + } + return Severity.INFO; + } + + @Override + public Map getContext() { + return Map.of( + "runId", runId, + "repository", repository, + "revision", revision, + "status", status, + "exception", exception + ); + } } diff --git a/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterStartedEvent.java b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterStartedEvent.java index 601c5fcb..0a5152c7 100644 --- a/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterStartedEvent.java +++ b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterStartedEvent.java @@ -1,13 +1,48 @@ package it.gov.innovazione.ndc.eventhandler.event; +import it.gov.innovazione.ndc.alerter.entities.EventCategory; +import it.gov.innovazione.ndc.alerter.entities.Severity; +import it.gov.innovazione.ndc.alerter.event.AlertableEvent; import it.gov.innovazione.ndc.model.harvester.Repository; import lombok.Builder; import lombok.Data; +import java.util.Map; + @Builder @Data -public class HarvesterStartedEvent { +public class HarvesterStartedEvent implements AlertableEvent { private final String runId; private final Repository repository; private final String revision; + + @Override + public String getName() { + return "Run " + runId + " started"; + } + + @Override + public String getDescription() { + return "Harvester run " + runId + " started"; + } + + @Override + public EventCategory getCategory() { + return EventCategory.APPLICATION; + } + + @Override + public Severity getSeverity() { + return Severity.INFO; + } + + @Override + public Map getContext() { + return Map.of( + "repository", repository, + "revision", revision, + "runId", runId + ); + } + } diff --git a/src/main/java/it/gov/innovazione/ndc/eventhandler/handler/AlerterEnabledEventListener.java b/src/main/java/it/gov/innovazione/ndc/eventhandler/handler/AlerterEnabledEventListener.java new file mode 100644 index 00000000..b50efb79 --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/eventhandler/handler/AlerterEnabledEventListener.java @@ -0,0 +1,47 @@ +package it.gov.innovazione.ndc.eventhandler.handler; + +import it.gov.innovazione.ndc.alerter.data.ProfileService; +import it.gov.innovazione.ndc.eventhandler.NdcEventHandler; +import it.gov.innovazione.ndc.eventhandler.NdcEventWrapper; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +import static it.gov.innovazione.ndc.harvester.service.ActualConfigService.ConfigKey; + +@Service +@RequiredArgsConstructor +@Slf4j +public class AlerterEnabledEventListener implements NdcEventHandler { + + private static final Collection> SUPPORTED_EVENTS = List.of(ConfigService.ConfigEvent.class); + + private final ProfileService profileService; + + @Value("${alerter.mail-sender.backoff:PT1H}") + private Duration backoff; + + @Override + public boolean canHandle(NdcEventWrapper event) { + return SUPPORTED_EVENTS.contains(event.getPayload().getClass()); + } + + @Override + public void handle(NdcEventWrapper event) { + ConfigService.ConfigEvent payload = (ConfigService.ConfigEvent) event.getPayload(); + if (Objects.isNull(payload) || Objects.isNull(payload.getChanges())) { + return; + } + if (payload.isChange(ConfigKey.ALERTER_ENABLED, Boolean.TRUE)) { + log.info("Alerter enabled, setting all profiles last updated"); + profileService.setAllLastUpdated(backoff); + } + } +} diff --git a/src/main/java/it/gov/innovazione/ndc/eventhandler/handler/AlerterEventHandler.java b/src/main/java/it/gov/innovazione/ndc/eventhandler/handler/AlerterEventHandler.java new file mode 100644 index 00000000..780e9d1a --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/eventhandler/handler/AlerterEventHandler.java @@ -0,0 +1,40 @@ +package it.gov.innovazione.ndc.eventhandler.handler; + +import it.gov.innovazione.ndc.alerter.AlerterService; +import it.gov.innovazione.ndc.alerter.event.AlertableEvent; +import it.gov.innovazione.ndc.eventhandler.NdcEventHandler; +import it.gov.innovazione.ndc.eventhandler.NdcEventWrapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.Collection; +import java.util.List; + +@Slf4j +@Service +@RequiredArgsConstructor +public class AlerterEventHandler implements NdcEventHandler { + + private static final Collection> SUPPORTED_EVENTS = List.of(AlertableEvent.class); + private final AlerterService alerterService; + + @Override + public boolean canHandle(NdcEventWrapper event) { + return SUPPORTED_EVENTS.stream() + .anyMatch(supportedEvent -> supportedEvent.isAssignableFrom(event.getPayload().getClass())); + } + + @Override + @SuppressWarnings("unchecked") + public void handle(NdcEventWrapper event) { + if (event.getPayload() instanceof AlertableEvent) { + NdcEventWrapper alertableEvent = (NdcEventWrapper) event; + handleAlertableEvent(alertableEvent); + } + } + + private void handleAlertableEvent(NdcEventWrapper alertableEvent) { + alerterService.alert(alertableEvent.getPayload()); + } +} diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvester.java b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvester.java index 4edc8afb..2d07afcb 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvester.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvester.java @@ -1,6 +1,10 @@ package it.gov.innovazione.ndc.harvester.harvesters; +import it.gov.innovazione.ndc.alerter.entities.EventCategory; +import it.gov.innovazione.ndc.alerter.entities.Severity; +import it.gov.innovazione.ndc.alerter.event.DefaultAlertableEvent; import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.eventhandler.event.HarvestedFileTooBigEvent; import it.gov.innovazione.ndc.eventhandler.event.HarvestedFileTooBigEvent.ViolatingSemanticAsset; import it.gov.innovazione.ndc.harvester.SemanticAssetHarvester; @@ -10,7 +14,6 @@ import it.gov.innovazione.ndc.harvester.exception.SinglePathProcessingException; import it.gov.innovazione.ndc.harvester.harvesters.utils.PathUtils; import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath; -import it.gov.innovazione.ndc.harvester.service.ConfigService; import it.gov.innovazione.ndc.model.harvester.Repository; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -18,6 +21,7 @@ import java.io.File; import java.nio.file.Path; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -55,6 +59,18 @@ public void harvest(Repository repository, Path rootPath) { } catch (SinglePathProcessingException e) { Optional.ofNullable(HarvestExecutionContextUtils.getContext()) .ifPresent(context -> context.addHarvestingError(repository, e, path.getAllFiles())); + eventPublisher.publishAlertableEvent( + "harvester", + DefaultAlertableEvent.builder() + .name("Harvester Single Path Processing Error") + .description("Error processing " + type + " " + path + " in repo " + repository.getUrl()) + .category(EventCategory.SEMANTIC) + .severity(Severity.WARNING) + .context(Map.of( + "error", e.getMessage(), + "path", path.getTtlPath(), + "repo", repository.getUrl())) + .build()); log.error("Error processing {} {} in repo {}", type, path, repository.getUrl(), e); } } diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvester.java b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvester.java index 338a014e..b115c7a2 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvester.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvester.java @@ -1,11 +1,11 @@ package it.gov.innovazione.ndc.harvester.harvesters; import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.harvester.AgencyRepositoryService; import it.gov.innovazione.ndc.harvester.SemanticAssetType; import it.gov.innovazione.ndc.harvester.model.CvPath; import it.gov.innovazione.ndc.harvester.pathprocessors.ControlledVocabularyPathProcessor; -import it.gov.innovazione.ndc.harvester.service.ConfigService; import org.springframework.stereotype.Component; import java.nio.file.Path; diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/OntologyHarvester.java b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/OntologyHarvester.java index 77e0616b..f0df2f98 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/OntologyHarvester.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/OntologyHarvester.java @@ -1,11 +1,11 @@ package it.gov.innovazione.ndc.harvester.harvesters; import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.harvester.AgencyRepositoryService; import it.gov.innovazione.ndc.harvester.SemanticAssetType; import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath; import it.gov.innovazione.ndc.harvester.pathprocessors.OntologyPathProcessor; -import it.gov.innovazione.ndc.harvester.service.ConfigService; import org.springframework.stereotype.Component; import java.nio.file.Path; diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/SchemaHarvester.java b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/SchemaHarvester.java index 26e5a837..379f8f91 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/SchemaHarvester.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/SchemaHarvester.java @@ -1,11 +1,11 @@ package it.gov.innovazione.ndc.harvester.harvesters; import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.harvester.AgencyRepositoryService; import it.gov.innovazione.ndc.harvester.SemanticAssetType; import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath; import it.gov.innovazione.ndc.harvester.pathprocessors.SchemaPathProcessor; -import it.gov.innovazione.ndc.harvester.service.ConfigService; import org.springframework.stereotype.Component; import java.nio.file.Path; diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/service/ActualConfigService.java b/src/main/java/it/gov/innovazione/ndc/harvester/service/ActualConfigService.java index a1baade8..3cd4ff0c 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/service/ActualConfigService.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/service/ActualConfigService.java @@ -1,6 +1,7 @@ package it.gov.innovazione.ndc.harvester.service; import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.model.harvester.Repository; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -266,7 +267,8 @@ private void removeConfigKey(ConfigKey configKey, String writtenBy, @RequiredArgsConstructor public enum ConfigKey { MAX_FILE_SIZE_BYTES("The maximum file size in bytes of a file to be harvested", Validator.IS_LONG, Parser.TO_LONG), - GITHUB_ISSUER_ENABLED("Enable the GitHub issuer capability", Validator.IS_BOOLEAN, Parser.TO_BOOLEAN); + GITHUB_ISSUER_ENABLED("Enable the GitHub issuer capability", Validator.IS_BOOLEAN, Parser.TO_BOOLEAN), + ALERTER_ENABLED("Enable the Alerter capability", Validator.IS_BOOLEAN, Parser.TO_BOOLEAN); private final String description; private final Validator validator; diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/service/ConfigReaderService.java b/src/main/java/it/gov/innovazione/ndc/harvester/service/ConfigReaderService.java index cc09495b..1d1b62e4 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/service/ConfigReaderService.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/service/ConfigReaderService.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import org.springframework.stereotype.Service; diff --git a/src/main/java/it/gov/innovazione/ndc/model/harvester/Repository.java b/src/main/java/it/gov/innovazione/ndc/model/harvester/Repository.java index a8092b8b..23d7c349 100644 --- a/src/main/java/it/gov/innovazione/ndc/model/harvester/Repository.java +++ b/src/main/java/it/gov/innovazione/ndc/model/harvester/Repository.java @@ -1,8 +1,8 @@ package it.gov.innovazione.ndc.model.harvester; import com.fasterxml.jackson.annotation.JsonIgnore; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.harvester.service.ActualConfigService; -import it.gov.innovazione.ndc.harvester.service.ConfigService; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; @@ -32,7 +32,6 @@ public class Repository { private Map config; @JsonIgnore private Map> rightsHolders; - private List maintainers; @Override public String toString() { diff --git a/src/main/java/it/gov/innovazione/ndc/service/AlerterMailSender.java b/src/main/java/it/gov/innovazione/ndc/service/AlerterMailSender.java new file mode 100644 index 00000000..146fbb45 --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/service/AlerterMailSender.java @@ -0,0 +1,126 @@ +package it.gov.innovazione.ndc.service; + +import it.gov.innovazione.ndc.alerter.data.EventService; +import it.gov.innovazione.ndc.alerter.data.ProfileService; +import it.gov.innovazione.ndc.alerter.data.UserService; +import it.gov.innovazione.ndc.alerter.dto.EventDto; +import it.gov.innovazione.ndc.alerter.dto.ProfileDto; +import it.gov.innovazione.ndc.alerter.dto.UserDto; +import it.gov.innovazione.ndc.alerter.entities.EventCategory; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Instant; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static it.gov.innovazione.ndc.harvester.service.ActualConfigService.ConfigKey.ALERTER_ENABLED; +import static org.apache.commons.collections4.ListUtils.emptyIfNull; + +@Component +@RequiredArgsConstructor +@Slf4j +public class AlerterMailSender { + + private final EmailService emailService; + private final ProfileService profileService; + private final EventService eventService; + private final UserService userService; + private final ConfigService configService; + + @Scheduled(fixedDelayString = "${alerter.mail-sender.fixed-delay:60000}") + void getEventsAndAlert() { + Collection profiles = profileService.findAll(); + for (ProfileDto profileDto : profiles) { + + Instant now = Instant.now(); + Instant lastAlertedAt = profileDto.getLastAlertedAt(); + + eventService.getEventsNewerThan(lastAlertedAt).stream() + .filter(eventDto -> eventDto.getCreatedAt().plusSeconds(profileDto.getAggregationTime()).isBefore(now)) + .filter(eventDto -> isSeverityHigherThanOrEqualsToMinSeverity(eventDto, profileDto)) + .filter(eventDto -> isApplicableToProfile(eventDto, profileDto)) + .collect(Collectors.groupingBy(EventDto::getCategory)) + .forEach((key, value) -> sendMessages(key, value, profileDto)); + } + } + + private boolean isAlerterEnabled() { + return (boolean) configService.fromGlobal(ALERTER_ENABLED).orElse(false); + } + + private boolean isApplicableToProfile(EventDto eventDto, ProfileDto profileDto) { + return emptyIfNull(profileDto.getEventCategories()).contains(eventDto.getCategory().name()); + } + + private void sendMessages(EventCategory category, List eventDtos, ProfileDto profileDto) { + + if (eventDtos.isEmpty()) { + return; + } + + if (!isAlerterEnabled()) { + log.warn("Alerter is disabled, no mails will be sent, following events will be stored in the database." + + "Events: {}", + eventDtos.stream() + .map(EventDto::toString) + .collect(Collectors.joining(", "))); + return; + } + + log.info("Sending email for detected {} events, to users with profile {} for category: {}", + eventDtos.size(), profileDto.getName(), category); + List recipients = userService.findAll().stream() + .filter(user -> StringUtils.equals(user.getProfile(), profileDto.getName())) + .collect(Collectors.toList()); + if (recipients.isEmpty()) { + log.warn("No recipients found for profile {}, " + + "for this profile no mails will be sent. " + + "It might still be possible these events will be notified to other profiles. " + + "Events: {}", + profileDto.getName(), + eventDtos.stream() + .map(EventDto::toString) + .collect(Collectors.joining(", "))); + return; + } + for (UserDto recipient : recipients) { + emailService.sendEmail(recipient.getEmail(), + "[SCHEMAGOV] [" + category + "] Alerter: Report degli eventi", + getMessageBody(eventDtos, recipient, profileDto)); + } + profileService.setLastUpdated(profileDto.getId()); + } + + private String getMessageBody(List eventDtos, UserDto recipient, ProfileDto profileDto) { + StringBuilder message = new StringBuilder("Ciao " + recipient.getName() + " " + recipient.getSurname() + ",\n\n" + + "Di seguito i dettagli degli errori riscontrati:\n"); + int i = 1; + for (EventDto eventDto : eventDtos) { + message.append(getDetailsForEvent(i, eventDto, recipient, profileDto)); + i++; + } + message.append("Origine: Generata automaticamente dall'harvester.\n\n"); + message.append("Cordiali saluti,\n\nIl team di supporto di Schemagov"); + return message.toString(); + } + + private String getDetailsForEvent(int i, EventDto eventDto, UserDto recipient, ProfileDto profileDto) { + return i + ". Titolo: " + eventDto.getName() + "\n" + + "Descrizione: " + eventDto.getDescription() + "\n" + + "Severity: " + eventDto.getSeverity() + "\n" + + "Contesto: " + eventDto.getContext() + "\n" + + "Creato da: " + eventDto.getCreatedBy() + "\n" + + "Creato il: " + eventDto.getCreatedAt() + "\n\n"; + } + + private boolean isSeverityHigherThanOrEqualsToMinSeverity(EventDto eventDto, ProfileDto profileDto) { + return eventDto.getSeverity().ordinal() >= profileDto.getMinSeverity().ordinal(); + } + +} diff --git a/src/main/java/it/gov/innovazione/ndc/service/EmailService.java b/src/main/java/it/gov/innovazione/ndc/service/EmailService.java new file mode 100644 index 00000000..6376ddf4 --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/service/EmailService.java @@ -0,0 +1,26 @@ +package it.gov.innovazione.ndc.service; + +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.mail.SimpleMailMessage; +import org.springframework.mail.javamail.JavaMailSender; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +class EmailService { + + private final JavaMailSender javaMailSender; + @Value("${alerter.mail.sender}") + private final String from; + + void sendEmail(String to, String subject, String text) { + SimpleMailMessage message = new SimpleMailMessage(); + message.setFrom(from); + message.setTo(to); + message.setSubject(subject); + message.setText(text); + javaMailSender.send(message); + } + +} diff --git a/src/main/java/it/gov/innovazione/ndc/service/GithubService.java b/src/main/java/it/gov/innovazione/ndc/service/GithubService.java index cb0911b4..520bb987 100644 --- a/src/main/java/it/gov/innovazione/ndc/service/GithubService.java +++ b/src/main/java/it/gov/innovazione/ndc/service/GithubService.java @@ -1,10 +1,10 @@ package it.gov.innovazione.ndc.service; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.harvester.context.HarvestExecutionContext; import it.gov.innovazione.ndc.harvester.context.HarvestExecutionContextUtils; import it.gov.innovazione.ndc.harvester.service.ActualConfigService; -import it.gov.innovazione.ndc.harvester.service.ConfigService; import it.gov.innovazione.ndc.harvester.service.RepositoryService; import it.gov.innovazione.ndc.model.harvester.Repository; import lombok.Data; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index a8c693f5..aa4c82fc 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -35,3 +35,10 @@ spring.flyway.baseline-on-migrate=true github.personal-access-token=${GITHUB_PERSONAL_ACCESS_TOKEN:} server.error.include-message=always +alerter.mail.sender=${ALERTER_MAIL_SENDER:servicedesk-schema@istat.it} +spring.mail.host=${ALERTER_SMTP_SERVER:mail.smtpbucket.com} +spring.mail.port=${ALERTER_SMTP_PORT:8025} +spring.mail.username=${ALERTER_SMTP_USER:servicedesk-schema@istat.it} +spring.mail.password=${ALERTER_SMTP_PASSWORD:} +spring.mail.properties.mail.smtp.auth=${ALERTER_SMTP_AUTH:true} +spring.mail.properties.mail.smtp.starttls.enable=${ALERTER_SMTP_STARTTLS:true} diff --git a/src/test/java/it/gov/innovazione/ndc/controller/HarvestJobControllerTest.java b/src/test/java/it/gov/innovazione/ndc/controller/HarvestJobControllerTest.java index c61777da..aff7c805 100644 --- a/src/test/java/it/gov/innovazione/ndc/controller/HarvestJobControllerTest.java +++ b/src/test/java/it/gov/innovazione/ndc/controller/HarvestJobControllerTest.java @@ -1,5 +1,6 @@ package it.gov.innovazione.ndc.controller; +import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; import it.gov.innovazione.ndc.harvester.HarvesterJob; import it.gov.innovazione.ndc.harvester.HarvesterService; import org.junit.jupiter.api.Test; @@ -17,14 +18,16 @@ class HarvestJobControllerTest { HarvesterJob harvesterJob; @Mock HarvesterService harvesterService; + @Mock + NdcEventPublisher eventPublisher; @InjectMocks HarvestJobController harvestJobController; @Test void shouldStartHarvestForSpecifiedRepositories() { String repoUrls = "http://github.com/repo,http://github.com/repo2"; - harvestJobController.harvestRepositories(repoUrls, null, false); - verify(harvesterJob).harvest(repoUrls, null, false); + harvestJobController.harvestRepositories(repoUrls, "", false); + verify(harvesterJob).harvest(repoUrls, "", false); } @Test diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvesterTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvesterTest.java index 4ae58cd6..c62bc50d 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvesterTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvesterTest.java @@ -1,12 +1,12 @@ package it.gov.innovazione.ndc.harvester.harvesters; import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.harvester.SemanticAssetType; import it.gov.innovazione.ndc.harvester.context.HarvestExecutionContext; import it.gov.innovazione.ndc.harvester.context.HarvestExecutionContextUtils; import it.gov.innovazione.ndc.harvester.exception.InvalidAssetException; import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath; -import it.gov.innovazione.ndc.harvester.service.ConfigService; import it.gov.innovazione.ndc.model.harvester.Repository; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvesterTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvesterTest.java index 937f28a9..6d0892b6 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvesterTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvesterTest.java @@ -1,9 +1,9 @@ package it.gov.innovazione.ndc.harvester.harvesters; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.harvester.AgencyRepositoryService; import it.gov.innovazione.ndc.harvester.model.CvPath; import it.gov.innovazione.ndc.harvester.pathprocessors.ControlledVocabularyPathProcessor; -import it.gov.innovazione.ndc.harvester.service.ConfigService; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/OntologyHarvesterTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/OntologyHarvesterTest.java index 4f5c4847..dede1450 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/OntologyHarvesterTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/OntologyHarvesterTest.java @@ -1,9 +1,9 @@ package it.gov.innovazione.ndc.harvester.harvesters; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.harvester.AgencyRepositoryService; import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath; import it.gov.innovazione.ndc.harvester.pathprocessors.OntologyPathProcessor; -import it.gov.innovazione.ndc.harvester.service.ConfigService; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/SchemaHarvesterTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/SchemaHarvesterTest.java index 87e376b7..c8b93439 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/SchemaHarvesterTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/SchemaHarvesterTest.java @@ -1,9 +1,9 @@ package it.gov.innovazione.ndc.harvester.harvesters; +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.harvester.AgencyRepositoryService; import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath; import it.gov.innovazione.ndc.harvester.pathprocessors.SchemaPathProcessor; -import it.gov.innovazione.ndc.harvester.service.ConfigService; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks;