From f000d41567a94f4d09e4b040d7063c6857b48ef7 Mon Sep 17 00:00:00 2001 From: anistouri Date: Wed, 24 Jan 2024 16:21:04 +0100 Subject: [PATCH] add instrumentation setup (#127) * add instrumentation setup * Replace the file size with the number of buses * Separation of failed import and export * Adding the number of buses for import and export Signed-off-by: TOURI ANIS * Change jauge meter by DistributionSummary meter to have the max value for buses * Fix checkstyle Signed-off-by: Slimane AMAR --------- Signed-off-by: TOURI ANIS Signed-off-by: Slimane AMAR Co-authored-by: Slimane AMAR --- pom.xml | 6 +- .../server/NetworkConversionController.java | 8 ++- .../server/NetworkConversionObserver.java | 71 +++++++++++++++++++ .../server/NetworkConversionService.java | 23 ++++-- .../server/dto/ExportNetworkInfos.java | 2 + src/main/resources/application-local.yml | 6 ++ .../server/NetworkConversionTest.java | 5 ++ 7 files changed, 110 insertions(+), 11 deletions(-) create mode 100644 src/main/java/com/powsybl/network/conversion/server/NetworkConversionObserver.java diff --git a/pom.xml b/pom.xml index a3ea4bc0..b8c739c2 100644 --- a/pom.xml +++ b/pom.xml @@ -170,13 +170,17 @@ org.springframework.boot spring-boot-starter-actuator - runtime org.springframework.cloud spring-cloud-stream-binder-rabbit runtime + + io.micrometer + micrometer-registry-prometheus + runtime + diff --git a/src/main/java/com/powsybl/network/conversion/server/NetworkConversionController.java b/src/main/java/com/powsybl/network/conversion/server/NetworkConversionController.java index 63eba652..807b8414 100644 --- a/src/main/java/com/powsybl/network/conversion/server/NetworkConversionController.java +++ b/src/main/java/com/powsybl/network/conversion/server/NetworkConversionController.java @@ -48,6 +48,9 @@ public class NetworkConversionController { @Autowired private NetworkConversionService networkConversionService; + @Autowired + private NetworkConversionObserver networkConversionObserver; + @PostMapping(value = "/networks") @Operation(summary = "Get a case file from its name and import it into the store") public ResponseEntity importCase(@Parameter(description = "Case UUID") @RequestParam("caseUuid") UUID caseUuid, @@ -82,8 +85,7 @@ public ResponseEntity exportNetwork(@Parameter(description = "Network UU @org.springframework.web.bind.annotation.RequestBody(required = false) Map formatParameters ) throws IOException { LOGGER.debug("Exporting network {}...", networkUuid); - - ExportNetworkInfos exportNetworkInfos = networkConversionService.exportNetwork(networkUuid, variantId, format, formatParameters); + ExportNetworkInfos exportNetworkInfos = networkConversionObserver.observeExport(format, () -> networkConversionService.exportNetwork(networkUuid, variantId, format, formatParameters)); HttpHeaders header = new HttpHeaders(); header.setContentDisposition(ContentDisposition.builder("attachment").filename(exportNetworkInfos.getNetworkName(), StandardCharsets.UTF_8).build()); return ResponseEntity.ok().headers(header).contentType(MediaType.APPLICATION_OCTET_STREAM).body(exportNetworkInfos.getNetworkData()); @@ -109,7 +111,7 @@ public ResponseEntity getCaseImportParameters(@Parameter @Operation(summary = "Export a cgmes network from the network-store") public ResponseEntity exportCgmesSv(@Parameter(description = "Network UUID") @PathVariable("networkUuid") UUID networkUuid) throws XMLStreamException { LOGGER.debug("Exporting network {}...", networkUuid); - ExportNetworkInfos exportNetworkInfos = networkConversionService.exportCgmesSv(networkUuid); + ExportNetworkInfos exportNetworkInfos = networkConversionObserver.observeExport("CGMES", () -> networkConversionService.exportCgmesSv(networkUuid)); HttpHeaders header = new HttpHeaders(); header.setContentDisposition(ContentDisposition.builder("attachment").filename(exportNetworkInfos.getNetworkName(), StandardCharsets.UTF_8).build()); return ResponseEntity.ok().headers(header).contentType(MediaType.APPLICATION_OCTET_STREAM).body(exportNetworkInfos.getNetworkData()); diff --git a/src/main/java/com/powsybl/network/conversion/server/NetworkConversionObserver.java b/src/main/java/com/powsybl/network/conversion/server/NetworkConversionObserver.java new file mode 100644 index 00000000..c055e4e6 --- /dev/null +++ b/src/main/java/com/powsybl/network/conversion/server/NetworkConversionObserver.java @@ -0,0 +1,71 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package com.powsybl.network.conversion.server; + +import com.powsybl.iidm.network.Network; +import com.powsybl.network.conversion.server.dto.ExportNetworkInfos; +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationRegistry; +import lombok.NonNull; +import org.springframework.stereotype.Service; + +/** + * @author Anis Touri + */ + +@Service +public class NetworkConversionObserver { + + private static final String OBSERVATION_PREFIX = "app.conversion."; + private static final String FORMAT_TAG_NAME = "format"; + + private static final String IMPORT_OBSERVATION_NAME = OBSERVATION_PREFIX + "import"; + private static final String NUMBER_BUSES_IMPORTED_METER_NAME = IMPORT_OBSERVATION_NAME + ".buses"; + + private static final String EXPORT_OBSERVATION_NAME = OBSERVATION_PREFIX + "export"; + private static final String NUMBER_BUSES_EXPORTED_METER_NAME = EXPORT_OBSERVATION_NAME + ".buses"; + + private final ObservationRegistry observationRegistry; + + private final MeterRegistry meterRegistry; + + public NetworkConversionObserver(@NonNull ObservationRegistry observationRegistry, @NonNull MeterRegistry meterRegistry) { + this.observationRegistry = observationRegistry; + this.meterRegistry = meterRegistry; + } + + public ExportNetworkInfos observeExport(String format, Observation.CheckedCallable callable) throws E { + Observation observation = createObservation(EXPORT_OBSERVATION_NAME, format); + ExportNetworkInfos exportInfos = observation.observeChecked(callable); + if (exportInfos != null) { + recordNumberBuses(NUMBER_BUSES_EXPORTED_METER_NAME, format, exportInfos.getNumberBuses()); + } + return exportInfos; + } + + public Network observeImport(String format, Observation.CheckedCallable callable) throws E { + Network network = createObservation(IMPORT_OBSERVATION_NAME, format).observeChecked(callable); + if (network != null) { + recordNumberBuses(NUMBER_BUSES_IMPORTED_METER_NAME, format, network.getBusView().getBusStream().count()); + } + return network; + } + + private Observation createObservation(String name, String format) { + return Observation.createNotStarted(name, observationRegistry) + .lowCardinalityKeyValue(FORMAT_TAG_NAME, format); + } + + private void recordNumberBuses(String meterName, String format, long numberBuses) { + DistributionSummary.builder(meterName) + .tags(FORMAT_TAG_NAME, format) + .register(meterRegistry) + .record(numberBuses); + } +} diff --git a/src/main/java/com/powsybl/network/conversion/server/NetworkConversionService.java b/src/main/java/com/powsybl/network/conversion/server/NetworkConversionService.java index 05fe9d9e..1edc79c5 100644 --- a/src/main/java/com/powsybl/network/conversion/server/NetworkConversionService.java +++ b/src/main/java/com/powsybl/network/conversion/server/NetworkConversionService.java @@ -96,6 +96,8 @@ public class NetworkConversionService { private final NotificationService notificationService; + private final NetworkConversionObserver networkConversionObserver; + private final ObjectMapper objectMapper; @Autowired @@ -105,11 +107,13 @@ public NetworkConversionService(@Value("${powsybl.services.case-server.base-uri: NetworkStoreService networkStoreService, EquipmentInfosService equipmentInfosService, NetworkConversionExecutionService networkConversionExecutionService, - NotificationService notificationService) { + NotificationService notificationService, + NetworkConversionObserver networkConversionObserver) { this.networkStoreService = networkStoreService; this.equipmentInfosService = equipmentInfosService; this.networkConversionExecutionService = networkConversionExecutionService; this.notificationService = notificationService; + this.networkConversionObserver = networkConversionObserver; RestTemplateBuilder restTemplateBuilder = new RestTemplateBuilder(); caseServerRest = restTemplateBuilder.build(); @@ -190,13 +194,16 @@ NetworkInfos importCase(UUID caseUuid, String variantId, UUID reportUuid, Map startTime = new AtomicReference<>(System.nanoTime()); + CaseInfos caseInfos = getCaseInfos(caseUuid); + String format = caseInfos.getFormat(); Network network; + Reporter finalReporter = reporter; if (!importParameters.isEmpty()) { Properties importProperties = new Properties(); importProperties.putAll(importParameters); - network = networkStoreService.importNetwork(dataSource, reporter, importProperties, false); + network = networkConversionObserver.observeImport(format, () -> networkStoreService.importNetwork(dataSource, finalReporter, importProperties, false)); } else { - network = networkStoreService.importNetwork(dataSource, reporter, false); + network = networkConversionObserver.observeImport(format, () -> networkStoreService.importNetwork(dataSource, finalReporter, false)); } UUID networkUuid = networkStoreService.getNetworkUuid(network); LOGGER.trace("Import network '{}' : {} seconds", networkUuid, TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime.get())); @@ -306,7 +313,8 @@ ExportNetworkInfos exportNetwork(UUID networkUuid, String variantId, networkName += ".zip"; networkData = createZipFile(listNames.toArray(new String[0]), memDataSource).toByteArray(); } - return new ExportNetworkInfos(networkName, networkData); + long networkSize = network.getBusView().getBusStream().count(); + return new ExportNetworkInfos(networkName, networkData, networkSize); } ByteArrayOutputStream createZipFile(String[] listNames, MemDataSource dataSource) throws IOException { @@ -376,7 +384,8 @@ public ExportNetworkInfos exportCgmesSv(UUID networkUuid) throws XMLStreamExcept writer.close(); } } - return new ExportNetworkInfos(network.getNameOrId(), outputStream.toByteArray()); + long networkSize = network.getBusView().getBusStream().count(); + return new ExportNetworkInfos(network.getNameOrId(), outputStream.toByteArray(), networkSize); } private static CgmesExportContext createContext(Network network) { @@ -393,8 +402,8 @@ NetworkInfos importCgmesCase(UUID caseUuid, List boundaries) { return importCase(caseUuid, null, UUID.randomUUID(), new HashMap<>()); } else { // import using the given boundaries CaseDataSourceClient dataSource = new CgmesCaseDataSourceClient(caseServerRest, caseUuid, boundaries); - var network = networkStoreService.importNetwork(dataSource); - var networkUuid = networkStoreService.getNetworkUuid(network); + Network network = networkConversionObserver.observeImport("CGMES", () -> networkStoreService.importNetwork(dataSource)); + UUID networkUuid = networkStoreService.getNetworkUuid(network); return new NetworkInfos(networkUuid, network.getId()); } } diff --git a/src/main/java/com/powsybl/network/conversion/server/dto/ExportNetworkInfos.java b/src/main/java/com/powsybl/network/conversion/server/dto/ExportNetworkInfos.java index 9a299505..91ffc8bf 100644 --- a/src/main/java/com/powsybl/network/conversion/server/dto/ExportNetworkInfos.java +++ b/src/main/java/com/powsybl/network/conversion/server/dto/ExportNetworkInfos.java @@ -23,4 +23,6 @@ public class ExportNetworkInfos { private byte[] networkData; + private long numberBuses; + } diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml index 73bd1696..819c39e5 100644 --- a/src/main/resources/application-local.yml +++ b/src/main/resources/application-local.yml @@ -23,4 +23,10 @@ gridsuite: base-uri: http://localhost:8087 report-server: base-uri: http://localhost:5028 + +management: + endpoints: + web: + exposure: + include: prometheus, health, info diff --git a/src/test/java/com/powsybl/network/conversion/server/NetworkConversionTest.java b/src/test/java/com/powsybl/network/conversion/server/NetworkConversionTest.java index acf034c1..3dfbd956 100644 --- a/src/test/java/com/powsybl/network/conversion/server/NetworkConversionTest.java +++ b/src/test/java/com/powsybl/network/conversion/server/NetworkConversionTest.java @@ -132,6 +132,7 @@ public void test() throws Exception { any(HttpEntity.class), eq(String.class), eq(UUID.fromString(caseUuid)))) .willReturn(ResponseEntity.ok("testCase")); + given(caseServerRest.getForEntity(eq("/v1/cases/" + caseUuid + "/infos"), any())).willReturn(ResponseEntity.ok(new CaseInfos(UUID.fromString(caseUuid.toString()), "testCase", "XIIDM"))); MvcResult mvcResult = mvc.perform(post("/v1/networks") .param("caseUuid", caseUuid) @@ -421,6 +422,7 @@ public void testImportCgmesCase() throws Exception { any(HttpEntity.class), eq(String.class), eq(caseUuid))) .willReturn(ResponseEntity.ok("testCase")); + given(caseServerRest.getForEntity(eq("/v1/cases/" + caseUuid + "/infos"), any())).willReturn(ResponseEntity.ok(new CaseInfos(UUID.fromString(caseUuid.toString()), "testCase", "XIIDM"))); MvcResult mvcResult = mvc.perform(post("/v1/networks/cgmes") .param("caseUuid", caseUuid.toString()) @@ -464,6 +466,7 @@ public void testSendReport() throws Exception { any(HttpEntity.class), eq(String.class), eq(caseUuid))) .willReturn(ResponseEntity.ok("testCase")); + given(caseServerRest.getForEntity(eq("/v1/cases/" + caseUuid + "/infos"), any())).willReturn(ResponseEntity.ok(new CaseInfos(UUID.fromString(caseUuid.toString()), "testCase", "XIIDM"))); MvcResult mvcResult = mvc.perform(post("/v1/networks") .param("caseUuid", caseUuid.toString()) @@ -494,6 +497,7 @@ public void testImportWithError() { given(networkStoreClient.getNetworkUuid(network)).willReturn(networkUuid); given(reportServerRest.exchange(eq("/v1/reports/" + reportUuid), eq(HttpMethod.PUT), any(HttpEntity.class), eq(ReporterModel.class))) .willReturn(new ResponseEntity<>(HttpStatus.OK)); + given(caseServerRest.getForEntity(eq("/v1/cases/" + caseUuid + "/infos"), any())).willReturn(ResponseEntity.ok(new CaseInfos(UUID.fromString(caseUuid.toString()), "testCase", "XIIDM"))); String message = assertThrows(NetworkConversionException.class, () -> networkConversionService.importCase(caseUuid, null, reportUuid, EMPTY_PARAMETERS)).getMessage(); assertTrue(message.contains(String.format("The save of network '%s' has failed", networkUuid))); @@ -520,6 +524,7 @@ public void testFlushNetworkWithError() { any(HttpEntity.class), eq(String.class), eq(caseUuid))) .willReturn(ResponseEntity.ok("testCase")); + given(caseServerRest.getForEntity(eq("/v1/cases/" + caseUuid + "/infos"), any())).willReturn(ResponseEntity.ok(new CaseInfos(UUID.fromString(caseUuid.toString()), "testCase", "XIIDM"))); String message = assertThrows(NetworkConversionException.class, () -> networkConversionService.importCase(caseUuid, null, reportUuid, EMPTY_PARAMETERS)).getMessage(); assertTrue(message.contains(String.format("The save of network '%s' has failed", networkUuid)));