add instrumentation setup (#127)

* add instrumentation setup * Replace the file size with the number of buses * Separation of failed import and export * Adding the number of buses for import and export Signed-off-by: TOURI ANIS <[email protected]> * Change jauge meter by DistributionSummary meter to have the max value for buses * Fix checkstyle Signed-off-by: Slimane AMAR <[email protected]> --------- Signed-off-by: TOURI ANIS <[email protected]> Signed-off-by: Slimane AMAR <[email protected]> Co-authored-by: Slimane AMAR <[email protected]>
powsybl · Jan 24, 2024 · f000d41 · f000d41
1 parent e0c0c09
commit f000d41
Show file tree

Hide file tree

Showing 7 changed files with 110 additions and 11 deletions.
diff --git a/pom.xml b/pom.xml
@@ -170,13 +170,17 @@
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-actuator</artifactId>
-            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>org.springframework.cloud</groupId>
             <artifactId>spring-cloud-stream-binder-rabbit</artifactId>
             <scope>runtime</scope>
         </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-registry-prometheus</artifactId>
+            <scope>runtime</scope>
+        </dependency>
 
         <!-- Test dependencies -->
         <dependency>

diff --git a/src/main/java/com/powsybl/network/conversion/server/NetworkConversionController.java b/src/main/java/com/powsybl/network/conversion/server/NetworkConversionController.java
@@ -48,6 +48,9 @@ public class NetworkConversionController {
     @Autowired
     private NetworkConversionService networkConversionService;
 
+    @Autowired
+    private NetworkConversionObserver networkConversionObserver;
+
     @PostMapping(value = "/networks")
     @Operation(summary = "Get a case file from its name and import it into the store")
     public ResponseEntity<NetworkInfos> importCase(@Parameter(description = "Case UUID") @RequestParam("caseUuid") UUID caseUuid,
@@ -82,8 +85,7 @@ public ResponseEntity<byte[]> exportNetwork(@Parameter(description = "Network UU
                                                 @org.springframework.web.bind.annotation.RequestBody(required = false) Map<String, Object> formatParameters
                                                 ) throws IOException {
         LOGGER.debug("Exporting network {}...", networkUuid);
-
-        ExportNetworkInfos exportNetworkInfos = networkConversionService.exportNetwork(networkUuid, variantId, format, formatParameters);
+        ExportNetworkInfos exportNetworkInfos = networkConversionObserver.observeExport(format, () -> networkConversionService.exportNetwork(networkUuid, variantId, format, formatParameters));
         HttpHeaders header = new HttpHeaders();
         header.setContentDisposition(ContentDisposition.builder("attachment").filename(exportNetworkInfos.getNetworkName(), StandardCharsets.UTF_8).build());
         return ResponseEntity.ok().headers(header).contentType(MediaType.APPLICATION_OCTET_STREAM).body(exportNetworkInfos.getNetworkData());
@@ -109,7 +111,7 @@ public ResponseEntity<ImportExportFormatMeta> getCaseImportParameters(@Parameter
     @Operation(summary = "Export a cgmes network from the network-store")
     public ResponseEntity<byte[]> exportCgmesSv(@Parameter(description = "Network UUID") @PathVariable("networkUuid") UUID networkUuid) throws XMLStreamException {
         LOGGER.debug("Exporting network {}...", networkUuid);
-        ExportNetworkInfos exportNetworkInfos = networkConversionService.exportCgmesSv(networkUuid);
+        ExportNetworkInfos exportNetworkInfos = networkConversionObserver.observeExport("CGMES", () -> networkConversionService.exportCgmesSv(networkUuid));
         HttpHeaders header = new HttpHeaders();
         header.setContentDisposition(ContentDisposition.builder("attachment").filename(exportNetworkInfos.getNetworkName(), StandardCharsets.UTF_8).build());
         return ResponseEntity.ok().headers(header).contentType(MediaType.APPLICATION_OCTET_STREAM).body(exportNetworkInfos.getNetworkData());

diff --git a/src/main/java/com/powsybl/network/conversion/server/NetworkConversionObserver.java b/src/main/java/com/powsybl/network/conversion/server/NetworkConversionObserver.java
@@ -0,0 +1,71 @@
+/**
+ * Copyright (c) 2024, RTE (http://www.rte-france.com)
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+package com.powsybl.network.conversion.server;
+
+import com.powsybl.iidm.network.Network;
+import com.powsybl.network.conversion.server.dto.ExportNetworkInfos;
+import io.micrometer.core.instrument.DistributionSummary;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+import lombok.NonNull;
+import org.springframework.stereotype.Service;
+
+/**
+ * @author Anis Touri <anis.touri at rte-france.com>
+ */
+
+@Service
+public class NetworkConversionObserver {
+
+    private static final String OBSERVATION_PREFIX = "app.conversion.";
+    private static final String FORMAT_TAG_NAME = "format";
+
+    private static final String IMPORT_OBSERVATION_NAME = OBSERVATION_PREFIX + "import";
+    private static final String NUMBER_BUSES_IMPORTED_METER_NAME = IMPORT_OBSERVATION_NAME + ".buses";
+
+    private static final String EXPORT_OBSERVATION_NAME = OBSERVATION_PREFIX + "export";
+    private static final String NUMBER_BUSES_EXPORTED_METER_NAME = EXPORT_OBSERVATION_NAME + ".buses";
+
+    private final ObservationRegistry observationRegistry;
+
+    private final MeterRegistry meterRegistry;
+
+    public NetworkConversionObserver(@NonNull ObservationRegistry observationRegistry, @NonNull MeterRegistry meterRegistry) {
+        this.observationRegistry = observationRegistry;
+        this.meterRegistry = meterRegistry;
+    }
+
+    public <E extends Throwable> ExportNetworkInfos observeExport(String format, Observation.CheckedCallable<ExportNetworkInfos, E> callable) throws E {
+        Observation observation = createObservation(EXPORT_OBSERVATION_NAME, format);
+        ExportNetworkInfos exportInfos = observation.observeChecked(callable);
+        if (exportInfos != null) {
+            recordNumberBuses(NUMBER_BUSES_EXPORTED_METER_NAME, format, exportInfos.getNumberBuses());
+        }
+        return exportInfos;
+    }
+
+    public <E extends Throwable> Network observeImport(String format, Observation.CheckedCallable<Network, E> callable) throws E {
+        Network network = createObservation(IMPORT_OBSERVATION_NAME, format).observeChecked(callable);
+        if (network != null) {
+            recordNumberBuses(NUMBER_BUSES_IMPORTED_METER_NAME, format, network.getBusView().getBusStream().count());
+        }
+        return network;
+    }
+
+    private Observation createObservation(String name, String format) {
+        return Observation.createNotStarted(name, observationRegistry)
+                .lowCardinalityKeyValue(FORMAT_TAG_NAME, format);
+    }
+
+    private void recordNumberBuses(String meterName, String format, long numberBuses) {
+        DistributionSummary.builder(meterName)
+                .tags(FORMAT_TAG_NAME, format)
+                .register(meterRegistry)
+                .record(numberBuses);
+    }
+}
diff --git a/src/main/java/com/powsybl/network/conversion/server/NetworkConversionService.java b/src/main/java/com/powsybl/network/conversion/server/NetworkConversionService.java
@@ -96,6 +96,8 @@ public class NetworkConversionService {
 
     private final NotificationService notificationService;
 
+    private final NetworkConversionObserver networkConversionObserver;
+
     private final ObjectMapper objectMapper;
 
     @Autowired
@@ -105,11 +107,13 @@ public NetworkConversionService(@Value("${powsybl.services.case-server.base-uri:
                                     NetworkStoreService networkStoreService,
                                     EquipmentInfosService equipmentInfosService,
                                     NetworkConversionExecutionService networkConversionExecutionService,
-                                    NotificationService notificationService) {
+                                    NotificationService notificationService,
+                                    NetworkConversionObserver networkConversionObserver) {
         this.networkStoreService = networkStoreService;
         this.equipmentInfosService = equipmentInfosService;
         this.networkConversionExecutionService = networkConversionExecutionService;
         this.notificationService = notificationService;
+        this.networkConversionObserver = networkConversionObserver;
 
         RestTemplateBuilder restTemplateBuilder = new RestTemplateBuilder();
         caseServerRest = restTemplateBuilder.build();
@@ -190,13 +194,16 @@ NetworkInfos importCase(UUID caseUuid, String variantId, UUID reportUuid, Map<St
         }
 
         AtomicReference<Long> startTime = new AtomicReference<>(System.nanoTime());
+        CaseInfos caseInfos = getCaseInfos(caseUuid);
+        String format = caseInfos.getFormat();
         Network network;
+        Reporter finalReporter = reporter;
         if (!importParameters.isEmpty()) {
             Properties importProperties = new Properties();
             importProperties.putAll(importParameters);
-            network = networkStoreService.importNetwork(dataSource, reporter, importProperties, false);
+            network = networkConversionObserver.observeImport(format, () -> networkStoreService.importNetwork(dataSource, finalReporter, importProperties, false));
         } else {
-            network = networkStoreService.importNetwork(dataSource, reporter, false);
+            network = networkConversionObserver.observeImport(format, () -> networkStoreService.importNetwork(dataSource, finalReporter, false));
         }
         UUID networkUuid = networkStoreService.getNetworkUuid(network);
         LOGGER.trace("Import network '{}' : {} seconds", networkUuid, TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime.get()));
@@ -306,7 +313,8 @@ ExportNetworkInfos exportNetwork(UUID networkUuid, String variantId,
             networkName += ".zip";
             networkData = createZipFile(listNames.toArray(new String[0]), memDataSource).toByteArray();
         }
-        return new ExportNetworkInfos(networkName, networkData);
+        long networkSize = network.getBusView().getBusStream().count();
+        return new ExportNetworkInfos(networkName, networkData, networkSize);
     }
 
     ByteArrayOutputStream createZipFile(String[] listNames, MemDataSource dataSource) throws IOException {
@@ -376,7 +384,8 @@ public ExportNetworkInfos exportCgmesSv(UUID networkUuid) throws XMLStreamExcept
                 writer.close();
             }
         }
-        return new ExportNetworkInfos(network.getNameOrId(), outputStream.toByteArray());
+        long networkSize = network.getBusView().getBusStream().count();
+        return new ExportNetworkInfos(network.getNameOrId(), outputStream.toByteArray(), networkSize);
     }
 
     private static CgmesExportContext createContext(Network network) {
@@ -393,8 +402,8 @@ NetworkInfos importCgmesCase(UUID caseUuid, List<BoundaryInfos> boundaries) {
             return importCase(caseUuid, null, UUID.randomUUID(), new HashMap<>());
         } else {  // import using the given boundaries
             CaseDataSourceClient dataSource = new CgmesCaseDataSourceClient(caseServerRest, caseUuid, boundaries);
-            var network = networkStoreService.importNetwork(dataSource);
-            var networkUuid = networkStoreService.getNetworkUuid(network);
+            Network network = networkConversionObserver.observeImport("CGMES", () -> networkStoreService.importNetwork(dataSource));
+            UUID networkUuid = networkStoreService.getNetworkUuid(network);
             return new NetworkInfos(networkUuid, network.getId());
         }
     }

diff --git a/src/main/java/com/powsybl/network/conversion/server/dto/ExportNetworkInfos.java b/src/main/java/com/powsybl/network/conversion/server/dto/ExportNetworkInfos.java
@@ -23,4 +23,6 @@ public class ExportNetworkInfos {
 
     private byte[] networkData;
 
+    private long numberBuses;
+
 }
diff --git a/src/main/resources/application-local.yml b/src/main/resources/application-local.yml
@@ -23,4 +23,10 @@ gridsuite:
       base-uri: http://localhost:8087
     report-server:
       base-uri: http://localhost:5028
+
+management:
+  endpoints:
+    web:
+      exposure:
+        include: prometheus, health, info
 
diff --git a/src/test/java/com/powsybl/network/conversion/server/NetworkConversionTest.java b/src/test/java/com/powsybl/network/conversion/server/NetworkConversionTest.java
@@ -132,6 +132,7 @@ public void test() throws Exception {
                 any(HttpEntity.class),
                 eq(String.class), eq(UUID.fromString(caseUuid))))
                 .willReturn(ResponseEntity.ok("testCase"));
+            given(caseServerRest.getForEntity(eq("/v1/cases/" + caseUuid + "/infos"), any())).willReturn(ResponseEntity.ok(new CaseInfos(UUID.fromString(caseUuid.toString()), "testCase", "XIIDM")));
 
             MvcResult mvcResult = mvc.perform(post("/v1/networks")
                 .param("caseUuid", caseUuid)
@@ -421,6 +422,7 @@ public void testImportCgmesCase() throws Exception {
             any(HttpEntity.class),
             eq(String.class), eq(caseUuid)))
             .willReturn(ResponseEntity.ok("testCase"));
+        given(caseServerRest.getForEntity(eq("/v1/cases/" + caseUuid + "/infos"), any())).willReturn(ResponseEntity.ok(new CaseInfos(UUID.fromString(caseUuid.toString()), "testCase", "XIIDM")));
 
         MvcResult mvcResult = mvc.perform(post("/v1/networks/cgmes")
                 .param("caseUuid", caseUuid.toString())
@@ -464,6 +466,7 @@ public void testSendReport() throws Exception {
             any(HttpEntity.class),
             eq(String.class), eq(caseUuid)))
             .willReturn(ResponseEntity.ok("testCase"));
+        given(caseServerRest.getForEntity(eq("/v1/cases/" + caseUuid + "/infos"), any())).willReturn(ResponseEntity.ok(new CaseInfos(UUID.fromString(caseUuid.toString()), "testCase", "XIIDM")));
 
         MvcResult mvcResult = mvc.perform(post("/v1/networks")
             .param("caseUuid", caseUuid.toString())
@@ -494,6 +497,7 @@ public void testImportWithError() {
         given(networkStoreClient.getNetworkUuid(network)).willReturn(networkUuid);
         given(reportServerRest.exchange(eq("/v1/reports/" + reportUuid), eq(HttpMethod.PUT), any(HttpEntity.class), eq(ReporterModel.class)))
                 .willReturn(new ResponseEntity<>(HttpStatus.OK));
+        given(caseServerRest.getForEntity(eq("/v1/cases/" + caseUuid + "/infos"), any())).willReturn(ResponseEntity.ok(new CaseInfos(UUID.fromString(caseUuid.toString()), "testCase", "XIIDM")));
 
         String message = assertThrows(NetworkConversionException.class, () -> networkConversionService.importCase(caseUuid, null, reportUuid, EMPTY_PARAMETERS)).getMessage();
         assertTrue(message.contains(String.format("The save of network '%s' has failed", networkUuid)));
@@ -520,6 +524,7 @@ public void testFlushNetworkWithError() {
             any(HttpEntity.class),
             eq(String.class), eq(caseUuid)))
             .willReturn(ResponseEntity.ok("testCase"));
+        given(caseServerRest.getForEntity(eq("/v1/cases/" + caseUuid + "/infos"), any())).willReturn(ResponseEntity.ok(new CaseInfos(UUID.fromString(caseUuid.toString()), "testCase", "XIIDM")));
 
         String message = assertThrows(NetworkConversionException.class, () -> networkConversionService.importCase(caseUuid, null, reportUuid, EMPTY_PARAMETERS)).getMessage();
         assertTrue(message.contains(String.format("The save of network '%s' has failed", networkUuid)));
Original file line number	Diff line number	Diff line change
Expand Up		@@ -23,4 +23,6 @@ public class ExportNetworkInfos {

		private byte[] networkData;

		private long numberBuses;

		}