From 5f739187ac84e7eff04fd30fe9857e313da9f4f0 Mon Sep 17 00:00:00 2001 From: Romuald Lemesle Date: Fri, 8 Mar 2024 09:22:07 +0100 Subject: [PATCH] [backend] Telemetry --- openbas-api/pom.xml | 46 ++++++- .../io/openbas/config/SessionManager.java | 20 ++- .../io/openbas/rest/exercise/ExerciseApi.java | 10 +- .../scheduler/PlatformJobDefinitions.java | 7 + .../openbas/scheduler/PlatformTriggers.java | 23 ++-- .../scheduler/jobs/TelemetryExecutionJob.java | 23 ++++ .../telemetry/OpenTelemetryConfig.java | 122 ++++++++++++++++++ .../telemetry/OpenTelemetryService.java | 59 +++++++++ .../io/openbas/telemetry/ServiceCounter.java | 44 +++++++ .../java/io/openbas/telemetry/Tracing.java | 14 ++ .../io/openbas/telemetry/TracingAspect.java | 41 ++++++ .../exporter/FileLogRecordExporter.java | 73 +++++++++++ .../exporter/FileMetricExporter.java | 59 +++++++++ .../src/main/resources/application.properties | 3 + openbas-dev/docker-compose-telemetry.yml | 69 ++++++++++ 15 files changed, 592 insertions(+), 21 deletions(-) create mode 100644 openbas-api/src/main/java/io/openbas/scheduler/jobs/TelemetryExecutionJob.java create mode 100644 openbas-api/src/main/java/io/openbas/telemetry/OpenTelemetryConfig.java create mode 100644 openbas-api/src/main/java/io/openbas/telemetry/OpenTelemetryService.java create mode 100644 openbas-api/src/main/java/io/openbas/telemetry/ServiceCounter.java create mode 100644 openbas-api/src/main/java/io/openbas/telemetry/Tracing.java create mode 100644 openbas-api/src/main/java/io/openbas/telemetry/TracingAspect.java create mode 100644 openbas-api/src/main/java/io/openbas/telemetry/exporter/FileLogRecordExporter.java create mode 100644 openbas-api/src/main/java/io/openbas/telemetry/exporter/FileMetricExporter.java create mode 100644 openbas-dev/docker-compose-telemetry.yml diff --git a/openbas-api/pom.xml b/openbas-api/pom.xml index c954fc1c86..ce6085c9c4 100644 --- a/openbas-api/pom.xml +++ b/openbas-api/pom.xml @@ -22,6 +22,8 @@ 2.5.0 1.4 9.2.1 + 1.35.0 + 1.23.1-alpha @@ -62,6 +64,18 @@ + + + + io.opentelemetry + opentelemetry-bom + ${opentelemetry.version} + pom + import + + + + io.openbas @@ -199,6 +213,33 @@ cron-utils ${cron-utils.version} + + + io.opentelemetry + opentelemetry-api + + + io.opentelemetry + opentelemetry-sdk + + + io.opentelemetry + opentelemetry-sdk-logs + + + io.opentelemetry.semconv + opentelemetry-semconv + ${opentelemetry-semconv.version} + + + io.opentelemetry + opentelemetry-exporter-logging + + + io.opentelemetry + opentelemetry-exporter-otlp + + org.springframework.boot @@ -216,11 +257,6 @@ spring-security-test test - - com.h2database - h2 - test - diff --git a/openbas-api/src/main/java/io/openbas/config/SessionManager.java b/openbas-api/src/main/java/io/openbas/config/SessionManager.java index 578012bf1e..403d1d62b6 100644 --- a/openbas-api/src/main/java/io/openbas/config/SessionManager.java +++ b/openbas-api/src/main/java/io/openbas/config/SessionManager.java @@ -1,6 +1,9 @@ package io.openbas.config; import io.openbas.database.model.User; +import jakarta.servlet.http.HttpSession; +import jakarta.servlet.http.HttpSessionEvent; +import jakarta.servlet.http.HttpSessionListener; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.security.core.Authentication; @@ -9,14 +12,12 @@ import org.springframework.security.oauth2.core.user.OAuth2User; import org.springframework.security.web.authentication.preauth.PreAuthenticatedAuthenticationToken; -import jakarta.servlet.http.HttpSession; -import jakarta.servlet.http.HttpSessionEvent; -import jakarta.servlet.http.HttpSessionListener; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.stream.Stream; +import static java.util.stream.Collectors.groupingBy; import static org.springframework.security.web.context.HttpSessionSecurityContextRepository.SPRING_SECURITY_CONTEXT_KEY; @Configuration @@ -104,4 +105,17 @@ public void refreshUserSessions(User databaseUser) { public void invalidateUserSession(String userId) { getUserSessions(userId).forEach(HttpSession::invalidate); } + + // Should have one session by user + public long getUserSessionsCount() { + int value = sessions.values() + .stream() + .map(this::extractPrincipal) + .filter((Optional::isPresent)) + .map((Optional::get)) + .collect(groupingBy(OpenBASPrincipal::getId)) + .keySet() + .size(); + return value; + } } diff --git a/openbas-api/src/main/java/io/openbas/rest/exercise/ExerciseApi.java b/openbas-api/src/main/java/io/openbas/rest/exercise/ExerciseApi.java index 4aea45ff71..cb3e4e61cc 100644 --- a/openbas-api/src/main/java/io/openbas/rest/exercise/ExerciseApi.java +++ b/openbas-api/src/main/java/io/openbas/rest/exercise/ExerciseApi.java @@ -15,6 +15,7 @@ import io.openbas.rest.exercise.form.*; import io.openbas.rest.helper.RestBehavior; import io.openbas.service.*; +import io.openbas.telemetry.Tracing; import jakarta.annotation.Resource; import jakarta.servlet.http.HttpServletResponse; import jakarta.transaction.Transactional; @@ -424,6 +425,7 @@ public Exercise removeExerciseTeamPlayers(@PathVariable String exerciseId, @Path // region exercises @Transactional(rollbackOn = Exception.class) @PostMapping("/api/exercises") + @Tracing(name = "EXERCISE CREATION", layer = "api", operation = "create") public Exercise createExercise(@Valid @RequestBody ExerciseCreateInput input) { Exercise exercise = new Exercise(); exercise.setUpdateAttributes(input); @@ -611,7 +613,7 @@ public List exercises() { : exerciseRepository.findAllGranted(currentUser().getId()); return fromIterable(exercises).stream().map(ExerciseSimple::fromExercise).toList(); } - // endregion +// endregion // region communication @GetMapping("/api/exercises/{exerciseId}/communications") @@ -624,7 +626,7 @@ public Iterable exerciseCommunications(@PathVariable String exerc } @GetMapping("/api/communications/attachment") - // @PreAuthorize("isExerciseObserver(#exerciseId)") +// @PreAuthorize("isExerciseObserver(#exerciseId)") public void downloadAttachment(@RequestParam String file, HttpServletResponse response) throws IOException { FileContainer fileContainer = fileService.getFileContainer(file).orElseThrow(); response.addHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + fileContainer.getName()); @@ -632,7 +634,7 @@ public void downloadAttachment(@RequestParam String file, HttpServletResponse re response.setStatus(HttpServletResponse.SC_OK); fileContainer.getInputStream().transferTo(response.getOutputStream()); } - // endregion +// endregion // region import/export @GetMapping("/api/exercises/{exerciseId}/export") @@ -763,5 +765,5 @@ public void exerciseImport(@RequestPart("file") MultipartFile file) throws Excep importService.handleFileImport(file); } - // endregion +// endregion } diff --git a/openbas-api/src/main/java/io/openbas/scheduler/PlatformJobDefinitions.java b/openbas-api/src/main/java/io/openbas/scheduler/PlatformJobDefinitions.java index b6dcdfa8e3..af059ffa8b 100644 --- a/openbas-api/src/main/java/io/openbas/scheduler/PlatformJobDefinitions.java +++ b/openbas-api/src/main/java/io/openbas/scheduler/PlatformJobDefinitions.java @@ -3,6 +3,7 @@ import io.openbas.scheduler.jobs.ComchecksExecutionJob; import io.openbas.scheduler.jobs.InjectsExecutionJob; import io.openbas.scheduler.jobs.ScenarioExecutionJob; +import io.openbas.scheduler.jobs.TelemetryExecutionJob; import org.quartz.JobBuilder; import org.quartz.JobDetail; import org.springframework.context.annotation.Bean; @@ -30,4 +31,10 @@ public JobDetail getScenarioExecution() { return JobBuilder.newJob(ScenarioExecutionJob.class) .storeDurably().withIdentity(jobKey("ScenarioExecutionJob")).build(); } + + @Bean + public JobDetail getTelemetryExecutionTrigger() { + return JobBuilder.newJob(TelemetryExecutionJob.class) + .storeDurably().withIdentity(jobKey("TelemetryExecutionJob")).build(); + } } diff --git a/openbas-api/src/main/java/io/openbas/scheduler/PlatformTriggers.java b/openbas-api/src/main/java/io/openbas/scheduler/PlatformTriggers.java index b5bbbf76cd..b212b52743 100644 --- a/openbas-api/src/main/java/io/openbas/scheduler/PlatformTriggers.java +++ b/openbas-api/src/main/java/io/openbas/scheduler/PlatformTriggers.java @@ -1,7 +1,7 @@ package io.openbas.scheduler; +import lombok.RequiredArgsConstructor; import org.quartz.Trigger; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; @@ -10,19 +10,15 @@ import static org.quartz.TriggerBuilder.newTrigger; @Component +@RequiredArgsConstructor public class PlatformTriggers { - private PlatformJobDefinitions platformJobs; - - @Autowired - public void setPlatformJobs(PlatformJobDefinitions platformJobs) { - this.platformJobs = platformJobs; - } + private final PlatformJobDefinitions platformJobs; @Bean public Trigger injectsExecutionTrigger() { return newTrigger() - .forJob(platformJobs.getInjectsExecution()) + .forJob(this.platformJobs.getInjectsExecution()) .withIdentity("InjectsExecutionTrigger") .withSchedule(cronSchedule("0 0/1 * * * ?")) // Every minute align on clock .build(); @@ -31,7 +27,7 @@ public Trigger injectsExecutionTrigger() { @Bean public Trigger comchecksExecutionTrigger() { return newTrigger() - .forJob(platformJobs.getComchecksExecution()) + .forJob(this.platformJobs.getComchecksExecution()) .withIdentity("ComchecksExecutionTrigger") .withSchedule(repeatMinutelyForever()) .build(); @@ -45,4 +41,13 @@ public Trigger scenarioExecutionTrigger() { .withSchedule(repeatMinutelyForever()) .build(); } + + @Bean + public Trigger telemetryExecutionTrigger() { + return newTrigger() + .forJob(this.platformJobs.getTelemetryExecutionTrigger()) + .withIdentity("TelemetryExecutionTrigger") + .withSchedule(cronSchedule("0 0/1 * * * ?")) // Every 1 hours + .build(); + } } diff --git a/openbas-api/src/main/java/io/openbas/scheduler/jobs/TelemetryExecutionJob.java b/openbas-api/src/main/java/io/openbas/scheduler/jobs/TelemetryExecutionJob.java new file mode 100644 index 0000000000..7658ce6347 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/scheduler/jobs/TelemetryExecutionJob.java @@ -0,0 +1,23 @@ +package io.openbas.scheduler.jobs; + +import io.openbas.telemetry.OpenTelemetryService; +import lombok.RequiredArgsConstructor; +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.springframework.stereotype.Component; + +@Component +@DisallowConcurrentExecution +@RequiredArgsConstructor +public class TelemetryExecutionJob implements Job { + + private final OpenTelemetryService openTelemetryService; + + @Override + public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { + this.openTelemetryService.registerMetric(); + } + +} diff --git a/openbas-api/src/main/java/io/openbas/telemetry/OpenTelemetryConfig.java b/openbas-api/src/main/java/io/openbas/telemetry/OpenTelemetryConfig.java new file mode 100644 index 0000000000..3d722154d5 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/telemetry/OpenTelemetryConfig.java @@ -0,0 +1,122 @@ +package io.openbas.telemetry; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.context.propagation.TextMapPropagator; +import io.opentelemetry.exporter.logging.LoggingMetricExporter; +import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.semconv.ResourceAttributes; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import org.springframework.context.annotation.Bean; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static java.util.Objects.requireNonNull; + +@Service +@RequiredArgsConstructor +public class OpenTelemetryConfig { + + private final Environment env; + + @Bean + public OpenTelemetry openTelemetry() { + Resource resource; + try { + resource = Resource.getDefault() + .toBuilder() + .put(ResourceAttributes.SERVICE_NAME, requireNonNull(this.env.getProperty("info.app.name"))) + .put(ResourceAttributes.SERVICE_VERSION, requireNonNull(this.env.getProperty("info.app.version"))) + .put(ResourceAttributes.SERVICE_INSTANCE_ID, UUID.randomUUID().toString()) + .putAll(Attributes.of(ResourceAttributes.HOST_IP, List.of(InetAddress.getLocalHost().getHostAddress()))) + .build(); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + + // -- METRICS -- + + // Log exporter + SdkMeterProviderBuilder sdkMeterProviderBuilder = SdkMeterProvider.builder() + .registerMetricReader( + PeriodicMetricReader.builder(LoggingMetricExporter.create()).setInterval(1, TimeUnit.MINUTES).build()); + + // File exporter: FIXME: use log system with retention like logstash +// sdkMeterProviderBuilder.registerMetricReader( +// PeriodicMetricReader.builder(FileMetricExporter.create(this.env.getProperty("telemetry.file"))).build() +// ); + + // OTLP exporter + boolean exporterOtlpEndpointEnabled = Boolean.TRUE.equals( + this.env.getProperty("telemetry.exporter.otlp.enabled", Boolean.class) + ); + if (exporterOtlpEndpointEnabled) { + sdkMeterProviderBuilder.registerMetricReader(PeriodicMetricReader.builder( + OtlpHttpMetricExporter.builder().build() // Take default endpoint uri + ).build()); + } + + SdkMeterProvider sdkMeterProvider = sdkMeterProviderBuilder + .setResource(resource) + .build(); + + // -- SPANS -- + + // Log exporter + SdkTracerProviderBuilder sdkTracerProviderBuilder = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create())); + + // OTLP exporter + if (exporterOtlpEndpointEnabled) { + sdkTracerProviderBuilder.addSpanProcessor( + SimpleSpanProcessor.create(OtlpGrpcSpanExporter.builder().build()) + ); + } + + SdkTracerProvider sdkTracerProvider = sdkTracerProviderBuilder + .setResource(resource) + .build(); + + return OpenTelemetrySdk.builder() + .setMeterProvider(sdkMeterProvider) + .setTracerProvider(sdkTracerProvider) + .setPropagators(ContextPropagators.create( + TextMapPropagator.composite(W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance()) + )) + .build(); + } + + @Bean + public Meter meter(@NotNull final OpenTelemetry openTelemetry) { + return openTelemetry.getMeter("openbasApi-meter"); + } + + @Bean + public Tracer tracer(@NotNull final OpenTelemetry openTelemetry) { + return openTelemetry.getTracer("openbasApi-tracer"); + } + +} + diff --git a/openbas-api/src/main/java/io/openbas/telemetry/OpenTelemetryService.java b/openbas-api/src/main/java/io/openbas/telemetry/OpenTelemetryService.java new file mode 100644 index 0000000000..88a969483f --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/telemetry/OpenTelemetryService.java @@ -0,0 +1,59 @@ +package io.openbas.telemetry; + +import io.opentelemetry.api.metrics.DoubleGaugeBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.Meter; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + + +@Service +@RequiredArgsConstructor +public class OpenTelemetryService { + + private static final String PREFIX_PRODUCT = "openbas.app."; + + private final Meter meter; + private final ServiceCounter serviceCounter; + + private DoubleHistogram sessionHistogram; + private DoubleHistogram simulationHistogram; + private DoubleGaugeBuilder memoryGauge; + private DoubleGaugeBuilder cpuGauge; + + + @PostConstruct + public void init() { + this.sessionHistogram = this.meter + .histogramBuilder(PREFIX_PRODUCT + "sessions") + .setDescription("Number of active sessions") + .setUnit("count") + .build(); + this.simulationHistogram = this.meter + .histogramBuilder(PREFIX_PRODUCT + "simulations") + .setDescription("Number of simulations played") + .setUnit("count") + .build(); + this.memoryGauge = this.meter + .gaugeBuilder(PREFIX_PRODUCT + "memory") + .setDescription("Memory usage") + .setUnit("GB"); + this.cpuGauge = this.meter + .gaugeBuilder(PREFIX_PRODUCT + "cpu") + .setDescription("CPU usage") + .setUnit("percentage"); + } + + public void registerMetric() { + this.sessionHistogram.record(this.serviceCounter.getActiveSessions()); + this.simulationHistogram.record(this.serviceCounter.getSimulationPlayed()); + this.memoryGauge.buildWithCallback(measurement -> { + measurement.record(this.serviceCounter.getMemoryUsage()); + }); + this.cpuGauge.buildWithCallback(measurement -> { + measurement.record(this.serviceCounter.getCpuUsage()); + }); + } + +} diff --git a/openbas-api/src/main/java/io/openbas/telemetry/ServiceCounter.java b/openbas-api/src/main/java/io/openbas/telemetry/ServiceCounter.java new file mode 100644 index 0000000000..55d79b5af3 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/telemetry/ServiceCounter.java @@ -0,0 +1,44 @@ +package io.openbas.telemetry; + +import com.sun.management.OperatingSystemMXBean; +import io.openbas.config.SessionManager; +import io.openbas.database.repository.ExerciseRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; + +@Service +@RequiredArgsConstructor +public class ServiceCounter { + + private final SessionManager sessionManager; + + private final ExerciseRepository exerciseRepository; + + // -- SECURITY -- + + public long getActiveSessions() { + return this.sessionManager.getUserSessionsCount(); + } + + // -- SYSTEM -- + + public double getMemoryUsage() { + MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); + return (double) memoryMXBean.getHeapMemoryUsage().getUsed() / 1073741824; + } + + public double getCpuUsage() { + OperatingSystemMXBean osBean = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class); + return osBean.getProcessCpuLoad(); + } + + // -- SIMULATION -- + + public long getSimulationPlayed() { + return this.exerciseRepository.count(); + } + +} diff --git a/openbas-api/src/main/java/io/openbas/telemetry/Tracing.java b/openbas-api/src/main/java/io/openbas/telemetry/Tracing.java new file mode 100644 index 0000000000..92804cafe8 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/telemetry/Tracing.java @@ -0,0 +1,14 @@ +package io.openbas.telemetry; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface Tracing { + String name() default ""; + String layer() default ""; + String operation() default ""; +} diff --git a/openbas-api/src/main/java/io/openbas/telemetry/TracingAspect.java b/openbas-api/src/main/java/io/openbas/telemetry/TracingAspect.java new file mode 100644 index 0000000000..dcd880d266 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/telemetry/TracingAspect.java @@ -0,0 +1,41 @@ +package io.openbas.telemetry; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import lombok.RequiredArgsConstructor; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.reflect.MethodSignature; +import org.springframework.stereotype.Component; + +import java.lang.reflect.Method; + +import static io.openbas.config.SessionHelper.currentUser; + +@Aspect +@Component +@RequiredArgsConstructor +public class TracingAspect { + + private final Tracer tracer; + + @Around("@annotation(io.openbas.telemetry.Tracing)") + public Object tracing(ProceedingJoinPoint proceedingJoinPoint) throws Throwable { + MethodSignature signature = (MethodSignature) proceedingJoinPoint.getSignature(); + Method method = signature.getMethod(); + Tracing tracing = method.getAnnotation(Tracing.class); + + Span span = tracer.spanBuilder(tracing.name()) + .setAttribute("USER", currentUser().getId()) + .setAttribute("LAYER", tracing.layer()) + .setAttribute("OPERATION", tracing.operation()) + .startSpan(); + try (Scope ignored = span.makeCurrent()) { + return proceedingJoinPoint.proceed(); + } finally { + span.end(); + } + } +} diff --git a/openbas-api/src/main/java/io/openbas/telemetry/exporter/FileLogRecordExporter.java b/openbas-api/src/main/java/io/openbas/telemetry/exporter/FileLogRecordExporter.java new file mode 100644 index 0000000000..3a9c5c1ec6 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/telemetry/exporter/FileLogRecordExporter.java @@ -0,0 +1,73 @@ +package io.openbas.telemetry.exporter; + +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.logs.data.LogRecordData; +import io.opentelemetry.sdk.logs.export.LogRecordExporter; +import jakarta.validation.constraints.NotBlank; +import lombok.Builder; + +import java.io.FileWriter; +import java.io.IOException; +import java.time.Instant; +import java.util.Collection; + +public class FileLogRecordExporter implements LogRecordExporter { + + private final String filePath; + + public FileLogRecordExporter(@NotBlank final String filePath) { + this.filePath = filePath; + } + + public static FileLogRecordExporter create(@NotBlank final String filePath) { + return new FileLogRecordExporter(filePath); + } + + @Override + public CompletableResultCode export(Collection logs) { + try (FileWriter writer = new FileWriter(this.filePath, true)) { + for (LogRecordData log : logs) { + LogOutput logOutput = LogOutput.builder() + .body(log.getBody().asString()) + .date( + Instant.ofEpochSecond( + 0L, + log.getObservedTimestampEpochNanos() + ).toString() + ) + .build(); + writer.write(logOutput + "\n"); + } + } catch (IOException e) { + return CompletableResultCode.ofFailure(); + } + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public void close() { + LogRecordExporter.super.close(); + } + + @Builder + public static class LogOutput { + + private String body; + private String date; + + public String toString() { + return this.date + " " + this.body; + } + } + +} diff --git a/openbas-api/src/main/java/io/openbas/telemetry/exporter/FileMetricExporter.java b/openbas-api/src/main/java/io/openbas/telemetry/exporter/FileMetricExporter.java new file mode 100644 index 0000000000..9dc6cb9a38 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/telemetry/exporter/FileMetricExporter.java @@ -0,0 +1,59 @@ +package io.openbas.telemetry.exporter; + +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.export.MetricExporter; +import jakarta.validation.constraints.NotBlank; +import org.jetbrains.annotations.NotNull; + +import java.io.FileWriter; +import java.io.IOException; +import java.util.Collection; + +public class FileMetricExporter implements MetricExporter { + + private final String filePath; + private final AggregationTemporality aggregationTemporality = AggregationTemporality.CUMULATIVE; + + public FileMetricExporter(@NotBlank final String filePath) { + this.filePath = filePath; + } + + public static FileMetricExporter create(@NotBlank final String filePath) { + return new FileMetricExporter(filePath); + } + + @Override + public CompletableResultCode export(Collection metrics) { + try (FileWriter writer = new FileWriter(filePath, true)) { + for (MetricData metric : metrics) { + writer.write(metric + "\n"); + } + } catch (IOException e) { + return CompletableResultCode.ofFailure(); + } + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public void close() { + MetricExporter.super.close(); + } + + @Override + public AggregationTemporality getAggregationTemporality(@NotNull InstrumentType instrumentType) { + return aggregationTemporality; + } +} diff --git a/openbas-api/src/main/resources/application.properties b/openbas-api/src/main/resources/application.properties index 2cd0701b68..c85ee6ae4a 100644 --- a/openbas-api/src/main/resources/application.properties +++ b/openbas-api/src/main/resources/application.properties @@ -118,6 +118,9 @@ logging.logback.rollingpolicy.file-name-pattern=${LOG_FILE}.-%d{yyyy-MM-dd}.%i logging.logback.rollingpolicy.max-file-size=10MB logging.logback.rollingpolicy.max-history=7 +# Telemetry +logging.level.io.opentelemetry.exporter.logging=info + ############# # INJECTORS # ############# diff --git a/openbas-dev/docker-compose-telemetry.yml b/openbas-dev/docker-compose-telemetry.yml new file mode 100644 index 0000000000..bf0bfc831f --- /dev/null +++ b/openbas-dev/docker-compose-telemetry.yml @@ -0,0 +1,69 @@ +version: '3' +services: + openbas-telemetry-otlp: + container_name: openbas-telemetry-otlp + image: otel/opentelemetry-collector:0.96.0 + restart: unless-stopped + command: [ "--config=/etc/config/config.yaml" ] + volumes: + - C:/Users/RomualdLemesle/Desktop/config/otlp-config.yaml:/etc/config/config.yaml + - C:/Users/RomualdLemesle/Desktop/config/otlp-output.log:/otlp-output.log + ports: + - "4318:4318" + - "8889:8889" + openbas-telemetry-prometheus: + container_name: openbas-telemetry-prometheus + image: prom/prometheus + restart: unless-stopped + volumes: + - C:/Users/RomualdLemesle/Desktop/config/prometheus-config.yml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + - "9464:9464" + openbas-telemetry-jaeger: + container_name: openbas-telemetry-jaeger + image: jaegertracing/all-in-one + restart: unless-stopped + ports: + - "16686:16686" + - "4317:4317" + openbas-telemetry-elasticsearch: + container_name: openbas-telemetry-elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:8.13.2 + environment: + - discovery.type=single-node + - xpack.ml.enabled=false + - xpack.security.enabled=false + - "ES_JAVA_OPTS=-Xms2G -Xmx2G" + restart: unless-stopped + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + ports: + - 9200:9200 + - 9300:9300 + openbas-telemetry-kibana: + container_name: openbas-telemetry-kibana + image: docker.elastic.co/kibana/kibana:8.13.2 + environment: + - ELASTICSEARCH_HOSTS=http://openbas-telemetry-elasticsearch:9200 + restart: unless-stopped + ports: + - 5601:5601 + depends_on: + - openbas-telemetry-elasticsearch + openbas-telemetry-filebeat: + container_name: openbas-telemetry-filebeat + image: docker.elastic.co/beats/filebeat:8.13.2 + user: root + volumes: + - C:/Users/RomualdLemesle/Desktop/config/filebeat-config.yml:/usr/share/filebeat/filebeat.yml + - C:/Users/RomualdLemesle/Desktop/config/otlp-output.log:/otlp-output.log + command: [ "/usr/share/filebeat/filebeat", "-e", "-c", "/usr/share/filebeat/filebeat.yml", "--strict.perms=false" ] + restart: unless-stopped + depends_on: + - openbas-telemetry-elasticsearch