Merge pull request #99 from ls1intum/develop

LocalCI Metrics
ls1intum · Jan 10, 2024 · 04c3874 · 04c3874
2 parents 1f21f5b + 8f4b3a8
commit 04c3874
Show file tree

Hide file tree

Showing 27 changed files with 437 additions and 19 deletions.
diff --git a/src/main/java/de/tum/cit/ase/domain/LocalCIStatus.java b/src/main/java/de/tum/cit/ase/domain/LocalCIStatus.java
@@ -0,0 +1,89 @@
+package de.tum.cit.ase.domain;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import jakarta.persistence.*;
+
+@Entity
+@Table(name = "local_ci_status")
+public class LocalCIStatus {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @Column(name = "is_finished")
+    private boolean isFinished;
+
+    @Column(name = "queued_jobs")
+    private int queuedJobs;
+
+    @Column(name = "total_jobs")
+    private int totalJobs;
+
+    @Column(name = "time_in_minutes")
+    private int timeInMinutes;
+
+    @Column(name = "avg_jobs_per_minute")
+    private double avgJobsPerMinute;
+
+    @OneToOne
+    @JoinColumn(name = "simulation_run_id", nullable = false)
+    @JsonIgnore
+    private SimulationRun simulationRun;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public boolean isFinished() {
+        return isFinished;
+    }
+
+    public void setFinished(boolean finished) {
+        isFinished = finished;
+    }
+
+    public int getQueuedJobs() {
+        return queuedJobs;
+    }
+
+    public void setQueuedJobs(int queuedJobs) {
+        this.queuedJobs = queuedJobs;
+    }
+
+    public int getTotalJobs() {
+        return totalJobs;
+    }
+
+    public void setTotalJobs(int totalJobs) {
+        this.totalJobs = totalJobs;
+    }
+
+    public int getTimeInMinutes() {
+        return timeInMinutes;
+    }
+
+    public void setTimeInMinutes(int timeInMinutes) {
+        this.timeInMinutes = timeInMinutes;
+    }
+
+    public double getAvgJobsPerMinute() {
+        return avgJobsPerMinute;
+    }
+
+    public void setAvgJobsPerMinute(double avgJobsPerMinute) {
+        this.avgJobsPerMinute = avgJobsPerMinute;
+    }
+
+    public SimulationRun getSimulationRun() {
+        return simulationRun;
+    }
+
+    public void setSimulationRun(SimulationRun simulationRun) {
+        this.simulationRun = simulationRun;
+    }
+}
diff --git a/src/main/java/de/tum/cit/ase/domain/SimulationRun.java b/src/main/java/de/tum/cit/ase/domain/SimulationRun.java
@@ -35,6 +35,9 @@ public class SimulationRun {
     @OneToMany(cascade = CascadeType.REMOVE, mappedBy = "simulationRun")
     private Set<LogMessage> logMessages;
 
+    @OneToOne(cascade = CascadeType.REMOVE, mappedBy = "simulationRun", fetch = FetchType.EAGER)
+    private LocalCIStatus localCIStatus;
+
     @Transient
     private ArtemisAccountDTO adminAccount;
 
@@ -113,6 +116,14 @@ public void setEndDateTime(ZonedDateTime endDateTime) {
         this.endDateTime = endDateTime;
     }
 
+    public LocalCIStatus getLocalCIStatus() {
+        return localCIStatus;
+    }
+
+    public void setLocalCIStatus(LocalCIStatus localCIStatus) {
+        this.localCIStatus = localCIStatus;
+    }
+
     public enum Status {
         QUEUED,
         RUNNING,

diff --git a/src/main/java/de/tum/cit/ase/repository/LocalCIStatusRepository.java b/src/main/java/de/tum/cit/ase/repository/LocalCIStatusRepository.java
@@ -0,0 +1,16 @@
+package de.tum.cit.ase.repository;
+
+import de.tum.cit.ase.domain.LocalCIStatus;
+import jakarta.transaction.Transactional;
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface LocalCIStatusRepository extends JpaRepository<LocalCIStatus, Long> {
+    @Modifying
+    @Transactional
+    @Query(value = "delete from LocalCIStatus status where status.isFinished = false")
+    void deleteAllNotFinished();
+}
diff --git a/src/main/java/de/tum/cit/ase/repository/SimulationRunRepository.java b/src/main/java/de/tum/cit/ase/repository/SimulationRunRepository.java
@@ -2,6 +2,7 @@
 
 import de.tum.cit.ase.domain.SimulationRun;
 import java.util.List;
+import java.util.Optional;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
@@ -16,5 +17,5 @@ public interface SimulationRunRepository extends JpaRepository<SimulationRun, Lo
     List<SimulationRun> findAllBySimulationId(@Param("simulationId") long simulationId);
 
     @Query(value = "select run from SimulationRun run left join fetch run.stats s left join fetch run.logMessages l where run.id = :#{#id}")
-    SimulationRun findByIdWithStatsAndLogMessages(long id);
+    Optional<SimulationRun> findByIdWithStatsAndLogMessages(long id);
 }
diff --git a/src/main/java/de/tum/cit/ase/service/LocalCIStatusService.java b/src/main/java/de/tum/cit/ase/service/LocalCIStatusService.java
@@ -0,0 +1,72 @@
+package de.tum.cit.ase.service;
+
+import de.tum.cit.ase.domain.LocalCIStatus;
+import de.tum.cit.ase.domain.SimulationRun;
+import de.tum.cit.ase.repository.LocalCIStatusRepository;
+import de.tum.cit.ase.service.artemis.interaction.SimulatedArtemisAdmin;
+import de.tum.cit.ase.web.websocket.SimulationWebsocketService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Service;
+
+@Service
+public class LocalCIStatusService {
+
+    private final Logger log = LoggerFactory.getLogger(LocalCIStatusService.class);
+    private final LocalCIStatusRepository localCIStatusRepository;
+    private final SimulationWebsocketService websocketService;
+
+    public LocalCIStatusService(LocalCIStatusRepository localCIStatusRepository, SimulationWebsocketService websocketService) {
+        this.localCIStatusRepository = localCIStatusRepository;
+        this.websocketService = websocketService;
+        cleanup();
+    }
+
+    public LocalCIStatus createLocalCIStatus(SimulationRun simulationRun) {
+        LocalCIStatus status = new LocalCIStatus();
+        status.setSimulationRun(simulationRun);
+        status.setFinished(false);
+        status.setAvgJobsPerMinute(0);
+        status.setQueuedJobs(0);
+        status.setTotalJobs(0);
+        status.setTimeInMinutes(0);
+        return localCIStatusRepository.save(status);
+    }
+
+    @Async
+    public void subscribeToLocalCIStatus(SimulationRun simulationRun, SimulatedArtemisAdmin admin, long courseId) {
+        log.info("Subscribing to local CI status for simulation run {}", simulationRun.getId());
+        LocalCIStatus status = createLocalCIStatus(simulationRun);
+
+        int numberOfQueuedJobs = admin.getBuildQueue(courseId).size();
+        status.setTotalJobs(numberOfQueuedJobs);
+        status.setQueuedJobs(numberOfQueuedJobs);
+        status = localCIStatusRepository.save(status);
+        websocketService.sendRunLocalCIUpdate(simulationRun.getId(), status);
+
+        do {
+            try {
+                Thread.sleep(1000 * 60);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+            log.debug("Updating local CI status for simulation run {}", simulationRun.getId());
+            numberOfQueuedJobs = admin.getBuildQueue(courseId).size();
+            status.setQueuedJobs(numberOfQueuedJobs);
+            status.setTimeInMinutes(status.getTimeInMinutes() + 1);
+            status.setAvgJobsPerMinute((double) (status.getTotalJobs() - status.getQueuedJobs()) / status.getTimeInMinutes());
+            status = localCIStatusRepository.save(status);
+            websocketService.sendRunLocalCIUpdate(simulationRun.getId(), status);
+        } while (numberOfQueuedJobs > 0);
+        status.setFinished(true);
+        status = localCIStatusRepository.save(status);
+        websocketService.sendRunLocalCIUpdate(simulationRun.getId(), status);
+        log.info("Finished subscribing to local CI status for simulation run {}", simulationRun.getId());
+    }
+
+    private void cleanup() {
+        log.info("Cleaning up local CI status");
+        localCIStatusRepository.deleteAllNotFinished();
+    }
+}
diff --git a/src/main/java/de/tum/cit/ase/service/PrometheusService.java b/src/main/java/de/tum/cit/ase/service/PrometheusService.java
@@ -77,7 +77,7 @@ public List<MetricValue> getCpuUsageVcs(SimulationRun run) {
      * @return A list of CPU usage values. An empty list if no Prometheus instance is configured for the CI system.
      */
     public List<MetricValue> getCpuUsageCi(SimulationRun run) {
-        log.info("Getting CI CPU usage for {}", run);
+        log.debug("Getting CI CPU usage for {}", run);
         var instance = artemisConfiguration.getPrometheusInstanceCi(run.getSimulation().getServer());
         if (instance == null || instance.isBlank()) {
             log.warn("No Prometheus instance configured for CI on {}", run.getSimulation().getServer());
@@ -94,7 +94,7 @@ public List<MetricValue> getCpuUsageCi(SimulationRun run) {
      * @return The response from Prometheus.
      */
     public QueryResponse executeQuery(String query, ZonedDateTime start, ZonedDateTime end) {
-        log.info("Querying Prometheus: {}", query);
+        log.debug("Querying Prometheus: {}", query);
         if (webClient == null) {
             setupWebclient();
         }

diff --git a/src/main/java/de/tum/cit/ase/service/artemis/ArtemisConfiguration.java b/src/main/java/de/tum/cit/ase/service/artemis/ArtemisConfiguration.java
@@ -22,6 +22,9 @@ public class ArtemisConfiguration {
     @Value("${artemis.local.prometheus-instances.ci}")
     private String localPrometheusInstanceCi;
 
+    @Value("${artemis.local.is-local}")
+    private boolean localIsLocal;
+
     @Value("${artemis.ts1.url}")
     private String test1Url;
 
@@ -37,6 +40,9 @@ public class ArtemisConfiguration {
     @Value("${artemis.ts1.prometheus-instances.ci}")
     private String test1PrometheusInstanceCi;
 
+    @Value("${artemis.ts1.is-local}")
+    private boolean test1IsLocal;
+
     @Value("${artemis.ts3.url}")
     private String test3Url;
 
@@ -52,6 +58,9 @@ public class ArtemisConfiguration {
     @Value("${artemis.ts3.prometheus-instances.ci}")
     private String test3PrometheusInstanceCi;
 
+    @Value("${artemis.ts3.is-local}")
+    private boolean test3IsLocal;
+
     @Value("${artemis.staging.url}")
     private String stagingUrl;
 
@@ -67,6 +76,9 @@ public class ArtemisConfiguration {
     @Value("${artemis.staging.prometheus-instances.ci}")
     private String stagingPrometheusInstanceCi;
 
+    @Value("${artemis.staging.is-local}")
+    private boolean stagingIsLocal;
+
     @Value("${artemis.staging2.url}")
     private String staging2Url;
 
@@ -82,6 +94,9 @@ public class ArtemisConfiguration {
     @Value("${artemis.staging2.prometheus-instances.ci}")
     private String staging2PrometheusInstanceCi;
 
+    @Value("${artemis.staging2.is-local}")
+    private boolean staging2IsLocal;
+
     @Value("${artemis.production.url}")
     private String productionUrl;
 
@@ -97,6 +112,9 @@ public class ArtemisConfiguration {
     @Value("${artemis.production.prometheus-instances.ci}")
     private String productionPrometheusInstanceCi;
 
+    @Value("${artemis.production.is-local}")
+    private boolean productionIsLocal;
+
     public String getUrl(ArtemisServer server) {
         return switch (server) {
             case LOCAL -> localUrl;
@@ -151,4 +169,15 @@ public String getPrometheusInstanceCi(ArtemisServer server) {
             case PRODUCTION -> productionPrometheusInstanceCi;
         };
     }
+
+    public boolean getIsLocal(ArtemisServer server) {
+        return switch (server) {
+            case LOCAL -> localIsLocal;
+            case TS1 -> test1IsLocal;
+            case TS3 -> test3IsLocal;
+            case STAGING -> stagingIsLocal;
+            case STAGING2 -> staging2IsLocal;
+            case PRODUCTION -> productionIsLocal;
+        };
+    }
 }
diff --git a/src/main/java/de/tum/cit/ase/service/artemis/interaction/SimulatedArtemisAdmin.java b/src/main/java/de/tum/cit/ase/service/artemis/interaction/SimulatedArtemisAdmin.java
@@ -10,6 +10,7 @@
 import io.reactivex.rxjava3.core.Scheduler;
 import io.reactivex.rxjava3.schedulers.Schedulers;
 import java.time.ZonedDateTime;
+import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import org.slf4j.LoggerFactory;
@@ -463,4 +464,14 @@ public void deleteExam(long courseId, long examId) {
             .toBodilessEntity()
             .block();
     }
+
+    public List<DomainObject> getBuildQueue(long courseId) {
+        return webClient
+            .get()
+            .uri(uriBuilder -> uriBuilder.pathSegment("api", "build-job-queue", "queued", String.valueOf(courseId)).build())
+            .retrieve()
+            .bodyToFlux(DomainObject.class)
+            .collectList()
+            .block();
+    }
 }
diff --git a/src/main/java/de/tum/cit/ase/service/simulation/SimulationDataService.java b/src/main/java/de/tum/cit/ase/service/simulation/SimulationDataService.java
@@ -77,7 +77,7 @@ public SimulationRun getSimulationRun(long id) {
     }
 
     public SimulationRun getSimulationRunWithStatsAndLogs(long id) {
-        return simulationRunRepository.findByIdWithStatsAndLogMessages(id);
+        return simulationRunRepository.findByIdWithStatsAndLogMessages(id).orElseThrow();
     }
 
     public void deleteSimulation(long id) {
@@ -166,7 +166,7 @@ public void cancelActiveRun(long runId) {
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
             }
-            run = simulationRunRepository.findById(runId).orElseThrow();
+            run = simulationRunRepository.findByIdWithStatsAndLogMessages(runId).orElseThrow();
 
             run.setStatus(SimulationRun.Status.CANCELLED);
             run.setEndDateTime(now());