Skip to content

Commit

Permalink
Creating Build Metrics for Log is updated and build is likely stuck (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
Waschndolos authored Nov 26, 2023
1 parent 7e3f6e6 commit 4f5e028
Show file tree
Hide file tree
Showing 10 changed files with 279 additions and 12 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ public class ExecutorCollector extends Collector {

private static final Logger LOGGER = LoggerFactory.getLogger(ExecutorCollector.class);

public ExecutorCollector() {
}

@Override
public List<MetricFamilySamples> collect() {
LOGGER.debug("Collecting executor metrics for prometheus");
Expand All @@ -30,13 +33,13 @@ public List<MetricFamilySamples> collect() {

List<MetricCollector<LoadStatistics.LoadStatisticsSnapshot, ? extends Collector>> collectors = new ArrayList<>();

collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_AVAILABLE_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_BUSY_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_CONNECTING_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_DEFINED_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_IDLE_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_ONLINE_GAUGE, labelNameArray, prefix));
collectors.add(factory.createExecutorCollector(CollectorType.EXECUTORS_QUEUE_LENGTH_GAUGE, labelNameArray, prefix));
collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_AVAILABLE_GAUGE, labelNameArray, prefix));
collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_BUSY_GAUGE, labelNameArray, prefix));
collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_CONNECTING_GAUGE, labelNameArray, prefix));
collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_DEFINED_GAUGE, labelNameArray, prefix));
collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_IDLE_GAUGE, labelNameArray, prefix));
collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_ONLINE_GAUGE, labelNameArray, prefix));
collectors.add(factory.createLoadStatisticsCollector(CollectorType.EXECUTORS_QUEUE_LENGTH_GAUGE, labelNameArray, prefix));

LOGGER.debug("getting load statistics for Executors");

Expand Down
11 changes: 11 additions & 0 deletions src/main/java/org/jenkinsci/plugins/prometheus/JobCollector.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public class JobCollector extends Collector {
private MetricCollector<Job<?, ?>, ? extends Collector> nbBuildsGauge;
private MetricCollector<Job<?, ?>, ? extends Collector> buildDiscardGauge;
private MetricCollector<Job<?, ?>, ? extends Collector> currentRunDurationGauge;
private MetricCollector<Job<?,?>, ? extends Collector> logUpdatedGauge;

private static class BuildMetrics {

Expand All @@ -44,6 +45,8 @@ private static class BuildMetrics {
public MetricCollector<Run<?, ?>, ? extends Collector> jobBuildTestsSkipped;
public MetricCollector<Run<?, ?>, ? extends Collector> jobBuildTestsFailing;

public MetricCollector<Run<?,?>, ? extends Collector> jobBuildLikelyStuck;

private final String buildPrefix;

public BuildMetrics(String buildPrefix) {
Expand All @@ -60,6 +63,7 @@ public void initCollectors(String[] labelNameArray, String[] labelStageNameArray
this.jobBuildTestsSkipped = factory.createRunCollector(CollectorType.SKIPPED_TESTS_GAUGE, labelNameArray, buildPrefix);
this.jobBuildTestsFailing = factory.createRunCollector(CollectorType.FAILED_TESTS_GAUGE, labelNameArray, buildPrefix);
this.stageSummary = factory.createRunCollector(CollectorType.STAGE_SUMMARY, labelStageNameArray, buildPrefix);
this.jobBuildLikelyStuck = factory.createRunCollector(CollectorType.BUILD_LIKELY_STUCK_GAUGE, labelNameArray, buildPrefix);
}
}

Expand Down Expand Up @@ -129,6 +133,8 @@ public List<MetricFamilySamples> collect() {

currentRunDurationGauge = factory.createJobCollector(CollectorType.CURRENT_RUN_DURATION_GAUGE, labelBaseNameArray);

logUpdatedGauge = factory.createJobCollector(CollectorType.JOB_LOG_UPDATED_GAUGE, labelBaseNameArray);

if (PrometheusConfiguration.get().isPerBuildMetrics()) {
labelNameArray = Arrays.copyOf(labelNameArray, labelNameArray.length + 1);
labelNameArray[labelNameArray.length - 1] = "number";
Expand Down Expand Up @@ -164,6 +170,7 @@ public List<MetricFamilySamples> collect() {
addSamples(samples, nbBuildsGauge.collect(), "Adding [{}] samples from gauge ({})");
addSamples(samples, buildDiscardGauge.collect(), "Adding [{}] samples from gauge ({})");
addSamples(samples, currentRunDurationGauge.collect(), "Adding [{}] samples from gauge ({})");
addSamples(samples, logUpdatedGauge.collect(), "Adding [{}] samples from gauge ({})");
addSamples(samples, lastBuildMetrics);
if (PrometheusConfiguration.get().isPerBuildMetrics()) {
addSamples(samples, perBuildMetrics);
Expand All @@ -190,6 +197,7 @@ private void addSamples(List<MetricFamilySamples> allSamples, BuildMetrics build
addSamples(allSamples, buildMetrics.jobBuildTestsTotal.collect(), "Adding [{}] samples from gauge ({})");
addSamples(allSamples, buildMetrics.jobBuildTestsSkipped.collect(), "Adding [{}] samples from gauge ({})");
addSamples(allSamples, buildMetrics.jobBuildTestsFailing.collect(), "Adding [{}] samples from gauge ({})");
addSamples(allSamples, buildMetrics.jobBuildLikelyStuck.collect(), "Adding [{}] samples from gauge ({})");
addSamples(allSamples, buildMetrics.stageSummary.collect(), "Adding [{}] samples from summary ({})");
}

Expand Down Expand Up @@ -217,6 +225,8 @@ protected void appendJobMetrics(Job<?, ?> job) {
jobHealthScoreGauge.calculateMetric(job, baseLabelValueArray);
buildDiscardGauge.calculateMetric(job, baseLabelValueArray);
currentRunDurationGauge.calculateMetric(job, baseLabelValueArray);
logUpdatedGauge.calculateMetric(job, baseLabelValueArray);

processRun(job, lastBuild, baseLabelValueArray, lastBuildMetrics);

Run<?, ?> run = lastBuild;
Expand Down Expand Up @@ -278,6 +288,7 @@ private void processRun(Job<?, ?> job, Run<?, ?> run, String[] buildLabelValueAr
buildMetrics.jobBuildTestsTotal.calculateMetric(run, buildLabelValueArray);
buildMetrics.jobBuildTestsSkipped.calculateMetric(run, buildLabelValueArray);
buildMetrics.jobBuildTestsFailing.calculateMetric(run, buildLabelValueArray);
buildMetrics.jobBuildLikelyStuck.calculateMetric(run,buildLabelValueArray);

Check warning on line 291 in src/main/java/org/jenkinsci/plugins/prometheus/JobCollector.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Not covered lines

Lines 228-291 are not covered by tests
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import org.jenkinsci.plugins.prometheus.collectors.executors.ExecutorCollectorFactory;
import org.jenkinsci.plugins.prometheus.collectors.jenkins.JenkinsCollectorFactory;
import org.jenkinsci.plugins.prometheus.collectors.jobs.JobCollectorFactory;
import org.json.Cookie;

import java.nio.file.FileStore;

Expand Down Expand Up @@ -52,7 +51,7 @@ public CollectorFactory() {
return jenkinsCollectorFactory.createCollector(type, labelNames);
}

public MetricCollector<LoadStatistics.LoadStatisticsSnapshot, ? extends Collector> createExecutorCollector(CollectorType type, String[] labelNames, String prefix) {
public MetricCollector<LoadStatistics.LoadStatisticsSnapshot, ? extends Collector> createLoadStatisticsCollector(CollectorType type, String[] labelNames, String prefix) {
return executorCollectorFactory.createCollector(type, labelNames, prefix);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ public enum CollectorType {
BUILD_RESULT_ORDINAL_GAUGE("build_result_ordinal"),
BUILD_START_GAUGE("build_start_time_milliseconds"),
BUILD_SUCCESSFUL_COUNTER("success_build_count"),
BUILD_LIKELY_STUCK_GAUGE("likely_stuck"),

FAILED_TESTS_GAUGE("build_tests_failing"),
SKIPPED_TESTS_GAUGE("last_build_tests_skipped"),
STAGE_SUMMARY("stage_duration_milliseconds_summary"),
Expand Down Expand Up @@ -53,9 +55,7 @@ public enum CollectorType {
COVERAGE_FILE_MISSED("coverage_file_missed"),
COVERAGE_FILE_TOTAL("coverage_file_total"),



;
JOB_LOG_UPDATED_GAUGE("job_log_updated");

private final String name;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ public BuildCollectorFactory() {
return saveBuildCollector(new StageSummary(labelNames, namespace, subsystem, prefix));
case TOTAL_TESTS_GAUGE:
return saveBuildCollector(new TotalTestsGauge(labelNames, namespace, subsystem, prefix));
case BUILD_LIKELY_STUCK_GAUGE:
return saveBuildCollector(new BuildLikelyStuckGauge(labelNames, namespace, subsystem, prefix));
default:
return new NoOpMetricCollector<>();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package org.jenkinsci.plugins.prometheus.collectors.builds;

import hudson.model.Executor;
import hudson.model.Run;
import io.prometheus.client.Gauge;
import io.prometheus.client.SimpleCollector;
import org.jenkinsci.plugins.prometheus.collectors.CollectorType;

public class BuildLikelyStuckGauge extends BuildsMetricCollector<Run<?, ?>, Gauge> {

protected BuildLikelyStuckGauge(String[] labelNames, String namespace, String subsystem, String prefix) {
super(labelNames, namespace, subsystem, prefix);
}

@Override
protected CollectorType getCollectorType() {
return CollectorType.BUILD_LIKELY_STUCK_GAUGE;
}

@Override
protected String getHelpText() {
return "Provides a hint if a build is likely stuck. Uses Jenkins function Executor#isLikelyStuck.";
}

@Override
protected SimpleCollector.Builder<?, Gauge> getCollectorBuilder() {
return Gauge.build();
}

@Override
public void calculateMetric(Run<?, ?> run, String[] labelValues) {
if (run == null || !run.isBuilding()) {
return;
}

Executor executor = run.getExecutor();
if (executor == null) {
return;
}

double likelyStuckDoubleValue = executor.isLikelyStuck() ? 1.0 : 0.0;
this.collector.labels(labelValues).set(likelyStuckDoubleValue);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ public JobCollectorFactory() {
return saveBuildCollector(new BuildDiscardGauge(labelNames, namespace, subsystem));
case CURRENT_RUN_DURATION_GAUGE:
return saveBuildCollector(new CurrentRunDurationGauge(labelNames, namespace, subsystem));
case JOB_LOG_UPDATED_GAUGE:
return saveBuildCollector(new LogUpdatedGauge(labelNames, namespace, subsystem));
default:
return new NoOpMetricCollector<>();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package org.jenkinsci.plugins.prometheus.collectors.jobs;

import hudson.model.Job;
import io.prometheus.client.Gauge;
import io.prometheus.client.SimpleCollector;
import org.jenkinsci.plugins.prometheus.collectors.CollectorType;
import org.jenkinsci.plugins.prometheus.collectors.builds.BuildsMetricCollector;

public class LogUpdatedGauge extends BuildsMetricCollector<Job<?, ?>, Gauge> {

protected LogUpdatedGauge(String[] labelNames, String namespace, String subsystem) {
super(labelNames, namespace, subsystem);
}

@Override
protected CollectorType getCollectorType() {
return CollectorType.JOB_LOG_UPDATED_GAUGE;
}

@Override
protected String getHelpText() {
return "Provides a hint if a job is still logging. Uses Jenkins function Job#isLogUpdated";
}

@Override
protected SimpleCollector.Builder<?, Gauge> getCollectorBuilder() {
return Gauge.build();
}

@Override
public void calculateMetric(Job<?, ?> jenkinsObject, String[] labelValues) {

if (jenkinsObject != null && jenkinsObject.isBuilding()) {

Check warning on line 33 in src/main/java/org/jenkinsci/plugins/prometheus/collectors/jobs/LogUpdatedGauge.java

View check run for this annotation

ci.jenkins.io / Code Coverage

Partially covered line

Line 33 is only partially covered, one branch is missing
boolean logUpdated = jenkinsObject.isLogUpdated();
this.collector.labels(labelValues).set(logUpdated ? 1.0 : 0.0);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package org.jenkinsci.plugins.prometheus.collectors.builds;

import hudson.model.Executor;
import io.prometheus.client.Collector;
import org.jenkinsci.plugins.prometheus.collectors.testutils.MockedRunCollectorTest;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;

import java.util.List;

import static org.mockito.Mockito.*;


public class BuildLikelyStuckGaugeTest extends MockedRunCollectorTest {


@Test
public void testNothingCalculatedWhenRunIsNull() {
BuildLikelyStuckGauge sut = new BuildLikelyStuckGauge(getLabelNames(), getNamespace(), getSubSystem(), "");

sut.calculateMetric(null, getLabelValues());

List<Collector.MetricFamilySamples> collect = sut.collect();

Assertions.assertEquals(1, collect.size());
Assertions.assertEquals(0, collect.get(0).samples.size());

}

@Test
public void testNothingCalculatedWhenJobIsNotBuilding() {
Mockito.when(mock.isBuilding()).thenReturn(false);

BuildLikelyStuckGauge sut = new BuildLikelyStuckGauge(getLabelNames(), getNamespace(), getSubSystem(), "");

sut.calculateMetric(mock, getLabelValues());

List<Collector.MetricFamilySamples> collect = sut.collect();

Assertions.assertEquals(1, collect.size());
Assertions.assertEquals(0, collect.get(0).samples.size());

}

@Test
public void testNothingCalculatedWhenNoExecutorFound() {
Mockito.when(mock.isBuilding()).thenReturn(true);
Mockito.when(mock.getExecutor()).thenReturn(null);

BuildLikelyStuckGauge sut = new BuildLikelyStuckGauge(getLabelNames(), getNamespace(), getSubSystem(), "");

sut.calculateMetric(mock, getLabelValues());

List<Collector.MetricFamilySamples> collect = sut.collect();

Assertions.assertEquals(1, collect.size());
Assertions.assertEquals(0, collect.get(0).samples.size());
}

@Test
public void testBuildIsLikelyStuck() {
Mockito.when(mock.isBuilding()).thenReturn(true);
Executor mockedExecutor = mock(Executor.class);
when(mockedExecutor.isLikelyStuck()).thenReturn(true);
Mockito.when(mock.getExecutor()).thenReturn(mockedExecutor);

BuildLikelyStuckGauge sut = new BuildLikelyStuckGauge(getLabelNames(), getNamespace(), getSubSystem(), "");

sut.calculateMetric(mock, getLabelValues());

List<Collector.MetricFamilySamples> collect = sut.collect();

Assertions.assertEquals(1, collect.size());
Assertions.assertEquals(1, collect.get(0).samples.size());
Assertions.assertEquals(1.0, collect.get(0).samples.get(0).value);
}

@Test
public void testBuildIsNotLikelyStuck() {
Mockito.when(mock.isBuilding()).thenReturn(true);
Executor mockedExecutor = mock(Executor.class);
when(mockedExecutor.isLikelyStuck()).thenReturn(false);
Mockito.when(mock.getExecutor()).thenReturn(mockedExecutor);

BuildLikelyStuckGauge sut = new BuildLikelyStuckGauge(getLabelNames(), getNamespace(), getSubSystem(), "");

sut.calculateMetric(mock, getLabelValues());

List<Collector.MetricFamilySamples> collect = sut.collect();

Assertions.assertEquals(1, collect.size());
Assertions.assertEquals(1, collect.get(0).samples.size());
Assertions.assertEquals(0.0, collect.get(0).samples.get(0).value);
}
}
Loading

0 comments on commit 4f5e028

Please sign in to comment.