diff --git a/src/main/java/org/jenkinsci/plugins/prometheus/service/DefaultPrometheusMetrics.java b/src/main/java/org/jenkinsci/plugins/prometheus/service/DefaultPrometheusMetrics.java
index 1cf59d43a..02985572f 100644
--- a/src/main/java/org/jenkinsci/plugins/prometheus/service/DefaultPrometheusMetrics.java
+++ b/src/main/java/org/jenkinsci/plugins/prometheus/service/DefaultPrometheusMetrics.java
@@ -36,23 +36,35 @@ public class DefaultPrometheusMetrics implements PrometheusMetrics {
     private final AtomicReference<String> cachedMetrics;
 
     private DefaultPrometheusMetrics() {
-        CollectorRegistry collectorRegistry = CollectorRegistry.defaultRegistry;
+        this.collectorRegistry = CollectorRegistry.defaultRegistry;
         DefaultExports.initialize();
-        this.collectorRegistry = collectorRegistry;
         this.cachedMetrics = new AtomicReference<>("");
     }
 
     public static synchronized DefaultPrometheusMetrics get() {
-        if(INSTANCE == null) {
+        if (INSTANCE == null) {
             INSTANCE = new DefaultPrometheusMetrics();
         }
         return INSTANCE;
     }
 
     @Restricted(NoExternalUse.class)
-    private void registerCollector(@NonNull Collector collector) {
+    private synchronized void registerCollector(@NonNull Collector collector) {
         collectorRegistry.register(collector);
-        logger.debug(String.format("Collector %s registered", collector.getClass().getName()));
+        logger.debug("Collector {} registered", collector.getClass().getName());
+    }
+
+    @Restricted(NoExternalUse.class)
+    private synchronized void cleanUpCollector() {
+        this.collectorRegistry.clear();
+    }
+
+    // Issue #683
+    @Restricted(NoExternalUse.class)
+    @Initializer(after = InitMilestone.PLUGINS_STARTED, before = InitMilestone.EXTENSIONS_AUGMENTED)
+    public static void cleanupCollectorRegistry() {
+        DefaultPrometheusMetrics instance = get();
+        instance.cleanUpCollector();
     }
 
     @Restricted(NoExternalUse.class)