fix: keep a strong reference to the processing started latency gauge

micrometer-support/pom.xml

@@ -51,6 +51,11 @@
       <artifactId>assertj-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.awaitility</groupId>
       <artifactId>awaitility</artifactId>

micrometer-support/src/main/java/io/javaoperatorsdk/operator/monitoring/micrometer/MicrometerMetrics.java

@@ -22,6 +22,7 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 import io.fabric8.kubernetes.api.model.HasMetadata;
 import io.javaoperatorsdk.operator.OperatorException;
@@ -78,6 +79,7 @@ public class MicrometerMetrics implements Metrics {
   private final boolean collectPerResourceMetrics;
   private final MeterRegistry registry;
   private final Map<String, AtomicInteger> gauges = new ConcurrentHashMap<>();
+  private final Map<String, AtomicLong> longGauges = new ConcurrentHashMap<>();
   private final Cleaner cleaner;
 
   /**
@@ -153,10 +155,19 @@ public void controllerRegistered(Controller<? extends HasMetadata> controller) {
   public void eventProcessingStarted(Controller<? extends HasMetadata> controller) {
     final var configuration = controller.getConfiguration();
     final var name = configuration.getName();
-    final var tags = new ArrayList<Tag>(3);
-    addGVKTags(GroupVersionKind.gvkFor(configuration.getResourceClass()), tags, false);
-    registry.gauge(
-        PROCESSING_STARTED_LATENCY + name, tags, ManagementFactory.getRuntimeMXBean().getUptime());
+    // the registry only holds a weak reference to the gauged object, so it has to be kept alive
+    // here, otherwise the gauge reports NaN as soon as the value is garbage collected
+    longGauges
+        .computeIfAbsent(
+            PROCESSING_STARTED_LATENCY + name,
+            key -> {
+              final var tags = new ArrayList<Tag>(3);
+              addGVKTags(GroupVersionKind.gvkFor(configuration.getResourceClass()), tags, false);
+              var holder = new AtomicLong();
+              registry.gauge(key, tags, holder);
+              return holder;
+            })
+        .set(ManagementFactory.getRuntimeMXBean().getUptime());
   }
 
   @Override

micrometer-support/src/main/java/io/javaoperatorsdk/operator/monitoring/micrometer/MicrometerMetricsV2.java

@@ -20,6 +20,7 @@
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Consumer;
 import java.util.function.Function;
 
@@ -69,6 +70,7 @@ public class MicrometerMetricsV2 implements Metrics {
 
   private final MeterRegistry registry;
   private final Map<String, AtomicInteger> gauges = new ConcurrentHashMap<>();
+  private final Map<String, AtomicLong> longGauges = new ConcurrentHashMap<>();
   private final Map<String, Timer> executionTimers = new ConcurrentHashMap<>();
   private final Function<Timer.Builder, Timer.Builder> timerConfig;
   private final boolean includeNamespaceTag;
@@ -150,10 +152,19 @@ public void controllerRegistered(Controller<? extends HasMetadata> controller) {
   @Override
   public void eventProcessingStarted(Controller<? extends HasMetadata> controller) {
     final var name = controller.getConfiguration().getName();
-    final var tags = new ArrayList<Tag>();
-    addControllerNameTag(name, tags);
-    registry.gauge(
-        PROCESSING_STARTED_LATENCY_GAUGE, tags, ManagementFactory.getRuntimeMXBean().getUptime());
+    // the registry only holds a weak reference to the gauged object, so it has to be kept alive
+    // here, otherwise the gauge reports NaN as soon as the value is garbage collected
+    longGauges
+        .computeIfAbsent(
+            processingStartedLatencyGaugeRefKey(name),
+            key -> {
+              final var tags = new ArrayList<Tag>();
+              addControllerNameTag(name, tags);
+              var holder = new AtomicLong();
+              registry.gauge(PROCESSING_STARTED_LATENCY_GAUGE, tags, holder);
+              return holder;
+            })
+        .set(ManagementFactory.getRuntimeMXBean().getUptime());
   }
 
   private String numberOfResourcesRefName(String name) {
@@ -289,6 +300,10 @@ private static String controllerQueueSizeGaugeRefKey(String controllerName) {
     return RECONCILIATIONS_QUEUE_SIZE_GAUGE + "." + controllerName;
   }
 
+  private static String processingStartedLatencyGaugeRefKey(String controllerName) {
+    return PROCESSING_STARTED_LATENCY_GAUGE + "." + controllerName;
+  }
+
   public static String getControllerName(Map<String, Object> metadata) {
     return (String) metadata.get(Constants.CONTROLLER_NAME);
   }

micrometer-support/src/test/java/io/javaoperatorsdk/operator/monitoring/micrometer/ProcessingStartedLatencyGaugeTest.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.monitoring.micrometer;
+
+import org.junit.jupiter.api.Test;
+
+import io.fabric8.kubernetes.api.model.ConfigMap;
+import io.javaoperatorsdk.operator.api.config.ControllerConfiguration;
+import io.javaoperatorsdk.operator.processing.Controller;
+import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class ProcessingStartedLatencyGaugeTest {
+
+  private static final String CONTROLLER_NAME = "testcontroller";
+
+  @Test
+  void latencyGaugeKeepsItsValue() {
+    var registry = new SimpleMeterRegistry();
+    var metrics = MicrometerMetricsV2.newBuilder(registry).build();
+
+    metrics.eventProcessingStarted(controller());
+
+    var gauge = registry.find(MicrometerMetricsV2.PROCESSING_STARTED_LATENCY_GAUGE).gauge();
+    assertThat(gauge).isNotNull();
+    assertThat(gauge.value()).isNotNaN().isPositive();
+
+    // the registry holds only a weak reference to the gauged object
+    forceGarbageCollection();
+
+    assertThat(gauge.value()).isNotNaN().isPositive();
+  }
+
+  @Test
+  void repeatedCallsUpdateTheSameGauge() {
+    var registry = new SimpleMeterRegistry();
+    var metrics = MicrometerMetricsV2.newBuilder(registry).build();
+
+    metrics.eventProcessingStarted(controller());
+    metrics.eventProcessingStarted(controller());
+
+    assertThat(registry.find(MicrometerMetricsV2.PROCESSING_STARTED_LATENCY_GAUGE).gauges())
+        .hasSize(1);
+  }
+
+  @SuppressWarnings("unchecked")
+  private static Controller<ConfigMap> controller() {
+    Controller<ConfigMap> controller = mock(Controller.class);
+    ControllerConfiguration<ConfigMap> configuration = mock(ControllerConfiguration.class);
+    when(controller.getConfiguration()).thenReturn(configuration);
+    when(configuration.getName()).thenReturn(CONTROLLER_NAME);
+    return controller;
+  }
+
+  private static void forceGarbageCollection() {
+    for (int i = 0; i < 5; i++) {
+      System.gc();
+    }
+  }
+}