IGNITE-28885 Add JMX Prometheus Exporter to Ducktests for Monitoring

modules/ducktests/pom.xml

@@ -109,6 +109,15 @@
             <artifactId>simpleclient_httpserver</artifactId>
             <version>${prometheus.version}</version>
         </dependency>
+
+        <!-- JMX Prometheus Exporter (javaagent for Prometheus metrics) -->
+        <dependency>
+            <groupId>io.prometheus.jmx</groupId>
+            <artifactId>jmx_prometheus_javaagent</artifactId>
+            <version>1.0.1</version>
+            <scope>runtime</scope>
+            <type>jar</type>
+        </dependency>
     </dependencies>
 
     <build>

modules/ducktests/tests/docker/requirements.txt

@@ -16,3 +16,4 @@
 filelock==3.8.2
 ducktape==0.13.0
 looseversion==1.3.0
+requests>=2.25.0

modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py

@@ -37,6 +37,8 @@
 from ignitetest.services.utils.background_thread import BackgroundThreadService
 from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue
 from ignitetest.services.utils.ignite_spec import resolve_spec, SHARED_PREPARED_FILE
+from ignitetest.services.utils.jmx_exporter import is_jmx_exporter_enabled, get_jmx_exporter_yml_content, \
+    JMX_EXPORTER_JAR_PATH, JMX_EXPORTER_YML_NAME, jmx_agent_jvm_opt
 from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin, JmxClient
 from ignitetest.services.utils.jvm_utils import JvmProcessMixin, JvmVersionMixin
 from ignitetest.services.utils.log_utils import monitor_log
@@ -177,13 +179,53 @@ def clean_node(self, node, **kwargs):
 
         node.account.ssh("rm -rf -- %s" % self.persistent_root, allow_fail=False)
 
+    def _copy_jmx_exporter_jar(self, node):
+        """
+        Copy jmx_exporter.jar to the node if JMX Exporter is enabled.
+        Locates the jar from Maven's target/libs directory.
+
+        The jar is always built on the host machine in modules/ducktests/target/libs/
+        regardless of whether tests run in Docker or direct mode.
+        """
+        if not is_jmx_exporter_enabled(self.globals):
+            return
+
+        # Path to jar from Maven build (relative to project root)
+        # This path is the same for both Docker and non-Docker modes
+        # Project root is 5 levels up from this file:
+        # utils -> services -> ignitetest -> tests -> ducktests
+        root = __file__
+        for _ in range(5):
+            root = os.path.dirname(root)
+        jar_from = os.path.join(root, "target/libs/jmx_prometheus_javaagent-1.0.1.jar")
+
+        if not os.path.exists(jar_from):
+            self.logger.warning(
+                "JMX Exporter jar not found at %s. Skip copying. "
+                "JMX Exporter requires running 'mvn package' before tests.", jar_from
+            )
+            return
+
+        # Copy jar to persistent_root (universal path for both modes)
+        # In Docker: /mnt/service (default), in non-Docker: /mnt/service (default)
+        jar_to = os.path.join(self.persistent_root, "jmx_exporter.jar")
+        node.account.copy_to(jar_from, jar_to)
+
+        self.logger.debug(
+            "Copied JMX Exporter jar from %s to %s on node %s",
+            jar_from, jar_to, node.account.hostname
+        )
+
     def init_persistent(self, node):
         """
         Init persistent directory.
         :param node: Ignite service node.
         """
         super().init_persistent(node)
 
+        # Copy jmx_exporter.jar if enabled
+        self._copy_jmx_exporter_jar(node)
+
         self._prepare_configs(node)
 
     def init_shared(self, node):
@@ -231,6 +273,13 @@ def _prepare_configs(self, node):
 
             self.logger.debug("Config %s for node %s: %s" % (name, node.account.hostname, config_txt))
 
+        # Create jmx_exporter.yml config if enabled
+        if is_jmx_exporter_enabled(self.globals):
+            yml_on_node = os.path.join(self.config_dir, JMX_EXPORTER_YML_NAME)
+            yml_content = get_jmx_exporter_yml_content()
+            node.account.create_file(yml_on_node, yml_content)
+            self.logger.debug("JMX Exporter config %s created on node %s" % (yml_on_node, node.account.hostname))
+
         setattr(node, "consistent_id", node.account.externally_routable_ip)
 
     def worker(self, idx, node, **kwargs):

modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py

@@ -36,6 +36,8 @@
 from ignitetest.services.utils.metrics.metrics import is_opencensus_metrics_enabled, configure_opencensus_metrics, \
     is_jmx_metrics_enabled, configure_jmx_metrics
 from ignitetest.services.utils.jmx_remote.jmx_remote_params import get_jmx_remote_params
+from ignitetest.services.utils.jmx_exporter import get_jmx_exporter_params, \
+    jmx_agent_jvm_opt, JMX_EXPORTER_YML_NAME
 from ignitetest.utils.ignite_test import JFR_ENABLED, SAFEPOINT_LOGS_ENABLED
 from ignitetest.utils.version import DEV_BRANCH
 
@@ -130,6 +132,15 @@ def __get_default_jvm_opts(self):
                                                    "-Dcom.sun.management.jmxremote.authenticate=false",
                                                    "-Dcom.sun.management.jmxremote.ssl=false"])
 
+        jmx_exp_cfg = get_jmx_exporter_params(self.service.context.globals)
+        if jmx_exp_cfg.enabled:
+            # Конфиг jmx_exporter.yml будет создан в config_dir на ноде
+            yml_on_node = os.path.join(self.service.config_dir, JMX_EXPORTER_YML_NAME)
+            # JMX Exporter jar is copied to persistent_root/jmx_exporter.jar in ignite_aware.py
+            jar_on_node = os.path.join(self.service.persistent_root, "jmx_exporter.jar")
+            default_jvm_opts = merge_jvm_settings(default_jvm_opts,
+                                                  [jmx_agent_jvm_opt(jmx_exp_cfg.port, yml_on_node, jar_on_node)])
+
         return default_jvm_opts
 
     def config_templates(self):

modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py

@@ -0,0 +1,90 @@
+"""
+JMX Prometheus Exporter helper for ducktests.
+
+Starts jmx_prometheus_javaagent as a JVM agent to expose JVM/Ignite metrics
+on :{port}/metrics in Prometheus text format for external scraping.
+
+Works for both Docker and remote nodes:
+ - jmx_exporter.jar is expected at JMX_EXPORTER_JAR_PATH on each node
+ - jmx_exporter.yml is created in the node's config directory at startup
+"""
+
+import os
+from dataclasses import dataclass
+from importlib.resources import files
+
+JMX_EXPORTER_JAR_PATH = "/opt/jmx_exporter.jar"
+
+JMX_EXPORTER_YML_NAME = "jmx_exporter.yml"
+
+
+@dataclass(frozen=True)
+class JmxExporterParams:
+    """
+    JMX Prometheus Exporter settings from globals config.
+
+    Example in globals:
+        {
+            "jmx_exporter": {
+                "enabled": true,
+                "port": 8083
+            }
+        }
+    """
+    enabled: bool
+    port: int
+
+
+def get_jmx_exporter_params(globals_cfg: dict = None) -> JmxExporterParams:
+    """
+    Read JMX Exporter parameters from globals configuration.
+    """
+    if globals_cfg is None:
+        globals_cfg = {}
+
+    cfg = globals_cfg.get("jmx_exporter", {})
+    if not isinstance(cfg, dict):
+        return JmxExporterParams(enabled=True, port=8083)
+
+    return JmxExporterParams(
+        enabled=bool(cfg.get("enabled", True)),
+        port=int(cfg.get("port", 8083)),
+    )
+
+
+def is_jmx_exporter_enabled(globals_cfg: dict = None) -> bool:
+    """
+    Check if JMX Exporter is enabled in globals.
+    """
+    return get_jmx_exporter_params(globals_cfg).enabled
+
+
+def get_jmx_exporter_yml_content() -> str:
+    """
+    Return the bundled jmx_exporter.yml content as a string.
+    """
+    file = (files(__package__)
+           .joinpath("jmx_exporter.yml"))
+    return file.read_text(encoding="utf-8")
+
+
+def jmx_agent_jvm_opt(port: int = 8083, config_path: str = None, jar_path: str = None) -> str:
+    """
+    Build -javaagent JVM option for jmx_prometheus_javaagent.
+
+    :param port: HTTP port for /metrics endpoint
+    :param config_path: Absolute path to jmx_exporter.yml ON THE NODE
+                        (e.g., /mnt/service/config/jmx_exporter.yml)
+    :param jar_path: Absolute path to jmx_exporter.jar ON THE NODE
+                     (e.g., /opt/jmx_exporter.jar or /mnt/service/jmx_exporter.jar)
+                     If None, uses default JMX_EXPORTER_JAR_PATH
+    :return: JVM argument string, e.g.
+             -javaagent:/opt/jmx_exporter.jar=8083:/mnt/service/config/jmx_exporter.yml
+    """
+    if config_path is None:
+        config_path = JMX_EXPORTER_YML_NAME
+
+    if jar_path is None:
+        jar_path = JMX_EXPORTER_JAR_PATH
+
+    return f"-javaagent:{jar_path}={port}:{config_path}"

modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/jmx_exporter.yml

@@ -0,0 +1,66 @@
+# JVM Memory & GC metrics for Prometheus scraping
+# Sufficient to reproduce jconsole "Memory" tab in Grafana.
+#
+# Exposed on :8083/metrics
+
+lowercaseOutputName: true
+whitelistObjectNames:
+  - "java.lang:type=Memory"
+  - "java.lang:type=MemoryPool,*"
+  - "java.lang:type=GarbageCollector,*"
+
+rules:
+  # ------------------------------------------------------------------
+  # Heap / Non-Heap overview  (java.lang:type=Memory)
+  #   Attributes: heapMemoryUsage (CompositeData: used/init/max/committed)
+  #               nonHeapMemoryUsage (CompositeData)
+  # ------------------------------------------------------------------
+  - pattern: 'java.lang<type=Memory><heapMemoryUsage>(\\w+)'
+    name: "jvm_memory_heap_bytes_$1"
+    type: GAUGE
+    help: "JVM heap memory — used / init / max / committed"
+
+  - pattern: 'java.lang<type=Memory><nonHeapMemoryUsage>(\\w+)'
+    name: "jvm_memory_nonheap_bytes_$1"
+    type: GAUGE
+    help: "JVM non-heap memory — used / init / max / committed"
+
+  # ------------------------------------------------------------------
+  # Per-pool usage  (java.lang:type=MemoryPool,name=<pool>)
+  #   Attributes: usage (CompositeData: used/init/max/committed)
+  #               collectionUsage (CompositeData, post-GC usage)
+  # ------------------------------------------------------------------
+  - pattern: 'java.lang<type=MemoryPool,name=(.*?)><usage>(\\w+)'
+    name: "jvm_memory_pool_usage_bytes"
+    labels:
+      pool: "$1"
+      attribute: "$2"
+    type: GAUGE
+    help: "JVM memory pool usage — used / init / max / committed"
+
+  - pattern: 'java.lang<type=MemoryPool,name=(.*?)><collectionUsage>(\\w+)'
+    name: "jvm_memory_pool_collection_bytes"
+    labels:
+      pool: "$1"
+      attribute: "$2"
+    type: GAUGE
+    help: "JVM memory pool post-GC usage — used / init / max / committed"
+
+  # ------------------------------------------------------------------
+  # Garbage Collectors  (java.lang:type=GarbageCollector,name=<gc>)
+  #   Attributes: collectionCount, collectionTime
+  # ------------------------------------------------------------------
+  - pattern: "java.lang<type=GarbageCollector,name=(.*?)><collectionCount>"
+    name: "jvm_gc_collection_count_total"
+    labels:
+      gc: "$1"
+    type: COUNTER
+    help: "Total number of GC collections"
+
+  - pattern: "java.lang<type=GarbageCollector,name=(.*?)><collectionTime>"
+    name: "jvm_gc_collection_time_seconds_total"
+    labels:
+      gc: "$1"
+    type: COUNTER
+    help: "Total GC collection time (ms → seconds via divisor)"
+    valueFactor: 0.001

modules/ducktests/tests/ignitetest/tests/jmx_metrics_test.py

@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains tests for JMX Prometheus Exporter metrics collection.
+"""
+
+import requests
+import time
+from ignitetest.services.ignite import IgniteService
+from ignitetest.services.utils.ignite_configuration import IgniteConfiguration
+from ignitetest.utils import cluster
+from ignitetest.utils.ignite_test import IgniteTest
+from ignitetest.utils.version import DEV_BRANCH, IgniteVersion
+
+
+class JmxMetricsTest(IgniteTest):
+    """
+    Tests JMX Prometheus Exporter metrics collection
+    """
+
+    @cluster(num_nodes=1)
+    def test_jmx_metrics_collection(self):
+        """
+        Test that JMX Exporter collects JVM memory and GC metrics
+        """
+        ignite = IgniteService(
+            self.test_context, 
+            IgniteConfiguration(version=IgniteVersion(DEV_BRANCH)),
+            num_nodes=1
+        )
+        ignite.start()
+
+        node = ignite.nodes[0]
+
+        # Get the node's IP address
+        node_ip = node.account.externally_routable_ip
+
+        # JMX Exporter exposes metrics on port 8083
+        metrics_url = f"http://{node_ip}:8083/metrics"
+
+        self.logger.info(f"Fetching metrics from {metrics_url}")
+
+        # Wait for JMX Exporter to be ready (status 200)
+        metrics_text = self._wait_for_metrics(metrics_url, timeout=60)
+        self.logger.info(f"JMX Exporter is ready. Received metrics (first 2000 chars):\n{metrics_text[:2000]}")
+
+        # Check for expected JVM memory metrics (in default JMX Exporter format)
+        assert "jvm_memory_used_bytes" in metrics_text, f"Missing jvm_memory_used_bytes metric"
+        assert "jvm_memory_committed_bytes" in metrics_text, f"Missing jvm_memory_committed_bytes metric"
+
+        # Check for GC metrics (in default JMX Exporter format)
+        assert "jvm_gc_collection_seconds" in metrics_text, f"Missing jvm_gc_collection_seconds metric"
+
+        self.logger.info("All expected JMX metrics are present!")
+
+        ignite.stop()
+
+    def _wait_for_metrics(self, url, timeout=60):
+        """
+        Wait for metrics endpoint to return status 200.
+        :param url: URL to metrics endpoint
+        :param timeout: Maximum wait time in seconds
+        :return: Metrics text
+        """
+        start_time = time.time()
+
+        while time.time() - start_time < timeout:
+            try:
+                response = requests.get(url, timeout=5)
+                if response.status_code == 200:
+                    return response.text
+            except requests.exceptions.RequestException as e:
+                self.logger.debug(f"Waiting for metrics: {e}")
+
+            time.sleep(1)
+
+        raise TimeoutError(f"Failed to get metrics from {url} within {timeout} seconds")