diff --git a/modules/ducktests/pom.xml b/modules/ducktests/pom.xml index 9f50b860f59ec..869f6dbdf2a9f 100644 --- a/modules/ducktests/pom.xml +++ b/modules/ducktests/pom.xml @@ -109,6 +109,15 @@ simpleclient_httpserver ${prometheus.version} + + + + io.prometheus.jmx + jmx_prometheus_javaagent + 1.0.1 + runtime + jar + diff --git a/modules/ducktests/tests/docker/requirements.txt b/modules/ducktests/tests/docker/requirements.txt index aba4f25a86d1d..e0fe52a947cc3 100644 --- a/modules/ducktests/tests/docker/requirements.txt +++ b/modules/ducktests/tests/docker/requirements.txt @@ -16,3 +16,4 @@ filelock==3.8.2 ducktape==0.13.0 looseversion==1.3.0 +requests>=2.25.0 diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py index 7d475844a2ed1..3bd09b056bc51 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py @@ -37,6 +37,8 @@ from ignitetest.services.utils.background_thread import BackgroundThreadService from ignitetest.services.utils.concurrent import CountDownLatch, AtomicValue from ignitetest.services.utils.ignite_spec import resolve_spec, SHARED_PREPARED_FILE +from ignitetest.services.utils.jmx_exporter import is_jmx_exporter_enabled, get_jmx_exporter_yml_content, \ + JMX_EXPORTER_JAR_PATH, JMX_EXPORTER_YML_NAME, jmx_agent_jvm_opt from ignitetest.services.utils.jmx_utils import ignite_jmx_mixin, JmxClient from ignitetest.services.utils.jvm_utils import JvmProcessMixin, JvmVersionMixin from ignitetest.services.utils.log_utils import monitor_log @@ -177,6 +179,43 @@ def clean_node(self, node, **kwargs): node.account.ssh("rm -rf -- %s" % self.persistent_root, allow_fail=False) + def _copy_jmx_exporter_jar(self, node): + """ + Copy jmx_exporter.jar to the node if JMX Exporter is enabled. + Locates the jar from Maven's target/libs directory. + + The jar is always built on the host machine in modules/ducktests/target/libs/ + regardless of whether tests run in Docker or direct mode. + """ + if not is_jmx_exporter_enabled(self.globals): + return + + # Path to jar from Maven build (relative to project root) + # This path is the same for both Docker and non-Docker modes + # Project root is 5 levels up from this file: + # utils -> services -> ignitetest -> tests -> ducktests + root = __file__ + for _ in range(5): + root = os.path.dirname(root) + jar_from = os.path.join(root, "target/libs/jmx_prometheus_javaagent-1.0.1.jar") + + if not os.path.exists(jar_from): + self.logger.warning( + "JMX Exporter jar not found at %s. Skip copying. " + "JMX Exporter requires running 'mvn package' before tests.", jar_from + ) + return + + # Copy jar to persistent_root (universal path for both modes) + # In Docker: /mnt/service (default), in non-Docker: /mnt/service (default) + jar_to = os.path.join(self.persistent_root, "jmx_exporter.jar") + node.account.copy_to(jar_from, jar_to) + + self.logger.debug( + "Copied JMX Exporter jar from %s to %s on node %s", + jar_from, jar_to, node.account.hostname + ) + def init_persistent(self, node): """ Init persistent directory. @@ -184,6 +223,9 @@ def init_persistent(self, node): """ super().init_persistent(node) + # Copy jmx_exporter.jar if enabled + self._copy_jmx_exporter_jar(node) + self._prepare_configs(node) def init_shared(self, node): @@ -231,6 +273,13 @@ def _prepare_configs(self, node): self.logger.debug("Config %s for node %s: %s" % (name, node.account.hostname, config_txt)) + # Create jmx_exporter.yml config if enabled + if is_jmx_exporter_enabled(self.globals): + yml_on_node = os.path.join(self.config_dir, JMX_EXPORTER_YML_NAME) + yml_content = get_jmx_exporter_yml_content() + node.account.create_file(yml_on_node, yml_content) + self.logger.debug("JMX Exporter config %s created on node %s" % (yml_on_node, node.account.hostname)) + setattr(node, "consistent_id", node.account.externally_routable_ip) def worker(self, idx, node, **kwargs): diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index a99aa163026f4..e91c8bb2adcfd 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -36,6 +36,8 @@ from ignitetest.services.utils.metrics.metrics import is_opencensus_metrics_enabled, configure_opencensus_metrics, \ is_jmx_metrics_enabled, configure_jmx_metrics from ignitetest.services.utils.jmx_remote.jmx_remote_params import get_jmx_remote_params +from ignitetest.services.utils.jmx_exporter import get_jmx_exporter_params, \ + jmx_agent_jvm_opt, JMX_EXPORTER_YML_NAME from ignitetest.utils.ignite_test import JFR_ENABLED, SAFEPOINT_LOGS_ENABLED from ignitetest.utils.version import DEV_BRANCH @@ -130,6 +132,15 @@ def __get_default_jvm_opts(self): "-Dcom.sun.management.jmxremote.authenticate=false", "-Dcom.sun.management.jmxremote.ssl=false"]) + jmx_exp_cfg = get_jmx_exporter_params(self.service.context.globals) + if jmx_exp_cfg.enabled: + # Конфиг jmx_exporter.yml будет создан в config_dir на ноде + yml_on_node = os.path.join(self.service.config_dir, JMX_EXPORTER_YML_NAME) + # JMX Exporter jar is copied to persistent_root/jmx_exporter.jar in ignite_aware.py + jar_on_node = os.path.join(self.service.persistent_root, "jmx_exporter.jar") + default_jvm_opts = merge_jvm_settings(default_jvm_opts, + [jmx_agent_jvm_opt(jmx_exp_cfg.port, yml_on_node, jar_on_node)]) + return default_jvm_opts def config_templates(self): diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py new file mode 100644 index 0000000000000..ba62f804ef854 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/__init__.py @@ -0,0 +1,90 @@ +""" +JMX Prometheus Exporter helper for ducktests. + +Starts jmx_prometheus_javaagent as a JVM agent to expose JVM/Ignite metrics +on :{port}/metrics in Prometheus text format for external scraping. + +Works for both Docker and remote nodes: + - jmx_exporter.jar is expected at JMX_EXPORTER_JAR_PATH on each node + - jmx_exporter.yml is created in the node's config directory at startup +""" + +import os +from dataclasses import dataclass +from importlib.resources import files + +JMX_EXPORTER_JAR_PATH = "/opt/jmx_exporter.jar" + +JMX_EXPORTER_YML_NAME = "jmx_exporter.yml" + + +@dataclass(frozen=True) +class JmxExporterParams: + """ + JMX Prometheus Exporter settings from globals config. + + Example in globals: + { + "jmx_exporter": { + "enabled": true, + "port": 8083 + } + } + """ + enabled: bool + port: int + + +def get_jmx_exporter_params(globals_cfg: dict = None) -> JmxExporterParams: + """ + Read JMX Exporter parameters from globals configuration. + """ + if globals_cfg is None: + globals_cfg = {} + + cfg = globals_cfg.get("jmx_exporter", {}) + if not isinstance(cfg, dict): + return JmxExporterParams(enabled=True, port=8083) + + return JmxExporterParams( + enabled=bool(cfg.get("enabled", True)), + port=int(cfg.get("port", 8083)), + ) + + +def is_jmx_exporter_enabled(globals_cfg: dict = None) -> bool: + """ + Check if JMX Exporter is enabled in globals. + """ + return get_jmx_exporter_params(globals_cfg).enabled + + +def get_jmx_exporter_yml_content() -> str: + """ + Return the bundled jmx_exporter.yml content as a string. + """ + file = (files(__package__) + .joinpath("jmx_exporter.yml")) + return file.read_text(encoding="utf-8") + + +def jmx_agent_jvm_opt(port: int = 8083, config_path: str = None, jar_path: str = None) -> str: + """ + Build -javaagent JVM option for jmx_prometheus_javaagent. + + :param port: HTTP port for /metrics endpoint + :param config_path: Absolute path to jmx_exporter.yml ON THE NODE + (e.g., /mnt/service/config/jmx_exporter.yml) + :param jar_path: Absolute path to jmx_exporter.jar ON THE NODE + (e.g., /opt/jmx_exporter.jar or /mnt/service/jmx_exporter.jar) + If None, uses default JMX_EXPORTER_JAR_PATH + :return: JVM argument string, e.g. + -javaagent:/opt/jmx_exporter.jar=8083:/mnt/service/config/jmx_exporter.yml + """ + if config_path is None: + config_path = JMX_EXPORTER_YML_NAME + + if jar_path is None: + jar_path = JMX_EXPORTER_JAR_PATH + + return f"-javaagent:{jar_path}={port}:{config_path}" diff --git a/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/jmx_exporter.yml b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/jmx_exporter.yml new file mode 100644 index 0000000000000..64fd70d633507 --- /dev/null +++ b/modules/ducktests/tests/ignitetest/services/utils/jmx_exporter/jmx_exporter.yml @@ -0,0 +1,66 @@ +# JVM Memory & GC metrics for Prometheus scraping +# Sufficient to reproduce jconsole "Memory" tab in Grafana. +# +# Exposed on :8083/metrics + +lowercaseOutputName: true +whitelistObjectNames: + - "java.lang:type=Memory" + - "java.lang:type=MemoryPool,*" + - "java.lang:type=GarbageCollector,*" + +rules: + # ------------------------------------------------------------------ + # Heap / Non-Heap overview (java.lang:type=Memory) + # Attributes: heapMemoryUsage (CompositeData: used/init/max/committed) + # nonHeapMemoryUsage (CompositeData) + # ------------------------------------------------------------------ + - pattern: 'java.lang(\\w+)' + name: "jvm_memory_heap_bytes_$1" + type: GAUGE + help: "JVM heap memory — used / init / max / committed" + + - pattern: 'java.lang(\\w+)' + name: "jvm_memory_nonheap_bytes_$1" + type: GAUGE + help: "JVM non-heap memory — used / init / max / committed" + + # ------------------------------------------------------------------ + # Per-pool usage (java.lang:type=MemoryPool,name=) + # Attributes: usage (CompositeData: used/init/max/committed) + # collectionUsage (CompositeData, post-GC usage) + # ------------------------------------------------------------------ + - pattern: 'java.lang(\\w+)' + name: "jvm_memory_pool_usage_bytes" + labels: + pool: "$1" + attribute: "$2" + type: GAUGE + help: "JVM memory pool usage — used / init / max / committed" + + - pattern: 'java.lang(\\w+)' + name: "jvm_memory_pool_collection_bytes" + labels: + pool: "$1" + attribute: "$2" + type: GAUGE + help: "JVM memory pool post-GC usage — used / init / max / committed" + + # ------------------------------------------------------------------ + # Garbage Collectors (java.lang:type=GarbageCollector,name=) + # Attributes: collectionCount, collectionTime + # ------------------------------------------------------------------ + - pattern: "java.lang" + name: "jvm_gc_collection_count_total" + labels: + gc: "$1" + type: COUNTER + help: "Total number of GC collections" + + - pattern: "java.lang" + name: "jvm_gc_collection_time_seconds_total" + labels: + gc: "$1" + type: COUNTER + help: "Total GC collection time (ms → seconds via divisor)" + valueFactor: 0.001 diff --git a/modules/ducktests/tests/ignitetest/tests/jmx_metrics_test.py b/modules/ducktests/tests/ignitetest/tests/jmx_metrics_test.py new file mode 100644 index 0000000000000..fb9702e6a2eec --- /dev/null +++ b/modules/ducktests/tests/ignitetest/tests/jmx_metrics_test.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains tests for JMX Prometheus Exporter metrics collection. +""" + +import requests +import time +from ignitetest.services.ignite import IgniteService +from ignitetest.services.utils.ignite_configuration import IgniteConfiguration +from ignitetest.utils import cluster +from ignitetest.utils.ignite_test import IgniteTest +from ignitetest.utils.version import DEV_BRANCH, IgniteVersion + + +class JmxMetricsTest(IgniteTest): + """ + Tests JMX Prometheus Exporter metrics collection + """ + + @cluster(num_nodes=1) + def test_jmx_metrics_collection(self): + """ + Test that JMX Exporter collects JVM memory and GC metrics + """ + ignite = IgniteService( + self.test_context, + IgniteConfiguration(version=IgniteVersion(DEV_BRANCH)), + num_nodes=1 + ) + ignite.start() + + node = ignite.nodes[0] + + # Get the node's IP address + node_ip = node.account.externally_routable_ip + + # JMX Exporter exposes metrics on port 8083 + metrics_url = f"http://{node_ip}:8083/metrics" + + self.logger.info(f"Fetching metrics from {metrics_url}") + + # Wait for JMX Exporter to be ready (status 200) + metrics_text = self._wait_for_metrics(metrics_url, timeout=60) + self.logger.info(f"JMX Exporter is ready. Received metrics (first 2000 chars):\n{metrics_text[:2000]}") + + # Check for expected JVM memory metrics (in default JMX Exporter format) + assert "jvm_memory_used_bytes" in metrics_text, f"Missing jvm_memory_used_bytes metric" + assert "jvm_memory_committed_bytes" in metrics_text, f"Missing jvm_memory_committed_bytes metric" + + # Check for GC metrics (in default JMX Exporter format) + assert "jvm_gc_collection_seconds" in metrics_text, f"Missing jvm_gc_collection_seconds metric" + + self.logger.info("All expected JMX metrics are present!") + + ignite.stop() + + def _wait_for_metrics(self, url, timeout=60): + """ + Wait for metrics endpoint to return status 200. + :param url: URL to metrics endpoint + :param timeout: Maximum wait time in seconds + :return: Metrics text + """ + start_time = time.time() + + while time.time() - start_time < timeout: + try: + response = requests.get(url, timeout=5) + if response.status_code == 200: + return response.text + except requests.exceptions.RequestException as e: + self.logger.debug(f"Waiting for metrics: {e}") + + time.sleep(1) + + raise TimeoutError(f"Failed to get metrics from {url} within {timeout} seconds")