From a22e1fc0baa8aa721132fbac57250456db878bdd Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov <160598371+comandeo-mongo@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:49:37 +0100 Subject: [PATCH 1/3] RUBY-3952 Run DriverBench under OpenTelemetry configurations The OpenTelemetry spec requires drivers to benchmark their tracing in four configurations (off, api-only, sdk-ratio, sdk-always) with the standard DriverBench infrastructure, and to record each one's overhead over the untraced baseline as a metric of its own. driver_bench:run now runs under the configuration named by DRIVER_BENCH_CONFIGURATION and can be narrowed with DRIVER_BENCH_TASKS. driver_bench:compare runs each configuration in its own process, interleaved and rotated across repetitions, and reports throughput loss, CPU time and allocations per operation relative to off. Results for other configurations are reported under " []" so that the baseline series is unchanged. --- profile/driver_bench/base.rb | 83 +++++++- profile/driver_bench/comparison.rb | 268 ++++++++++++++++++++++++ profile/driver_bench/configuration.rb | 174 +++++++++++++++ profile/driver_bench/rake/tasks.rake | 16 ++ profile/driver_bench/single_doc/base.rb | 6 + profile/driver_bench/suite.rb | 52 ++++- 6 files changed, 587 insertions(+), 12 deletions(-) create mode 100644 profile/driver_bench/comparison.rb create mode 100644 profile/driver_bench/configuration.rb diff --git a/profile/driver_bench/base.rb b/profile/driver_bench/base.rb index b65de595ac..d23d12c62c 100644 --- a/profile/driver_bench/base.rb +++ b/profile/driver_bench/base.rb @@ -3,6 +3,7 @@ require 'benchmark' require 'mongo' +require_relative 'configuration' require_relative 'percentiles' module Mongo @@ -41,11 +42,20 @@ def self.bench_name(benchmark_name = nil) # Instantiate a new micro-benchmark class. def initialize - @max_iterations = debug_mode? ? 10 : 100 - @min_time = debug_mode? ? 1 : 60 + @max_iterations = Integer(ENV['DRIVER_BENCH_MAX_ITERATIONS'] || (debug_mode? ? 10 : 100)) + @min_time = Float(ENV['DRIVER_BENCH_MIN_TIME'] || (debug_mode? ? 1 : 60)) @max_time = 300 # 5 minutes end + # The number of driver operations one iteration performs, or nil when + # the task has no meaningful per-operation unit. Per-operation metrics + # are only reported for tasks that define it. + # + # @return [ Integer | nil ] operations per iteration. + def ops_per_iteration + nil + end + def debug_mode? ENV['PERF_DEBUG'] end @@ -61,7 +71,8 @@ def run { name: self.class.bench_name, score: score, - percentiles: percentiles } + percentiles: percentiles, + per_op: per_op_metrics(percentiles) } end private @@ -82,9 +93,13 @@ def run_benchmark setup + @cpu_times = [] + @gc_times = [] + @allocations = [] + loop do before_task - timing = consider_gc { Benchmark.realtime { debug_mode? ? sleep(0.1) : do_task } } + timing = consider_gc { measure_iteration { debug_mode? ? sleep(0.1) : do_task } } after_task iteration_count += 1 @@ -104,9 +119,65 @@ def run_benchmark end end - # Instantiate a new client. + # Times one iteration, and records alongside the wall-clock time the + # process CPU time and the number of objects allocated. CPU time leaves + # out the time spent waiting on the server, and allocations are nearly + # deterministic, so both show driver-side cost with less noise than + # throughput. Process CPU time includes the driver's background threads + # (e.g. server monitoring), which cost the same in every configuration. + # + # GC time is recorded too, where the runtime reports it (Ruby 3.1+). + # When and how long the collector runs varies between processes more + # than the cost of tracing does, so CPU time without GC is the steadier + # measure of the driver's own work. + # + # @return [ Float ] the wall-clock time in seconds. + def measure_iteration(&block) + allocated = GC.stat(:total_allocated_objects) + gc = gc_time + cpu = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID) + timing = Benchmark.realtime(&block) + @cpu_times.push(Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID) - cpu) + @gc_times.push(gc_time - gc) if gc + @allocations.push(GC.stat(:total_allocated_objects) - allocated) + timing + end + + # @return [ Float | nil ] the total time spent in GC so far, in + # seconds, or nil when the runtime does not report it. + def gc_time + ms = GC.stat[:time] + ms && (ms / 1000.0) + end + + # Per-operation medians, for tasks that define ops_per_iteration. + # + # @return [ Hash ] metric name to value. + def per_op_metrics(timings) + return {} unless ops_per_iteration + + ops = ops_per_iteration.to_f + { + 'wall_us_per_op' => timings[50] / ops * 1_000_000, + 'cpu_us_per_op' => Percentiles.new(@cpu_times)[50] / ops * 1_000_000, + 'allocs_per_op' => Percentiles.new(@allocations)[50] / ops + }.merge(gc_metrics(ops)) + end + + def gc_metrics(ops) + return {} if @gc_times.empty? + + cpu_ex_gc = @cpu_times.zip(@gc_times).map { |cpu, gc| cpu - gc } + { + 'gc_us_per_op' => Percentiles.new(@gc_times)[50] / ops * 1_000_000, + 'cpu_ex_gc_us_per_op' => Percentiles.new(cpu_ex_gc)[50] / ops * 1_000_000 + } + end + + # Instantiate a new client, with the options of the configuration + # being measured. def new_client(uri = ENV['MONGODB_URI']) - Mongo::Client.new(uri) + Mongo::Client.new(uri, Configuration.current.client_options) end # Takes care of garbage collection considerations before diff --git a/profile/driver_bench/comparison.rb b/profile/driver_bench/comparison.rb new file mode 100644 index 0000000000..27f33ae8cd --- /dev/null +++ b/profile/driver_bench/comparison.rb @@ -0,0 +1,268 @@ +# frozen_string_literal: true + +require 'json' +require 'tmpdir' + +require_relative 'configuration' + +module Mongo + module DriverBench + # Runs DriverBench micro-benchmarks under several driver configurations + # and reports what each configuration costs relative to the baseline, as + # the OpenTelemetry spec's Benchmarking section requires. + # + # OpenTelemetry can only be installed into a process once, and the + # "api-only" configuration requires that the SDK was never loaded, so each + # configuration is measured in its own subprocess. + # + # Configurations are run interleaved -- the whole set, then the whole set + # again -- rather than one configuration to completion and then the next. + # The quantity being measured is a difference between two configurations, + # and a machine that slows down halfway through a run would otherwise put + # all of one configuration's samples in the fast half and all of another's + # in the slow half, which shows up as overhead that is not there. The + # order is also rotated by one position each repetition, so that no + # configuration always runs in the same slot. + # + # Parameterised by the environment: + # + # CONFIGURATIONS comma-separated configuration names (default: all) + # REPS repetitions of the whole set (default 1) + # MONGODB_URI server to benchmark + # DRIVER_BENCH_TASKS + # the micro-benchmarks to run, as for Suite (default: + # the two the spec requires) + # PERFORMANCE_RESULTS_FILE + # where the combined results are written + # DRIVER_BENCH_MAX_ITERATIONS, DRIVER_BENCH_MIN_TIME + # passed through to every run, see Base + # RUBY_YJIT_ENABLE + # run with YJIT; fails if this ruby was built without it + # + # @api private + class Comparison + DEFAULT_TASKS = 'small doc insertone,find one by id' + DEFAULT_RESULTS_FILE = 'perf-comparison.json' + + # Metrics carried from every child run into the comparison. Each is + # reduced to its median across repetitions. + CARRIED_METRICS = %w[ + score wall_us_per_op cpu_us_per_op cpu_ex_gc_us_per_op gc_us_per_op allocs_per_op + time-90% time-99% + ].freeze + + TABLE_FORMAT = '%-22s %-12s %9s %15s %11s %9s %9s %9s' + + def self.run! + new.run + end + + def initialize(env = ENV) + @env = env + @reps = Integer(env['REPS'] || 1) + @configurations = requested_configurations + @samples = Hash.new { |hash, key| hash[key] = Hash.new { |h, k| h[k] = [] } } + end + + # Runs every configuration, @reps times, and reports the comparison. + # + # @return [ String ] the summary table. + def run + raise 'the baseline configuration is required to compare against' unless + @configurations.any?(&:baseline?) + + # Checked here as well as in every child, to fail before the first + # of many long runs rather than inside it. + Configuration.check_jit! + + Dir.mktmpdir('driver-bench') do |dir| + 1.upto(@reps) do |rep| + @configurations.rotate(rep - 1).each { |configuration| measure(configuration, rep, dir) } + end + end + + save_perf_data(compile_perf_data) + summarize + end + + private + + def requested_configurations + names = (@env['CONFIGURATIONS'] || '').split(',').map(&:strip).reject(&:empty?) + return Configuration::ALL if names.empty? + + names.map { |name| Configuration[name] } + end + + def baseline + @baseline ||= @configurations.find(&:baseline?) + end + + # Runs one configuration once, in its own process, and records the + # metrics of every micro-benchmark it reported. + def measure(configuration, rep, dir) + results_file = File.join(dir, "#{configuration.name}-#{rep}.json") + + puts format("\n----- rep %d/%d: %s -----", rep, @reps, configuration.name) + unless system(child_env(configuration, results_file), 'bundle', 'exec', 'rake', 'driver_bench:run') + raise "configuration #{configuration.name} failed in rep #{rep}" + end + + JSON.parse(File.read(results_file)).each do |entry| + samples = @samples[[ configuration.task_name(entry['info']['test_name']), configuration.name ]] + entry['metrics'].each do |metric| + samples[metric['name']] << metric['value'] if CARRIED_METRICS.include?(metric['name']) + end + end + end + + def child_env(configuration, results_file) + { + Configuration::ENV_VAR => configuration.name, + 'PERFORMANCE_RESULTS_FILE' => results_file, + 'DRIVER_BENCH_TASKS' => @env['DRIVER_BENCH_TASKS'] || DEFAULT_TASKS + } + end + + # The median of one metric across repetitions. Unlike the spec's + # nearest-rank percentile, which suits its iteration counts, this + # averages the middle two of an even number of repetitions: nearest + # rank would always pick the lower one. + def median(task, configuration, metric) + values = @samples[[ task, configuration.name ]][metric].sort + return nil if values.empty? + + mid = values.length / 2 + values.length.odd? ? values[mid] : (values[mid - 1] + values[mid]) / 2.0 + end + + def spread(task, configuration) + values = @samples[[ task, configuration.name ]]['score'] + return nil if values.empty? + + values.minmax + end + + # Scores are throughput, so enabling a feature shows up as a loss. + # + # @return [ Float | nil ] the percentage of throughput given up, + # relative to the baseline. + def loss_for(task, configuration) + reference = median(task, baseline, 'score') + score = median(task, configuration, 'score') + return nil if reference.nil? || score.nil? || reference.zero? + + (reference - score) / reference * 100.0 + end + + # CPU time and allocations grow with cost, so the overhead is the + # difference over the baseline. + def added(task, configuration, metric) + reference = median(task, baseline, metric) + value = median(task, configuration, metric) + return nil if reference.nil? || value.nil? + + value - reference + end + + def cpu_overhead_pct(task, configuration, metric = 'cpu_us_per_op') + reference = median(task, baseline, metric) + delta = added(task, configuration, metric) + return nil if delta.nil? || reference.zero? + + delta / reference * 100.0 + end + + def tasks + @tasks ||= @samples.keys.map(&:first).uniq + end + + def compile_perf_data + tasks.flat_map do |task| + @configurations.filter_map do |configuration| + next if median(task, configuration, 'score').nil? + + { + 'info' => { + 'test_name' => configuration.perf_test_name(task), + 'args' => {} + }, + 'metrics' => metrics_for(task, configuration) + } + end + end + end + + def metrics_for(task, configuration) + metrics = CARRIED_METRICS.filter_map do |name| + value = median(task, configuration, name) + { 'name' => name, 'value' => value } unless value.nil? + end + min, max = spread(task, configuration) + metrics << { 'name' => 'score_min', 'value' => min } << { 'name' => 'score_max', 'value' => max } + return metrics if configuration.baseline? + + # The overheads are recorded as metrics of their own, rather than + # left to be derived by comparing two time series later, so that + # they can be watched for regressions directly and so that + # host-to-host variation cancels out of them. + { + 'overhead_pct' => loss_for(task, configuration), + 'cpu_overhead_pct' => cpu_overhead_pct(task, configuration), + 'cpu_us_added_per_op' => added(task, configuration, 'cpu_us_per_op'), + 'cpu_ex_gc_overhead_pct' => cpu_overhead_pct(task, configuration, 'cpu_ex_gc_us_per_op'), + 'cpu_ex_gc_us_added_per_op' => added(task, configuration, 'cpu_ex_gc_us_per_op'), + 'gc_us_added_per_op' => added(task, configuration, 'gc_us_per_op'), + 'allocs_added_per_op' => added(task, configuration, 'allocs_per_op') + }.each do |name, value| + metrics << { 'name' => name, 'value' => value } unless value.nil? + end + metrics + end + + def save_perf_data(data, file_name: @env['PERFORMANCE_RESULTS_FILE'] || DEFAULT_RESULTS_FILE) + File.write(file_name, data.to_json) + end + + def summarize + lines = [ format("\n===== Configuration comparison (%d rep%s, median) =====", + @reps, (@reps == 1) ? '' : 's') ] + lines << 'loss: throughput given up vs off; spread: min-max MB/s across reps; ' \ + 'cpu: CPU us added per op, excluding GC where reported; gc: GC us added per op; ' \ + 'allocs: objects added per op' + lines << '' + lines << format(TABLE_FORMAT, + task: 'micro-benchmark', config: 'config', loss: 'loss %', spread: 'spread MB/s', + cpu: 'cpu us/op', cpu_pct: 'cpu %', gc: 'gc us/op', allocs: 'allocs') + tasks.each do |task| + @configurations.reject(&:baseline?).each { |configuration| lines << row(task, configuration) } + end + + lines.join("\n") + end + + def row(task, configuration) + min, max = spread(task, configuration) + + format(TABLE_FORMAT, + task: task, config: configuration.name, + loss: signed(loss_for(task, configuration)), + spread: min ? format('%.4g-%.4g', min, max) : 'n/a', + cpu: signed(added(task, configuration, cpu_metric(task))), + cpu_pct: signed(cpu_overhead_pct(task, configuration, cpu_metric(task))), + gc: signed(added(task, configuration, 'gc_us_per_op')), + allocs: signed(added(task, configuration, 'allocs_per_op'), '%+.0f')) + end + + # CPU time excluding GC where the runtime reports GC time, total CPU + # time otherwise. + def cpu_metric(task) + median(task, baseline, 'cpu_ex_gc_us_per_op') ? 'cpu_ex_gc_us_per_op' : 'cpu_us_per_op' + end + + def signed(value, pattern = '%+.2f') + value.nil? ? 'n/a' : format(pattern, value) + end + end + end +end diff --git a/profile/driver_bench/configuration.rb b/profile/driver_bench/configuration.rb new file mode 100644 index 0000000000..a632ff88b8 --- /dev/null +++ b/profile/driver_bench/configuration.rb @@ -0,0 +1,174 @@ +# frozen_string_literal: true + +module Mongo + module DriverBench + # One driver configuration under which the DriverBench tasks can be run. + # + # A benchmark result is identified by the pair (task, configuration), so + # every configuration becomes its own time series and can be watched for + # regressions independently. Comparing a configuration against the + # baseline is what catches a performance regression in an opt-in feature + # such as OpenTelemetry: no benchmark that only ever runs the default + # configuration will execute that code at all. + # + # The configurations are the ones the OpenTelemetry spec's Benchmarking + # section requires. OpenTelemetry cannot be reconfigured once its SDK has + # been installed into a process, and the "api-only" configuration + # requires that the SDK was never loaded, so exactly one configuration + # can be measured per process. Comparison runs one subprocess per + # configuration. + # + # @api private + class Configuration + # The environment variable naming the configuration to run under. + ENV_VAR = 'DRIVER_BENCH_CONFIGURATION' + + # The configuration every other one is compared against. + BASELINE = 'off' + + # @return [ String ] the short name of the configuration, used to key + # results and to select the configuration from the environment. + attr_reader :name + + # @return [ String ] a human-readable description of the configuration. + attr_reader :description + + # @return [ Hash ] options to pass to every Mongo::Client the + # benchmarks construct. + attr_reader :client_options + + # @param name [ String ] the short name of the configuration. + # @param description [ String ] what the configuration measures. + # @param otel [ Symbol ] how much of OpenTelemetry to load: +:none+ for + # nothing, +:api+ for the API without an SDK, +:sdk+ for a configured + # SDK. + # @param client_options [ Hash ] options for every client. + # @param sampler_env [ Hash ] OpenTelemetry environment variables that + # select the sampler, applied before the SDK is configured. + def initialize(name:, description:, otel:, client_options: {}, sampler_env: {}) + @name = name + @description = description + @otel = otel + @client_options = client_options + @sampler_env = sampler_env + end + + # The test name to report a task's results under. perf.send only + # accepts integer args, so the configuration goes into the name. The + # baseline keeps the bare task name, so that its series continues the + # one recorded before configurations existed. + # + # @param task [ String ] the micro-benchmark name. + # + # @return [ String ] the reported test name. + def perf_test_name(task) + baseline? ? task : "#{task} [#{name}]" + end + + # The inverse of #perf_test_name. + # + # @param test_name [ String ] a reported test name. + # + # @return [ String ] the micro-benchmark name. + def task_name(test_name) + test_name.delete_suffix(" [#{name}]") + end + + # Whether this configuration is the baseline that others are compared + # against. + def baseline? + name == BASELINE + end + + # Loads and configures OpenTelemetry for this configuration. + # + # Must run before any Mongo::Client is constructed: a client decides + # whether tracing is active when it builds its tracer, and that + # decision depends on whether ::OpenTelemetry is defined. + def install! + case @otel + when :api then require 'opentelemetry-api' + when :sdk then install_sdk! + end + end + + # All configurations, baseline first. + ALL = [ + new(name: 'off', + description: 'tracing disabled; the baseline', + otel: :none), + + new(name: 'api-only', + description: 'tracing enabled; OpenTelemetry API without an SDK, so every tracing call is a no-op', + otel: :api, + client_options: { tracing: { enabled: true } }), + + new(name: 'sdk-ratio', + description: 'tracing enabled; SDK installed; TraceIdRatioBased sampler recording 1% of traces', + otel: :sdk, + client_options: { tracing: { enabled: true } }, + sampler_env: { 'OTEL_TRACES_SAMPLER' => 'traceidratio', + 'OTEL_TRACES_SAMPLER_ARG' => '0.01' }), + + new(name: 'sdk-always', + description: 'tracing enabled; SDK installed; every span sampled', + otel: :sdk, + client_options: { tracing: { enabled: true } }, + sampler_env: { 'OTEL_TRACES_SAMPLER' => 'always_on' }) + ].freeze + + # Looks up a configuration by name. + # + # @param name [ String ] the configuration name. + # + # @return [ Configuration ] the named configuration. + # + # @raise [ ArgumentError ] if no such configuration exists. + def self.[](name) + ALL.find { |configuration| configuration.name == name } || + raise(ArgumentError, + "unknown configuration #{name.inspect}; " \ + "known configurations are #{ALL.map(&:name).join(', ')}") + end + + # Fails when YJIT was asked for (RUBY_YJIT_ENABLE) but this ruby runs + # without it. A ruby built without YJIT only warns and carries on, and + # the benchmark would then quietly measure the interpreter, where the + # cost of tracing is several times higher. + def self.check_jit! + return unless %w[1 true yes].include?(ENV['RUBY_YJIT_ENABLE'].to_s.downcase) + return if defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled? + + raise "RUBY_YJIT_ENABLE is set, but #{RUBY_DESCRIPTION} runs without YJIT" + end + + # The configuration the current process is measuring, named by the + # DRIVER_BENCH_CONFIGURATION environment variable. + # + # @return [ Configuration ] the current configuration. + def self.current + @current ||= self[ENV[ENV_VAR] || BASELINE] + end + + private + + def install_sdk! + require 'opentelemetry-sdk' + + # The SDK reads its sampler and exporter from the environment when it + # is configured, so set them here rather than trusting the caller to: + # a configuration must mean the same thing however the suite was + # invoked. + @sampler_env.each { |key, value| ENV[key] = value } + + # No exporter, and so no span processor either, as the spec + # recommends. Sampled spans are still fully recorded without one; a + # processor would charge the SDK's export machinery to the driver and + # add variance. + ENV['OTEL_TRACES_EXPORTER'] = 'none' + + ::OpenTelemetry::SDK.configure + end + end + end +end diff --git a/profile/driver_bench/rake/tasks.rake b/profile/driver_bench/rake/tasks.rake index 16d038b85b..e8098375ec 100644 --- a/profile/driver_bench/rake/tasks.rake +++ b/profile/driver_bench/rake/tasks.rake @@ -40,6 +40,22 @@ namespace :driver_bench do Mongo::DriverBench::Suite.run! end + desc 'Compares DriverBench micro-benchmarks across driver configurations' + task compare: :data do + require_relative '../comparison' + + puts Mongo::DriverBench::Comparison.run! + end + + desc 'Lists the driver configurations the suite can run under' + task :configurations do + require_relative '../configuration' + + Mongo::DriverBench::Configuration::ALL.each do |configuration| + puts format('%-12s %s', configuration.name, configuration.description) + end + end + desc 'Runs the crypto benchmark' task :crypto do require_relative '../crypto/decrypt' diff --git a/profile/driver_bench/single_doc/base.rb b/profile/driver_bench/single_doc/base.rb index cf540a9a4e..a2c7e5b0d6 100644 --- a/profile/driver_bench/single_doc/base.rb +++ b/profile/driver_bench/single_doc/base.rb @@ -28,6 +28,12 @@ def scale 10_000 end + # Every single-doc task performs one driver operation per unit of + # scale. + def ops_per_iteration + scale + end + def teardown cleanup_client end diff --git a/profile/driver_bench/suite.rb b/profile/driver_bench/suite.rb index 40259872fb..f7d7708966 100644 --- a/profile/driver_bench/suite.rb +++ b/profile/driver_bench/suite.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative 'bson' +require_relative 'configuration' require_relative 'multi_doc' require_relative 'parallel' require_relative 'single_doc' @@ -46,23 +47,60 @@ def self.run! end def run + Configuration.check_jit! + configuration.install! + announce_configuration + perf_data = [] benches = Hash.new { |h, k| h[k] = [] } - ALL.each do |klass| + tasks.each do |klass| result = run_benchmark(klass) perf_data << compile_perf_data(result) append_to_benchmarks(klass, result, benches) end - perf_data += compile_benchmarks(benches) + # The composites average a fixed list of micro-benchmarks, so they are + # only meaningful when every micro-benchmark ran. + perf_data += compile_benchmarks(benches) if tasks == ALL save_perf_data(perf_data) summarize_perf_data(perf_data) end + # The micro-benchmarks to run. + # + # By default this is every micro-benchmark, but DRIVER_BENCH_TASKS + # narrows it to those whose names contain one of a comma-separated list + # of substrings (case-insensitive). + # + # @return [ Array ] the micro-benchmark classes to run. + def tasks + @tasks ||= begin + patterns = ENV['DRIVER_BENCH_TASKS'].to_s.split(',').map { |p| p.strip.downcase }.reject(&:empty?) + selected = ALL.select do |klass| + patterns.empty? || patterns.any? { |pattern| klass.bench_name.downcase.include?(pattern) } + end + raise 'no micro-benchmarks match DRIVER_BENCH_TASKS' if selected.empty? + + (selected == ALL) ? ALL : selected.freeze + end + end + private + def configuration + @configuration ||= Configuration.current + end + + def announce_configuration + puts format('===== DriverBench: %s =====', configuration.name) + puts configuration.description + puts RUBY_DESCRIPTION + puts format('%d of %d micro-benchmarks', tasks.length, ALL.length) unless tasks == ALL + puts + end + def run_benchmark(klass) print klass.bench_name, ': ' $stdout.flush @@ -77,16 +115,18 @@ def compile_perf_data(result) { 'name' => "time-#{percentile}%", 'value' => result[:percentiles][percentile] } end + per_op_data = result[:per_op].map { |name, value| { 'name' => name, 'value' => value } } { 'info' => { - 'test_name' => result[:name], + 'test_name' => configuration.perf_test_name(result[:name]), 'args' => {}, }, 'metrics' => [ { 'name' => 'score', 'value' => result[:score] }, - *percentile_data + *percentile_data, + *per_op_data ] } end @@ -107,7 +147,7 @@ def compile_benchmarks(benches) benches.map do |bench, score| { 'info' => { - 'test_name' => bench, + 'test_name' => configuration.perf_test_name(bench), 'args' => {} }, 'metrics' => [ @@ -119,7 +159,7 @@ def compile_benchmarks(benches) end def summarize_perf_data(data) - puts '===== Performance Results =====' + puts format('===== Performance Results (%s) =====', configuration.name) data.each do |item| puts format('%s : %4.4g', item['info']['test_name'], item['metrics'][0]['value']) next unless item['metrics'].length > 1 From ce5a63674792dcc5850967317508046fff21df6a Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov <160598371+comandeo-mongo@users.noreply.github.com> Date: Wed, 28 Oct 2026 10:49:37 +0100 Subject: [PATCH 2/3] RUBY-3952 Add a daily Evergreen task for the OpenTelemetry comparison Runs Small doc insertOne and Find one by ID under every configuration on one host with YJIT, and sends the overheads to the performance store. --- .evergreen/config.yml | 56 ++++++++++++++++++++++++++++++ .evergreen/config/common.yml.erb | 43 +++++++++++++++++++++++ .evergreen/config/standard.yml.erb | 13 +++++++ 3 files changed, 112 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 885a49c09c..4d116ce637 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -299,6 +299,44 @@ functions: params: file: "${PROJECT_DIRECTORY}/perf.json" + "run benchmark comparison": + # Runs the micro-benchmarks the OpenTelemetry spec requires under every + # configuration it requires, interleaved on one host, and reports what + # each configuration costs relative to the untraced baseline. Fewer + # iterations per run buy more interleaved repetitions in the same time. + # YJIT is on, as in production; the run fails if this ruby was built + # without it. + - command: shell.exec + type: test + params: + shell: bash + working_dir: "src" + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] + script: | + ${PREPARE_SHELL} + TEST_CMD="bundle exec rake driver_bench:compare" \ + REPS="${driver_bench_reps|8}" \ + DRIVER_BENCH_MAX_ITERATIONS="${driver_bench_max_iterations|20}" \ + DRIVER_BENCH_MIN_TIME="${driver_bench_min_time|20}" \ + RUBY_YJIT_ENABLE=1 \ + PERFORMANCE_RESULTS_FILE="$PROJECT_DIRECTORY/perf-comparison.json" \ + .evergreen/run-tests.sh + # Sent to the performance store so that the overhead metrics are watched + # for regressions, and kept as an artifact for comparison by hand. + - command: perf.send + params: + file: "${PROJECT_DIRECTORY}/perf-comparison.json" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ./src/perf-comparison.json + display_name: perf-comparison.json + remote_file: ${UPLOAD_BUCKET}/${version_id}/${build_id}/artifacts/${build_variant}/perf-comparison.json + content_type: application/json + permissions: public-read + bucket: mciuploads + "run tests with orchestration and drivers tools": - command: subprocess.exec type: test @@ -598,6 +636,11 @@ tasks: - name: "driver-bench" commands: - func: "run benchmarks" + # About 1.5 hours at the default eight reps. + - name: "driver-bench-otel" + exec_timeout_secs: 10800 + commands: + - func: "run benchmark comparison" - name: "test-csot" commands: - func: "run CSOT tests" @@ -1131,6 +1174,19 @@ buildvariants: tasks: - name: "driver-bench" + # Compares DriverBench tasks with OpenTelemetry disabled against the + # enabled configurations the OpenTelemetry spec requires. + - matrix_name: DriverBenchOTel + matrix_spec: + ruby: "ruby-4.0" + mongodb-version: "8.0" + topology: standalone + os: ubuntu2204 + display_name: DriverBench OTel + tasks: + - name: "driver-bench-otel" + batchtime: 1440 # run at most once a day + - matrix_name: "auth/ssl" matrix_spec: auth-and-ssl: ["auth-and-ssl", "noauth-and-nossl"] diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index 30ab63e802..b7c18ee9c6 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -296,6 +296,44 @@ functions: params: file: "${PROJECT_DIRECTORY}/perf.json" + "run benchmark comparison": + # Runs the micro-benchmarks the OpenTelemetry spec requires under every + # configuration it requires, interleaved on one host, and reports what + # each configuration costs relative to the untraced baseline. Fewer + # iterations per run buy more interleaved repetitions in the same time. + # YJIT is on, as in production; the run fails if this ruby was built + # without it. + - command: shell.exec + type: test + params: + shell: bash + working_dir: "src" + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] + script: | + ${PREPARE_SHELL} + TEST_CMD="bundle exec rake driver_bench:compare" \ + REPS="${driver_bench_reps|8}" \ + DRIVER_BENCH_MAX_ITERATIONS="${driver_bench_max_iterations|20}" \ + DRIVER_BENCH_MIN_TIME="${driver_bench_min_time|20}" \ + RUBY_YJIT_ENABLE=1 \ + PERFORMANCE_RESULTS_FILE="$PROJECT_DIRECTORY/perf-comparison.json" \ + .evergreen/run-tests.sh + # Sent to the performance store so that the overhead metrics are watched + # for regressions, and kept as an artifact for comparison by hand. + - command: perf.send + params: + file: "${PROJECT_DIRECTORY}/perf-comparison.json" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ./src/perf-comparison.json + display_name: perf-comparison.json + remote_file: ${UPLOAD_BUCKET}/${version_id}/${build_id}/artifacts/${build_variant}/perf-comparison.json + content_type: application/json + permissions: public-read + bucket: mciuploads + "run tests with orchestration and drivers tools": - command: subprocess.exec type: test @@ -595,6 +633,11 @@ tasks: - name: "driver-bench" commands: - func: "run benchmarks" + # About 1.5 hours at the default eight reps. + - name: "driver-bench-otel" + exec_timeout_secs: 10800 + commands: + - func: "run benchmark comparison" - name: "test-csot" commands: - func: "run CSOT tests" diff --git a/.evergreen/config/standard.yml.erb b/.evergreen/config/standard.yml.erb index 1f4e48ae20..1e0974fcf8 100644 --- a/.evergreen/config/standard.yml.erb +++ b/.evergreen/config/standard.yml.erb @@ -61,6 +61,19 @@ buildvariants: tasks: - name: "driver-bench" + # Compares DriverBench tasks with OpenTelemetry disabled against the + # enabled configurations the OpenTelemetry spec requires. + - matrix_name: DriverBenchOTel + matrix_spec: + ruby: <%= latest_ruby %> + mongodb-version: <%= latest_stable_mdb %> + topology: standalone + os: ubuntu2204 + display_name: DriverBench OTel + tasks: + - name: "driver-bench-otel" + batchtime: 1440 # run at most once a day + - matrix_name: "auth/ssl" matrix_spec: auth-and-ssl: ["auth-and-ssl", "noauth-and-nossl"] From 99f56178386940ed5902bd30d4b44e2ee7615e53 Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov <160598371+comandeo-mongo@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:16:59 +0100 Subject: [PATCH 3/3] RUBY-3952 Keep the plain DriverBench results unchanged Per-operation metrics are now reported only when the comparison asks for them, so the daily driver-bench task sends the same metrics to the performance store as before. Also correct the YJIT comment and restore the note on why the task needs AWS credentials. --- .evergreen/config.yml | 8 ++++++-- .evergreen/config/common.yml.erb | 8 ++++++-- profile/driver_bench/base.rb | 12 ++++++++++-- profile/driver_bench/comparison.rb | 3 ++- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 4d116ce637..1951361443 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -304,13 +304,17 @@ functions: # configuration it requires, interleaved on one host, and reports what # each configuration costs relative to the untraced baseline. Fewer # iterations per run buy more interleaved repetitions in the same time. - # YJIT is on, as in production; the run fails if this ruby was built - # without it. + # Every configuration runs with YJIT, which production deployments + # commonly enable; without it the interpreter's per-call cost inflates + # the tracing overhead. The run fails if this ruby was built without + # YJIT. - command: shell.exec type: test params: shell: bash working_dir: "src" + # Needed to download "latest"/"latest-build" from the private S3 + # bucket (DRIVERS-3628). Any other version needs no AWS access. include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] script: | ${PREPARE_SHELL} diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index b7c18ee9c6..4a80506185 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -301,13 +301,17 @@ functions: # configuration it requires, interleaved on one host, and reports what # each configuration costs relative to the untraced baseline. Fewer # iterations per run buy more interleaved repetitions in the same time. - # YJIT is on, as in production; the run fails if this ruby was built - # without it. + # Every configuration runs with YJIT, which production deployments + # commonly enable; without it the interpreter's per-call cost inflates + # the tracing overhead. The run fails if this ruby was built without + # YJIT. - command: shell.exec type: test params: shell: bash working_dir: "src" + # Needed to download "latest"/"latest-build" from the private S3 + # bucket (DRIVERS-3628). Any other version needs no AWS access. include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] script: | ${PREPARE_SHELL} diff --git a/profile/driver_bench/base.rb b/profile/driver_bench/base.rb index d23d12c62c..35b0775337 100644 --- a/profile/driver_bench/base.rb +++ b/profile/driver_bench/base.rb @@ -56,6 +56,13 @@ def ops_per_iteration nil end + # Whether to report per-operation metrics. Comparison asks for them; + # a plain suite run leaves them out, so that the results it sends to + # the performance store are the same as before they existed. + def per_op_metrics? + %w[1 true yes].include?(ENV['DRIVER_BENCH_PER_OP_METRICS'].to_s.downcase) + end + def debug_mode? ENV['PERF_DEBUG'] end @@ -150,11 +157,12 @@ def gc_time ms && (ms / 1000.0) end - # Per-operation medians, for tasks that define ops_per_iteration. + # Per-operation medians, for tasks that define ops_per_iteration, when + # asked for (see #per_op_metrics?). # # @return [ Hash ] metric name to value. def per_op_metrics(timings) - return {} unless ops_per_iteration + return {} unless per_op_metrics? && ops_per_iteration ops = ops_per_iteration.to_f { diff --git a/profile/driver_bench/comparison.rb b/profile/driver_bench/comparison.rb index 27f33ae8cd..37b884db97 100644 --- a/profile/driver_bench/comparison.rb +++ b/profile/driver_bench/comparison.rb @@ -120,7 +120,8 @@ def child_env(configuration, results_file) { Configuration::ENV_VAR => configuration.name, 'PERFORMANCE_RESULTS_FILE' => results_file, - 'DRIVER_BENCH_TASKS' => @env['DRIVER_BENCH_TASKS'] || DEFAULT_TASKS + 'DRIVER_BENCH_TASKS' => @env['DRIVER_BENCH_TASKS'] || DEFAULT_TASKS, + 'DRIVER_BENCH_PER_OP_METRICS' => '1' } end