diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 885a49c09c..1951361443 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -299,6 +299,48 @@ functions: params: file: "${PROJECT_DIRECTORY}/perf.json" + "run benchmark comparison": + # Runs the micro-benchmarks the OpenTelemetry spec requires under every + # configuration it requires, interleaved on one host, and reports what + # each configuration costs relative to the untraced baseline. Fewer + # iterations per run buy more interleaved repetitions in the same time. + # Every configuration runs with YJIT, which production deployments + # commonly enable; without it the interpreter's per-call cost inflates + # the tracing overhead. The run fails if this ruby was built without + # YJIT. + - command: shell.exec + type: test + params: + shell: bash + working_dir: "src" + # Needed to download "latest"/"latest-build" from the private S3 + # bucket (DRIVERS-3628). Any other version needs no AWS access. + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] + script: | + ${PREPARE_SHELL} + TEST_CMD="bundle exec rake driver_bench:compare" \ + REPS="${driver_bench_reps|8}" \ + DRIVER_BENCH_MAX_ITERATIONS="${driver_bench_max_iterations|20}" \ + DRIVER_BENCH_MIN_TIME="${driver_bench_min_time|20}" \ + RUBY_YJIT_ENABLE=1 \ + PERFORMANCE_RESULTS_FILE="$PROJECT_DIRECTORY/perf-comparison.json" \ + .evergreen/run-tests.sh + # Sent to the performance store so that the overhead metrics are watched + # for regressions, and kept as an artifact for comparison by hand. + - command: perf.send + params: + file: "${PROJECT_DIRECTORY}/perf-comparison.json" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ./src/perf-comparison.json + display_name: perf-comparison.json + remote_file: ${UPLOAD_BUCKET}/${version_id}/${build_id}/artifacts/${build_variant}/perf-comparison.json + content_type: application/json + permissions: public-read + bucket: mciuploads + "run tests with orchestration and drivers tools": - command: subprocess.exec type: test @@ -598,6 +640,11 @@ tasks: - name: "driver-bench" commands: - func: "run benchmarks" + # About 1.5 hours at the default eight reps. + - name: "driver-bench-otel" + exec_timeout_secs: 10800 + commands: + - func: "run benchmark comparison" - name: "test-csot" commands: - func: "run CSOT tests" @@ -1131,6 +1178,19 @@ buildvariants: tasks: - name: "driver-bench" + # Compares DriverBench tasks with OpenTelemetry disabled against the + # enabled configurations the OpenTelemetry spec requires. + - matrix_name: DriverBenchOTel + matrix_spec: + ruby: "ruby-4.0" + mongodb-version: "8.0" + topology: standalone + os: ubuntu2204 + display_name: DriverBench OTel + tasks: + - name: "driver-bench-otel" + batchtime: 1440 # run at most once a day + - matrix_name: "auth/ssl" matrix_spec: auth-and-ssl: ["auth-and-ssl", "noauth-and-nossl"] diff --git a/.evergreen/config/common.yml.erb b/.evergreen/config/common.yml.erb index 30ab63e802..4a80506185 100644 --- a/.evergreen/config/common.yml.erb +++ b/.evergreen/config/common.yml.erb @@ -296,6 +296,48 @@ functions: params: file: "${PROJECT_DIRECTORY}/perf.json" + "run benchmark comparison": + # Runs the micro-benchmarks the OpenTelemetry spec requires under every + # configuration it requires, interleaved on one host, and reports what + # each configuration costs relative to the untraced baseline. Fewer + # iterations per run buy more interleaved repetitions in the same time. + # Every configuration runs with YJIT, which production deployments + # commonly enable; without it the interpreter's per-call cost inflates + # the tracing overhead. The run fails if this ruby was built without + # YJIT. + - command: shell.exec + type: test + params: + shell: bash + working_dir: "src" + # Needed to download "latest"/"latest-build" from the private S3 + # bucket (DRIVERS-3628). Any other version needs no AWS access. + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] + script: | + ${PREPARE_SHELL} + TEST_CMD="bundle exec rake driver_bench:compare" \ + REPS="${driver_bench_reps|8}" \ + DRIVER_BENCH_MAX_ITERATIONS="${driver_bench_max_iterations|20}" \ + DRIVER_BENCH_MIN_TIME="${driver_bench_min_time|20}" \ + RUBY_YJIT_ENABLE=1 \ + PERFORMANCE_RESULTS_FILE="$PROJECT_DIRECTORY/perf-comparison.json" \ + .evergreen/run-tests.sh + # Sent to the performance store so that the overhead metrics are watched + # for regressions, and kept as an artifact for comparison by hand. + - command: perf.send + params: + file: "${PROJECT_DIRECTORY}/perf-comparison.json" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: ./src/perf-comparison.json + display_name: perf-comparison.json + remote_file: ${UPLOAD_BUCKET}/${version_id}/${build_id}/artifacts/${build_variant}/perf-comparison.json + content_type: application/json + permissions: public-read + bucket: mciuploads + "run tests with orchestration and drivers tools": - command: subprocess.exec type: test @@ -595,6 +637,11 @@ tasks: - name: "driver-bench" commands: - func: "run benchmarks" + # About 1.5 hours at the default eight reps. + - name: "driver-bench-otel" + exec_timeout_secs: 10800 + commands: + - func: "run benchmark comparison" - name: "test-csot" commands: - func: "run CSOT tests" diff --git a/.evergreen/config/standard.yml.erb b/.evergreen/config/standard.yml.erb index 1f4e48ae20..1e0974fcf8 100644 --- a/.evergreen/config/standard.yml.erb +++ b/.evergreen/config/standard.yml.erb @@ -61,6 +61,19 @@ buildvariants: tasks: - name: "driver-bench" + # Compares DriverBench tasks with OpenTelemetry disabled against the + # enabled configurations the OpenTelemetry spec requires. + - matrix_name: DriverBenchOTel + matrix_spec: + ruby: <%= latest_ruby %> + mongodb-version: <%= latest_stable_mdb %> + topology: standalone + os: ubuntu2204 + display_name: DriverBench OTel + tasks: + - name: "driver-bench-otel" + batchtime: 1440 # run at most once a day + - matrix_name: "auth/ssl" matrix_spec: auth-and-ssl: ["auth-and-ssl", "noauth-and-nossl"] diff --git a/profile/driver_bench/base.rb b/profile/driver_bench/base.rb index b65de595ac..35b0775337 100644 --- a/profile/driver_bench/base.rb +++ b/profile/driver_bench/base.rb @@ -3,6 +3,7 @@ require 'benchmark' require 'mongo' +require_relative 'configuration' require_relative 'percentiles' module Mongo @@ -41,11 +42,27 @@ def self.bench_name(benchmark_name = nil) # Instantiate a new micro-benchmark class. def initialize - @max_iterations = debug_mode? ? 10 : 100 - @min_time = debug_mode? ? 1 : 60 + @max_iterations = Integer(ENV['DRIVER_BENCH_MAX_ITERATIONS'] || (debug_mode? ? 10 : 100)) + @min_time = Float(ENV['DRIVER_BENCH_MIN_TIME'] || (debug_mode? ? 1 : 60)) @max_time = 300 # 5 minutes end + # The number of driver operations one iteration performs, or nil when + # the task has no meaningful per-operation unit. Per-operation metrics + # are only reported for tasks that define it. + # + # @return [ Integer | nil ] operations per iteration. + def ops_per_iteration + nil + end + + # Whether to report per-operation metrics. Comparison asks for them; + # a plain suite run leaves them out, so that the results it sends to + # the performance store are the same as before they existed. + def per_op_metrics? + %w[1 true yes].include?(ENV['DRIVER_BENCH_PER_OP_METRICS'].to_s.downcase) + end + def debug_mode? ENV['PERF_DEBUG'] end @@ -61,7 +78,8 @@ def run { name: self.class.bench_name, score: score, - percentiles: percentiles } + percentiles: percentiles, + per_op: per_op_metrics(percentiles) } end private @@ -82,9 +100,13 @@ def run_benchmark setup + @cpu_times = [] + @gc_times = [] + @allocations = [] + loop do before_task - timing = consider_gc { Benchmark.realtime { debug_mode? ? sleep(0.1) : do_task } } + timing = consider_gc { measure_iteration { debug_mode? ? sleep(0.1) : do_task } } after_task iteration_count += 1 @@ -104,9 +126,66 @@ def run_benchmark end end - # Instantiate a new client. + # Times one iteration, and records alongside the wall-clock time the + # process CPU time and the number of objects allocated. CPU time leaves + # out the time spent waiting on the server, and allocations are nearly + # deterministic, so both show driver-side cost with less noise than + # throughput. Process CPU time includes the driver's background threads + # (e.g. server monitoring), which cost the same in every configuration. + # + # GC time is recorded too, where the runtime reports it (Ruby 3.1+). + # When and how long the collector runs varies between processes more + # than the cost of tracing does, so CPU time without GC is the steadier + # measure of the driver's own work. + # + # @return [ Float ] the wall-clock time in seconds. + def measure_iteration(&block) + allocated = GC.stat(:total_allocated_objects) + gc = gc_time + cpu = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID) + timing = Benchmark.realtime(&block) + @cpu_times.push(Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID) - cpu) + @gc_times.push(gc_time - gc) if gc + @allocations.push(GC.stat(:total_allocated_objects) - allocated) + timing + end + + # @return [ Float | nil ] the total time spent in GC so far, in + # seconds, or nil when the runtime does not report it. + def gc_time + ms = GC.stat[:time] + ms && (ms / 1000.0) + end + + # Per-operation medians, for tasks that define ops_per_iteration, when + # asked for (see #per_op_metrics?). + # + # @return [ Hash ] metric name to value. + def per_op_metrics(timings) + return {} unless per_op_metrics? && ops_per_iteration + + ops = ops_per_iteration.to_f + { + 'wall_us_per_op' => timings[50] / ops * 1_000_000, + 'cpu_us_per_op' => Percentiles.new(@cpu_times)[50] / ops * 1_000_000, + 'allocs_per_op' => Percentiles.new(@allocations)[50] / ops + }.merge(gc_metrics(ops)) + end + + def gc_metrics(ops) + return {} if @gc_times.empty? + + cpu_ex_gc = @cpu_times.zip(@gc_times).map { |cpu, gc| cpu - gc } + { + 'gc_us_per_op' => Percentiles.new(@gc_times)[50] / ops * 1_000_000, + 'cpu_ex_gc_us_per_op' => Percentiles.new(cpu_ex_gc)[50] / ops * 1_000_000 + } + end + + # Instantiate a new client, with the options of the configuration + # being measured. def new_client(uri = ENV['MONGODB_URI']) - Mongo::Client.new(uri) + Mongo::Client.new(uri, Configuration.current.client_options) end # Takes care of garbage collection considerations before diff --git a/profile/driver_bench/comparison.rb b/profile/driver_bench/comparison.rb new file mode 100644 index 0000000000..37b884db97 --- /dev/null +++ b/profile/driver_bench/comparison.rb @@ -0,0 +1,269 @@ +# frozen_string_literal: true + +require 'json' +require 'tmpdir' + +require_relative 'configuration' + +module Mongo + module DriverBench + # Runs DriverBench micro-benchmarks under several driver configurations + # and reports what each configuration costs relative to the baseline, as + # the OpenTelemetry spec's Benchmarking section requires. + # + # OpenTelemetry can only be installed into a process once, and the + # "api-only" configuration requires that the SDK was never loaded, so each + # configuration is measured in its own subprocess. + # + # Configurations are run interleaved -- the whole set, then the whole set + # again -- rather than one configuration to completion and then the next. + # The quantity being measured is a difference between two configurations, + # and a machine that slows down halfway through a run would otherwise put + # all of one configuration's samples in the fast half and all of another's + # in the slow half, which shows up as overhead that is not there. The + # order is also rotated by one position each repetition, so that no + # configuration always runs in the same slot. + # + # Parameterised by the environment: + # + # CONFIGURATIONS comma-separated configuration names (default: all) + # REPS repetitions of the whole set (default 1) + # MONGODB_URI server to benchmark + # DRIVER_BENCH_TASKS + # the micro-benchmarks to run, as for Suite (default: + # the two the spec requires) + # PERFORMANCE_RESULTS_FILE + # where the combined results are written + # DRIVER_BENCH_MAX_ITERATIONS, DRIVER_BENCH_MIN_TIME + # passed through to every run, see Base + # RUBY_YJIT_ENABLE + # run with YJIT; fails if this ruby was built without it + # + # @api private + class Comparison + DEFAULT_TASKS = 'small doc insertone,find one by id' + DEFAULT_RESULTS_FILE = 'perf-comparison.json' + + # Metrics carried from every child run into the comparison. Each is + # reduced to its median across repetitions. + CARRIED_METRICS = %w[ + score wall_us_per_op cpu_us_per_op cpu_ex_gc_us_per_op gc_us_per_op allocs_per_op + time-90% time-99% + ].freeze + + TABLE_FORMAT = '%-22s %-12s %9s %15s %11s %9s %9s %9s' + + def self.run! + new.run + end + + def initialize(env = ENV) + @env = env + @reps = Integer(env['REPS'] || 1) + @configurations = requested_configurations + @samples = Hash.new { |hash, key| hash[key] = Hash.new { |h, k| h[k] = [] } } + end + + # Runs every configuration, @reps times, and reports the comparison. + # + # @return [ String ] the summary table. + def run + raise 'the baseline configuration is required to compare against' unless + @configurations.any?(&:baseline?) + + # Checked here as well as in every child, to fail before the first + # of many long runs rather than inside it. + Configuration.check_jit! + + Dir.mktmpdir('driver-bench') do |dir| + 1.upto(@reps) do |rep| + @configurations.rotate(rep - 1).each { |configuration| measure(configuration, rep, dir) } + end + end + + save_perf_data(compile_perf_data) + summarize + end + + private + + def requested_configurations + names = (@env['CONFIGURATIONS'] || '').split(',').map(&:strip).reject(&:empty?) + return Configuration::ALL if names.empty? + + names.map { |name| Configuration[name] } + end + + def baseline + @baseline ||= @configurations.find(&:baseline?) + end + + # Runs one configuration once, in its own process, and records the + # metrics of every micro-benchmark it reported. + def measure(configuration, rep, dir) + results_file = File.join(dir, "#{configuration.name}-#{rep}.json") + + puts format("\n----- rep %d/%d: %s -----", rep, @reps, configuration.name) + unless system(child_env(configuration, results_file), 'bundle', 'exec', 'rake', 'driver_bench:run') + raise "configuration #{configuration.name} failed in rep #{rep}" + end + + JSON.parse(File.read(results_file)).each do |entry| + samples = @samples[[ configuration.task_name(entry['info']['test_name']), configuration.name ]] + entry['metrics'].each do |metric| + samples[metric['name']] << metric['value'] if CARRIED_METRICS.include?(metric['name']) + end + end + end + + def child_env(configuration, results_file) + { + Configuration::ENV_VAR => configuration.name, + 'PERFORMANCE_RESULTS_FILE' => results_file, + 'DRIVER_BENCH_TASKS' => @env['DRIVER_BENCH_TASKS'] || DEFAULT_TASKS, + 'DRIVER_BENCH_PER_OP_METRICS' => '1' + } + end + + # The median of one metric across repetitions. Unlike the spec's + # nearest-rank percentile, which suits its iteration counts, this + # averages the middle two of an even number of repetitions: nearest + # rank would always pick the lower one. + def median(task, configuration, metric) + values = @samples[[ task, configuration.name ]][metric].sort + return nil if values.empty? + + mid = values.length / 2 + values.length.odd? ? values[mid] : (values[mid - 1] + values[mid]) / 2.0 + end + + def spread(task, configuration) + values = @samples[[ task, configuration.name ]]['score'] + return nil if values.empty? + + values.minmax + end + + # Scores are throughput, so enabling a feature shows up as a loss. + # + # @return [ Float | nil ] the percentage of throughput given up, + # relative to the baseline. + def loss_for(task, configuration) + reference = median(task, baseline, 'score') + score = median(task, configuration, 'score') + return nil if reference.nil? || score.nil? || reference.zero? + + (reference - score) / reference * 100.0 + end + + # CPU time and allocations grow with cost, so the overhead is the + # difference over the baseline. + def added(task, configuration, metric) + reference = median(task, baseline, metric) + value = median(task, configuration, metric) + return nil if reference.nil? || value.nil? + + value - reference + end + + def cpu_overhead_pct(task, configuration, metric = 'cpu_us_per_op') + reference = median(task, baseline, metric) + delta = added(task, configuration, metric) + return nil if delta.nil? || reference.zero? + + delta / reference * 100.0 + end + + def tasks + @tasks ||= @samples.keys.map(&:first).uniq + end + + def compile_perf_data + tasks.flat_map do |task| + @configurations.filter_map do |configuration| + next if median(task, configuration, 'score').nil? + + { + 'info' => { + 'test_name' => configuration.perf_test_name(task), + 'args' => {} + }, + 'metrics' => metrics_for(task, configuration) + } + end + end + end + + def metrics_for(task, configuration) + metrics = CARRIED_METRICS.filter_map do |name| + value = median(task, configuration, name) + { 'name' => name, 'value' => value } unless value.nil? + end + min, max = spread(task, configuration) + metrics << { 'name' => 'score_min', 'value' => min } << { 'name' => 'score_max', 'value' => max } + return metrics if configuration.baseline? + + # The overheads are recorded as metrics of their own, rather than + # left to be derived by comparing two time series later, so that + # they can be watched for regressions directly and so that + # host-to-host variation cancels out of them. + { + 'overhead_pct' => loss_for(task, configuration), + 'cpu_overhead_pct' => cpu_overhead_pct(task, configuration), + 'cpu_us_added_per_op' => added(task, configuration, 'cpu_us_per_op'), + 'cpu_ex_gc_overhead_pct' => cpu_overhead_pct(task, configuration, 'cpu_ex_gc_us_per_op'), + 'cpu_ex_gc_us_added_per_op' => added(task, configuration, 'cpu_ex_gc_us_per_op'), + 'gc_us_added_per_op' => added(task, configuration, 'gc_us_per_op'), + 'allocs_added_per_op' => added(task, configuration, 'allocs_per_op') + }.each do |name, value| + metrics << { 'name' => name, 'value' => value } unless value.nil? + end + metrics + end + + def save_perf_data(data, file_name: @env['PERFORMANCE_RESULTS_FILE'] || DEFAULT_RESULTS_FILE) + File.write(file_name, data.to_json) + end + + def summarize + lines = [ format("\n===== Configuration comparison (%d rep%s, median) =====", + @reps, (@reps == 1) ? '' : 's') ] + lines << 'loss: throughput given up vs off; spread: min-max MB/s across reps; ' \ + 'cpu: CPU us added per op, excluding GC where reported; gc: GC us added per op; ' \ + 'allocs: objects added per op' + lines << '' + lines << format(TABLE_FORMAT, + task: 'micro-benchmark', config: 'config', loss: 'loss %', spread: 'spread MB/s', + cpu: 'cpu us/op', cpu_pct: 'cpu %', gc: 'gc us/op', allocs: 'allocs') + tasks.each do |task| + @configurations.reject(&:baseline?).each { |configuration| lines << row(task, configuration) } + end + + lines.join("\n") + end + + def row(task, configuration) + min, max = spread(task, configuration) + + format(TABLE_FORMAT, + task: task, config: configuration.name, + loss: signed(loss_for(task, configuration)), + spread: min ? format('%.4g-%.4g', min, max) : 'n/a', + cpu: signed(added(task, configuration, cpu_metric(task))), + cpu_pct: signed(cpu_overhead_pct(task, configuration, cpu_metric(task))), + gc: signed(added(task, configuration, 'gc_us_per_op')), + allocs: signed(added(task, configuration, 'allocs_per_op'), '%+.0f')) + end + + # CPU time excluding GC where the runtime reports GC time, total CPU + # time otherwise. + def cpu_metric(task) + median(task, baseline, 'cpu_ex_gc_us_per_op') ? 'cpu_ex_gc_us_per_op' : 'cpu_us_per_op' + end + + def signed(value, pattern = '%+.2f') + value.nil? ? 'n/a' : format(pattern, value) + end + end + end +end diff --git a/profile/driver_bench/configuration.rb b/profile/driver_bench/configuration.rb new file mode 100644 index 0000000000..a632ff88b8 --- /dev/null +++ b/profile/driver_bench/configuration.rb @@ -0,0 +1,174 @@ +# frozen_string_literal: true + +module Mongo + module DriverBench + # One driver configuration under which the DriverBench tasks can be run. + # + # A benchmark result is identified by the pair (task, configuration), so + # every configuration becomes its own time series and can be watched for + # regressions independently. Comparing a configuration against the + # baseline is what catches a performance regression in an opt-in feature + # such as OpenTelemetry: no benchmark that only ever runs the default + # configuration will execute that code at all. + # + # The configurations are the ones the OpenTelemetry spec's Benchmarking + # section requires. OpenTelemetry cannot be reconfigured once its SDK has + # been installed into a process, and the "api-only" configuration + # requires that the SDK was never loaded, so exactly one configuration + # can be measured per process. Comparison runs one subprocess per + # configuration. + # + # @api private + class Configuration + # The environment variable naming the configuration to run under. + ENV_VAR = 'DRIVER_BENCH_CONFIGURATION' + + # The configuration every other one is compared against. + BASELINE = 'off' + + # @return [ String ] the short name of the configuration, used to key + # results and to select the configuration from the environment. + attr_reader :name + + # @return [ String ] a human-readable description of the configuration. + attr_reader :description + + # @return [ Hash ] options to pass to every Mongo::Client the + # benchmarks construct. + attr_reader :client_options + + # @param name [ String ] the short name of the configuration. + # @param description [ String ] what the configuration measures. + # @param otel [ Symbol ] how much of OpenTelemetry to load: +:none+ for + # nothing, +:api+ for the API without an SDK, +:sdk+ for a configured + # SDK. + # @param client_options [ Hash ] options for every client. + # @param sampler_env [ Hash ] OpenTelemetry environment variables that + # select the sampler, applied before the SDK is configured. + def initialize(name:, description:, otel:, client_options: {}, sampler_env: {}) + @name = name + @description = description + @otel = otel + @client_options = client_options + @sampler_env = sampler_env + end + + # The test name to report a task's results under. perf.send only + # accepts integer args, so the configuration goes into the name. The + # baseline keeps the bare task name, so that its series continues the + # one recorded before configurations existed. + # + # @param task [ String ] the micro-benchmark name. + # + # @return [ String ] the reported test name. + def perf_test_name(task) + baseline? ? task : "#{task} [#{name}]" + end + + # The inverse of #perf_test_name. + # + # @param test_name [ String ] a reported test name. + # + # @return [ String ] the micro-benchmark name. + def task_name(test_name) + test_name.delete_suffix(" [#{name}]") + end + + # Whether this configuration is the baseline that others are compared + # against. + def baseline? + name == BASELINE + end + + # Loads and configures OpenTelemetry for this configuration. + # + # Must run before any Mongo::Client is constructed: a client decides + # whether tracing is active when it builds its tracer, and that + # decision depends on whether ::OpenTelemetry is defined. + def install! + case @otel + when :api then require 'opentelemetry-api' + when :sdk then install_sdk! + end + end + + # All configurations, baseline first. + ALL = [ + new(name: 'off', + description: 'tracing disabled; the baseline', + otel: :none), + + new(name: 'api-only', + description: 'tracing enabled; OpenTelemetry API without an SDK, so every tracing call is a no-op', + otel: :api, + client_options: { tracing: { enabled: true } }), + + new(name: 'sdk-ratio', + description: 'tracing enabled; SDK installed; TraceIdRatioBased sampler recording 1% of traces', + otel: :sdk, + client_options: { tracing: { enabled: true } }, + sampler_env: { 'OTEL_TRACES_SAMPLER' => 'traceidratio', + 'OTEL_TRACES_SAMPLER_ARG' => '0.01' }), + + new(name: 'sdk-always', + description: 'tracing enabled; SDK installed; every span sampled', + otel: :sdk, + client_options: { tracing: { enabled: true } }, + sampler_env: { 'OTEL_TRACES_SAMPLER' => 'always_on' }) + ].freeze + + # Looks up a configuration by name. + # + # @param name [ String ] the configuration name. + # + # @return [ Configuration ] the named configuration. + # + # @raise [ ArgumentError ] if no such configuration exists. + def self.[](name) + ALL.find { |configuration| configuration.name == name } || + raise(ArgumentError, + "unknown configuration #{name.inspect}; " \ + "known configurations are #{ALL.map(&:name).join(', ')}") + end + + # Fails when YJIT was asked for (RUBY_YJIT_ENABLE) but this ruby runs + # without it. A ruby built without YJIT only warns and carries on, and + # the benchmark would then quietly measure the interpreter, where the + # cost of tracing is several times higher. + def self.check_jit! + return unless %w[1 true yes].include?(ENV['RUBY_YJIT_ENABLE'].to_s.downcase) + return if defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled? + + raise "RUBY_YJIT_ENABLE is set, but #{RUBY_DESCRIPTION} runs without YJIT" + end + + # The configuration the current process is measuring, named by the + # DRIVER_BENCH_CONFIGURATION environment variable. + # + # @return [ Configuration ] the current configuration. + def self.current + @current ||= self[ENV[ENV_VAR] || BASELINE] + end + + private + + def install_sdk! + require 'opentelemetry-sdk' + + # The SDK reads its sampler and exporter from the environment when it + # is configured, so set them here rather than trusting the caller to: + # a configuration must mean the same thing however the suite was + # invoked. + @sampler_env.each { |key, value| ENV[key] = value } + + # No exporter, and so no span processor either, as the spec + # recommends. Sampled spans are still fully recorded without one; a + # processor would charge the SDK's export machinery to the driver and + # add variance. + ENV['OTEL_TRACES_EXPORTER'] = 'none' + + ::OpenTelemetry::SDK.configure + end + end + end +end diff --git a/profile/driver_bench/rake/tasks.rake b/profile/driver_bench/rake/tasks.rake index 16d038b85b..e8098375ec 100644 --- a/profile/driver_bench/rake/tasks.rake +++ b/profile/driver_bench/rake/tasks.rake @@ -40,6 +40,22 @@ namespace :driver_bench do Mongo::DriverBench::Suite.run! end + desc 'Compares DriverBench micro-benchmarks across driver configurations' + task compare: :data do + require_relative '../comparison' + + puts Mongo::DriverBench::Comparison.run! + end + + desc 'Lists the driver configurations the suite can run under' + task :configurations do + require_relative '../configuration' + + Mongo::DriverBench::Configuration::ALL.each do |configuration| + puts format('%-12s %s', configuration.name, configuration.description) + end + end + desc 'Runs the crypto benchmark' task :crypto do require_relative '../crypto/decrypt' diff --git a/profile/driver_bench/single_doc/base.rb b/profile/driver_bench/single_doc/base.rb index cf540a9a4e..a2c7e5b0d6 100644 --- a/profile/driver_bench/single_doc/base.rb +++ b/profile/driver_bench/single_doc/base.rb @@ -28,6 +28,12 @@ def scale 10_000 end + # Every single-doc task performs one driver operation per unit of + # scale. + def ops_per_iteration + scale + end + def teardown cleanup_client end diff --git a/profile/driver_bench/suite.rb b/profile/driver_bench/suite.rb index 40259872fb..f7d7708966 100644 --- a/profile/driver_bench/suite.rb +++ b/profile/driver_bench/suite.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative 'bson' +require_relative 'configuration' require_relative 'multi_doc' require_relative 'parallel' require_relative 'single_doc' @@ -46,23 +47,60 @@ def self.run! end def run + Configuration.check_jit! + configuration.install! + announce_configuration + perf_data = [] benches = Hash.new { |h, k| h[k] = [] } - ALL.each do |klass| + tasks.each do |klass| result = run_benchmark(klass) perf_data << compile_perf_data(result) append_to_benchmarks(klass, result, benches) end - perf_data += compile_benchmarks(benches) + # The composites average a fixed list of micro-benchmarks, so they are + # only meaningful when every micro-benchmark ran. + perf_data += compile_benchmarks(benches) if tasks == ALL save_perf_data(perf_data) summarize_perf_data(perf_data) end + # The micro-benchmarks to run. + # + # By default this is every micro-benchmark, but DRIVER_BENCH_TASKS + # narrows it to those whose names contain one of a comma-separated list + # of substrings (case-insensitive). + # + # @return [ Array ] the micro-benchmark classes to run. + def tasks + @tasks ||= begin + patterns = ENV['DRIVER_BENCH_TASKS'].to_s.split(',').map { |p| p.strip.downcase }.reject(&:empty?) + selected = ALL.select do |klass| + patterns.empty? || patterns.any? { |pattern| klass.bench_name.downcase.include?(pattern) } + end + raise 'no micro-benchmarks match DRIVER_BENCH_TASKS' if selected.empty? + + (selected == ALL) ? ALL : selected.freeze + end + end + private + def configuration + @configuration ||= Configuration.current + end + + def announce_configuration + puts format('===== DriverBench: %s =====', configuration.name) + puts configuration.description + puts RUBY_DESCRIPTION + puts format('%d of %d micro-benchmarks', tasks.length, ALL.length) unless tasks == ALL + puts + end + def run_benchmark(klass) print klass.bench_name, ': ' $stdout.flush @@ -77,16 +115,18 @@ def compile_perf_data(result) { 'name' => "time-#{percentile}%", 'value' => result[:percentiles][percentile] } end + per_op_data = result[:per_op].map { |name, value| { 'name' => name, 'value' => value } } { 'info' => { - 'test_name' => result[:name], + 'test_name' => configuration.perf_test_name(result[:name]), 'args' => {}, }, 'metrics' => [ { 'name' => 'score', 'value' => result[:score] }, - *percentile_data + *percentile_data, + *per_op_data ] } end @@ -107,7 +147,7 @@ def compile_benchmarks(benches) benches.map do |bench, score| { 'info' => { - 'test_name' => bench, + 'test_name' => configuration.perf_test_name(bench), 'args' => {} }, 'metrics' => [ @@ -119,7 +159,7 @@ def compile_benchmarks(benches) end def summarize_perf_data(data) - puts '===== Performance Results =====' + puts format('===== Performance Results (%s) =====', configuration.name) data.each do |item| puts format('%s : %4.4g', item['info']['test_name'], item['metrics'][0]['value']) next unless item['metrics'].length > 1