Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions .evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,48 @@ functions:
params:
file: "${PROJECT_DIRECTORY}/perf.json"

"run benchmark comparison":
# Runs the micro-benchmarks the OpenTelemetry spec requires under every
# configuration it requires, interleaved on one host, and reports what
# each configuration costs relative to the untraced baseline. Fewer
# iterations per run buy more interleaved repetitions in the same time.
# Every configuration runs with YJIT, which production deployments
# commonly enable; without it the interpreter's per-call cost inflates
# the tracing overhead. The run fails if this ruby was built without
# YJIT.
- command: shell.exec
type: test
params:
shell: bash
working_dir: "src"
# Needed to download "latest"/"latest-build" from the private S3
# bucket (DRIVERS-3628). Any other version needs no AWS access.
include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
script: |
${PREPARE_SHELL}
TEST_CMD="bundle exec rake driver_bench:compare" \
REPS="${driver_bench_reps|8}" \
DRIVER_BENCH_MAX_ITERATIONS="${driver_bench_max_iterations|20}" \
DRIVER_BENCH_MIN_TIME="${driver_bench_min_time|20}" \
RUBY_YJIT_ENABLE=1 \
PERFORMANCE_RESULTS_FILE="$PROJECT_DIRECTORY/perf-comparison.json" \
.evergreen/run-tests.sh
# Sent to the performance store so that the overhead metrics are watched
# for regressions, and kept as an artifact for comparison by hand.
- command: perf.send
params:
file: "${PROJECT_DIRECTORY}/perf-comparison.json"
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: ./src/perf-comparison.json
display_name: perf-comparison.json
remote_file: ${UPLOAD_BUCKET}/${version_id}/${build_id}/artifacts/${build_variant}/perf-comparison.json
content_type: application/json
permissions: public-read
bucket: mciuploads

"run tests with orchestration and drivers tools":
- command: subprocess.exec
type: test
Expand Down Expand Up @@ -598,6 +640,11 @@ tasks:
- name: "driver-bench"
commands:
- func: "run benchmarks"
# About 1.5 hours at the default eight reps.
- name: "driver-bench-otel"
exec_timeout_secs: 10800
commands:
- func: "run benchmark comparison"
- name: "test-csot"
commands:
- func: "run CSOT tests"
Expand Down Expand Up @@ -1131,6 +1178,19 @@ buildvariants:
tasks:
- name: "driver-bench"

# Compares DriverBench tasks with OpenTelemetry disabled against the
# enabled configurations the OpenTelemetry spec requires.
- matrix_name: DriverBenchOTel
matrix_spec:
ruby: "ruby-4.0"
mongodb-version: "8.0"
topology: standalone
os: ubuntu2204
display_name: DriverBench OTel
tasks:
- name: "driver-bench-otel"
batchtime: 1440 # run at most once a day

- matrix_name: "auth/ssl"
matrix_spec:
auth-and-ssl: ["auth-and-ssl", "noauth-and-nossl"]
Expand Down
47 changes: 47 additions & 0 deletions .evergreen/config/common.yml.erb
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,48 @@ functions:
params:
file: "${PROJECT_DIRECTORY}/perf.json"

"run benchmark comparison":
# Runs the micro-benchmarks the OpenTelemetry spec requires under every
# configuration it requires, interleaved on one host, and reports what
# each configuration costs relative to the untraced baseline. Fewer
# iterations per run buy more interleaved repetitions in the same time.
# Every configuration runs with YJIT, which production deployments
# commonly enable; without it the interpreter's per-call cost inflates
# the tracing overhead. The run fails if this ruby was built without
# YJIT.
- command: shell.exec
type: test
params:
shell: bash
working_dir: "src"
# Needed to download "latest"/"latest-build" from the private S3
# bucket (DRIVERS-3628). Any other version needs no AWS access.
include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
script: |
${PREPARE_SHELL}
TEST_CMD="bundle exec rake driver_bench:compare" \
REPS="${driver_bench_reps|8}" \
DRIVER_BENCH_MAX_ITERATIONS="${driver_bench_max_iterations|20}" \
DRIVER_BENCH_MIN_TIME="${driver_bench_min_time|20}" \
RUBY_YJIT_ENABLE=1 \
PERFORMANCE_RESULTS_FILE="$PROJECT_DIRECTORY/perf-comparison.json" \
.evergreen/run-tests.sh
# Sent to the performance store so that the overhead metrics are watched
# for regressions, and kept as an artifact for comparison by hand.
- command: perf.send
params:
file: "${PROJECT_DIRECTORY}/perf-comparison.json"
- command: s3.put
params:
aws_key: ${aws_key}
aws_secret: ${aws_secret}
local_file: ./src/perf-comparison.json
display_name: perf-comparison.json
remote_file: ${UPLOAD_BUCKET}/${version_id}/${build_id}/artifacts/${build_variant}/perf-comparison.json
content_type: application/json
permissions: public-read
bucket: mciuploads

"run tests with orchestration and drivers tools":
- command: subprocess.exec
type: test
Expand Down Expand Up @@ -595,6 +637,11 @@ tasks:
- name: "driver-bench"
commands:
- func: "run benchmarks"
# About 1.5 hours at the default eight reps.
- name: "driver-bench-otel"
exec_timeout_secs: 10800
commands:
- func: "run benchmark comparison"
- name: "test-csot"
commands:
- func: "run CSOT tests"
Expand Down
13 changes: 13 additions & 0 deletions .evergreen/config/standard.yml.erb
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,19 @@ buildvariants:
tasks:
- name: "driver-bench"

# Compares DriverBench tasks with OpenTelemetry disabled against the
# enabled configurations the OpenTelemetry spec requires.
- matrix_name: DriverBenchOTel
matrix_spec:
ruby: <%= latest_ruby %>
mongodb-version: <%= latest_stable_mdb %>
topology: standalone
os: ubuntu2204
display_name: DriverBench OTel
tasks:
- name: "driver-bench-otel"
batchtime: 1440 # run at most once a day

- matrix_name: "auth/ssl"
matrix_spec:
auth-and-ssl: ["auth-and-ssl", "noauth-and-nossl"]
Expand Down
91 changes: 85 additions & 6 deletions profile/driver_bench/base.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
require 'benchmark'
require 'mongo'

require_relative 'configuration'
require_relative 'percentiles'

module Mongo
Expand Down Expand Up @@ -41,11 +42,27 @@ def self.bench_name(benchmark_name = nil)

# Instantiate a new micro-benchmark class.
def initialize
@max_iterations = debug_mode? ? 10 : 100
@min_time = debug_mode? ? 1 : 60
@max_iterations = Integer(ENV['DRIVER_BENCH_MAX_ITERATIONS'] || (debug_mode? ? 10 : 100))
@min_time = Float(ENV['DRIVER_BENCH_MIN_TIME'] || (debug_mode? ? 1 : 60))
@max_time = 300 # 5 minutes
end

# The number of driver operations one iteration performs, or nil when
# the task has no meaningful per-operation unit. Per-operation metrics
# are only reported for tasks that define it.
#
# @return [ Integer | nil ] operations per iteration.
def ops_per_iteration
nil
end

# Whether to report per-operation metrics. Comparison asks for them;
# a plain suite run leaves them out, so that the results it sends to
# the performance store are the same as before they existed.
def per_op_metrics?
%w[1 true yes].include?(ENV['DRIVER_BENCH_PER_OP_METRICS'].to_s.downcase)
end

def debug_mode?
ENV['PERF_DEBUG']
end
Expand All @@ -61,7 +78,8 @@ def run

{ name: self.class.bench_name,
score: score,
percentiles: percentiles }
percentiles: percentiles,
per_op: per_op_metrics(percentiles) }
end

private
Expand All @@ -82,9 +100,13 @@ def run_benchmark

setup

@cpu_times = []
@gc_times = []
@allocations = []

loop do
before_task
timing = consider_gc { Benchmark.realtime { debug_mode? ? sleep(0.1) : do_task } }
timing = consider_gc { measure_iteration { debug_mode? ? sleep(0.1) : do_task } }
after_task

iteration_count += 1
Expand All @@ -104,9 +126,66 @@ def run_benchmark
end
end

# Instantiate a new client.
# Times one iteration, and records alongside the wall-clock time the
# process CPU time and the number of objects allocated. CPU time leaves
# out the time spent waiting on the server, and allocations are nearly
# deterministic, so both show driver-side cost with less noise than
# throughput. Process CPU time includes the driver's background threads
# (e.g. server monitoring), which cost the same in every configuration.
#
# GC time is recorded too, where the runtime reports it (Ruby 3.1+).
# When and how long the collector runs varies between processes more
# than the cost of tracing does, so CPU time without GC is the steadier
# measure of the driver's own work.
#
# @return [ Float ] the wall-clock time in seconds.
def measure_iteration(&block)
allocated = GC.stat(:total_allocated_objects)
gc = gc_time
cpu = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
timing = Benchmark.realtime(&block)
@cpu_times.push(Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID) - cpu)
@gc_times.push(gc_time - gc) if gc
@allocations.push(GC.stat(:total_allocated_objects) - allocated)
timing
end

# @return [ Float | nil ] the total time spent in GC so far, in
# seconds, or nil when the runtime does not report it.
def gc_time
ms = GC.stat[:time]
ms && (ms / 1000.0)
end

# Per-operation medians, for tasks that define ops_per_iteration, when
# asked for (see #per_op_metrics?).
#
# @return [ Hash<String, Float> ] metric name to value.
def per_op_metrics(timings)
return {} unless per_op_metrics? && ops_per_iteration

ops = ops_per_iteration.to_f
{
'wall_us_per_op' => timings[50] / ops * 1_000_000,
'cpu_us_per_op' => Percentiles.new(@cpu_times)[50] / ops * 1_000_000,
'allocs_per_op' => Percentiles.new(@allocations)[50] / ops
}.merge(gc_metrics(ops))
end

def gc_metrics(ops)
return {} if @gc_times.empty?

cpu_ex_gc = @cpu_times.zip(@gc_times).map { |cpu, gc| cpu - gc }
{
'gc_us_per_op' => Percentiles.new(@gc_times)[50] / ops * 1_000_000,
'cpu_ex_gc_us_per_op' => Percentiles.new(cpu_ex_gc)[50] / ops * 1_000_000
}
end

# Instantiate a new client, with the options of the configuration
# being measured.
def new_client(uri = ENV['MONGODB_URI'])
Mongo::Client.new(uri)
Mongo::Client.new(uri, Configuration.current.client_options)
end

# Takes care of garbage collection considerations before
Expand Down
Loading
Loading