feat: DH-21975: Gc benchmarking

.github/matrix/gc-train-1000-100.sh

@@ -0,0 +1,53 @@
+# Compare GC configurations across cycle times and profiles
+# TSV columns: run_label, docker_image, test_package, test_class_list,
+#              test_iterations, scale_row_count, distribution, config_options
+EXPECTED_COMBOS=32
+
+IMG=ghcr.io/stanbrub/server:jvm25
+PKG=io.deephaven.benchmark.tests.train
+CLS='*Train'
+ITERS=1
+ROWSM=1
+DIST=random
+
+BASE="-Xms48g -Xmx48g -XX:+UseStringDeduplication -XX:+UseCompactObjectHeaders -DServerStateTracker.reportIntervalMillis=1000"
+HUGE="-XX:+AlwaysPreTouch -XX:+UseTransparentHugePages"
+
+G1="-XX:+UseG1GC"
+ZGC="-XX:-UseG1GC -XX:+UseZGC"
+SHEN="-XX:-UseG1GC -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational"
+PARA="-XX:-UseG1GC -XX:+UseParallelGC"
+
+R="$IMG\t$PKG\t$CLS\t$ITERS\t$ROWSM\t$DIST"
+
+# emit <label> <gc> <cycle_ms> <inc_load_target> — outputs regular and _huge variants
+run() {
+  local opts="$2 $BASE -DPeriodicUpdateGraph.targetCycleDurationMillis=$3 -Dbench.incLoadTarget=$4"
+  echo -e "$1\t$R\t$opts"
+  echo -e "${1}_huge\t$R\t$opts $HUGE"
+}
+
+# 1000ms cycle
+run gc_g1gc_1000_p100_j25  "$G1"   1000  1.00
+run gc_zgc_1000_p100_j25   "$ZGC"  1000  1.00
+run gc_shen_1000_p100_j25  "$SHEN" 1000  1.00
+run gc_para_1000_p100_j25  "$PARA" 1000  1.00
+
+# 100ms cycle, 100% load target
+run gc_g1gc_100_p100_j25   "$G1"   100   1.00
+run gc_zgc_100_p100_j25    "$ZGC"  100   1.00
+run gc_shen_100_p100_j25   "$SHEN" 100   1.00
+run gc_para_100_p100_j25   "$PARA" 100   1.00
+
+# 100ms cycle, 90% load target
+run gc_g1gc_100_p90_j25    "$G1"   100   0.90
+run gc_zgc_100_p90_j25     "$ZGC"  100   0.90
+run gc_shen_100_p90_j25    "$SHEN" 100   0.90
+run gc_para_100_p90_j25    "$PARA" 100   0.90
+
+# 100ms cycle, 80% load target
+run gc_g1gc_100_p80_j25    "$G1"   100   0.80
+run gc_zgc_100_p80_j25     "$ZGC"  100   0.80
+run gc_shen_100_p80_j25    "$SHEN" 100   0.80
+run gc_para_100_p80_j25    "$PARA" 100   0.80
+

.github/matrix/gc-train-heap-fail.sh

@@ -0,0 +1,57 @@
+# Sweep heap size to find failure points per GC and test class
+# TSV columns: run_label, docker_image, test_package, test_class_list,
+#              test_iterations, scale_row_count, distribution, config_options
+
+IMG=ghcr.io/stanbrub/server:jvm25
+PKG=io.deephaven.benchmark.tests.train
+ITERS=1
+ROWSM=10
+DIST=random
+STEP=1
+
+BASE="-XX:+AlwaysPreTouch -XX:+UseTransparentHugePages -XX:+UseStringDeduplication -XX:+UseCompactObjectHeaders -DServerStateTracker.reportIntervalMillis=1000"
+CYCLE="-DPeriodicUpdateGraph.targetCycleDurationMillis=1000 -Dbench.incLoadTarget=1.00"
+
+declare -A GCS=(
+  [g1gc]="-XX:+UseG1GC"
+  [zgc]="-XX:-UseG1GC -XX:+UseZGC"
+  [shen]="-XX:-UseG1GC -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational"
+  [para]="-XX:-UseG1GC -XX:+UseParallelGC"
+)
+
+# Per-class, per-GC heap ranges: "class  gc  min  max"
+COMBOS=(				# Failure Success
+  "FilterTrain       g1gc   22  24"	#     20G     21G
+  "FilterTrain       zgc     3   5"	#      1G      2G
+  "FilterTrain       shen   18  19"	#     19G     17G
+  "FilterTrain       para    7   7"	#      3G      4G
+  "FormulaTrain      g1gc    2   4"	#              1G
+  "FormulaTrain      zgc     2   4"	#              1G
+  "FormulaTrain      shen    5   7"	#      3G      4G
+  "FormulaTrain      para    2   4"	#              1G
+  "NaturalJoinTrain  g1gc   18  20"	#     16G     17G
+  "NaturalJoinTrain  zgc    17  19"	#     15G     16G
+  "NaturalJoinTrain  shen   19  21"	#     17G     18G
+  "NaturalJoinTrain  para   21  23"	#     19G     20G
+  "AggByTrain        g1gc    2   4"	#              1G
+  "AggByTrain        zgc     2   4"	#              1G
+  "AggByTrain        shen    5   7"	#      3G      4G
+  "AggByTrain        para   18  20"	#     16G     17G
+  "OrderedTrain      g1gc    4   6"	#      2G      3G
+  "OrderedTrain      zgc     4   6"	#      2G      3G
+  "OrderedTrain      shen    4   6"	#      2G      3G
+  "OrderedTrain      para    4   6"	#      2G      3G
+  "UpdateByTrain     g1gc   16  18"	#     14G     15G
+  "UpdateByTrain     zgc    17  19"	#     15G     16G
+  "UpdateByTrain     shen   18  20"	#     16G     17G
+  "UpdateByTrain     para   21  23"	#     19G     20G
+)
+
+for combo in "${COMBOS[@]}"; do
+  read -r cls gc min_heap max_heap <<< "$combo"
+  for h in $(seq ${max_heap} -${STEP} ${min_heap}); do
+    opts="${GCS[$gc]} -Xms${h}g -Xmx${h}g $BASE $CYCLE"
+    echo -e "gc_${gc}_${cls}_h${h}_j25\t$IMG\t$PKG\t*${cls}\t$ITERS\t$ROWSM\t$DIST\t$opts"
+  done
+done
+

.github/resources/adhoc-benchmark-docker-compose.yml

@@ -9,7 +9,9 @@ services:
       - ./data:/data
       - ./minio:/minio
     environment:
+      - "JAVA_OPTS=-XX:+UseG1GC"
       - "START_OPTS=-DAuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler ${CONFIG_OPTS}"
+      - "DEEPHAVEN_HOST_OS_DIR=${ENV_DEEPHAVEN_HOST_OS_DIR}"
 
   redpanda:
     security_opt:

.github/scripts/adhoc.sh

@@ -27,6 +27,7 @@ rm -f ${OUTPUT_NAME}; touch ${OUTPUT_NAME}
 # Get the label part of an image/branch name
 # ex. edge@sha256:15ab331629805076cdf5ed6666186c6b578298ab493a980779338d153214640e
 # ex. user123:1111-my-pull-request
+# ex. ghcr.io/stanbrub/server:jvm25
 # ex. 0.36.0 or edge
 getSetLabel() {
   PREFIX="$1"
@@ -53,11 +54,11 @@ deleteMetal() {
     RESP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE -H "Authorization: Bearer ${TOKEN}" "https://api.phoenixnap.com/bmc/v1/servers/${DEVICE_ID}")
     CURL_EXIT=$?
     if (( ${CURL_EXIT} != 0 )); then
-        echo "Failed deleting server ${DEVICE_NAME} with exit code ${CURL_EXIT}"
+        echo "::error::Failed deleting server ${DEVICE_NAME} with exit code ${CURL_EXIT}"
         exit 1
     fi
     if (( ${RESP_CODE} != 404 && (${RESP_CODE} < 200 || ${RESP_CODE} >= 300) )); then
-        echo "Failed deleting server ${DEVICE_NAME} with http code ${RESP_CODE}"
+        echo "::error::Failed deleting server ${DEVICE_NAME} with http code ${RESP_CODE}"
         exit 1
     fi
     echo "Successfully deleted server ${DEVICE_NAME}"
@@ -121,7 +122,7 @@ if [[ ${ACTION} == "deploy-metal" ]]; then
   IP_ADDRESS=$(jq -r '.publicIpAddresses[0] // empty' <<< "${RESPONSE}")
   DEVICE_ID=$(jq -r '.id // empty' <<< "${RESPONSE}")
   if [[ -z "$IP_ADDRESS" || -z "$DEVICE_ID" ]]; then
-    echo "Failed Getting IP Address and Device ID"
+    echo "::error::Failed Getting IP Address and Device ID"
     exit 1
   else
     echo "Got Address ${IP_ADDRESS} and Device Id ${DEVICE_ID}"
@@ -144,7 +145,7 @@ if [[ ${ACTION} == "deploy-metal" ]]; then
 
   DURATION=$(($(date +%s) - ${BEGIN_SECS}))
   if [[ ${STATUS} -eq 0 ]]; then
-    echo "Failed to provision device ${ACTOR} after ${DURATION} seconds"
+    echo "::error::Failed to provision device ${ACTOR} after ${DURATION} seconds"
     echo "Last provision attempt status: ${SERVER_STATUS}"
     exit 1
   fi

.github/scripts/build-benchmark-artifact-remote.sh

@@ -21,10 +21,18 @@ fi
 
 title () { echo; echo $1; }
 
+rm -rf ${RUN_DIR}/results ${RUN_DIR}/logs
+
+if [ -f "${RUN_DIR}/standard-tests.jar" ]; then
+  echo "Benchmark artifact already built. Skipping."
+  exit 0
+fi
+
 title "- Building Remote Benchmark Artifact on ${HOST} -"
 
 title "-- Building and Verifying --"
 cd ${GIT_DIR}
+sleep 10  # Some JVM command lines start DH slower than others
 mvn verify
 
 title "-- Cleanup After Build  --"

.github/scripts/build-docker-image-remote.sh

@@ -25,6 +25,11 @@ fi
 
 title () { echo; echo $1; }
 
+if docker image inspect deephaven/server:benchmark-local &>/dev/null 2>&1; then
+  echo "Docker image already built. Skipping."
+  exit 0
+fi
+
 title "- Setting up Remote Docker Image on ${HOST} -"
 
 title "-- Building Deephaven Docker Image --"

.github/scripts/build-matrix.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+
+# Copyright (c) 2026-2026 Deephaven Data Labs and Patent Pending
+
+# Runs a matrix script and converts its TSV output to a JSON matrix for GitHub Actions.
+# The matrix script outputs tab-separated rows with fields:
+#   run_label, docker_image, test_package, test_class_list,
+#   test_iterations, scale_row_count, distribution, config_options
+# Scale_row_count is in millions and auto-scaled to actual.
+# Test_iterations is auto-forced to odd.
+#
+# Usage: build-matrix.sh <matrix-script>
+
+FILE="$1"
+
+if [[ -z "$FILE" ]]; then
+  echo "Usage: build-matrix.sh <matrix-script>"
+  exit 1
+fi
+
+if [[ ! -f "$FILE" ]]; then
+  echo "::error::Matrix script not found: ${FILE}"
+  exit 1
+fi
+
+TSV=$(bash "$FILE")
+
+RESULT=$(echo "$TSV" | jq -Rsc '
+  ["run_label","docker_image","test_package","test_class_list",
+   "test_iterations","scale_row_count","distribution","config_options"] as $h |
+  split("\n") | map(select(length > 0)) |
+  [.[] | split("\t") | [range(length) as $i | {($h[$i]): .[$i]}] | add |
+    if .scale_row_count then
+      .scale_row_count = ((.scale_row_count | tonumber) * 1000000 | tostring)
+    else . end |
+    if .test_iterations then
+      .test_iterations = ((.test_iterations | tonumber) as $n |
+        if ($n % 2) == 0 then ($n + 1) else $n end | tostring)
+    else . end
+  ]
+')
+
+COUNT=$(echo "$RESULT" | jq 'length')
+EXPECTED=$(grep -m1 '^EXPECTED_COMBOS=' "$FILE" | cut -d= -f2 || true)
+
+if [[ -n "$EXPECTED" && "$COUNT" -ne "$EXPECTED" ]]; then
+  echo "::error::Expected ${EXPECTED} combinations but got ${COUNT}. Update EXPECTED_COMBOS in ${FILE}."; exit 1
+fi
+echo "Matrix combinations: ${COUNT}"
+
+MATRIX_OUT=$(echo "$RESULT" | jq -c .)
+if [[ -n "$GITHUB_OUTPUT" ]]; then
+  echo "matrix=${MATRIX_OUT}" >> "$GITHUB_OUTPUT"
+  echo "combo_count=${COUNT}" >> "$GITHUB_OUTPUT"
+  echo "### Matrix: ${COUNT} combinations" >> "$GITHUB_STEP_SUMMARY"
+else
+  echo "matrix=${MATRIX_OUT}"
+fi
+

.github/scripts/build-server-distribution-remote.sh

@@ -28,23 +28,32 @@ fi
 
 title () { echo; echo $1; }
 
+DEEPHAVEN_VERSION_FILE=${GIT_DIR}/deephaven-core/build/version
+if [ -f "${DEEPHAVEN_VERSION_FILE}" ]; then
+  echo "Server distribution already built. Skipping."
+  exit 0
+fi
+
 OWNER=$(sed 's/'"${BRANCH_DELIM}"'.*//g' <<< "${DOCKER_IMG}")
 BRANCH_NAME=$(sed 's/.*'"${BRANCH_DELIM}"'//g' <<< "${DOCKER_IMG}")
 echo "OWNER: ${OWNER}"
 echo "BRANCH: ${BRANCH_NAME}"
 
 title "-- Cloning deephaven-core --"
 cd ${GIT_DIR}
-rm -rf deephaven-core
-# Do not use --single-branch here, because it does not allow checkout by commit hash 
-git clone https://github.com/${OWNER}/deephaven-core.git
+if [ ! -d "deephaven-core" ]; then
+  # Do not use --single-branch here, because it does not allow checkout by commit hash 
+  git clone https://github.com/${OWNER}/deephaven-core.git
+fi
 cd deephaven-core
+git fetch origin
 git checkout ${BRANCH_NAME}
 
 title "-- Cloning deephaven-server-docker --"
 cd ${GIT_DIR}
-rm -rf deephaven-server-docker
-git clone -b main --single-branch https://github.com/deephaven/deephaven-server-docker.git
+if [ ! -d "deephaven-server-docker" ]; then
+  git clone -b main --single-branch https://github.com/deephaven/deephaven-server-docker.git
+fi
 
 title "-- Assembling Python Deephaven Core Server --"
 cd ${GIT_DIR}/deephaven-core

.github/scripts/clear-test-server-remote.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+
+# Copyright (c) 2026-2026 Deephaven Data Labs and Patent Pending
+
+# Clear all build artifacts and data on the remote server to prepare for a fresh setup
+
+GIT_DIR=${HOME}/git
+RUN_DIR=${HOME}/run
+DEEPHAVEN_DIR=${HOME}/deephaven
+
+title () { echo; echo $1; }
+
+title "- Clearing Remote Server on $(hostname) -"
+
+title "-- Removing Git Repositories --"
+sudo rm -rf ${GIT_DIR}
+
+if command -v docker &>/dev/null; then
+  title "-- Stopping Docker Containers --"
+  sudo docker ps -q | xargs --no-run-if-empty -n 1 sudo docker kill
+  title "-- Removing Docker Containers --"
+  sudo docker ps -a -q | xargs --no-run-if-empty -n 1 sudo docker rm --force
+  title "-- Removing Docker Images --"
+  sudo docker images -a -q | xargs --no-run-if-empty -n 1 sudo docker rmi --force
+  title "-- Pruning Docker Volumes --"
+  sudo docker system prune --volumes --force
+fi
+
+title "-- Removing Run Directory --"
+sudo rm -rf ${RUN_DIR}
+
+title "-- Removing Deephaven Directory --"
+sudo rm -rf ${DEEPHAVEN_DIR}

.github/scripts/manage-deephaven-remote.sh

@@ -35,19 +35,16 @@ if [[ ${CONFIG_OPTS} == "<default>" ]]; then
   CONFIG_OPTS="-Xmx24g"
 fi
 echo "CONFIG_OPTS=${CONFIG_OPTS}" > .env
+echo "ENV_DEEPHAVEN_HOST_OS_DIR=${DEEPHAVEN_DIR}" >> .env
 
-IS_BRANCH="false"
-if [[ ${DOCKER_IMG} == *"@sha"*":"* ]]; then
-  IS_BRANCH="false"
+if [[ ${DOCKER_IMG} == ghcr.io/* ]]; then
+  echo "DOCKER_IMG=${DOCKER_IMG}" >> .env
+  docker compose pull
 elif [[ ${DOCKER_IMG} == *":"* ]]; then
-  IS_BRANCH="true"
-fi
-
-if [[ ${IS_BRANCH} == "false" ]]; then
+  echo "DOCKER_IMG=deephaven/server:benchmark-local" >> .env
+else
   echo "DOCKER_IMG=ghcr.io/deephaven/server:${DOCKER_IMG}" >> .env
   docker compose pull
-else 
-  echo "DOCKER_IMG=deephaven/server:benchmark-local" >> .env
 fi
 
 if [[ ${DIRECTIVE} == 'start' ]]; then

.github/scripts/run-benchmarks-remote.sh

@@ -43,7 +43,7 @@ title "-- Running Benchmarks --"
 set +f
 cd ${RUN_DIR}
 cat ${RUN_TYPE}-scale-benchmark.properties | sed 's|${baseRowCount}|'"${ROW_COUNT}|g" | sed 's|${baseDistrib}|'"${DISTRIB}|g" | sed 's|${userHome}|'"${HOME}|g" > scale-benchmark.properties
-JAVA_OPTS=$(echo -Dbenchmark.profile=scale-benchmark.properties -jar deephaven-benchmark-*-standalone.jar -cp standard-tests.jar)
+JAVA_OPTS=$(echo -Xmx4g -Dbenchmark.profile=scale-benchmark.properties -jar deephaven-benchmark-*-standalone.jar -cp standard-tests.jar)
 set -f
 
 if [ "${TAG_NAME}" = "Any" ]; then

.github/scripts/run-ssh-local.sh

@@ -5,7 +5,7 @@ set -o pipefail
 set -o nounset
 set -f
 
-# Copyright (c) 2023-2024 Deephaven Data Labs and Patent Pending
+# Copyright (c) 2023-2026 Deephaven Data Labs and Patent Pending
 
 # Executes a local script on a remote server while storing output relative to the 
 # local working directory. Also, this script wraps arguments provided for the
@@ -26,4 +26,19 @@ for i in ${@:5}; do
   args+=("'"$i"'")
 done
 
-ssh -o 'ServerAliveInterval 60' ${USER}@${HOST} 'bash -s' -- "${args[@]}" < ${SCRIPT_DIR}/${SCRIPT_NAME}.sh |& tee logs/${SCRIPT_NAME}.log
+MAX_RETRIES=5
+RETRY_DELAY=15
+for ((attempt=1; attempt<=MAX_RETRIES; attempt++)); do
+  if ssh -o 'ConnectTimeout 10' ${USER}@${HOST} true; then
+    break
+  fi
+  if [[ $attempt -lt $MAX_RETRIES ]]; then
+    echo "SSH connection attempt $attempt failed. Retrying in ${RETRY_DELAY}s..."
+    sleep $RETRY_DELAY
+  else
+    echo "SSH connection failed after $MAX_RETRIES attempts"
+    exit 1
+  fi
+done
+
+ssh -o 'ConnectTimeout 10' -o 'ServerAliveInterval 60' ${USER}@${HOST} 'bash -s' -- "${args[@]}" < ${SCRIPT_DIR}/${SCRIPT_NAME}.sh |& tee logs/${SCRIPT_NAME}.log