diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt index a71030887c43f..dff27afb0c44e 100644 --- a/extra/mariabackup/CMakeLists.txt +++ b/extra/mariabackup/CMakeLists.txt @@ -114,3 +114,28 @@ ADD_DEPENDENCIES(mbstream GenError) IF(MSVC) SET_TARGET_PROPERTIES(mbstream PROPERTIES LINK_FLAGS setargv.obj) ENDIF() + + +######################################################################## +# mariadb-backup-server: BACKUP SERVER-compatible shell wrapper +######################################################################## +# A drop-in mariadb-backup-compatible POSIX-sh wrapper that translates the +# CLI into server-side BACKUP SERVER SQL. Experimental; OFF by default +# Installed as a mariadb-backup-server; clients opt in via symlink/alias +# (see extra/mariabackup/scripts/README.md). +OPTION(WITH_MARIABACKUP_WRAPPER + "Install the BACKUP SERVER shell wrapper (mariadb-backup-server)" OFF) +ADD_FEATURE_INFO(MARIABACKUP_WRAPPER WITH_MARIABACKUP_WRAPPER + "BACKUP SERVER-compatible mariadb-backup shell wrapper") + +IF(WITH_MARIABACKUP_WRAPPER AND NOT WIN32) + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/scripts/mariadb-backup-server.sh + ${CMAKE_CURRENT_BINARY_DIR}/mariadb-backup-server COPYONLY) + INSTALL_SCRIPT(${CMAKE_CURRENT_BINARY_DIR}/mariadb-backup-server + DESTINATION ${INSTALL_BINDIR} COMPONENT Backup) + + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/scripts/mbstream-server.sh + ${CMAKE_CURRENT_BINARY_DIR}/mbstream-server COPYONLY) + INSTALL_SCRIPT(${CMAKE_CURRENT_BINARY_DIR}/mbstream-server + DESTINATION ${INSTALL_BINDIR} COMPONENT Backup) +ENDIF() diff --git a/extra/mariabackup/scripts/README.md b/extra/mariabackup/scripts/README.md new file mode 100644 index 0000000000000..f9253022a36a0 --- /dev/null +++ b/extra/mariabackup/scripts/README.md @@ -0,0 +1,263 @@ +# mariadb-backup-server.sh — a BACKUP SERVER wrapper + +`mariadb-backup-server.sh` makes the server-side `BACKUP SERVER` command look +like the old `mariadb-backup` tool. You call it with the familiar +`--backup` / `--prepare` / `--copy-back` options; under the hood +it just runs `BACKUP SERVER TO ''` over a normal `mariadb` +connection and lets the server do the work. It's plain POSIX `sh`, +so it runs anywhere `mariadb-backup` does. + +You need a server that supports `BACKUP SERVER`, the `mariadb` client on +`PATH`, and an account allowed to run `BACKUP SERVER`. The parent of the +target directory has to exist and be writable. + +## Installing / enabling it + +The wrapper is experimental and off by default. Build with +`-DWITH_MARIABACKUP_WRAPPER=ON` and it installs next to the real +binaries, under its own names so it never shadows them: + +``` +/usr/bin/mariadb-backup # the C++ binary, unchanged +/usr/bin/mariadb-backup-server # this wrapper +/usr/bin/mbstream # the C++ binary, unchanged +/usr/bin/mbstream-server # the tar-based mbstream shim (for --stream) +``` + +To send your existing `mariadb-backup` (and `mbstream`) calls through the +wrapper instead, point the names at them yourself — an alias, or a symlink +earlier in `PATH`: + +```sh +alias mariadb-backup=mariadb-backup-server +alias mbstream=mbstream-server +# or +ln -s /usr/bin/mariadb-backup-server ~/bin/mariadb-backup +ln -s /usr/bin/mbstream-server ~/bin/mbstream +``` + +`mbstream-server` is only needed for `--stream` backups (it extracts the +wrapper's tar stream). + +## Backing up + +```sh +mariadb-backup-server --backup --target-dir=/backup/full +``` + +That runs `BACKUP SERVER TO '/backup/full'`. Use `--parallel=N` to ask for N +concurrent streams (`... N CONCURRENT`; N=1 is the default and changes +nothing). + +Connection options: +`--user`, `--password`, `--host`, `--port`, `--socket`, `--defaults-file`, +`--defaults-extra-file` and their short forms are passed straight to +the `mariadb` client. + +`--throttle`, `--no-lock` and `--safe-slave-backup` are accepted and ignored; + +When the backup finishes the wrapper drops a `backup-prepare.cnf` into the +target dir, next to the server's own `backup.cnf`. It records where `mariadbd` +lives and the InnoDB layout, so `--prepare` can recover the backup later +without you respelling all of that. + +### Streaming to stdout + +```sh +mariadb-backup-server --backup --stream=tar > /backup/full.tar +``` + +`--stream` runs `BACKUP SERVER WITH [N CONCURRENT] ''` instead of +`... TO ''`. The server hands each stream's tar to ``, the wrapper +collects the parts and writes them to its stdout, so you can redirect to a file +or pipe onwards. + +Two things follow from how `BACKUP SERVER` streams, and both differ from +`mariadb-backup`: + +- **Local only.** The stream command runs *inside the server*, so its output +lands on the server's filesystem. The wrapper can only pick it up when it runs +on the same host (a shared filesystem), as the user the server writes as +(typically `mysql`), or with a `--target-dir` the server can write. A remote +`--host` cannot stream this way. + +- **tar only.** The server emits tar, never `xbstream`. Any `--stream=` +value (including `xbstream`) is accepted but the output is always tar. + +`--target-dir` is optional here; when given it is just the scratch directory for +the per-stream parts (otherwise a `mktemp` dir is used). + +The output is the per-stream tar entries concatenated, with +`backup-prepare.cnf` appended as the trailing archive. + +```sh +mkdir restore && tar -xf /backup/full.tar -C restore +``` + +After extraction the directory holds the data files, the server's `backup.cnf` +*and* the wrapper's `backup-prepare.cnf`, so it can be prepared exactly like a +directory backup: + +```sh +mariadb-backup-server --prepare --target-dir=restore +``` + +### `mbstream.sh` — the extraction shim + +The real `mbstream`/`xbstream` binary cannot read the wrapper's stream, because +that stream is plain tar, not the `xbstream` format. So a companion shim, +`mbstream.sh`, ships next to `mariadb-backup-server.sh` and maps the `mbstream` CLI onto +`tar`, letting existing pipelines (and tests) that call `mbstream` keep working +unchanged: + +```sh +mbstream-server -x -C restore < /backup/full.tar # tar -x -C restore +mbstream-server -c -C dir file1 file2 # tar -c -C dir file1 file2 +``` + +Notes: + +- **Extraction is a plain `tar -x`** — the stream has a single +end-of-archive marker + +- **mbstream-only options are accepted and ignored** — +`-p` / `--parallel`, compression flags, `-v` : So legacy +invocations don't break. + +- It understands `-x` (extract, from stdin) and `-c` (create, to stdout), +with `-C `; anything else is treated as a file operand or ignored. + +- It is a thin tar wrapper, so it only understands the wrapper's tar streams — +do not point it at a real `xbstream` archive, and do not feed the wrapper's +output to the real `mbstream`. + +## Preparing + +```sh +mariadb-backup-server --prepare --target-dir=/backup/full +``` + +Prepare makes the backup bootable: it starts `mariadbd --bootstrap` on the +target directory, replays the archived redo up to the backup's end LSN, then +replaces the archived log with a fresh `ib_logfile0` so a normal server can +start on it. Both `backup.cnf` and `backup-prepare.cnf` have to be there (they +are, if this wrapper took the backup). + +Options: + +- `--use-memory=N` — buffer pool size during recovery. +- `--innodb-*` and `--tmpdir` are forwarded to the bootstrap server. +- `--defaults-file` / `--defaults-extra-file`, and encryption options such as + `--file-key-management*` / `--loose-file-key-management*` / `--plugin-load-add`, + are layered onto the bootstrap (as an extra defaults file / extra options) so + you can supply anything `backup-prepare.cnf` did not record. + +The `mariadbd` used for the bootstrap is the path recorded in +`backup-prepare.cnf` at backup time if that binary exists; otherwise `mariadbd` +from `PATH`. (Recorded-first matters: it is the same version that took the +backup, so it can always parse the backed-up tablespace format.) + +## Restoring + +With the backup prepared and the server stopped, put it into the datadir: + +```sh +mariadb-backup-server --copy-back --target-dir=/backup/full --datadir=/var/lib/mysql +mariadb-backup-server --move-back --target-dir=/backup/full --datadir=/var/lib/mysql +``` + +`--copy-back` leaves the backup where it is; `--move-back` renames the files +across, which is quicker on the same filesystem but consumes the backup. Either +way the datadir is created if missing, and the wrapper won't write into a +non-empty datadir unless you add `--force-non-empty-directories`. + +If the source server kept its Aria logs outside the datadir, pass the same +`--aria-log-dir-path=` you use on the server. The wrapper creates that +directory and moves the restored `aria_log_control` / `aria_log.*` files into +it, so the server finds them on restart: + +```sh +mariadb-backup-server --copy-back --target-dir=/backup/full --datadir=/var/lib/mysql \ + --aria-log-dir-path=/var/lib/aria_logs +``` + +Neither fixes ownership, so finish up with: + +```sh +chown -R mysql:mysql /var/lib/mysql +systemctl start mariadb +``` + +## What it doesn't do + +These stop with an error instead of quietly producing an incomplete backup: + +- incremental backup/prepare: `--incremental-basedir`, `--incremental-dir`, `--apply-log-only` +- partial backup: `--databases`, `--tables`, `--tables-file`. This needs server-side +`backup_include`/`backup_exclude`, which don't exist yet +- compression and encryption of the output: `--compress`, `--encrypt` (error out) +- `--rollback-xa`: not supported (errors out) + +`--stream` is supported with the caveats above (local only, tar only, extract +with a plain `tar -x`); + +`--export` is accepted but not implemented: it warns and does a plain recovery +(no per-table `.cfg` files for IMPORT TABLESPACE). + +## Environment overrides + +The wrapped commands can be overridden via environment variables, +mainly for testing: + +- `MARIADB`: the client used to talk to the server (default `mariadb`), +e.g. `MARIADB='mariadb --protocol=tcp'`. + +- `MARIADBD`: the server binary the `--prepare` bootstrap runs (by default the +path recorded in `backup-prepare.cnf`, else `mariadbd` on `PATH`). When set it +overrides that resolution. To run it under `rr`, put `rr` in `MARIADBD` and let +`rr`'s own `_RR_TRACE_DIR` choose where the trace goes, e.g. +`_RR_TRACE_DIR=/dev/shm/rr MARIADBD='rr record mariadbd' ...`. + +- `TAR`: the tar implementation (default `tar`), +e.g. `TAR=bsdtar` (libarchive-tools). +Used by `--stream` and by `mbstream-server`. + +## The two .cnf files + +`backup.cnf` is written by the server and tells `--prepare` what parts of redo to replay: + +```ini +[server] +# checkpoint=54088 +innodb_log_recovery_start=54088 # recovery starts scanning here +innodb_log_recovery_target=56337 # the backup's end LSN; recovery stops here +``` + +`backup-prepare.cnf` is written by the wrapper and handed to the prepare +bootstrap as its defaults file: + +```ini +# mariadbd=/usr/sbin/mariadbd +[mariadbd] +innodb_page_size=16384 +innodb_data_file_path=ibdata1:12M:autoextend +innodb_undo_tablespaces=3 +innodb_checksum_algorithm=full_crc32 +innodb_log_file_size=100663296 +``` + +If the server is encrypted it also records how to load the key-management +plugin again, so an encrypted backup can be prepared without extra input. +For `file_key_management` it captures every plugin variable (the same way +`mariadb-backup` writes them into `backup-my.cnf`) + +```ini +plugin-load-add=file_key_management +innodb_encrypt_log=ON +loose-file-key-management +loose-file_key_management_filename=/etc/mysql/keys.enc +loose-file_key_management_filekey=FILE:/etc/mysql/keyfile.key +loose-file_key_management_encryption_algorithm=aes_cbc +loose-file_key_management_digest=sha1 +loose-file_key_management_use_pbkdf2=1 +``` diff --git a/extra/mariabackup/scripts/mariadb-backup-server.sh b/extra/mariabackup/scripts/mariadb-backup-server.sh new file mode 100755 index 0000000000000..57a64109b4159 --- /dev/null +++ b/extra/mariabackup/scripts/mariadb-backup-server.sh @@ -0,0 +1,366 @@ +#!/bin/sh +me=${0##*/} +die() { + echo "$me: $*" >&2 + exit 1 +} + +# The wrapped commands can be overridden for testing, e.g. +# MARIADB='mariadb --protocol=tcp', TAR=bsdtar. + +# To run the prepare bootstrap under rr, include it in MARIADBD +# (rr's own _RR_TRACE_DIR controls where the trace is written). +# e.g. _RR_TRACE_DIR=/dev/shm/rr MARIADBD='rr record mariadbd' ... +: "${MARIADB:=mariadb}" +: "${TAR:=tar}" + +MODE= +TARGET_DIR= +DATADIR= +PARALLEL= +USE_MEMORY= +FORCE_NON_EMPTY= +EXPORT= +ROLLBACK_XA= +MARIADB_OPTS= +INNODB_OPTS= +MYSQLD_EXTRA= +PREPARE_DEFAULTS= +ARIA_LOG_DIR= +STREAM= + +while [ $# -gt 0 ]; do + case $1 in + --backup) MODE=backup ;; + --prepare|--apply-log) MODE=prepare ;; + --copy-back) MODE=copy-back ;; + --move-back) MODE=move-back ;; + + --target-dir=*) TARGET_DIR=${1#*=} ;; + --datadir=*) DATADIR=${1#*=} ;; + --aria-log-dir-path=*) ARIA_LOG_DIR=${1#*=} ;; + --use-memory=*) USE_MEMORY=${1#*=} ;; + --parallel=*) PARALLEL=${1#*=} ;; + + --export) EXPORT=1 ;; + --rollback-xa) ROLLBACK_XA=1 ;; + --force-non-empty-directories) FORCE_NON_EMPTY=1 ;; + + --innodb|--innodb=*|--innodb-*|--innodb_*|--skip-innodb-*|--skip_innodb_*) + INNODB_OPTS="$INNODB_OPTS $1" ;; + --tmpdir=*) MYSQLD_EXTRA="$MYSQLD_EXTRA $1" ;; + -t) MYSQLD_EXTRA="$MYSQLD_EXTRA --tmpdir=$2"; shift ;; + -t*) MYSQLD_EXTRA="$MYSQLD_EXTRA --tmpdir=${1#-t}" ;; + --incremental-basedir=*|--incremental-dir=*) + die "incremental backup/prepare is not supported" ;; + --apply-log-only) + die "--apply-log-only is not supported" ;; + --databases=*|--databases-exclude=*|--tables=*|--tables-exclude=*|--tables-file=*) + die "partial backup needs server-side backup_include/backup_exclude, which doesn't exist yet" ;; + # BACKUP SERVER only ever produces tar + --stream|--stream=*) STREAM=1 ;; + --compress|--compress=*|--compress-threads=*) die "--compress is not supported" ;; + --encrypt|--encrypt=*) die "--encrypt is not supported" ;; + --innobackupex) die "innobackupex mode is not supported" ;; + + # Defaults files feed the backup client + --defaults-file=*|--defaults-extra-file=*) + MARIADB_OPTS="$MARIADB_OPTS $1" + PREPARE_DEFAULTS="$PREPARE_DEFAULTS --defaults-extra-file=${1#*=}" ;; + # Encryption options are forwarded to the prepare bootstrap. + --file-key-management*|--plugin-load-add=*|--loose-file-key-management*|\ + --aria-encrypt-tables*|--encrypt-tmp-disk-tables*) + PREPARE_DEFAULTS="$PREPARE_DEFAULTS $1" ;; + + --user=*|--password=*|--host=*|--port=*|--socket=*|\ + --defaults-group=*|\ + --secure-auth|--skip-secure-auth|--ssl|--ssl-verify-server-cert|\ + --ssl-ca=*|--ssl-capath=*|--ssl-cert=*|--ssl-cipher=*|\ + --ssl-crl=*|--ssl-crlpath=*|--ssl-key=*|--tls-version=*) + MARIADB_OPTS="$MARIADB_OPTS $1" ;; + -p) + if [ -n "${2-}" ] && case $2 in -*) false ;; *) true ;; esac; then + MARIADB_OPTS="$MARIADB_OPTS -p$2"; shift + else + MARIADB_OPTS="$MARIADB_OPTS -p" + fi ;; + -u|-P|-S) MARIADB_OPTS="$MARIADB_OPTS $1 $2"; shift ;; + -H) MARIADB_OPTS="$MARIADB_OPTS --host=$2"; shift ;; + -p*|-u*|-P*|-S*) MARIADB_OPTS="$MARIADB_OPTS $1" ;; + -H*) MARIADB_OPTS="$MARIADB_OPTS --host=${1#-H}" ;; + + -h) DATADIR=$2; shift ;; + -h*) DATADIR=${1#-h} ;; + + # Everything else is accepted and ignored: + *) ;; + esac + shift +done + +# In stream mode --target-dir is optional: +# it is only a scratch directory for the per-stream tar parts +# (a mktemp dir is used when it is omitted). +[ -n "$STREAM" ] || [ -n "$TARGET_DIR" ] || die "--target-dir required" + +# Run the client with the connection options we collected. +ask() { $MARIADB $MARIADB_OPTS -BN -e "$1" 2>/dev/null; } + +# Print the backup-prepare.cnf contents to stdout. +# It captures everything --prepare's offline bootstrap needs: +# where mariadbd lives, the InnoDB parameters, and how to +# reload the encryption key plugin. Used both for a +# directory backup (written into the target dir) +# and a streamed backup (appended to the stream so it lands +# in the extracted directory). + +write_prepare_cnf() { + _mariadbd= + _pidfile=$(ask "SELECT @@global.pid_file") + if [ -n "$_pidfile" ] && [ -r "$_pidfile" ]; then + _pid=$(cat "$_pidfile" 2>/dev/null) + [ -n "$_pid" ] && _mariadbd=$(readlink -f "/proc/$_pid/exe" 2>/dev/null) + fi + if [ -z "$_mariadbd" ]; then + _basedir=$(ask "SELECT @@global.basedir") + for _c in "$_basedir/sbin/mariadbd" "$_basedir/bin/mariadbd" \ + "$_basedir/sbin/mysqld" "$_basedir/bin/mysqld"; do + [ -x "$_c" ] && { _mariadbd=$_c; break; } + done + fi + + _page_size=$(ask "SELECT @@global.innodb_page_size") + _data_file_path=$(ask "SELECT @@global.innodb_data_file_path") + _undo_ts=$(ask "SELECT @@global.innodb_undo_tablespaces") + _checksum=$(ask "SELECT @@global.innodb_checksum_algorithm") + _log_file_size=$(ask "SELECT @@global.innodb_log_file_size") + + _enc=$(ask "SELECT LOWER(plugin_name) FROM information_schema.PLUGINS + WHERE plugin_type='ENCRYPTION' AND plugin_status='ACTIVE' LIMIT 1") + + [ -n "$_mariadbd" ] && echo "# mariadbd=$_mariadbd" + echo "[mariadbd]" + [ -n "$_page_size" ] && echo "innodb_page_size=$_page_size" + [ -n "$_data_file_path" ] && echo "innodb_data_file_path=$_data_file_path" + [ -n "$_undo_ts" ] && echo "innodb_undo_tablespaces=$_undo_ts" + [ -n "$_checksum" ] && echo "innodb_checksum_algorithm=$_checksum" + [ -n "$_log_file_size" ] && echo "innodb_log_file_size=$_log_file_size" + + if [ -n "$_enc" ]; then + echo "plugin-load-add=$_enc" + _plugin_dir=$(ask "SELECT @@global.plugin_dir") + [ -n "$_plugin_dir" ] && echo "plugin-dir=$_plugin_dir" + case $(ask "SELECT @@global.innodb_encrypt_log") in + 1|ON) echo "innodb_encrypt_log=ON" ;; + esac + if [ "$_enc" = file_key_management ]; then + echo "loose-file-key-management" + ask "SHOW VARIABLES LIKE 'file_key_management%'" | + while read -r _fkm_name _fkm_value; do + [ -n "$_fkm_name" ] && echo "loose-$_fkm_name=$_fkm_value" + done + fi + fi +} + + +# prepare +if [ "$MODE" = prepare ]; then + [ -z "$ROLLBACK_XA" ] || die "--rollback-xa is not supported" + [ -d "$TARGET_DIR" ] || die "no such directory: $TARGET_DIR" + [ -f "$TARGET_DIR/backup.cnf" ] || die "backup.cnf not found in $TARGET_DIR" + + cnf=$TARGET_DIR/backup-prepare.cnf + [ -f "$cnf" ] || die "$cnf missing - was this backup made by the wrapper?" + [ -z "$EXPORT" ] || echo "$me: --export not implemented, doing a plain recovery" >&2 + + # Prefer the binary recorded at backup time + # else, fall back to mariadbd on PATH only if the + # recorded one is missing. + # MARIADBD overrides the recorded/PATH binary + if [ -n "${MARIADBD-}" ]; then + mariadbd=$MARIADBD + else + mariadbd=$(sed -n 's/^# *mariadbd=//p' "$cnf" | tail -n1) + [ -n "$mariadbd" ] && [ -x "$mariadbd" ] || mariadbd=mariadbd + fi + + # backup.cnf tells us the LSN window recovery should replay. + start=$(grep '^innodb_log_recovery_start' "$TARGET_DIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + target=$(grep '^innodb_log_recovery_target' "$TARGET_DIR/backup.cnf" | cut -d= -f2 | tr -d ' ') + + opts="--datadir=$TARGET_DIR --innodb=FORCE" + [ -n "$start" ] && opts="$opts --innodb-log-recovery-start=$start" + [ -n "$target" ] && opts="$opts --innodb-log-recovery-target=$target" + [ -n "$USE_MEMORY" ] && opts="$opts --innodb-buffer-pool-size=$USE_MEMORY" + opts="$opts$INNODB_OPTS$MYSQLD_EXTRA" + + # Recovery stops at the backup's end LSN but leaves the + # archived log (ib_.log) behind, which a normal server + # won't boot from. After recovery we build a fresh ib_logfile0 + # (header + a checkpoint at the end LSN) and put it in place + # of the archived log. + input=/dev/null + newlog=$TARGET_DIR/ib_logfile0.new + if [ -n "$target" ]; then + lsn=$(printf '%016x' "$target") + rm -f "$newlog" + input=$(mktemp) + cat > "$input" <&2 + exit 0 +fi + + +# copy-back / move-back +if [ "$MODE" = copy-back ] || [ "$MODE" = move-back ]; then + [ -d "$TARGET_DIR" ] || die "no such directory: $TARGET_DIR" + [ -f "$TARGET_DIR/backup.cnf" ] || die "backup.cnf not found in $TARGET_DIR" + [ -n "$DATADIR" ] || die "--datadir required for --$MODE" + + # mariadb-backup creates the datadir if it's missing; do the same. + [ -d "$DATADIR" ] || mkdir -p "$DATADIR" || die "cannot create datadir: $DATADIR" + + # Refuse a non-empty datadir (dotfiles included) unless told otherwise. + if [ -z "$FORCE_NON_EMPTY" ]; then + for f in "$DATADIR"/* "$DATADIR"/.[!.]* "$DATADIR"/..?*; do + { [ -e "$f" ] || [ -L "$f" ]; } && + die "datadir not empty: $DATADIR (pass --force-non-empty-directories)" + done + fi + + if [ "$MODE" = copy-back ]; then + echo "$me: copying $TARGET_DIR -> $DATADIR" >&2 + cp -R "$TARGET_DIR"/. "$DATADIR"/ || die "copy-back failed" + else + echo "$me: moving $TARGET_DIR -> $DATADIR" >&2 + for f in "$TARGET_DIR"/* "$TARGET_DIR"/.[!.]* "$TARGET_DIR"/..?*; do + { [ -e "$f" ] || [ -L "$f" ]; } || continue + mv "$f" "$DATADIR"/ || die "move-back failed" + done + fi + + # Aria logs live in --aria-log-dir-path when set; + # backup placed them at the datadir root, + # so relocate them there and create the directory the server + # expects on restart. + if [ -n "$ARIA_LOG_DIR" ] && [ "$ARIA_LOG_DIR" != "$DATADIR" ]; then + mkdir -p "$ARIA_LOG_DIR" || die "cannot create aria-log-dir-path: $ARIA_LOG_DIR" + for f in "$DATADIR"/aria_log_control "$DATADIR"/aria_log.*; do + [ -e "$f" ] && { mv "$f" "$ARIA_LOG_DIR"/ || die "aria log relocate failed"; } + done + fi + + echo "Restore completed: $DATADIR" >&2 + echo "$me: now run: chown -R mysql:mysql $DATADIR && start the server" >&2 + exit 0 +fi + + +# backup (streaming) +# "BACKUP SERVER WITH [N CONCURRENT] ''" runs +# inside the server feeding that stream's tar to the command's stdin. +if [ -n "$STREAM" ]; then + if [ -n "$TARGET_DIR" ]; then + scratch=$TARGET_DIR + mkdir -p "$scratch" || die "cannot create scratch dir: $scratch" + keep_scratch=1 + else + scratch=$(mktemp -d "${TMPDIR:-/tmp}/mariabackup-stream.XXXXXX") \ + || die "cannot create scratch directory" + keep_scratch= + fi + + cleanup_stream() { + rm -f "$scratch"/.mariabackup-stream.sh "$scratch"/*.tar \ + "$scratch"/backup-prepare.cnf + [ -n "$keep_scratch" ] || rmdir "$scratch" 2>/dev/null + } + + # The server appends the stream index as $1 and writes that stream's tar + # to our stdin; we drop each one into the scratch dir under .tar. + helper=$scratch/.mariabackup-stream.sh + cat > "$helper" < "$scratch/\$1.tar" +EOF + + # Invoke via "/bin/sh " so the script needs no execute bit + # (scratch may sit on a noexec filesystem such as /dev/shm). + sql="BACKUP SERVER WITH" + case $PARALLEL in # --parallel=N -> "N CONCURRENT" (1 is default) + ''|*[!0-9]*) ;; + *) [ "$PARALLEL" -gt 1 ] && sql="$sql $PARALLEL CONCURRENT" ;; + esac + sql="$sql '/bin/sh $helper'" + + # Keep stdout clean for the tar: send any client output to stderr. + $MARIADB $MARIADB_OPTS -e "$sql" >&2 \ + || { cleanup_stream; die "BACKUP SERVER failed"; } + + # Concatenate the per-stream tars to stdout in index order, + # then append backup-prepare.cnf as one more tar archive + # so it lands in the extracted directory alongside the + # server's backup.cnf. The per-stream tars carry no + # end-of-archive marker; only this trailing + # backup-prepare.cnf adds the single end marker, + # so the whole stream extracts with a plain "tar -x". + n=1 + while [ -f "$scratch/$n.tar" ]; do + cat "$scratch/$n.tar" || { cleanup_stream; die "streaming to stdout failed"; } + n=$((n + 1)) + done + + echo "$me: appending backup-prepare.cnf to the stream" >&2 + write_prepare_cnf > "$scratch/backup-prepare.cnf" \ + || { cleanup_stream; die "could not build backup-prepare.cnf"; } + $TAR -C "$scratch" -cf - backup-prepare.cnf \ + || { cleanup_stream; die "could not append backup-prepare.cnf to the stream"; } + + cleanup_stream + echo "$me: streamed $((n - 1)) tar stream(s) plus backup-prepare.cnf to stdout" >&2 + exit 0 +fi + + +# --- backup (directory) ----------------------------------------------------- +parent=$(dirname "$TARGET_DIR") +[ -d "$parent" ] || die "parent directory does not exist: $parent" +[ -w "$parent" ] || die "parent directory is not writable: $parent" + +sql="BACKUP SERVER TO '$TARGET_DIR'" +case $PARALLEL in # --parallel=N -> "N CONCURRENT" (1 is the default) + ''|*[!0-9]*) ;; + *) [ "$PARALLEL" -gt 1 ] && sql="$sql $PARALLEL CONCURRENT" ;; +esac +$MARIADB $MARIADB_OPTS -e "$sql" || die "BACKUP SERVER failed" + +# Create backup-prepare.cnf with everything --prepare's offline +# bootstrap needs: where mariadbd is, the InnoDB parameters, +# and how to load the key plugin again. +[ -f "$TARGET_DIR/backup.cnf" ] || exit 0 + +echo "$me: writing backup-prepare.cnf" >&2 +write_prepare_cnf > "$TARGET_DIR/backup-prepare.cnf" diff --git a/extra/mariabackup/scripts/mbstream-server.sh b/extra/mariabackup/scripts/mbstream-server.sh new file mode 100755 index 0000000000000..7ba2bbabe7e6d --- /dev/null +++ b/extra/mariabackup/scripts/mbstream-server.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# mbstream-compatible for the BACKUP SERVER wrapper. +# +# This maps the mbstream CLI to `tar`: +# +# mbstream -x -C < archive -> tar -x -C +# mbstream -c > archive -> tar -c +# +# mbstream-only options are accepted and ignored so +# existing invocations keep working unchanged. +me=${0##*/} +die() { + echo "$me: $*" >&2 + exit 1 +} + +# The tar implementation can be overridden for testing. +# e.g. TAR=bsdtar. +: "${TAR:=tar}" + +mode= +dir=. +files= + +while [ $# -gt 0 ]; do + case $1 in + -x|--extract) mode=x ;; + -c|--create) mode=c ;; + -C|--directory) dir=$2; shift ;; + -C*) dir=${1#-C} ;; + --directory=*) dir=${1#*=} ;; + + # mbstream-only flags that have no tar equivalent: drop them. + -p|--parallel) shift ;; + -p*|--parallel=*) ;; + --decompress|--compress) ;; + -v|--verbose) ;; + + --) shift; break ;; + # Reject anything we do not recognise rather than + # silently ignoring it (e.g. GNU tar's -b takes an argument + # that would otherwise be mistaken for a file operand). + # This is an mbstream-on-tar shim, not tar. + -*) die "unsupported option: $1" ;; + *) files="$files $1" ;; + esac + shift +done + +while [ $# -gt 0 ]; do + files="$files $1" + shift +done + +case $mode in + # The wrapper concatenates the per-stream tar entries + # with no end-of-archive marker between them; only the + # trailing backup-prepare.cnf adds the single end marker. + x) exec $TAR -x -f - -C "$dir" ;; + c) + [ -n "$files" ] || files=. + exec $TAR -c -f - -C "$dir" $files ;; + *) die "expected -x (extract) or -c (create)" ;; +esac diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 80b202aa4c03e..59f386a86edae 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -166,6 +166,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/opt_hints.cc ../sql/opt_hints.h ../sql/opt_trace_ddl_info.cc ../sql/opt_trace_ddl_info.h ../sql/sql_path.cc + ../sql/sql_backup.cc ${GEN_SOURCES} ${MYSYS_LIBWRAP_SOURCE} ) diff --git a/mysql-test/collections/buildbot_suites.bat b/mysql-test/collections/buildbot_suites.bat index e673afd50c926..cde563ac97877 100644 --- a/mysql-test/collections/buildbot_suites.bat +++ b/mysql-test/collections/buildbot_suites.bat @@ -6,6 +6,7 @@ innodb,^ versioning,^ plugins,^ mariabackup,^ +backup,^ roles,^ auth_gssapi,^ mysql_sha2,^ diff --git a/mysql-test/include/have_mariabackup_combination.combinations b/mysql-test/include/have_mariabackup_combination.combinations new file mode 100644 index 0000000000000..12bc42487d520 --- /dev/null +++ b/mysql-test/include/have_mariabackup_combination.combinations @@ -0,0 +1,3 @@ +[CLIENT] + +[SERVER] diff --git a/mysql-test/include/have_mariabackup_combination.inc b/mysql-test/include/have_mariabackup_combination.inc new file mode 100644 index 0000000000000..a2b79d6310efc --- /dev/null +++ b/mysql-test/include/have_mariabackup_combination.inc @@ -0,0 +1,5 @@ +if ($MTR_COMBINATION_SERVER) +{ + --source include/have_mariabackup_wrapper.inc +} + diff --git a/mysql-test/include/have_mariabackup_wrapper.inc b/mysql-test/include/have_mariabackup_wrapper.inc new file mode 100644 index 0000000000000..d9d8ee82e74ee --- /dev/null +++ b/mysql-test/include/have_mariabackup_wrapper.inc @@ -0,0 +1,38 @@ +# Redirect `$XTRABACKUP` so existing test invocations like +# +# --exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf \ +# --backup --target-dir=$targetdir +# +# run through extra/mariabackup/scripts/mariadb-backup-server.sh — the BACKUP +# SERVER compatibility wrapper — without any change to the test body. +# +# --source include/have_mariabackup_wrapper.inc +# # ... rest of the test, using $XTRABACKUP as usual ... +# +# $XTRABACKUP : now points at mariadb-backup-server.sh + +--source include/not_windows.inc + +--let MARIABACKUP_WRAPPER=$MYSQL_TEST_DIR/../extra/mariabackup/scripts/mariadb-backup-server.sh +--let MBSTREAM_WRAPPER=$MYSQL_TEST_DIR/../extra/mariabackup/scripts/mbstream-server.sh + +# The wrapper shells out to the bare `mariadb` client, which mtr does not put +# on PATH. Prepend the build's client directories so it resolves. A `let` with +# no leading $ is exported to the environment of later --exec commands. +--let PATH=$MYSQL_BINDIR/client:$MYSQL_BINDIR/client_release:$MYSQL_BINDIR/client_debug:$MYSQL_BINDIR/bin:$PATH + +--error 0,1 +perl; +my $w = $ENV{MARIABACKUP_WRAPPER}; +my $m = $ENV{MBSTREAM_WRAPPER}; +exit 1 unless $w && -x $w && $m && -x $m; +exit 0; +EOF + +if ($errno) +{ + --skip mariadb-backup-server.sh wrapper unavailable (script or sh missing) +} + +--let XTRABACKUP=$MARIABACKUP_WRAPPER +--let XBSTREAM=$MBSTREAM_WRAPPER diff --git a/mysql-test/main/backup_server.result b/mysql-test/main/backup_server.result new file mode 100644 index 0000000000000..7a064d7e5fe11 --- /dev/null +++ b/mysql-test/main/backup_server.result @@ -0,0 +1,6 @@ +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; diff --git a/mysql-test/main/backup_server.test b/mysql-test/main/backup_server.test new file mode 100644 index 0000000000000..50cf326d39838 --- /dev/null +++ b/mysql-test/main/backup_server.test @@ -0,0 +1,10 @@ +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--rmdir $MYSQLTEST_VARDIR/some_directory diff --git a/mysql-test/main/backup_server_locking.result b/mysql-test/main/backup_server_locking.result new file mode 100644 index 0000000000000..6e49a09e3e873 --- /dev/null +++ b/mysql-test/main/backup_server_locking.result @@ -0,0 +1,17 @@ +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +BACKUP STAGE START; +connect backup,localhost,root; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR 70100: Query was interrupted: execution time limit 0.1 sec exceeded +connection default; +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't execute the command as you have a BACKUP STAGE active +BACKUP STAGE END; +connection backup; +SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +ERROR HY000: Can't create directory 'MYSQLTEST_VARDIR/some_directory' (Errcode: 17 "File exists") +disconnect backup; +connection default; diff --git a/mysql-test/main/backup_server_locking.test b/mysql-test/main/backup_server_locking.test new file mode 100644 index 0000000000000..4e769b61b2f20 --- /dev/null +++ b/mysql-test/main/backup_server_locking.test @@ -0,0 +1,31 @@ +--source include/not_embedded.inc +--source include/count_sessions.inc + +--mkdir $MYSQLTEST_VARDIR/some_directory +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE START; +--connect (backup,localhost,root) +--error ER_STATEMENT_TIMEOUT +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +--connection default + +--error ER_BACKUP_LOCK_IS_ACTIVE +evalp BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; + +BACKUP STAGE END; +--connection backup +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error 21 +evalp SET STATEMENT max_statement_time=0.1 FOR +BACKUP SERVER TO '$MYSQLTEST_VARDIR/some_directory'; +--disconnect backup +--connection default + +--rmdir $MYSQLTEST_VARDIR/some_directory + +--source include/wait_until_count_sessions.inc diff --git a/mysql-test/main/grant_backup_server.result b/mysql-test/main/grant_backup_server.result new file mode 100644 index 0000000000000..a4d85292dee45 --- /dev/null +++ b/mysql-test/main/grant_backup_server.result @@ -0,0 +1,27 @@ +CREATE USER user1@localhost IDENTIFIED BY ''; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON test.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the RELOAD privilege(s) for this operation +disconnect con1; +connection default; +GRANT RELOAD ON test.* TO user1@localhost; +ERROR HY000: Incorrect usage of DB GRANT and GLOBAL PRIVILEGES +GRANT RELOAD ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO 'some_directory'; +ERROR 42000: Access denied; you need (at least one of) the SELECT privilege(s) for this operation +disconnect con1; +connection default; +GRANT SELECT ON *.* TO user1@localhost; +connect con1,localhost,user1; +BACKUP SERVER TO '$datadir/some_directory'; +ERROR HY000: Incorrect arguments to BACKUP SERVER TO +disconnect con1; +connection default; +DROP USER user1@localhost; diff --git a/mysql-test/main/grant_backup_server.test b/mysql-test/main/grant_backup_server.test new file mode 100644 index 0000000000000..726abb19908bb --- /dev/null +++ b/mysql-test/main/grant_backup_server.test @@ -0,0 +1,29 @@ +--source include/not_embedded.inc +CREATE USER user1@localhost IDENTIFIED BY ''; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON test.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +--error ER_WRONG_USAGE +GRANT RELOAD ON test.* TO user1@localhost; +GRANT RELOAD ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +BACKUP SERVER TO 'some_directory'; +--disconnect con1 +--connection default +GRANT SELECT ON *.* TO user1@localhost; +--connect con1,localhost,user1 +--let $datadir=`select @@datadir` +--error ER_WRONG_ARGUMENTS +evalp BACKUP SERVER TO '$datadir/some_directory'; +--disconnect con1 +--connection default +DROP USER user1@localhost; diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result index 4e6593e67af42..93a89adcfef96 100644 --- a/mysql-test/main/mysqld--help.result +++ b/mysql-test/main/mysqld--help.result @@ -2039,7 +2039,7 @@ performance-schema-max-socket-classes 10 performance-schema-max-socket-instances -1 performance-schema-max-sql-text-length 1024 performance-schema-max-stage-classes 170 -performance-schema-max-statement-classes 227 +performance-schema-max-statement-classes 228 performance-schema-max-statement-stack 10 performance-schema-max-table-handles -1 performance-schema-max-table-instances -1 diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 9aadfa5456d61..a0e751d5434d1 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -180,6 +180,7 @@ END main- archive- atomic- + backup- binlog- binlog_encryption- binlog_in_engine- diff --git a/mysql-test/suite/backup/backup_innodb,debug.rdiff b/mysql-test/suite/backup/backup_innodb,debug.rdiff new file mode 100644 index 0000000000000..592e1bf5e57ad --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_innodb.result ++++ backup_innodb,debug.result +@@ -20,7 +20,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER TO 'target_directory' 4 CONCURRENT; ++connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; ++INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; ++connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/backup/backup_innodb.combinations b/mysql-test/suite/backup/backup_innodb.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_innodb.result b/mysql-test/suite/backup/backup_innodb.result new file mode 100644 index 0000000000000..bff88f510bfe9 --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.result @@ -0,0 +1,60 @@ +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; +BACKUP SERVER TO '$target_directory'; +ROLLBACK; +SELECT * FROM t; +a b +1 +BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH '/bin/false'' at line 1 +BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'WITH 4 CONCURRENT '/bin/false'' at line 1 +BACKUP SERVER WITH '/bin/false'; +ERROR HY000: IO Write error: (...) BACKUP SERVER +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER TO 'target_directory' 4 CONCURRENT; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/some_directory/backup.cnf --datadir=MYSQLTEST_VARDIR/some_directory +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +i +1 +1 +2 +3 +5 +SELECT * FROM at1; +i +2 +3 +5 +7 +DROP TABLE t, at0, at1; +ERROR HY000: Table 't' is read only +SELECT * FROM at0; +ERROR 42S02: Table 'test.at0' doesn't exist +SELECT * FROM at1; +ERROR 42S02: Table 'test.at1' doesn't exist +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_innodb.test b/mysql-test/suite/backup/backup_innodb.test new file mode 100644 index 0000000000000..c58cff8eec21a --- /dev/null +++ b/mysql-test/suite/backup/backup_innodb.test @@ -0,0 +1,104 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/maybe_debug.inc + +--disable_query_log +call mtr.add_suppression("mariadbd.*: IO Write error:"); +--enable_query_log + +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (2), (3), (5), (7); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (1), (1), (2), (3), (5); + +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +BEGIN; +INSERT INTO t SET a=1; + +--let $target_directory=/tmp/some_directory$MTR_COMBINATION_ARCHIVED +# comment out the following line (and replace all "rmdir" with "exec rm -fr") +# to test cross-filesystem copy +--let $target_directory=$MYSQLTEST_VARDIR/some_directory + +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory + +evalp BACKUP SERVER TO '$target_directory'; +--rmdir $target_directory +ROLLBACK; +# BACKUP SERVER will implicitly commit the current transaction +SELECT * FROM t; + +--error ER_PARSE_ERROR +evalp BACKUP SERVER TO '$target_directory' WITH '/bin/false'; +--error ER_PARSE_ERROR +evalp BACKUP SERVER TO '$target_directory' WITH 4 CONCURRENT '/bin/false'; +--replace_regex /\(.*\)/(...)/ +--error ER_IO_WRITE_ERROR +BACKUP SERVER WITH '/bin/false'; + +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +if ($have_debug) { +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +--replace_result $target_directory target_directory +send_eval BACKUP SERVER TO '$target_directory' 4 CONCURRENT; +--connection default +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +SET DEBUG_SYNC='now SIGNAL resume'; +--connection backup +--reap +} +if (!$have_debug) { +--replace_result $target_directory target_directory +eval BACKUP SERVER TO '$target_directory' 4 CONCURRENT; +} + +--disconnect backup +--connection default +ROLLBACK; +SELECT * FROM t; +DELETE FROM t; +DROP TABLE at0, at1; + +if ($MARIADB_UPGRADE_EXE) { +let target_directory=$target_directory; +perl; +open IN, "<", "$ENV{MYSQLTEST_VARDIR}/my.cnf"; +open OUT, ">>", "$ENV{target_directory}/backup.cnf"; +print OUT while (); +close(IN); +close(OUT); +EOF +} +if (!$MARIADB_UPGRADE_EXE) { + --exec cat $MYSQLTEST_VARDIR/my.cnf >> $target_directory/backup.cnf +} +--let $restart_parameters=--defaults-file=$target_directory/backup.cnf --datadir=$target_directory +--source include/restart_mysqld.inc + +SELECT * FROM t; +# A nonzero innodb_log_recovery_target makes InnoDB read-only. +--error ER_OPEN_AS_READONLY +DELETE FROM t; +# Non-InnoDB tables are read-write. +SELECT * FROM at0; +SELECT * FROM at1; +--error ER_OPEN_AS_READONLY +DROP TABLE t, at0, at1; +--error ER_NO_SUCH_TABLE +SELECT * FROM at0; +--error ER_NO_SUCH_TABLE +SELECT * FROM at1; + +--let $restart_parameters= +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; + +--rmdir $target_directory diff --git a/mysql-test/suite/backup/backup_stream,debug.rdiff b/mysql-test/suite/backup/backup_stream,debug.rdiff new file mode 100644 index 0000000000000..8130727323dec --- /dev/null +++ b/mysql-test/suite/backup/backup_stream,debug.rdiff @@ -0,0 +1,16 @@ +--- backup_stream.result ++++ backup_stream,debug.result +@@ -8,7 +8,13 @@ + BEGIN; + DELETE FROM t; + connect backup,localhost,root; ++SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; + BACKUP SERVER WITH 2 CONCURRENT 'stream.bat'; ++connection default; ++SET DEBUG_SYNC='now WAIT_FOR start'; ++INSERT INTO t(a) SELECT * FROM seq_1_to_30000; ++SET DEBUG_SYNC='now SIGNAL resume'; ++connection backup; + disconnect backup; + connection default; + ROLLBACK; diff --git a/mysql-test/suite/backup/backup_stream.combinations b/mysql-test/suite/backup/backup_stream.combinations new file mode 100644 index 0000000000000..7fd419bba7692 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.combinations @@ -0,0 +1,4 @@ +[archived] +innodb_log_archive=ON +[circular] +innodb_log_archive=OFF diff --git a/mysql-test/suite/backup/backup_stream.result b/mysql-test/suite/backup/backup_stream.result new file mode 100644 index 0000000000000..e12a2f683c286 --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.result @@ -0,0 +1,35 @@ +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +INSERT INTO t SET a=1; +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (1); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (0); +BEGIN; +DELETE FROM t; +connect backup,localhost,root; +BACKUP SERVER WITH 2 CONCURRENT 'stream.bat'; +disconnect backup; +connection default; +ROLLBACK; +SELECT * FROM t; +a b +1 +DELETE FROM t; +DROP TABLE at0, at1; +# restart: --defaults-file=MYSQLTEST_VARDIR/backup_stream/backup.cnf --datadir=MYSQLTEST_VARDIR/backup_stream +SELECT * FROM at0; +i +0 +SELECT * FROM at1; +i +1 +SELECT * FROM t; +a b +1 +DELETE FROM t; +ERROR HY000: Table 't' is read only +# restart +SELECT * FROM t; +a b +DROP TABLE t; diff --git a/mysql-test/suite/backup/backup_stream.test b/mysql-test/suite/backup/backup_stream.test new file mode 100644 index 0000000000000..0ad21d49a663c --- /dev/null +++ b/mysql-test/suite/backup/backup_stream.test @@ -0,0 +1,85 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/maybe_debug.inc + +CREATE TABLE t(a INT PRIMARY KEY, b CHAR(255) DEFAULT '' NOT NULL, INDEX(b)) +ENGINE=INNODB; +INSERT INTO t SET a=1; + +CREATE TABLE at1(i INTEGER) ENGINE=Aria TRANSACTIONAL=1; +INSERT INTO at1 VALUES (1); +CREATE TABLE at0 (i INTEGER) ENGINE=Aria TRANSACTIONAL=0; +INSERT INTO at0 VALUES (0); + +--remove_files_wildcard $MYSQL_TMP_DIR/. stream.bat +--write_file $MYSQL_TMP_DIR/stream.bat +#!/bin/sh +exec cat > $MYSQL_TMP_DIR/$1.tar +EOF + +--let $script=$MYSQL_TMP_DIR/stream.bat +if (!$MARIADB_UPGRADE_EXE) { +# Because we may run on Linux /dev/shm which may be mounted as noexec, +# we cannot rely on chmod +x, but must explicitly invoke a shell on the script. +--let $script=/bin/sh $script +} +if ($MARIADB_UPGRADE_EXE) +{ +--exec echo "@cat > %MYSQL_TMP_DIR%\%1.tar" > $MYSQL_TMP_DIR/stream.bat +} + +BEGIN; +DELETE FROM t; + +--connect backup,localhost,root +if ($have_debug) { +SET DEBUG_SYNC='innodb_backup_start SIGNAL start WAIT_FOR resume'; +--replace_result $script stream.bat +send_eval BACKUP SERVER WITH 2 CONCURRENT '$script'; +--connection default +SET DEBUG_SYNC='now WAIT_FOR start'; +INSERT INTO t(a) SELECT * FROM seq_1_to_30000; +SET DEBUG_SYNC='now SIGNAL resume'; +--connection backup +--reap +} +if (!$have_debug) { +--replace_result $script stream.bat +eval BACKUP SERVER WITH 2 CONCURRENT '$script'; +} + +--remove_file $MYSQL_TMP_DIR/stream.bat +--disconnect backup +--connection default +ROLLBACK; +SELECT * FROM t; +DELETE FROM t; +DROP TABLE at0, at1; + +--let $target_directory=$MYSQLTEST_VARDIR/backup_stream +# Clean up after a previous failed test, in case we are retrying. +--error 0,1 +--rmdir $target_directory +--mkdir $target_directory +--exec tar xf $MYSQL_TMP_DIR/1.tar -C $target_directory +--exec tar xf $MYSQL_TMP_DIR/2.tar -C $target_directory + +--exec cat $MYSQLTEST_VARDIR/my.cnf >> $target_directory/backup.cnf + +--let $restart_parameters=--defaults-file=$target_directory/backup.cnf --datadir=$target_directory +--source include/restart_mysqld.inc +SELECT * FROM at0; +SELECT * FROM at1; +SELECT * FROM t; +--error ER_OPEN_AS_READONLY +DELETE FROM t; + +--remove_file $MYSQL_TMP_DIR/1.tar +--remove_file $MYSQL_TMP_DIR/2.tar + +--let $restart_parameters= +--source include/restart_mysqld.inc +SELECT * FROM t; +DROP TABLE t; + +--rmdir $target_directory diff --git a/mysql-test/suite/backup/suite.pm b/mysql-test/suite/backup/suite.pm new file mode 100644 index 0000000000000..7d36a22655efa --- /dev/null +++ b/mysql-test/suite/backup/suite.pm @@ -0,0 +1,19 @@ +package My::Suite::Backup; + +@ISA = qw(My::Suite); +use My::Find; +use File::Basename; +use strict; + +return "Not run for embedded server" if $::opt_embedded_server; + +my $have_cat = index(`echo meowl|cat 2>&1`,"meowl") >= 0; +my $have_tar = `tar --version 2>&1` =~ /tar .*\d\.\d/; + +sub skip_combinations { + my %skip; + $skip{'backup_stream.test'} = 'needs cat,tar' unless $have_cat && $have_tar; + %skip; +} + +bless { }; diff --git a/mysql-test/suite/mariabackup/aria_encrypted.test b/mysql-test/suite/mariabackup/aria_encrypted.test index 6ed3e0d08eff3..017a0b249dcfe 100644 --- a/mysql-test/suite/mariabackup/aria_encrypted.test +++ b/mysql-test/suite/mariabackup/aria_encrypted.test @@ -1,6 +1,7 @@ --source include/have_file_key_management.inc --source include/have_innodb.inc --source include/have_sequence.inc +--source include/have_mariabackup_combination.inc --echo # --echo # MDEV-38246 aria_read index failed on encrypted database during backup @@ -16,3 +17,4 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir; drop table if exists t1,t2; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.inc b/mysql-test/suite/mariabackup/aria_log_dir_path.inc new file mode 100644 index 0000000000000..1c7d8d12679d8 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.inc @@ -0,0 +1,104 @@ +--source include/have_maria.inc + +--echo # +--echo # MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used +--echo # + +--let $datadir=`SELECT @@datadir` +--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup + +if ($ARIA_LOGDIR_MARIADB == '') +{ + --let $ARIA_LOGDIR_MARIADB=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +if ($ARIA_LOGDIR_FS == '') +{ + --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +--let $server_parameters=--aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=$ARIA_LOGDIR_MARIADB + + +--echo # Restart mariadbd with the test specific parameters +--mkdir $ARIA_LOGDIR_FS +--let $restart_parameters=$server_parameters +--source include/restart_mysqld.inc + + +--echo # Create and populate an Aria table (and Aria logs) +CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; +DELIMITER $$; +BEGIN NOT ATOMIC + FOR id IN 0..9 DO + INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); + END FOR; +END; +$$ +DELIMITER ;$$ + + +--echo # Testing aria log files before --backup +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + +--echo # mariadb-backup --backup +--disable_result_log +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir +--enable_result_log + + +--echo # mariadb-backup --prepare +--disable_result_log +--exec $XTRABACKUP --prepare --target-dir=$targetdir +--enable_result_log + +--echo # shutdown server +--disable_result_log +--source include/shutdown_mysqld.inc +--echo # remove datadir +--rmdir $datadir +--echo # remove aria-log-dir-path +--rmdir $ARIA_LOGDIR_FS + + +--echo # mariadb-backup --copy-back +--let $mariadb_backup_parameters=--defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$datadir --target-dir=$targetdir --parallel=2 --throttle=1 --aria-log-dir-path=$ARIA_LOGDIR_MARIADB +--replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--exec echo "# with parameters: $mariadb_backup_parameters" +--exec $XTRABACKUP $mariadb_backup_parameters + + +--echo # starting server +--let $restart_parameters=$server_parameters +--source include/start_mysqld.inc +--enable_result_log +--rmdir $targetdir + + +--echo # Check that the table is there after --copy-back +SELECT COUNT(*) from t1; +DROP TABLE t1; + + +--echo # Testing aria log files after --copy-back +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +#--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + + +--echo # Restarting mariadbd with default parameters +--let $restart_parameters= +--source include/restart_mysqld.inc +--rmdir $ARIA_LOGDIR_FS diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.test b/mysql-test/suite/mariabackup/aria_log_dir_path.test index 40bc39446bf00..e479458edfddb 100644 --- a/mysql-test/suite/mariabackup/aria_log_dir_path.test +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.test @@ -1,105 +1,2 @@ ---source include/have_maria.inc - ---echo # ---echo # MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used ---echo # - ---let $datadir=`SELECT @@datadir` ---let $targetdir=$MYSQLTEST_VARDIR/tmp/backup - -if ($ARIA_LOGDIR_MARIADB == '') -{ - --let $ARIA_LOGDIR_MARIADB=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path -} - -if ($ARIA_LOGDIR_FS == '') -{ - --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path -} - ---let $server_parameters=--aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=$ARIA_LOGDIR_MARIADB - - ---echo # Restart mariadbd with the test specific parameters ---mkdir $ARIA_LOGDIR_FS ---let $restart_parameters=$server_parameters ---source include/restart_mysqld.inc - - ---echo # Create and populate an Aria table (and Aria logs) -CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; -DELIMITER $$; -BEGIN NOT ATOMIC - FOR id IN 0..9 DO - INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); - END FOR; -END; -$$ -DELIMITER ;$$ - - ---echo # Testing aria log files before --backup -SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; ---file_exists $ARIA_LOGDIR_FS/aria_log_control ---file_exists $ARIA_LOGDIR_FS/aria_log.00000001 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000002 ---error 1 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000003 ---replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ -SHOW ENGINE aria logs; - ---echo # mariadb-backup --backup ---disable_result_log ---mkdir $targetdir ---exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir ---enable_result_log - - ---echo # mariadb-backup --prepare ---disable_result_log ---exec $XTRABACKUP --prepare --target-dir=$targetdir ---enable_result_log - ---echo # shutdown server ---disable_result_log ---source include/shutdown_mysqld.inc ---echo # remove datadir ---rmdir $datadir ---echo # remove aria-log-dir-path ---rmdir $ARIA_LOGDIR_FS - - ---echo # mariadb-backup --copy-back ---let $mariadb_backup_parameters=--defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$datadir --target-dir=$targetdir --parallel=2 --throttle=1 --aria-log-dir-path=$ARIA_LOGDIR_MARIADB ---replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR $MYSQLTEST_VARDIR MYSQLTEST_VARDIR ---exec echo "# with parameters: $mariadb_backup_parameters" ---exec $XTRABACKUP $mariadb_backup_parameters - - ---echo # starting server ---let $restart_parameters=$server_parameters ---source include/start_mysqld.inc ---enable_result_log ---rmdir $targetdir - - ---echo # Check that the table is there after --copy-back -SELECT COUNT(*) from t1; -DROP TABLE t1; - - ---echo # Testing aria log files after --copy-back -SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; ---file_exists $ARIA_LOGDIR_FS/aria_log_control -#--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000002 ---error 1 ---file_exists $ARIA_LOGDIR_FS/aria_log.00000003 ---replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ -SHOW ENGINE aria logs; - - ---echo # Restarting mariadbd with default parameters ---let $restart_parameters= ---source include/restart_mysqld.inc ---rmdir $ARIA_LOGDIR_FS +--source include/have_mariabackup_combination.inc +--source aria_log_dir_path.inc diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test index c8169959929b9..b2875f496daac 100644 --- a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test +++ b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test @@ -1,4 +1,4 @@ --let $ARIA_LOGDIR_MARIADB=../../tmp/backup_aria_log_dir_path_rel --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path_rel ---source aria_log_dir_path.test +--source aria_log_dir_path.inc diff --git a/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff b/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff new file mode 100644 index 0000000000000..f71d98dc11c82 --- /dev/null +++ b/mysql-test/suite/mariabackup/defer_space,SERVER.rdiff @@ -0,0 +1,10 @@ +--- defer_space.result ++++ defer_space,SERVER.result +@@ -21,7 +21,5 @@ + CREATE TABLE t1(c INT) ENGINE=INNODB; + # Corrupt the table + # restart +-# xtrabackup backup +-FOUND 10 /Header page consists of zero bytes*/ in backup.log + UNLOCK TABLES; + DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/defer_space.test b/mysql-test/suite/mariabackup/defer_space.test index 397a1ff5dc23c..bad0ed259600c 100644 --- a/mysql-test/suite/mariabackup/defer_space.test +++ b/mysql-test/suite/mariabackup/defer_space.test @@ -2,6 +2,7 @@ --source include/have_debug.inc --source include/not_embedded.inc --source include/no_valgrind_without_big.inc +--source include/have_mariabackup_combination.inc call mtr.add_suppression("InnoDB: Expected tablespace id .*"); --echo # Mariabackup --backup with page0 INIT_PAGE redo record @@ -51,6 +52,8 @@ close FILE or die "close"; EOF --source include/start_mysqld.inc +if (!$MTR_COMBINATION_SERVER) +{ echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; @@ -62,7 +65,8 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir= --let SEARCH_PATTERN=Header page consists of zero bytes* --let SEARCH_FILE=$backuplog --source include/search_pattern_in_file.inc -UNLOCK TABLES; -DROP TABLE t1; rmdir $targetdir; remove_file $backuplog; +} +UNLOCK TABLES; +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff b/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff new file mode 100644 index 0000000000000..4e2798c8a3b3d --- /dev/null +++ b/mysql-test/suite/mariabackup/full_backup,SERVER.rdiff @@ -0,0 +1,30 @@ +--- full_backup.result ++++ full_backup,SERVER.result +@@ -3,10 +3,6 @@ + SET GLOBAL innodb_max_purge_lag_wait=0; + # xtrabackup backup + NOT FOUND /InnoDB: Allocated tablespace ID/ in backup.log +-SELECT variable_value FROM information_schema.global_status +-WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +-variable_value +-0 + INSERT INTO t VALUES(2); + # xtrabackup prepare + # shutdown server +@@ -44,16 +40,3 @@ + # + # MDEV-34713: mariadb_upgrade_info should be backed up and restored + # +-CREATE TABLE t2(i INT) ENGINE INNODB; +-INSERT INTO t2 VALUES(100); +-# xtrabackup backup +-# xtrabackup prepare +-# shutdown server +-# remove datadir +-# xtrabackup move back +-# restart: --innodb_undo_tablespaces=0 +-FOUND 1 /^[0-9]+\.[0-9]+\.[0-9]+/ in mariadb_upgrade_info +-SELECT * FROM t2; +-i +-100 +-DROP TABLE t2; diff --git a/mysql-test/suite/mariabackup/full_backup.test b/mysql-test/suite/mariabackup/full_backup.test index 8a1ce756eeb17..1b75e4d2c051a 100644 --- a/mysql-test/suite/mariabackup/full_backup.test +++ b/mysql-test/suite/mariabackup/full_backup.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc --source include/innodb_page_size.inc CREATE TABLE t(i INT) ENGINE INNODB; @@ -8,8 +9,13 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --let $backup_log=$MYSQLTEST_VARDIR/tmp/backup.log --disable_result_log +# The new BACKUP SERVER wrapper ignores --innodb-log-write-ahead-size, so the +# invalid-value invocation that is expected to fail does not apply to it. +if (!$MTR_COMBINATION_SERVER) +{ --error 1 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --parallel=10 --innodb-log-write-ahead-size=4095 > $backup_log 2>&1; +} exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --parallel=10 --innodb-log-write-ahead-size=10000 --innodb_log_checkpoint_now=1 > $backup_log 2>&1; --enable_result_log @@ -19,8 +25,12 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir= --source include/search_pattern_in_file.inc --remove_file $backup_log +# Only the native mariabackup leaves dirty pages worth checking here. +if (!$MTR_COMBINATION_SERVER) +{ SELECT variable_value FROM information_schema.global_status WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +} INSERT INTO t VALUES(2); @@ -38,6 +48,8 @@ rmdir $targetdir; --echo # MDEV-27121 mariabackup incompatible with disabled dedicated --echo # undo log tablespaces --echo # +# The new BACKUP SERVER wrapper's prepare does not preserve prepared XA +# transactions across backup/restore, so this scenario does not apply to it. call mtr.add_suppression("InnoDB: innodb_undo_tablespaces=0 disables dedicated undo log tablespaces"); call mtr.add_suppression("InnoDB: Cannot change innodb_undo_tablespaces=0 because previous shutdown was not with innodb_fast_shutdown=0"); call mtr.add_suppression("Found 1 prepared XA transactions"); @@ -72,7 +84,8 @@ rmdir $targetdir; --echo # --echo # MDEV-34713: mariadb_upgrade_info should be backed up and restored --echo # - +if (!$MTR_COMBINATION_SERVER) +{ CREATE TABLE t2(i INT) ENGINE INNODB; INSERT INTO t2 VALUES(100); @@ -106,3 +119,4 @@ SELECT * FROM t2; DROP TABLE t2; --remove_file $_datadir/mariadb_upgrade_info rmdir $targetdir; +} diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test index 0da6774445722..fe7dec0160f1a 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.test +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -1,6 +1,6 @@ --source include/not_embedded.inc --source include/have_file_key_management.inc - +--source include/have_mariabackup_combination.inc --echo # --echo # MDEV-13416 mariabackup fails with EFAULT "Bad Address" --echo # diff --git a/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff b/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff new file mode 100644 index 0000000000000..921b39ff6cd58 --- /dev/null +++ b/mysql-test/suite/mariabackup/log_tables,SERVER.rdiff @@ -0,0 +1,16 @@ +--- mysql-test/suite/mariabackup/log_tables.result ++++ mysql-test/suite/mariabackup/log_tables,SERVER.result +@@ -10,7 +10,6 @@ + (command_type = "Query" OR command_type = "Execute") ; + event_time user_host thread_id server_id command_type argument + TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO t VALUES (1) +-# Insert new row into general_log table after it has been copied on BLOCK_DDL. + # Backup to dir. + # Xtrabackup prepare. + # shutdown server +@@ -22,5 +21,4 @@ + (command_type = "Query" OR command_type = "Execute") ; + event_time user_host thread_id server_id command_type argument + TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO t VALUES (1) +-TIMESTAMP USER_HOST THREAD_ID 1 Query INSERT INTO test.t VALUES (2) + DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/log_tables.test b/mysql-test/suite/mariabackup/log_tables.test index 707057a80e642..41c8581e2659f 100644 --- a/mysql-test/suite/mariabackup/log_tables.test +++ b/mysql-test/suite/mariabackup/log_tables.test @@ -1,6 +1,7 @@ # Test for copying log tables tail --source include/have_aria.inc --source include/have_debug.inc +--source include/have_mariabackup_combination.inc --let $targetdir=$MYSQLTEST_VARDIR/tmp/backup @@ -21,8 +22,11 @@ SELECT * FROM mysql.general_log WHERE argument LIKE "INSERT INTO %" AND (command_type = "Query" OR command_type = "Execute") ; +if (!$MTR_COMBINATION_SERVER) +{ --echo # Insert new row into general_log table after it has been copied on BLOCK_DDL. --let after_stage_block_ddl=INSERT INTO test.t VALUES (2) +} --echo # Backup to dir. --disable_result_log diff --git a/mysql-test/suite/mariabackup/mdev-18438.test b/mysql-test/suite/mariabackup/mdev-18438.test index a6ec45476ff2c..f05b06b147384 100644 --- a/mysql-test/suite/mariabackup/mdev-18438.test +++ b/mysql-test/suite/mariabackup/mdev-18438.test @@ -1,8 +1,14 @@ +--source include/have_mariabackup_combination.inc let $basedir=$MYSQLTEST_VARDIR/tmp/mdev-18438; mkdir $basedir; exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --extra-lsndir=$basedir/extra_lsndir --stream=xbstream > $basedir/stream.xb; mkdir $basedir/backup; +# The BACKUP SERVER wrapper ignores --extra-lsndir, so it never creates +# extra_lsndir; only the native tool does. +if (!$MTR_COMBINATION_SERVER) +{ rmdir $basedir/extra_lsndir; +} --disable_result_log exec $XBSTREAM -x -C $basedir/backup < $basedir/stream.xb; --enable_result_log diff --git a/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff b/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff new file mode 100644 index 0000000000000..dba5eb260296d --- /dev/null +++ b/mysql-test/suite/mariabackup/partition_notwin,SERVER.rdiff @@ -0,0 +1,8 @@ +--- partition_notwin.result ++++ partition_notwin,SERVER.result +@@ -7,5 +7,4 @@ + ) engine=myisam + partition by hash (id) + partitions 600; +-FOUND 1 /Error 24 on file ./test/t1#P#p\d+\.MY[DI] open during `test`.`t1` table copy: Too many open files/ in backup.log + drop table t1; diff --git a/mysql-test/suite/mariabackup/partition_notwin.test b/mysql-test/suite/mariabackup/partition_notwin.test index 10687e19935e6..85bafcfb5b83d 100644 --- a/mysql-test/suite/mariabackup/partition_notwin.test +++ b/mysql-test/suite/mariabackup/partition_notwin.test @@ -1,5 +1,6 @@ source include/not_windows.inc; source include/have_partition.inc; +source include/have_mariabackup_combination.inc; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $log=$MYSQL_TMP_DIR/backup.log; @@ -14,11 +15,21 @@ create table t1 ( partition by hash (id) partitions 600; -error 1; +let $errno = 1; +if ($MTR_COMBINATION_SERVER) +{ +let $errno = 0; +} +--error $errno exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir > $log 2>&1; + +# The "Too many open files" diagnostic only applies to native mariabackup. +if (!$MTR_COMBINATION_SERVER) +{ let SEARCH_FILE=$log; let SEARCH_PATTERN=Error 24 on file ./test/t1#P#p\d+\.MY[DI] open during `test`.`t1` table copy: Too many open files; source include/search_pattern_in_file.inc; +} rmdir $targetdir; #remove_file $log; diff --git a/mysql-test/suite/mariabackup/relative_path.test b/mysql-test/suite/mariabackup/relative_path.test index bd25a217e711e..60c287403165c 100644 --- a/mysql-test/suite/mariabackup/relative_path.test +++ b/mysql-test/suite/mariabackup/relative_path.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_mariabackup_combination.inc CREATE TABLE t(i INT) ENGINE INNODB; INSERT INTO t VALUES(1); diff --git a/mysql-test/suite/mariabackup/row_format_redundant.test b/mysql-test/suite/mariabackup/row_format_redundant.test index 5bae9218d840c..1820fec9ebfaa 100644 --- a/mysql-test/suite/mariabackup/row_format_redundant.test +++ b/mysql-test/suite/mariabackup/row_format_redundant.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_mariabackup_combination.inc --let $targetdir=$MYSQLTEST_VARDIR/tmp/backup diff --git a/mysql-test/suite/mariabackup/small_ibd.test b/mysql-test/suite/mariabackup/small_ibd.test index bb476b8771e98..9076abff6c4cf 100644 --- a/mysql-test/suite/mariabackup/small_ibd.test +++ b/mysql-test/suite/mariabackup/small_ibd.test @@ -1,4 +1,5 @@ --source include/innodb_page_size.inc +--source include/have_mariabackup_combination.inc # Check if ibd smaller than page size are skipped # It is possible, due to race conditions that new file diff --git a/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff b/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff new file mode 100644 index 0000000000000..d886111b06fb7 --- /dev/null +++ b/mysql-test/suite/mariabackup/undo_space_id,SERVER.rdiff @@ -0,0 +1,13 @@ +--- undo_space_id.result ++++ undo_space_id,SERVER.result +@@ -11,10 +11,3 @@ + undo001 + undo002 + DROP TABLE t1; +-# +-# MDEV-33980 mariadb-backup --backup is missing +-# retry logic for undo tablespaces +-# +-# xtrabackup backup +-# Display undo log files from target directory +-FOUND 5 /Retrying to read undo tablespace*/ in backup.log diff --git a/mysql-test/suite/mariabackup/undo_space_id.test b/mysql-test/suite/mariabackup/undo_space_id.test index 168740fc528ce..e239189040c89 100644 --- a/mysql-test/suite/mariabackup/undo_space_id.test +++ b/mysql-test/suite/mariabackup/undo_space_id.test @@ -1,5 +1,6 @@ --source include/have_innodb.inc --source include/have_debug.inc +--source include/have_mariabackup_combination.inc --echo # Create 2 UNDO TABLESPACE(UNDO001(space_id =3), UNDO002(space_id =4)) @@ -24,6 +25,8 @@ list_files $basedir undo*; DROP TABLE t1; rmdir $basedir; +if (!$MTR_COMBINATION_SERVER) +{ --echo # --echo # MDEV-33980 mariadb-backup --backup is missing --echo # retry logic for undo tablespaces @@ -42,3 +45,4 @@ list_files $basedir undo*; --source include/search_pattern_in_file.inc rmdir $basedir; remove_file $backuplog; +} diff --git a/mysql-test/suite/mariabackup/undo_truncate.test b/mysql-test/suite/mariabackup/undo_truncate.test index a23c9cf64ff6b..901bf73920a41 100644 --- a/mysql-test/suite/mariabackup/undo_truncate.test +++ b/mysql-test/suite/mariabackup/undo_truncate.test @@ -2,6 +2,7 @@ --source include/not_embedded.inc --source include/have_sequence.inc --source include/have_file_key_management.inc +--source include/have_mariabackup_combination.inc SET GLOBAL innodb_undo_log_truncate = 0; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff b/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff new file mode 100644 index 0000000000000..e785a0601fe94 --- /dev/null +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed,SERVER.rdiff @@ -0,0 +1,8 @@ +--- unencrypted_page_compressed.result ++++ unencrypted_page_compressed,SERVER.result +@@ -8,5 +8,4 @@ + # Corrupt the table + # restart: --skip-innodb-buffer-pool-load-at-startup + # xtrabackup backup +-FOUND 1 /Database page corruption detected.*/ in backup.log + drop table t1; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test index 68f22e69e0325..fc79a0a211e89 100644 --- a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); @@ -39,6 +40,8 @@ echo # xtrabackup backup; --disable_result_log let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; +if (!$MTR_COMBINATION_SERVER) +{ --error 1 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --core-file > $backuplog; --enable_result_log @@ -47,6 +50,7 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --let SEARCH_FILE=$backuplog --source include/search_pattern_in_file.inc remove_file $backuplog; +rmdir $targetdir; +} drop table t1; -rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/vector.test b/mysql-test/suite/mariabackup/vector.test index 4f60ddce10e9c..1889d8c412ec3 100644 --- a/mysql-test/suite/mariabackup/vector.test +++ b/mysql-test/suite/mariabackup/vector.test @@ -1,3 +1,4 @@ +--source include/have_mariabackup_combination.inc create table t1 (id int auto_increment primary key, v vector(5) not null, vector index (v)) engine=innodb; insert t1 (v) values (Vec_Fromtext('[0.418,0.809,0.823,0.598,0.033]')), (Vec_Fromtext('[0.687,0.789,0.496,0.574,0.917]')), diff --git a/mysql-test/suite/perfschema/r/max_program_zero.result b/mysql-test/suite/perfschema/r/max_program_zero.result index 047643e06988d..a0e486b2af9c0 100644 --- a/mysql-test/suite/perfschema/r/max_program_zero.result +++ b/mysql-test/suite/perfschema/r/max_program_zero.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 1 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/ortho_iter.result b/mysql-test/suite/perfschema/r/ortho_iter.result index 56c22c8453d8e..589704f4056de 100644 --- a/mysql-test/suite/perfschema/r/ortho_iter.result +++ b/mysql-test/suite/perfschema/r/ortho_iter.result @@ -251,7 +251,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/privilege_table_io.result b/mysql-test/suite/perfschema/r/privilege_table_io.result index 0c82be9f05810..b518052613308 100644 --- a/mysql-test/suite/perfschema/r/privilege_table_io.result +++ b/mysql-test/suite/perfschema/r/privilege_table_io.result @@ -57,7 +57,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_idle.result b/mysql-test/suite/perfschema/r/start_server_disable_idle.result index d0665e3bf4c65..12b956d90769c 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_idle.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_idle.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_stages.result b/mysql-test/suite/perfschema/r/start_server_disable_stages.result index 2ef68328144ff..30ce9d12e56a0 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_stages.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_stages.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_statements.result b/mysql-test/suite/perfschema/r/start_server_disable_statements.result index 0ece2a0c52ed1..4bacdbdfedf68 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_statements.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_statements.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result index ededc09aac95d..3a6831eb2c7ff 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_transactions.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_transactions.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_disable_waits.result b/mysql-test/suite/perfschema/r/start_server_disable_waits.result index 23db9362161e4..a864576dfa979 100644 --- a/mysql-test/suite/perfschema/r/start_server_disable_waits.result +++ b/mysql-test/suite/perfschema/r/start_server_disable_waits.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_innodb.result b/mysql-test/suite/perfschema/r/start_server_innodb.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_innodb.result +++ b/mysql-test/suite/perfschema/r/start_server_innodb.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_index.result b/mysql-test/suite/perfschema/r/start_server_low_index.result index 11cade0a2132f..dbd36d2eaa92d 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_index.result +++ b/mysql-test/suite/perfschema/r/start_server_low_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result index 484550095202e..fab64f49d45e6 100644 --- a/mysql-test/suite/perfschema/r/start_server_low_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_low_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_account.result b/mysql-test/suite/perfschema/r/start_server_no_account.result index aab8d3eba9caa..e2cccdba19b43 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_account.result +++ b/mysql-test/suite/perfschema/r/start_server_no_account.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result index 4dfdab9de9f30..44b114013ace2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result index a8d0cbeca3855..27ecb59a40f17 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_cond_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_class.result b/mysql-test/suite/perfschema/r/start_server_no_file_class.result index fcc01880a7107..5189d36618b05 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result index c56201e7d0a80..533c02383c7b8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_file_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_file_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_host.result b/mysql-test/suite/perfschema/r/start_server_no_host.result index 662beb3b88a49..e328ea3d3c26a 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_host.result +++ b/mysql-test/suite/perfschema/r/start_server_no_host.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_index.result b/mysql-test/suite/perfschema/r/start_server_no_index.result index ccff0cb113faa..686f430cdc440 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_index.result +++ b/mysql-test/suite/perfschema/r/start_server_no_index.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mdl.result b/mysql-test/suite/perfschema/r/start_server_no_mdl.result index ebe64409deb0c..c2b6dc3d9eb74 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result index 01a217c534bfd..2a4cdcf8ddd37 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_memory_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_memory_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result index 1b3efda5210a9..9b77c7b897c47 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result index 599498915f3f1..3a62653efc18f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_mutex_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result index 73ac1acb9f145..7dd5842809135 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result +++ b/mysql-test/suite/perfschema/r/start_server_no_prepared_stmts_instances.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result index 04aa037b960e3..3598c722b9453 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result index e8156711eb3f2..7a60805d2d03f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_rwlock_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result index 2d17bd6f49203..76da2883ba6b2 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_actors.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result index 16afee28ee70b..58ac763394a96 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result +++ b/mysql-test/suite/perfschema/r/start_server_no_setup_objects.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result index 9de0006aa7abc..5a86d51a06231 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 0 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result index bcef0e5c01b05..75ab84f4b3cb8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_socket_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 0 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result index 1dda39dc79e92..bb750ebd34a26 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stage_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stage_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 0 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result index 95584521eceb2..6cf3f740fb1be 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result index da6c54b6bbafa..1f2716410a9bc 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_stages_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result index 09a9a544f5b13..f1078542f69ef 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result index 5ce9874799e8b..a6fc523df2c60 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_statements_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result index a170fe097fd23..304732cb37927 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_hdl.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 0 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result index 3f009de021d1a..3ef43495d0f22 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 0 diff --git a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result index db4fe3413106b..20cf1a531ca51 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_table_lock.result +++ b/mysql-test/suite/perfschema/r/start_server_no_table_lock.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result index b1b6e614c43ac..e8eb4589fce56 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_class.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_class.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result index e540f3ce78cbd..4dbf292fc3c10 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result +++ b/mysql-test/suite/perfschema/r/start_server_no_thread_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result index 80da238ba0ed7..09168c68f9fe8 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result index f5d32f0100968..9ec63e991deda 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_transactions_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_user.result b/mysql-test/suite/perfschema/r/start_server_no_user.result index cb249b4e242d3..861b7d897c9c5 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_user.result +++ b/mysql-test/suite/perfschema/r/start_server_no_user.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result index c419aad2995f3..710cef232da58 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result index 0b2bf39ff874f..df4351a8f358f 100644 --- a/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result +++ b/mysql-test/suite/perfschema/r/start_server_no_waits_history_long.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_off.result b/mysql-test/suite/perfschema/r/start_server_off.result index 17db0395d894b..f3bf7bf7fb15e 100644 --- a/mysql-test/suite/perfschema/r/start_server_off.result +++ b/mysql-test/suite/perfschema/r/start_server_off.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_on.result b/mysql-test/suite/perfschema/r/start_server_on.result index cff87457b3bef..09cba65c57ce0 100644 --- a/mysql-test/suite/perfschema/r/start_server_on.result +++ b/mysql-test/suite/perfschema/r/start_server_on.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/start_server_variables.result b/mysql-test/suite/perfschema/r/start_server_variables.result index f25f9ab69d1c3..b86116e099cdd 100644 --- a/mysql-test/suite/perfschema/r/start_server_variables.result +++ b/mysql-test/suite/perfschema/r/start_server_variables.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 10 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result index 442e212557b55..3e8469b881801 100644 --- a/mysql-test/suite/perfschema/r/statement_program_lost_inst.result +++ b/mysql-test/suite/perfschema/r/statement_program_lost_inst.result @@ -129,7 +129,7 @@ performance_schema_max_socket_classes 10 performance_schema_max_socket_instances 1000 performance_schema_max_sql_text_length 1024 performance_schema_max_stage_classes 170 -performance_schema_max_statement_classes 227 +performance_schema_max_statement_classes 228 performance_schema_max_statement_stack 2 performance_schema_max_table_handles 1000 performance_schema_max_table_instances 500 diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 97fd9a7c0ef61..f0834ea3a1e12 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1064,7 +1064,7 @@ NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 18446744073709551615 NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL -READ_ONLY YES +READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_LOG_RECOVERY_TARGET SESSION_VALUE NULL diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index e54e894e1d0fc..fd16d901d64fb 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -163,6 +163,7 @@ SET (SQL_SOURCE grant.cc sql_explain.cc sql_analyze_stmt.cc + sql_backup.cc sql_join_cache.cc create_options.cc multi_range_read.cc opt_histogram_json.cc diff --git a/sql/handler.h b/sql/handler.h index 3ab9e0bcd1a8e..ab4026cf57600 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1496,6 +1496,61 @@ struct transaction_participant ulonglong (*prepare_commit_versioned)(THD *thd, ulonglong *trx_id); }; +/** BACKUP SERVER target */ +struct backup_target +{ +#ifdef _WIN32 + /** Target directory path name, or nullptr if streaming */ + const char *path; +#else + /** Target directory descriptor, or -1 if streaming */ + int fd; +#endif +}; + +/** BACKUP SERVER worker specific context */ +struct backup_sink +{ +#ifdef _WIN32 + /** A value indicating an invalid stream */ + static constexpr HANDLE NO_STREAM{INVALID_HANDLE_VALUE}; + /** Target pipe, or NO_STREAM if path!=nullptr */ + HANDLE stream; +#else + /** A value indicating an invalid file descriptor or stream */ + static constexpr int NO_STREAM{-1}; + /** Target pipe, or NO_STREAM if copying to a directory */ + int stream; +#endif + /** storage engine context returned by handlerton::backup_start() */ + void *ha_data; +}; + +/** BACKUP SERVER execution phase */ +enum backup_phase +{ + /** finish backup, possibly after BACKUP_PHASE_ABORT */ + BACKUP_PHASE_FINISH= -2, + /** abort any operation */ + BACKUP_PHASE_ABORT= -1, + /** preparatory phase executed while holding no locks */ + BACKUP_PHASE_PREPARE_START= 0, + /** initial actual work phase; @see MDL_BACKUP_START */ + BACKUP_PHASE_START, + /** copy while new writes to non-transactional tables are blocked; + @see MDL_BACKUP_FLUSH */ + BACKUP_PHASE_NO_BEGIN_NON_TRANS, + /** copy while any writes to non-transactional tables are blocked; + @see MDL_BACKUP_WAIT_FLUSH */ + BACKUP_PHASE_NO_DML_NON_TRANS, + /** copy files while DDL is blocked; @see MDL_BACKUP_WAIT_DDL */ + BACKUP_PHASE_NO_DDL, + /** determine the logical time of the backup and copy any + remaining files while MDL_BACKUP_WAIT_COMMIT is active; + this is followed by BACKUP_PHASE_FINISH */ + BACKUP_PHASE_NO_COMMIT +}; + /* handlerton is a singleton structure - one instance per storage engine - to provide access to storage engine functionality that works on the @@ -1892,9 +1947,48 @@ struct handlerton : public transaction_participant /********************************************************************* backup **********************************************************************/ + + /** BACKUP STAGE START */ void (*prepare_for_backup)(void); + /** BACKUP STAGE END */ void (*end_backup)(void); + /** + Start of a BACKUP SERVER phase, + when no backup_step() or backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_PREPARE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to sink, or nullptr + @retval -1 on failure + */ + void *(*backup_start)(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink); + /** + Process a file that was collected in backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int (*backup_step)(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink); + /** + Finish a phase, once all calls for the current phase are completed. + @param thd current sesssion + @param target backup target + @param phase last phase on which backup_start() was successfully + invoked, or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success + */ + int (*backup_end)(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink); + /********************************************************************** WSREP specific **********************************************************************/ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index ead3f926f8cdf..dcdf55716e3a4 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3537,6 +3537,7 @@ SHOW_VAR com_status_vars[]= { {"assign_to_keycache", STMT_STATUS(SQLCOM_ASSIGN_TO_KEYCACHE)}, {"backup", STMT_STATUS(SQLCOM_BACKUP)}, {"backup_lock", STMT_STATUS(SQLCOM_BACKUP_LOCK)}, + {"backup_server", STMT_STATUS(SQLCOM_BACKUP_SERVER)}, {"begin", STMT_STATUS(SQLCOM_BEGIN)}, {"binlog", STMT_STATUS(SQLCOM_BINLOG_BASE64_EVENT)}, {"call_procedure", STMT_STATUS(SQLCOM_CALL)}, diff --git a/sql/sql_backup.cc b/sql/sql_backup.cc new file mode 100644 index 0000000000000..f4ad618b4694a --- /dev/null +++ b/sql/sql_backup.cc @@ -0,0 +1,886 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "mdl.h" +#include "mysys_err.h" +#include "sql_class.h" +#include "sql_backup.h" +#include "sql_backup_interface.h" +#include "sql_parse.h" +#include "my_atomic_wrapper.h" +#include "tpool.h" +#include "aligned.h" + +#if defined __linux__ || defined __FreeBSD__ +using copying_step= ssize_t(int,int,size_t,off_t*); +template +static ssize_t copy(int in_fd, int out_fd, off_t offset, off_t end) noexcept +{ + for (;;) + { + const size_t c{size_t(std::min(end - offset, INT_MAX >> 20 << 20))}; + ssize_t ret= step(in_fd, out_fd, c, &offset); + if (ret < 0) + { + if (nonblocking && errno == EAGAIN) + continue; + return ret; + } + if (offset == end) + return 0; + if (!ret) + return -1; + } +} + +# if 1 // disable to work around https://github.com/rr-debugger/rr/issues/4059 +/* Copy between files in a single (type of) file system */ +static inline ssize_t +copy_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return copy_file_range(in_fd, offset, out_fd, offset, count, 0); +} +# define cfr(src,dst,start,end) copy(src, dst, start, end) +# endif +#endif + +#ifdef _WIN32 +using tpool::pread; +using tpool::pwrite; +#else +# include +/** + Copy a file using a memory mapping. + @tparam stream true=write to a stream, false=pwrite to a file + @param in_fd source file + @param out_fd destination + @param o start offset + @param end last offset (exclusive) + @return error code + @retval 0 on success + @retval 1 if a memory mapping failed +*/ +template +static ssize_t mmap_copy(int in_fd, int out_fd, uint64_t o, uint64_t end) +{ +# if SIZEOF_SIZE_T < 8 + if (end != size_t(end)) + return 1; +# endif + const size_t count= size_t(end - o); + void *p= mmap(nullptr, count, PROT_READ, MAP_SHARED, in_fd, off_t(o)); + if (p == MAP_FAILED) + return 1; + ssize_t ret; + size_t c{count}; + for (const char *b= static_cast(p);; b+= ret, o+= uint64_t(ret)) + { + const size_t size{std::min(c, size_t(INT_MAX >> 20 << 20))}; + if (stream) + if ((ret= backup_stream_write(out_fd, b, size))) + break; + ret= stream ? size : pwrite(out_fd, b, size, off_t(o)); + if (ret < 0) + break; + c-= ret; + if (!c) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + munmap(p, c); + return ret; +} +#endif + +/** + Copy a file using positioned reads. + @tparam stream true=write to a stream, false=pwrite to a file + @param in_fd source file + @param out_fd destination + @param o start offset + @param end last offset (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +template +static ssize_t pread_write(IF_WIN(const native_file_handle&,int) in_fd, + IF_WIN(const native_file_handle&,int) out_fd, + uint64_t o, uint64_t end) + noexcept +{ + constexpr size_t READ_WRITE_SIZE= 65536; + char *b= static_cast(aligned_malloc(READ_WRITE_SIZE, 4096)); + if (!b) + return -1; + ssize_t ret; + for (uint64_t count{end - o};; o+= ret) + { + ret= pread(in_fd, b, + ssize_t(std::min(count, READ_WRITE_SIZE)), o); + if (ret > 0) + { + if (!stream) + ret= pwrite(out_fd, b, size_t(ret), o); + else if (backup_stream_write(out_fd, b, size_t(ret))) + { + ret= -1; + break; + } + } + if (ret < 0) + break; + count-= uint64_t(ret); + if (!count) + { + ret= 0; + break; + } + if (!ret) + { + ret= -1; + break; + } + } + aligned_free(b); + return ret; +} + +#ifdef __APPLE__ +/* The inline copy_entire_file() invokes fcopyfile() */ +#elif defined _WIN32 +/* CopyFileEx() should be used */ +#else +/** Copy a file (whole content). +@param src source file descriptor +@param dst target to append src to +@return error code (non-positive) +@retval 0 on success */ +extern "C" int copy_entire_file(int src, int dst) +{ + return copy_file(src, dst, 0, lseek(src, 0, SEEK_END)); +} +#endif + +/** Copy a portion of a file. +@param src source file descriptor +@param dst target to append src to +@param start first offset to copy +@param end last offset to copy (exclusive) +@return error code (non-positive) +@retval 0 on success */ +extern "C" int copy_file(IF_WIN(const native_file_handle&,int) src, + IF_WIN(const native_file_handle&,int) dst, + uint64_t start, uint64_t end) +{ + assert(end >= start); + ssize_t ret; +# ifdef cfr + if (!(ret= cfr(src, dst, off_t(start), off_t(end)))) + return int(ret); +# ifdef __linux__ + if (errno == EOPNOTSUPP || errno == EXDEV) +# endif +# endif +# ifdef __linux__ // starting with Linux 2.6.33, we can rely on sendfile(2) + return (start != 0 && off_t(start) != lseek(dst, start, SEEK_SET)) + ? -1 + : backup_stream_append_async(src, dst, start, end); +# else +# ifndef _WIN32 + if ((ret= mmap_copy(src, dst, start, end)) == 1) +# endif + ret= pread_write(src, dst, start, end); +# endif + assert(ret <= 0); + return int(ret); +} + +/** Append to the configuration file. +@param target backup target directory +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +extern "C" int backup_config_append(IF_WIN(const char*, int) target, + const char *config, size_t size) +{ + /* FIXME: append to a pre-created configuration file */ +#ifdef _WIN32 + HANDLE dst; + { + std::string path{target}; + path.append("/backup.cnf"); + dst= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (dst != INVALID_HANDLE_VALUE) + { + BOOL ok; + for (;;) + { + DWORD written; + ok= WriteFile(dst, config, DWORD(size), &written, nullptr); + if (ok || GetLastError() != ERROR_IO_PENDING) + break; + assert(written < DWORD(size)); + config+= written; + size-= size_t(written); + } + if (CloseHandle(dst) & ok) + return 0; + } + } +#else + int dst= openat(target, "backup.cnf", + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + return dst; + ssize_t ret; + for (; (ret= write(dst, config, size)) >= 0; config+= ret, size -= ret) + { + assert(size_t(ret) <= size); + if (!(size-= size_t(ret))) + { + ret= 0; + break; + } + } + if (!(close(dst) | ret)) + return 0; +#endif + my_error(ER_CANT_CREATE_FILE, MYF(0), "backup.cnf", errno); + return -1; +} + +/** backup context */ +struct backup_target_phase +{ + /** target directory or stream */ + backup_target target; + /** current phase of backup */ + backup_phase phase; + /** backup worker state (output stream) */ + backup_sink sink; + /** stream object for streaming backup */ + FILE *stream; + /** handlerton::backup_step return value in multi-threaded operation */ + int ret; + /** engine-specific backup context */ + std::unordered_map &ha_data; +}; + +/** + Inform a storage engine of an upcoming backup by invoking + handlerton::backup_start(BACKUP_PHASE_PREPARE_START) before + acquiring any locks. + @param thd current session + @param plugin storage engine + @return whether the operation failed +*/ +static my_bool backup_preparation(THD *thd, plugin_ref plugin, void*) noexcept +{ + const auto bs= plugin_hton(plugin)->backup_start; + return bs && bs(thd, nullptr, BACKUP_PHASE_PREPARE_START, nullptr); +} + +/** + Invoke handlerton::backup_start() on a storage engine, + when there are no pending handlerton::backup_step() in any thread. + @param thd current session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_start(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + assert(int{t.phase} >= 0 || t.phase == BACKUP_PHASE_FINISH); + if (hton->backup_start) + { + t.sink.ha_data= t.ha_data[hton]; + void *data= hton->backup_start(thd, &t.target, t.phase, &t.sink); + if (data == reinterpret_cast(-1)) + return true; + assert(!t.ha_data[hton] || t.ha_data[hton] == data); + t.ha_data[hton]= data; + } + return false; +} + +/** + Invoke handlerton::backup_end() on a storage engine, + when there are no pending handlerton::backup_step() in any thread. + @param thd current session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_end(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + if (hton->backup_end) + { + t.sink.ha_data= t.ha_data[hton]; + return hton->backup_end(thd, &t.target, t.phase, &t.sink); + } + return false; +} + +/** + Invoke handlerton::backup_step() on a storage engine in a thread + that may or may not be associated with a BACKUP SERVER connection, + between handlerton::backup_start() and handlerton::backup_end() + of the same backup_phase. + @param thd the BACKUP SERVER session + @param plugin storage engine + @param arg the backup_target_phase context + @return whether the operation failed +*/ +static my_bool backup_step(THD *thd, plugin_ref plugin, void *arg) noexcept +{ + const handlerton *hton= plugin_hton(plugin); + backup_target_phase &t{*static_cast(arg)}; + assert(int{t.phase} >= 0 || t.phase == BACKUP_PHASE_FINISH); + int res= 0; + if (hton->backup_step) + { + t.sink.ha_data= t.ha_data[hton]; + while ((res= hton->backup_step(thd, &t.target, t.phase, &t.sink))) + if (res < 0) + break; + } + return res != 0; +} + +/** Number of background tasks executing backup_step_callback */ +static Atomic_counter backup_step_callback_pending{0}; + +/** Invoke backup_step() in a background task */ +static void backup_step_callback(void *arg) noexcept +{ + backup_target_phase &t{*static_cast(arg)}; + assert(!t.ret); + t.ret= plugin_foreach_with_mask(nullptr, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, &t); +#ifndef NDEBUG + auto was_pending= +#endif + backup_step_callback_pending--; + assert(was_pending); +} + +/** + Execute all handlerton::backup_step() until completion or failure. + @param thd current connection + @param target_phase backup target and phase + @param threads number of execution threads + @param tp thread pool +*/ +static bool backup_steps(THD *thd, backup_target_phase *target_phase, + int threads, tpool::thread_pool *tp) +{ + assert(!backup_step_callback_pending); + if (threads == 1) + return plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + tpool::task *const tasks= + static_cast(alloca(threads * sizeof *tasks)); + backup_step_callback_pending= threads - 1; + for (int n{threads}; --n; ) + { + target_phase[n].phase= target_phase->phase; + tp->submit_task(new (&tasks[n]) tpool::task{backup_step_callback, + &target_phase[n]}); + } + bool fail= plugin_foreach_with_mask(thd, backup_step, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + while (backup_step_callback_pending) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + if (fail) + return fail; + + for (int n{threads}; --n; ) + if (target_phase[n].ret) + { + my_error(ER_UNKNOWN_ERROR, MYF(0)); + return true; + } + + return false; +} + +bool Sql_cmd_backup::execute(THD *thd) +{ + assert(!!target == !command); + + if (check_global_access(thd, RELOAD_ACL) || + check_global_access(thd, SELECT_ACL) || + (target && error_if_data_home_dir(target, "BACKUP SERVER TO"))) + return true; + + if (thd->current_backup_stage != BACKUP_FINISHED) + { + my_error(ER_BACKUP_LOCK_IS_ACTIVE, MYF(0)); + return true; + } + + bool fail{plugin_foreach_with_mask(thd, backup_preparation, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + nullptr)}; + if (fail) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return true; + } + + /* Block concurrent BACKUP SERVER and BACKUP STAGE */ + MDL_request mdl_request; + MDL_REQUEST_INIT(&mdl_request, MDL_key::BACKUP, "", "", MDL_BACKUP_START, + MDL_EXPLICIT); + + if (thd->mdl_context.acquire_lock(&mdl_request, + thd->variables.lock_wait_timeout)) + return true; + + tpool::thread_pool *tp= nullptr; + std::unordered_map ha_data{}; + backup_target_phase *target_phase= static_cast + (alloca(threads * sizeof *target_phase)); + if (threads > 1 && !(tp= tpool::create_thread_pool_generic())) + { + oor: + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + err_exit: + thd->mdl_context.release_lock(mdl_request.ticket); + delete tp; + return true; + } + + if (command) + { + char cmd[1024]; + for (int t{threads}; t; ) + { + if (snprintf(cmd, sizeof cmd, "%s %d", command, t) >= int(sizeof cmd)) + goto oor; + FILE *f= my_popen(cmd, "w"); + if (!f) + { + while (t < threads) + my_pclose(target_phase[t++].stream); + goto oor; + } +#ifdef _WIN32 + HANDLE sink= (HANDLE) _get_osfhandle(_fileno(f)); +#else + int sink= fileno(f); +#endif + new (&target_phase[--t]) + backup_target_phase{backup_target{IF_WIN(nullptr, -1)}, + BACKUP_PHASE_START, backup_sink{sink, nullptr}, f, 0, ha_data}; + } + } + else if (my_mkdir(target, 0755, MYF(MY_WME))) + goto err_exit; + else + { +#ifndef _WIN32 + const int dir{open(target, O_DIRECTORY)}; + if (dir < 0) + { + my_error(EE_CANT_MKDIR, MYF(ME_BELL), target, errno); + goto err_exit; + } +#endif + for (int t{threads}; t; ) + { + new (&target_phase[--t]) + backup_target_phase{backup_target{IF_WIN(target, dir)}, + BACKUP_PHASE_START, + backup_sink{backup_sink::NO_STREAM, nullptr}, nullptr, 0, ha_data}; + } + } + + static_assert(int{MDL_BACKUP_START} + 1 == int{MDL_BACKUP_FLUSH}, ""); + static_assert(int{MDL_BACKUP_START} + 2 == int{MDL_BACKUP_WAIT_FLUSH}, ""); + static_assert(int{MDL_BACKUP_START} + 3 == int{MDL_BACKUP_WAIT_DDL}, ""); + static_assert(int{MDL_BACKUP_START} + 4 == int{MDL_BACKUP_WAIT_COMMIT}, ""); + static_assert(int{BACKUP_PHASE_START} + 1 == + int{BACKUP_PHASE_NO_BEGIN_NON_TRANS}, ""); + static_assert(int{BACKUP_PHASE_START} + 2 == + int{BACKUP_PHASE_NO_DML_NON_TRANS}, ""); + static_assert(int{BACKUP_PHASE_START} + 3 == int{BACKUP_PHASE_NO_DDL}, ""); + static_assert(int{BACKUP_PHASE_START} + 4 == + int{BACKUP_PHASE_NO_COMMIT}, ""); + int phase= int{BACKUP_PHASE_START}; + goto backup_phase_start; + + for (; phase <= int{BACKUP_PHASE_NO_COMMIT}; phase++) + { + assert(!fail); + { + const enum_mdl_type mdl= + enum_mdl_type(phase - int{BACKUP_PHASE_START} + int{MDL_BACKUP_START}); + fail= + thd->mdl_context.upgrade_shared_lock(mdl_request.ticket, mdl, + thd->variables.lock_wait_timeout); + if (fail) + break; + } + backup_phase_start: + target_phase->phase= backup_phase(phase); + fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + if (fail) + break; + fail= backup_steps(thd, target_phase, threads, tp) || + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase); + if (fail) + break; + } + + /* The final part must not interfere with the use of the server datadir. + Release the locks. */ + thd->mdl_context.release_lock(mdl_request.ticket); + if (!fail) + { + target_phase->phase= BACKUP_PHASE_FINISH; + fail= plugin_foreach_with_mask(thd, backup_start, + MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase) || + backup_steps(thd, target_phase, threads, tp); + } + else + { + target_phase->phase= BACKUP_PHASE_ABORT; + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, target_phase); + target_phase->phase= BACKUP_PHASE_FINISH; + } + + fail= + plugin_foreach_with_mask(thd, backup_end, MYSQL_STORAGE_ENGINE_PLUGIN, + PLUGIN_IS_DELETED|PLUGIN_IS_READY, + target_phase) || fail; + delete tp; + + if (command) + for (int t= threads; t--; ) + my_pclose(target_phase[t].stream); +#ifndef _WIN32 + else + std::ignore= close(target_phase->target.fd); +#endif + + if (!fail) + my_ok(thd); + return fail; +} + +/** + Encode an octal string. + @param start first byte of buffer + @param end first byte after buffer + @param n number to encode +*/ +static void ustar_write_octal(char *start, char *end, uint64_t n) noexcept +{ + for (*--end= '\0'; --end >= start; n>>= 3) + *end= char('0' + (n & 7)); +} + +/** + Encode a quantity in 12 bytes. + @param start first byte of the buffer + @param n number to encode +*/ +static void ustar_write_dozen(char *start, uint64_t n) noexcept +{ + if (n < 1ULL << 33) + ustar_write_octal(start, start + 12, n); + else + { + const uint32_t head{my_htobe32(1U << 31)}; + n= my_htobe64(n); + memcpy(start, &head, 4); + memcpy(start + 4, &n, 8); + } +} + +/** Initialize a ustar block +@param buf GNU tape archiver --format=oldgnu header block +@param name name of the block +@param mode file access mode +@param size physical size of the following block */ +static void ustar_block_init(char *buf, const char *name, mode_t mode, + uint64_t size) noexcept +{ + strncpy(buf, name, 100); + ustar_write_octal(buf + 100, buf + 108, uint64_t(mode)); + ustar_write_octal(buf + 108, buf + 116, 0/* POSIX uid */); + ustar_write_octal(buf + 116, buf + 124, 0/* POSIX gid */); + ustar_write_dozen(buf + 124, size); + /* last modification time */ + ustar_write_octal(buf + 136, buf + 148, 0); + memset(buf + 148, ' ', 9); /* initial block checksum and dummy type */ + memset(buf + 157, '\0', 100); /* name of linked file (unused) */ + memcpy(buf + 257, "ustar ", 8); + strncpy(buf + 265, "root" /* POSIX owner name */, 32); + strncpy(buf + 297, "root" /* POSIX group name */, 512 - 297); + /* The caller will fill in the rest and invoke ustar_block_checksum() */ +} + +/** + Compute and write the POSIX tar block checksum. + @param buf POSIX tar block +*/ +static void ustar_block_checksum(char *buf) noexcept +{ + uint16_t sum{0}; + for (int i{0}; i < 512; i++) + sum+= uint16_t{uint8_t(buf[i])}; + ustar_write_octal(buf + 148, buf + 155, sum); +} + +/** + Write data to a stream. + @param stream backup stream + @param buf source buffer + @param size length of the buffer (usually an integer multiple of 512) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_write(IF_WIN(HANDLE, int) stream, const void *buf, + size_t size) +{ +#ifdef _WIN32 + for (DWORD sz= DWORD(size);;) + { + DWORD wrote; + if (WriteFile(stream, buf, sz, &wrote, nullptr)) + { + assert(wrote == sz); + return 0; + } + else if (GetLastError() != ERROR_IO_PENDING) + return -1; + buf= static_cast(buf) + wrote; + sz-= wrote; + } +#else + do + { + ssize_t wrote= write(stream, buf, size); + assert(wrote <= ssize_t(size)); + if (wrote < 0) + { + if (errno == EAGAIN) + continue; + return -1; + } + buf= static_cast(buf) + wrote; + size-= size_t(wrote); + } + while (size); +#endif + return 0; +} + +/** + Copy a prefix of a NUL terminated string to a buffer, NUL-padded. + @param b output buffer + @param s NUL terminated string + @param size size of buf, in bytes +*/ +static inline char *ustar_zeropad(char *b, const char *s, size_t size) noexcept +{ +#if defined __GNUC__ && __GNUC__ >= 8 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wstringop-truncation" +#endif + return strncpy(b, s, size); +#if defined __GNUC__ && __GNUC__ >= 8 +# pragma GCC diagnostic pop +#endif +} + +/** Start streaming a file. +@param stream backup stream +@param name file name +@param mode file access mode +@param size physical length of the file, in bytes +@param chunks payload chunks of a sparse file, or nullptr +@param n_chunks number of chunks; 0 unless sparse file +@return error code (non-positive) +@retval 0 on success */ +extern "C" +int backup_stream_start(IF_WIN(HANDLE, int) stream, + const char *name, mode_t mode, uint64_t size, + const struct backup_chunk *chunks, size_t n_chunks) +{ + assert(stream != backup_sink::NO_STREAM); + char buf[512]; + size_t s= strlen(name); + if (s > 100) + { + /* Write a block that contains the full name length, + followed by blocks that contain the full name, in + tar --format=oldgnu */ + ustar_block_init(buf, "././@LongLink", 0644, s); + ustar_block_checksum(buf); + if (int err= backup_stream_write(stream, buf, sizeof buf)) + return err; + const size_t whole{s & ~(sizeof buf)}; + if (whole) + if (int err= backup_stream_write(stream, name, whole)) + return err; + if (s - whole) + { + ustar_zeropad(buf, name + whole, sizeof buf); + if (int err= backup_stream_write(stream, buf, sizeof buf)) + return err; + } + } + + ustar_block_init(buf, name, mode, size); + if (!n_chunks) + buf[156]= '0'; + else + { + buf[156]= 'S'; + char *h= &buf[386]; + if (n_chunks > 4) + return -1; // FIXME; support more chunks + for (size_t i= 0; i < n_chunks; i++, h+= 24) + { + ustar_write_dozen(h, chunks[i].offset); + ustar_write_dozen(h + 12, chunks[i].length); + } + ustar_write_dozen(&buf[0x1e3], chunks[n_chunks - 1].offset + + chunks[n_chunks - 1].length); + } + ustar_block_checksum(buf); + return backup_stream_write(stream, buf, sizeof buf); +} + +/** Append to the configuration file. +@param target backup stream +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +extern "C" int backup_stream_config(IF_WIN(HANDLE, int) stream, + const char *config, size_t size) +{ + /* FIXME: append to a pre-created configuration file */ + if (int ret= + backup_stream_start(stream, "backup.cnf", 0644, size, nullptr, 0)) + return ret; + char buf[512]; + const size_t whole{size & ~((sizeof buf) - 1)}; + if (whole) + if (int err= backup_stream_write(stream, config, whole)) + return err; + if (size == whole) + return 0; + ustar_zeropad(buf, config + whole, sizeof buf); + return backup_stream_write(stream, buf, sizeof buf); +} + +/** + Append a file snippet to stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_append(IF_WIN(const native_file_handle&,int) src, + IF_WIN(HANDLE, int) stream, + uint64_t start, uint64_t end) +{ + assert(stream != backup_sink::NO_STREAM); + ssize_t ret; +#ifndef _WIN32 + if ((ret= mmap_copy(src, stream, start, end)) == 1) +#endif + ret= pread_write(src, stream, start, end); + return int(ret); +} + +#ifdef __linux__ +# include +/** Copy a file to a stream or to a regular file. */ +static inline ssize_t +send_step(int in_fd, int out_fd, size_t count, off_t *offset) noexcept +{ + return sendfile(out_fd, in_fd, offset, count); +} + +/** + Append an immutable snippet of a file to the stream, + allowing Linux sendfile(2) to be invoked. + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +extern "C" int backup_stream_append_async(int src, int stream, + uint64_t start, uint64_t end) +{ + assert(stream != backup_sink::NO_STREAM); + return int(copy(src, stream, off_t(start), off_t(end))); +} +#endif + +#ifdef _WIN32 +extern "C" int backup_stream_append_plain(HANDLE src, HANDLE stream, + uint64_t start, uint64_t end) +{ + return backup_stream_append(src, stream, start, end); +} +#endif diff --git a/sql/sql_backup.h b/sql/sql_backup.h new file mode 100644 index 0000000000000..2759e0d64b43d --- /dev/null +++ b/sql/sql_backup.h @@ -0,0 +1,53 @@ +/***************************************************************************** +Copyright (c) 2026 MariaDB plc. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +#pragma once + +/** BACKUP SERVER */ +class Sql_cmd_backup : public Sql_cmd +{ + /** target directory, or nullptr when streaming */ + const char *const target{nullptr}; + /** argument of my_popen() for streaming backup, or nullptr */ + const char *const command{nullptr}; + /** number of concurrent threads to use */ + const int threads; + +public: + /** + Constructor. + @param target name of target or scratch directory + @param threads number of concurrent threads to use + */ + Sql_cmd_backup(LEX_CSTRING target, int threads) : + target(target.str), threads(threads) {} + /** + Constructor. + @param threads number of concurrent threads to use + @param command nullptr, or a shell command for handling a backup stream + */ + Sql_cmd_backup(int threads, LEX_CSTRING command) : + command(command.str), threads(threads) {} + ~Sql_cmd_backup()= default; + + bool execute(THD *thd) override; + + enum_sql_command sql_command_code() const override + { + return SQLCOM_BACKUP_SERVER; + } +}; diff --git a/sql/sql_backup_interface.h b/sql/sql_backup_interface.h new file mode 100644 index 0000000000000..94892ba7591ab --- /dev/null +++ b/sql/sql_backup_interface.h @@ -0,0 +1,178 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +struct backup_target; + +/** A payload chunk in a sparse file that is being streamed */ +struct backup_chunk +{ + /** byte offset of the start of the payload, from the start of the file */ + uint64_t offset; + /** length of the hole */ + uint64_t length; +}; + +#ifdef _WIN32 +/* Use CopyFileEx() to copy entire files */ +struct native_file_handle; +#elif defined __APPLE__ +/* You should invoke fclonefileat(2) manually before attempting +copy_entire_file() or copy_file() */ +# include +# include +# include +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (negative) +@retval 0 on success */ +inline int copy_entire_file(int src, int dst) +{ + return fcopyfile(src, dst, NULL, COPYFILE_ALL | COPYFILE_CLONE); +} +#else +# ifdef __cplusplus +extern "C" +# endif +/** Copy an entire file. +@param src source file descriptor +@param dst target to append src to +@return error code (non-positive) +@retval 0 on success */ +int copy_entire_file(int src, int dst); +#endif + +#ifdef __cplusplus +extern "C" +#endif +/** Copy a portion of a file. +@param src source file descriptor +@param dst target to append src to +@param start first offset to copy +@param end last offset to copy (exclusive) +@return error code (non-positive) +@retval 0 on success */ +int copy_file(IF_WIN(const native_file_handle&,int) src, + IF_WIN(const native_file_handle&,int) dst, + uint64_t start, uint64_t end); + +#ifdef __cplusplus +extern "C" +#endif +/** Append to the configuration file. +@param target backup target directory +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +int backup_config_append(IF_WIN(const char*, int) target, + const char *config, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** Append to the configuration file. +@param target backup stream +@param config the configuration file snippet to append +@param size length of the snippet +@return error code (non-positive) +@retval 0 on success */ +int backup_stream_config(IF_WIN(HANDLE, int) stream, + const char *config, size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** Start streaming a file. +@param target backup target +@param name file name +@param mode file access mode +@param size physical length of the file, in bytes +@param chunks payload chunks of a sparse file, or nullptr +@param n_chunks number of chunks; 0 unless sparse file +@return error code (non-positive) +@retval 0 on success */ +int backup_stream_start(IF_WIN(HANDLE, int) stream, + const char *name, mode_t mode, uint64_t size, + const struct backup_chunk *chunks, size_t n_chunks); + +#ifdef __cplusplus +extern "C" +#endif +/** + Write data to a stream. + @param stream backup stream + @param buf source buffer + @param size length of the buffer (usually an integer multiple of 512) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_write(IF_WIN(HANDLE, int) stream, const void *buf, + size_t size); + +#ifdef __cplusplus +extern "C" +#endif +/** + Append a file snippet to the stream, + after a corresponding call to backup_stream_start(). + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_append(IF_WIN(const native_file_handle&,int) src, + IF_WIN(HANDLE, int) stream, + uint64_t start, uint64_t end); + +#ifdef __linux__ +# ifdef __cplusplus +extern "C" +# endif +/** + Append an immutable snippet of a file to the stream, + allowing Linux sendfile(2) to be invoked. + + Note that tar uses 512-byte blocks. If end-start is not a multiple of + 512 bytes, backup_stream_write() must be invoked to zero-pad the output. + @param src source file + @param stream backup stream + @param start first offset to copy + @param end last offset to copy (exclusive) + @return error code (non-positive) + @retval 0 on success +*/ +int backup_stream_append_async(int src, int stream, + uint64_t start, uint64_t end); +#elif defined _WIN32 +# define backup_stream_append_async backup_stream_append_plain +#else +# define backup_stream_append_async backup_stream_append +#endif + +#ifdef _WIN32 +# ifdef __cplusplus +extern "C" +# endif +int backup_stream_append_plain(HANDLE src, HANDLE stream, + uint64_t start, uint64_t end); +#else +# define backup_stream_append_plain backup_stream_append +#endif diff --git a/sql/sql_command.h b/sql/sql_command.h index 9c9166706a034..b8903399711f0 100644 --- a/sql/sql_command.h +++ b/sql/sql_command.h @@ -103,6 +103,7 @@ enum enum_sql_command { SQLCOM_SHOW_PACKAGE_BODY_CODE, SQLCOM_BACKUP, SQLCOM_BACKUP_LOCK, SQLCOM_SHOW_CREATE_SERVER, + SQLCOM_BACKUP_SERVER, /* When a command is added here, be sure it's also added in mysqld.cc diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index dc069695b23b6..4b67aed0f6475 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -781,6 +781,7 @@ void init_update_queries(void) sql_command_flags[SQLCOM_DROP_SERVER]|= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP]= CF_AUTO_COMMIT_TRANS; sql_command_flags[SQLCOM_BACKUP_LOCK]= CF_AUTO_COMMIT_TRANS; + sql_command_flags[SQLCOM_BACKUP_SERVER]= CF_AUTO_COMMIT_TRANS; /* The following statements can deal with temporary tables, @@ -5899,6 +5900,7 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt) case SQLCOM_CALL: case SQLCOM_REVOKE: case SQLCOM_GRANT: + case SQLCOM_BACKUP_SERVER: if (thd->variables.option_bits & OPTION_IF_EXISTS) lex->create_info.set(DDL_options_st::OPT_IF_EXISTS); DBUG_ASSERT(lex->m_sql_cmd != NULL); @@ -10249,7 +10251,7 @@ int test_if_data_home_dir(const char *dir) if (!dir) DBUG_RETURN(0); - (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH); + (void) fn_format(path, dir, "", "", MY_RETURN_REAL_PATH|MY_RESOLVE_SYMLINKS); DBUG_RETURN(path_starts_from_data_home_dir(path)); } diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index b2071b1c50b7f..08bf1fb7d84c8 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -50,6 +50,7 @@ #include "sql_alter.h" // Sql_cmd_alter_table* #include "sql_truncate.h" // Sql_cmd_truncate_table #include "sql_admin.h" // Sql_cmd_analyze/Check..._table +#include "sql_backup.h" #include "sql_partition_admin.h" // Sql_cmd_alter_table_*_part. #include "sql_handler.h" // Sql_cmd_handler_* #include "sql_signal.h" @@ -1473,7 +1474,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %type json_on_response %type json_type_constraint -%type json_key_unique_constraint +%type json_key_unique_constraint opt_concurrent %type json_predicate %type field_type field_type_all field_type_all_builtin @@ -15563,6 +15564,31 @@ backup_statements: /* Table list is empty for unlock */ Lex->sql_command= SQLCOM_BACKUP_LOCK; } + | SERVER_SYM TO_SYM TEXT_STRING_sys opt_concurrent + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3, $4); + } + | SERVER_SYM WITH opt_concurrent TEXT_STRING_sys + { + Lex->sql_command= SQLCOM_BACKUP_SERVER; + Lex->m_sql_cmd= new (thd->mem_root) Sql_cmd_backup($3, $4); + } + ; + +opt_concurrent: + /* empty */ + { $$= 1; } + | ulonglong_num CONCURRENT + { + $$= int($1); + if ($1 < 1 || $1 > 256) + { + my_error(ER_DATA_OUT_OF_RANGE, myf(0), "CONCURRENT", + "BACKUP SERVER"); + MYSQL_YYABORT; + } + } ; opt_delete_gtid_domain: diff --git a/sql/sys_vars.inl b/sql/sys_vars.inl index 0f8aa1eb63bf6..8e5f0b984e81e 100644 --- a/sql/sys_vars.inl +++ b/sql/sys_vars.inl @@ -2506,6 +2506,7 @@ public: bool session_update(THD *thd, set_var *var) override; }; +#ifdef HAVE_REPLICATION /* Class for replicate_events_marked_for_skip. We need a custom update function that ensures the slave is stopped when @@ -2647,6 +2648,7 @@ public: } const uchar *global_value_ptr(THD *thd, const LEX_CSTRING *base) const override; }; +#endif /* HAVE_REPLICATION */ /** diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 9e3a23b34ab46..d63751a16af08 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -185,6 +185,8 @@ SET(INNOBASE_SOURCES handler/handler0alter.cc handler/innodb_binlog.cc handler/i_s.cc + handler/backup_innodb.h + handler/backup_innodb.cc ibuf/ibuf0ibuf.cc include/btr0btr.h include/btr0btr.inl diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 577973db456b0..b2df0439426fc 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -2298,7 +2298,7 @@ void buf_page_free(fil_space_t *space, uint32_t page, mtr_t *mtr) if (block->index) btr_search_drop_page_hash_index(block, nullptr); #endif /* BTR_CUR_HASH_ADAPT */ - block->page.set_freed(block->page.state()); + block->page.set_freed(); mtr->memo_push(block, MTR_MEMO_PAGE_X_MODIFY); } @@ -2367,7 +2367,7 @@ buf_page_t *buf_page_get_zip(const page_id_t page_id) noexcept exclusive latch on this block and either in progress or invoking buf_pool_t::corrupted_evict(). - Let us aqcuire and release buf_pool.mutex to ensure that any + Let us acquire and release buf_pool.mutex to ensure that any buf_pool_t::corrupted_evict() will proceed before we reacquire the hash_lock that it could be waiting for. @@ -2501,7 +2501,12 @@ buf_block_t *buf_pool_t::unzip(buf_page_t *b, buf_pool_t::hash_chain &chain) case buf_page_t::REINIT + 1: break; default: - ut_ad(state < buf_page_t::READ_FIX); + /* + There may be a fake "write fix" if a thread is executing + between InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() on this page. + */ + ut_ad(!buf_page_t::is_read_fixed(state)); if (state < buf_page_t::UNFIXED + 1) { @@ -2520,7 +2525,7 @@ buf_block_t *buf_pool_t::unzip(buf_page_t *b, buf_pool_t::hash_chain &chain) goto wait_for_unfix; } - /* Ensure that another buf_page_get_low() or buf_page_t::page_fix() + /* Ensure that another buf_page_get_low() or buf_pool_t::page_fix() will wait for block->page.lock.x_unlock(). buf_relocate() will copy the state from b to block and replace b with block in page_hash. */ b->set_state(buf_page_t::READ_FIX); @@ -2879,7 +2884,17 @@ buf_page_get_gen( if (!nowait) { goto latch_waited; } else { - ut_ad(state < buf_page_t::READ_FIX); + /* If a thread is executing between + InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() + for this page, a fake "write + fix" may exist. We are free to + modify the page in the buffer + pool, but buf_page_t::flush() + will refuse to write it to the + file system. */ + ut_ad(!buf_page_t:: + is_read_fixed(state)); } /* fall through */ case RW_S_LATCH: @@ -2893,8 +2908,7 @@ buf_page_get_gen( } else { not_read_fixed: ut_ad(state > buf_page_t::FREED); - ut_ad(state < buf_page_t::READ_FIX - || state > buf_page_t::WRITE_FIX); + ut_ad(!buf_page_t::is_read_fixed(state)); if (UNIV_UNLIKELY(!block->page.frame && mode == BUF_PEEK_IF_IN_POOL)) { /* The BUF_PEEK_IF_IN_POOL mode is mainly used @@ -2956,7 +2970,8 @@ buf_page_get_gen( break; case RW_SX_LATCH: block->page.lock.u_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); break; default: ut_ad(rw_latch == RW_X_LATCH); @@ -3039,7 +3054,8 @@ buf_block_t *buf_page_optimistic_get(buf_block_t *block, goto fail; else { - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); if (modify_clock != block->modify_clock || block->page.is_freed()) { @@ -3198,24 +3214,33 @@ static buf_block_t *buf_page_create_low(page_id_t page_id, ulint zip_size, mysql_mutex_unlock(&buf_pool.mutex); bpage->lock.x_lock(); - const page_id_t id{bpage->id()}; - if (UNIV_UNLIKELY(id != page_id)) + if (UNIV_UNLIKELY(bpage->id() != page_id)) { - ut_ad(id.is_corrupted()); + ut_ad(bpage->id().is_corrupted()); ut_ad(bpage->is_freed()); + backtrack: bpage->unfix(); bpage->lock.x_unlock(); goto retry; } mysql_mutex_lock(&buf_pool.mutex); state= bpage->state(); - ut_ad(!bpage->is_io_fixed(state)); ut_ad(bpage->buf_fix_count(state)); } else state= bpage->state(); ut_ad(state > buf_page_t::FREED); + + if (UNIV_UNLIKELY(buf_page_t::is_write_fixed(state))) + /* A thread should be executing between + InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() on this page. + + We play it safe and will wait until the fake "write fix" + has been cleared. */ + goto backtrack; + ut_ad(state < buf_page_t::READ_FIX); /* In addition to our buffer-fix, there may be another that is held by a concurrent IORequest::read_complete() that had @@ -3831,8 +3856,7 @@ void buf_pool_t::validate() noexcept /* do nothing */ break; default: - if (f >= buf_page_t::READ_FIX - && f < buf_page_t::WRITE_FIX) { + if (buf_page_t::is_read_fixed(f)) { /* A read-fixed block is not necessarily in the page_hash yet. */ break; diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index fb4a7bc5d9958..b9afbb158250f 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -752,7 +752,7 @@ void buf_dblwr_t::flush_buffered_writes_completed(const IORequest &request) static_cast(frame))); ut_ad(lsn); ut_ad(lsn >= bpage->oldest_modification()); - if (lsn < e.request.node->space->get_create_lsn()) + if (lsn < e.request.node->space->create_lsn) { /* mtr_t::commit_shrink() must have been invoked between buf_dblwr_t::flush_buffered_writes() and diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index be4b0e473b26b..bb6e07fe38405 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -35,6 +35,7 @@ Created 11/11/1995 Heikki Tuuri #include "buf0buf.h" #include "buf0checksum.h" #include "buf0dblwr.h" +#include "backup_innodb.h" #include "srv0start.h" #include "page0zip.h" #include "fil0fil.h" @@ -244,7 +245,7 @@ void buf_flush_remove_pages(uint32_t id) noexcept { const auto s= bpage->state(); ut_ad(s >= buf_page_t::REMOVE_HASH); - ut_ad(s < buf_page_t::READ_FIX || s >= buf_page_t::WRITE_FIX); + ut_ad(!buf_page_t::is_read_fixed(s)); buf_page_t *prev= UT_LIST_GET_PREV(list, bpage); const page_id_t bpage_id(bpage->id()); @@ -786,7 +787,7 @@ bool buf_page_t::flush(fil_space_t *space) noexcept ut_ad((space->is_temporary()) == (space == fil_system.temp_space)); ut_ad(space->referenced()); - const auto s= state(); + uint32_t s{state()}; const lsn_t lsn= mach_read_from_8(my_assume_aligned<8> @@ -812,16 +813,40 @@ bool buf_page_t::flush(fil_space_t *space) noexcept return false; } - if (UNIV_UNLIKELY(lsn < space->get_create_lsn())) + if (UNIV_UNLIKELY(lsn < space->create_lsn)) { ut_ad(!space->is_temporary()); ut_ad(!space->is_being_imported()); goto freed; } - ut_d(const auto f=) zip.fix.fetch_add(WRITE_FIX - UNFIXED); - ut_ad(f >= UNFIXED); - ut_ad(f < READ_FIX); + do + { + ut_ad(s >= UNFIXED); + if (UNIV_UNLIKELY(s >= READ_FIX)) + { + /* + A thread must have successfully executed write_fix_try() after + fix(), executing between InnoDB_backup::backup_batch_start() + and InnoDB_backup::backup_batch_stop(). + */ + ut_ad(s > WRITE_FIX); + /* Backup skips the temporary tablespace */ + ut_ad(oldest_modification() > 2); + lock.u_unlock(true); + return false; + } + } + /* + compare_exchange_strong() ensures that the write-fix was set by us + and not a concurrent write_fix_try(). + */ + while (!zip.fix.compare_exchange_strong(s, s + (WRITE_FIX - UNFIXED), + std::memory_order_acquire, + std::memory_order_relaxed)); + + ut_ad(s >= UNFIXED); + ut_ad(s < READ_FIX); ut_ad((space == fil_system.temp_space) ? oldest_modification() == 2 : oldest_modification() > 2); @@ -1002,6 +1027,14 @@ uint32_t fil_space_t::flush_freed(bool writable) noexcept mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex); mysql_mutex_assert_not_owner(&buf_pool.mutex); + /* Note: There is no need to check if another thread is executing + between InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_end(), because we are only overwriting + freed (garbage) pages. If backup reads a torn page, it will also + have copied a corresponding FREE_PAGE record, which would be applied + on recovery. Besides, the freed page should never be reachable from + other pages that are part of the snapshot. */ + const bool punch_hole= chain.start->punch_hole == 1; if (!punch_hole && !srv_immediate_scrub_data_uncompressed) return 0; @@ -1084,7 +1117,7 @@ static ulint buf_flush_try_neighbors(fil_space_t *space, (FIL_PAGE_LSN + (bpage->zip.data ? bpage->zip.data : bpage->frame))); ut_ad(lsn >= bpage->oldest_modification()); - if (UNIV_UNLIKELY(lsn < space->get_create_lsn())) + if (UNIV_UNLIKELY(lsn < space->create_lsn)) { ut_a(!bpage->flush(space)); mysql_mutex_unlock(&buf_pool.mutex); @@ -1277,6 +1310,14 @@ ATTRIBUTE_COLD static size_t buf_flush_LRU_to_withdraw(size_t to_withdraw, return to_withdraw; } + +/** @return the first page number that is not being backed up */ +inline uint32_t fil_space_t::backup_page_end() const noexcept +{ + mysql_mutex_assert_owner(&buf_pool.mutex); + return backup_end.load(std::memory_order_acquire); +} + /** Flush dirty blocks from the end buf_pool.LRU, and move clean blocks to buf_pool.free. @param max maximum number of blocks to flush @@ -1294,6 +1335,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1343,11 +1385,12 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, if (state < buf_page_t::READ_FIX && bpage->lock.u_lock_try(true)) { - ut_ad(!bpage->is_io_fixed()); + ut_ad(!bpage->is_read_fixed()); /* tolerate write_fix_try() */ switch (bpage->oldest_modification()) { case 2: /* LRU flushing will always evict pages of the temporary tablespace, in buf_page_write_complete(). */ + ut_ad(!bpage->is_io_fixed()); ++n->evicted; break; case 1: @@ -1361,6 +1404,8 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, /* fall through */ case 0: bpage->lock.u_unlock(true); + /* Reload in case write_fix_try() had been undone meanwhile. */ + state= bpage->state(); goto evict; } /* Block is ready for flush. Dispatch an IO request. */ @@ -1382,7 +1427,9 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, mysql_mutex_lock(&buf_pool.mutex); goto no_space; } + mysql_mutex_lock(&buf_pool.mutex); + backup_page_end= space->backup_page_end(); buf_pool.stat.n_pages_written+= p.second; } else @@ -1411,7 +1458,8 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, break; } - if (neighbors && space->is_rotational() && UNIV_LIKELY(!to_withdraw) && + if (neighbors && UNIV_LIKELY(!(to_withdraw | backup_page_end)) && + space->is_rotational() && /* Skip neighbourhood flush from LRU list if we haven't yet reached half of the free page target. */ UT_LIST_GET_LEN(buf_pool.free) * 2 >= free_limit) @@ -1423,10 +1471,17 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, flush: if (UNIV_UNLIKELY(to_withdraw != 0)) to_withdraw= buf_flush_LRU_to_withdraw(to_withdraw, *bpage); - if (bpage->flush(space)) + const uint32_t page{bpage->id().page_no()}; + if (page < backup_page_end && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) + bpage->lock.u_unlock(true); + else if (bpage->flush(space)) + { ++n->flushed; - else - continue; + goto reacquire_mutex; + } + + continue; } goto reacquire_mutex; @@ -1486,6 +1541,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; uint32_t last_space_id= FIL_NULL; + uint32_t backup_page_end= 0; static_assert(FIL_NULL > SRV_TMP_SPACE_ID, "consistency"); static_assert(FIL_NULL > SRV_SPACE_ID_UPPER_BOUND, "consistency"); @@ -1527,7 +1583,9 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept if (!bpage->lock.u_lock_try(true)) goto skip; - ut_ad(!bpage->is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist; + we will check it in buf_page_t::flush() */ + ut_ad(!bpage->is_read_fixed()); if (bpage->oldest_modification() == 1) { @@ -1562,6 +1620,7 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept space= p.first; last_space_id= space_id; mysql_mutex_lock(&buf_pool.mutex); + backup_page_end= space ? space->backup_page_end() : 0; buf_pool.stat.n_pages_written+= p.second; mysql_mutex_lock(&buf_pool.flush_list_mutex); } @@ -1581,9 +1640,17 @@ static ulint buf_do_flush_list_batch(ulint max_n, lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); do { - if (neighbors && space->is_rotational()) + if (neighbors && UNIV_LIKELY(!backup_page_end) && + space->is_rotational()) count+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, count, max_n); + else if (page_id.page_no() < backup_page_end && + page_id.page_no() >= + backup_page_end - space->BACKUP_BATCH_SIZE) + { + bpage->lock.u_unlock(true); + continue; + } else if (bpage->flush(space)) ++count; else @@ -1693,6 +1760,7 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept if (written) buf_pool.stat.n_pages_written+= written; } + mysql_mutex_lock(&buf_pool.flush_list_mutex); for (buf_page_t *bpage= UT_LIST_GET_LAST(buf_pool.flush_list); bpage; ) @@ -1735,18 +1803,27 @@ bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) noexcept acquired= false; goto was_freed; } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + uint32_t page, backup_page_end; + page= bpage->id().page_no(); + backup_page_end= space->backup_page_end(); + if (UNIV_UNLIKELY(page < backup_page_end) && + page >= backup_page_end - space->BACKUP_BATCH_SIZE) + { + bpage->lock.u_unlock(true); + skip: + mysql_mutex_lock(&buf_pool.flush_list_mutex); + may_have_skipped= true; + goto done; + } + if (bpage->flush(space)) { ++n_flush; - if (!--max_n_flush) - { - mysql_mutex_lock(&buf_pool.mutex); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - may_have_skipped= true; - goto done; - } mysql_mutex_lock(&buf_pool.mutex); + if (!--max_n_flush) + goto skip; } } @@ -2062,6 +2139,7 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept this->end_lsn= end_lsn; if (!archive) { + archived_checkpoint= checkpoint; archived_lsn= end_lsn; checkpoint_completed: if (resize_log.m_file == log.m_file) @@ -2079,7 +2157,7 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept /* Make the previous archived log file read-only */ #ifdef _WIN32 resize_log.close(); - SetFileAttributesA(get_archive_path().c_str(), + SetFileAttributesA(get_archive_path(get_first_lsn() - capacity()).c_str(), FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_ARCHIVE); #else struct stat st; @@ -2089,9 +2167,10 @@ inline lsn_t log_t::write_checkpoint(lsn_t checkpoint, lsn_t end_lsn) noexcept st.st_mode= 0444; if (fchmod(resize_log.m_file, st.st_mode)) my_error(ER_ERROR_ON_CLOSE, MYF(ME_ERROR_LOG), - get_archive_path().c_str(), errno); + get_archive_path(get_first_lsn() - capacity()).c_str(), errno); resize_log.close(); #endif + innodb_backup_checkpoint(); } else goto checkpoint_completed; diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 95e2a6cbe0972..7b8bf9a3e06b9 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1271,20 +1271,15 @@ void buf_LRU_truncate_temp(uint32_t threshold) /* Set the extent descriptor page state as FREED */ for (uint32_t cur_xdes_page= xdes_calc_descriptor_page( 0, fil_system.temp_space->free_limit); - cur_xdes_page >= threshold;) + cur_xdes_page >= threshold; + cur_xdes_page-= uint32_t(srv_page_size)) { mtr_t mtr{nullptr}; mtr.start(); if (buf_block_t* block= buf_page_get_gen( page_id_t(SRV_TMP_SPACE_ID, cur_xdes_page), 0, RW_X_LATCH, nullptr, BUF_PEEK_IF_IN_POOL, &mtr)) - { - uint32_t state= block->page.state(); - ut_ad(state > buf_page_t::UNFIXED); - ut_ad(state < buf_page_t::READ_FIX); - block->page.set_freed(state); - } - cur_xdes_page-= uint32_t(srv_page_size); + block->page.set_freed(); mtr.commit(); } diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index d8976dec66974..49ffe82fcd49c 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -889,6 +889,10 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) dict_sys.lock(SRW_LOCK_CALL); + if (fil_system.have_all_spaces) { + goto done; + } + if (!spaces && !upgrade && !encryption_key_id_exists(FIL_DEFAULT_ENCRYPTION_KEY)) { max_space_id = dict_find_max_space_id(&pcur, &mtr); @@ -934,7 +938,15 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) newly created or rebuilt tables or partitions, but will otherwise ignore the flag. */ - if (fil_space_for_table_exists_in_mem(space_id, flags)) { + fil_space_t* space + = fil_space_for_table_exists_in_mem(space_id, flags); + + if (space) { + if (upgrade) { + space->get_size(); + } + next: + max_space_id = ut_max(max_space_id, space_id); continue; } @@ -959,11 +971,13 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) const bool not_dropped{!rec_get_deleted_flag(rec, 0)}; /* Check that the .ibd file exists. */ - if (fil_ibd_open(space_id, dict_tf_to_fsp_flags(flags), - not_dropped - ? fil_space_t::VALIDATE_NOTHING - : fil_space_t::MAYBE_MISSING, - name, filepath)) { + space = fil_ibd_open( + space_id, dict_tf_to_fsp_flags(flags), + not_dropped + ? fil_space_t::VALIDATE_NOTHING + : fil_space_t::MAYBE_MISSING, + name, filepath); + if (space && (!upgrade || space->get_size())) { } else if (!not_dropped) { } else if (srv_operation == SRV_OPERATION_NORMAL && srv_start_after_restore @@ -980,11 +994,11 @@ void dict_load_tablespaces(const std::set *spaces, bool upgrade) static_cast(len), field); } - max_space_id = ut_max(max_space_id, space_id); - ut_free(filepath); + goto next; } + fil_system.have_all_spaces = upgrade; done: mtr.commit(); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 47bfde7ebd23d..7973837028ebd 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -2035,6 +2035,7 @@ fil_ibd_create( fil_node_t* node = space->add(path, OS_FILE_CLOSED, size, false, true); space->set_stopped(); mysql_mutex_unlock(&fil_system.mutex); + lsn_t create_lsn; buf_block_t *header[2]; { @@ -2053,9 +2054,10 @@ fil_ibd_create( header[0]->page.lock.x_lock(); header[1]->page.lock.x_lock(); mtr.commit(); + create_lsn = mtr.commit_lsn(); /* Durably write the FILE_CREATE record (as well as records for the fsp_header_init() above before creating the file. */ - log_write_up_to(mtr.commit_lsn(), true); + log_write_up_to(create_lsn, true); } DEBUG_SYNC_C("fil_ibd_create_logged"); @@ -2136,6 +2138,7 @@ fil_ibd_create( mysql_mutex_lock(&fil_system.mutex); space->clear_stopped(); + space->create_lsn = create_lsn; node->handle = file; node->find_metadata(IF_WIN(,true)); if (++fil_system.n_open >= srv_max_n_open_files) { diff --git a/storage/innobase/handler/backup_innodb.cc b/storage/innobase/handler/backup_innodb.cc new file mode 100644 index 0000000000000..7890e1b60d9ab --- /dev/null +++ b/storage/innobase/handler/backup_innodb.cc @@ -0,0 +1,1382 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "my_global.h" +#include "sql_class.h" +#include "backup_innodb.h" +#include "sql_backup_interface.h" +#include "trx0trx.h" +#include "buf0flu.h" +#include "log0crypt.h" +#include "dict0load.h" +#include +#ifdef __linux__ +# include +# include +#endif + +/** Associate a transaction with the current session +@param thd session +@return InnoDB transaction */ +trx_t *check_trx_exists(THD *thd) noexcept; + +/** Try to write-fix a block. +@param s expected state() +@return new s (right before potentially setting the write-fix); +a write-fix was acquired if !is_freed(s) && !is_io_fixed(s) holds */ +inline uint32_t buf_page_t::write_fix_try(uint32_t s) noexcept +{ + /* The calling thread must hold a fix() */ + ut_ad(s > FREED); + /* + set_freed() or flush() may run concurrently. + compare_exchange_strong() ensures that the write-fix was set by us. + */ + while (!is_freed(s) && !is_io_fixed(s) && + !zip.fix.compare_exchange_strong(s, s + (WRITE_FIX - UNFIXED), + std::memory_order_acquire, + std::memory_order_relaxed)); + return s; +} + +/** Try to undo a successful write_fix_try(). */ +inline void buf_page_t::write_unfix_try() noexcept +{ + uint32_t s{state()}; + /* set_freed() may clear the write-fix before or during this loop. + We must use a compare-and-exchange loop. */ + while (is_write_fixed(s) && + !zip.fix.compare_exchange_weak(s, s - (WRITE_FIX - UNFIXED), + std::memory_order_relaxed, + std::memory_order_relaxed)); +} + +/** + Ensure that there are no page writes in progress. + @param end array of fil_space_t::BACKUP_BATCH_SIZE block descriptors + @param space_id tablespace identifier + @param end_page last page number that is being copied + @return pointer to the new end of the array, of write-fixed blocks +*/ +static buf_page_t **innodb_backup_batch_wait(buf_page_t **end, + uint32_t space_id, + uint32_t end_page) + noexcept +{ + const page_id_t start + {space_id, end_page & ~(fil_space_t::BACKUP_BATCH_SIZE - 1)}; + ut_ad(end_page - 1 > start.page_no()); + for (page_id_t id{space_id, end_page};; --id) + { + auto &chain= buf_pool.page_hash.cell_get(id.fold()); + page_hash_latch &hash_lock{buf_pool.page_hash.lock_get(chain)}; + hash_lock.lock_shared(); + buf_page_t *const b= *end= buf_pool.page_hash.get(id, chain); + if (b && b->oldest_modification_acquire() > 2) + { + uint32_t state{b->fix()}; + /* The above buffer-fix froze b in buf_pool.page_hash */ + hash_lock.unlock_shared(); + /* + We found out that this page was dirty, and eventually such pages + must be either freed or written back to the file system. + + The lock-free buf_page_t::write_fix_try() aims to block + concurrent asynchronous buf_page_t::flush(). It will not + block any access to the page in the buffer pool; we are + not holding any page latch. + */ + state= b->write_fix_try(state + 1); + if (UNIV_LIKELY(!b->is_io_fixed(state))) + { + if (b->is_freed(state)) + /* + Freed blocks will not be written back to the file system. + As noted in fil_space_t::flush_freed(), we do allow + concurrent FALLOC_FL_PUNCH_HOLE of PAGE_COMPRESSED pages + and NUL writes by immediate_scrub_data_uncompressed=ON + while the data is being backed up. This should be fine, + because those operations are only overwriting freed + (garbage) data. + */ + safe: + b->unfix(); + else + /* Schedule a call of b->write_unfix_try() and b->unfix(). */ + end++; + } + else if (!b->is_write_fixed(state)) + /* + The block is read-fixed. It is fine to have two concurrent + reads from the data file: one to the buffer pool and another + by the backup. + + Any subsequent write will be gated by acquiring + buf_pool.mutex and checking fil_space_t::backup_page_end(), + which will hold up any further writes until + fil_space_t::backup_stop() is invoked by + InnoDB_backup::backup_batch_stop(). + */ + goto safe; + else + { + /* + Wait for buf_page_t::flush() to be concluded by + buf_page_t::write_complete(). + */ + b->lock.u_lock(); + ut_ad(!b->is_io_fixed()); + b->lock.u_unlock(); + /* + The pending write was completed. Any subsequent write will + be gated by acquiring buf_pool.mutex and checking + fil_space_t::backup_page_end(), which will hold up any + further writes until fil_space_t::backup_stop() is invoked + by InnoDB_backup::backup_batch_stop(). + */ + goto safe; + } + } + else + hash_lock.unlock_shared(); + + if (id == start) + break; + } + return end; +} + +namespace +{ +/** Backup state; protected by log_sys.latch */ +class InnoDB_backup +{ + /** Backup context */ + struct context + { + /** Start LSN of the first backed up log file */ + lsn_t first_lsn; + /** Start LSN of the last log file, or LSN_MAX if not determined yet */ + lsn_t max_first_lsn; + /** Final LSN of the backup, or LSN_MAX if not determined yet */ + lsn_t last_lsn; + /** size of the first log file */ + uint64_t first_size; + /** Checkpoint at the start of the backup */ + lsn_t checkpoint; + /** Log record pointing to the checkpoint */ + lsn_t checkpoint_end_lsn; + /** the original state of innodb_log_archive before/after backup */ + bool archived; + /** whether end() was invoked */ + bool cleaned_up; + /** the start LSN of the last hard-linked file, or 0 */ + std::atomic last_hardlink; + + /** + Note that a log file was hard-linked. + @param lsn start LSN of a hard-linked file + */ + void note_hardlink(lsn_t lsn) noexcept + { + for (lsn_t last= last_hardlink.load(std::memory_order_relaxed); + last < lsn && !last_hardlink. + compare_exchange_weak(last, lsn, + std::memory_order_relaxed, + std::memory_order_relaxed); ) {} + } + + /** Ensure that the last, hard-linked log file is not shared with + the server data directory, by copying it until the final LSN + @param target backup target directory + @param hl last_hardlink + @return error code + @retval 0 on success + */ + ATTRIBUTE_COLD int de_hardlink(const backup_target &target, lsn_t hl) + noexcept + { +#ifdef _WIN32 + std::string src{target.path}; + src.push_back('/'); + std::string dst{src}; + src.append("ib_logfile101"); + log_sys.append_archive_name(dst, hl); + const char *const s_{src.c_str()}, *const d_{dst.c_str()}; + if (!MoveFileEx(d_, s_, 0)) + { + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), d_, s_, errno); + return 1; + } + HANDLE s, d; + for (;;) + { + s= CreateFile(s_, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (s != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), s_, errno); + return 1; + } + d= CreateFile(d_, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (d == INVALID_HANDLE_VALUE) + { + error_return: + my_osmaperr(GetLastError()); + std::ignore= CloseHandle(s); + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), s_, d_, errno); + return 1; + } +#else + std::string dst; + log_sys.append_archive_name(dst, hl); + const char *const d_{dst.c_str()}; + int d{-1}; + int err= ER_FILE_NOT_FOUND; + int s= openat(target.fd, d_, O_RDONLY); + if (s == -1) + { + error_return: + my_error(err, MYF(ME_ERROR_LOG), d_, errno); + if (s != -1) + std::ignore= close(s); + return 1; + } + err= ER_CANT_DELETE_FILE; + if (unlinkat(target.fd, d_, 0)) + goto error_return; + err= ER_CANT_CREATE_FILE; + d= openat(target.fd, d_, O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (d < 0) + goto error_return; +#endif + const uint64_t end{log_sys.START_OFFSET + last_lsn - hl}; + /* First, extend the file to a valid size. */ +#ifdef _WIN32 + int f; + { + LARGE_INTEGER li; + li.QuadPart= std::max(log_sys.FILE_SIZE_MIN, + (end + 4095) & ~4095ULL); + f= !SetFilePointerEx(d, li, nullptr, FILE_BEGIN) || !SetEndOfFile(d); + } +#else + int f= + ftruncate(d, std::max(log_sys.FILE_SIZE_MIN, + (end + 4095) & ~4095LL)); +#endif + if (!f && + !(f= copy_file(s, d, log_sys.START_OFFSET + + (hl == first_lsn) * (checkpoint - hl), end)) && + hl == first_lsn) + { + uint64_t cp_buf[8]{}; + write_checkpoint_buf(cp_buf, + checkpoint_end_lsn - hl + log_sys.START_OFFSET); + f= write_checkpoint(d, cp_buf); + } + if (IF_WIN(!CloseHandle(d), close(d)) | f) + goto error_return; + std::ignore= IF_WIN(CloseHandle(s), close(s)); +#ifdef _WIN32 + if (!DeleteFile(s_)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_DELETE_FILE, MYF(ME_ERROR_LOG), s_, errno); + return 1; + } +#endif + return 0; + } + + /** + Finish a backup. + @param target backup target + @param sink backup worker context + @return error code + @retval 0 on success + */ + int cleanup(const backup_target &target, const backup_sink &sink) noexcept + { + const lsn_t hl{last_hardlink.load(std::memory_order_relaxed)}; + if (hl == LSN_MAX) + return 0; + log_sys.latch.rd_lock(); + const lsn_t current_first_lsn{log_sys.get_first_lsn()}; + log_sys.latch.rd_unlock(); + if (hl == current_first_lsn) + { + ut_ad(sink.stream == sink.NO_STREAM); + if (int fail= de_hardlink(target, hl)) + return fail; + } + return write_config(target, sink); + } + }; + + /** pointer to backup context, or nullptr if no backup is active */ + context *ctx; + + /** the original innodb_log_file_size, or 0 */ + uint64_t old_size; + + /** collection of files and sizes to be copied */ + std::vector queue; + /** collection of completed log archive files to be + hard-linked, copied, or moved */ + std::vector logs; + +public: + /** + Start of BACKUP SERVER: collect all files to be backed up + @param thd current session + @return ctx + @retval -1 on failure + */ + void *init(THD *thd) noexcept + { + log_sys.latch.wr_lock(); + ut_ad(!ctx); + ut_ad(queue.empty()); + if (!logs.empty()) + { + /* A new BACKUP SERVER is being invoked before a previous one + had been fully finalized. Clean up any log files. */ + delete_logs(); + logs.clear(); + } + + const bool fail{log_sys.backup_start(&old_size, thd)}; + + if (!fail) + { + lsn_t start_end; + const lsn_t start= +#if 1 /* TODO: for incremental backup, allow the start to be specified */ + log_sys.get_latest_checkpoint(start_end); +#else + log_sys.archived_checkpoint; + start_end= log_sys.archived_lsn; +#endif + ctx= new context{ + log_sys.get_first_lsn(), LSN_MAX, LSN_MAX, log_sys.file_size, + start, start_end, !old_size, false, 0 + }; + + /* Collect all tablespaces that have been created before our + start checkpoint. Newer tablespaces will be recovered by the + innodb_log_archive=ON recovery. + + If a tablespace is deleted before step() is invoked, the file + will not be copied, and a FILE_DELETE record in the log will + ensure correct recovery. + + If a tablespace is renamed between this and end(), the recovery + of a FILE_RENAME record will ensure the correct file name, + no matter which name was used by step(). */ + mysql_mutex_lock(&fil_system.mutex); + for (fil_space_t &space : fil_system.space_list) + if (space.id < SRV_SPACE_ID_UPPER_BOUND && + !space.is_being_imported() && !space.is_stopping()) + { + /* FIXME: how to initialize create_lsn for old files, to + have efficient incremental backup? + fil_node_t::read_page0() cannot assign it from + FIL_PAGE_LSN because that would not reflect the file + creation but for example allocating or freeing a page. + + The easy parts of initializing space->create_lsn are + as follows: + (1) In log_parse_file() when processing FILE_CREATE + (2) In deferred_spaces.create() + (3) In fil_ibd_create() outside recovery */ + uint64_t s{space.id}; +#if 1 /* MDEV-39694 FIXME: recover FILE_CREATE by creating files */ + if (space.create_lsn < start) +#endif + s|= uint64_t{std::min(space.size, space.free_limit)} << 32; + queue.emplace_back(s); + } + mysql_mutex_unlock(&fil_system.mutex); + } + log_sys.latch.wr_unlock(); + DEBUG_SYNC(thd, "innodb_backup_start"); + return fail ? reinterpret_cast(-1) : ctx; + } + + /** + Process a file that was collected at init(). + This may be invoked from multiple concurrent threads. + @param target backup target + @param phase backup phase + @param sink backup worker context + @return number of files remaining, or negative on error + @retval 0 on completion + */ + int step(const backup_target &target, backup_phase phase, + const backup_sink &sink) noexcept + { + uint64_t id_limit{0}; + lsn_t lsn{0}; + log_sys.latch.wr_lock(); + const lsn_t first{log_sys.get_first_lsn()}; + ut_ad(sink.ha_data); + ut_ad(ctx ? ctx == sink.ha_data + : phase == BACKUP_PHASE_FINISH || phase == BACKUP_PHASE_NO_COMMIT); + ut_ad(static_cast(sink.ha_data)->last_lsn == LSN_MAX + ? phase == BACKUP_PHASE_START : !ctx); + size_t size{queue.size()}; + ut_ad(!size || phase == BACKUP_PHASE_START); + if (!logs.empty()) + { + lsn= logs.back(); + logs.pop_back(); + if (!size) + size= logs.size(); + } + else if (size) + { + ut_ad(phase == BACKUP_PHASE_START); + size--; + id_limit= queue.back(); + queue.pop_back(); + } + log_sys.latch.wr_unlock(); + + if (lsn) + { + if (UNIV_UNLIKELY(lsn > first)) + /* Wait for checkpoint_complete(). */ + buf_flush_sync_batch(lsn, true); + if (replicate(lsn, target, sink, lsn < first)) + return -1; + } + else if (!id_limit); + else if (fil_space_t *space= fil_space_t::get(uint32_t(id_limit))) + { + int res= -1; + uint32_t start{0}, limit{uint32_t(id_limit >> 32)}; +#ifdef _WIN32 + if (sink.stream == sink.NO_STREAM) + { + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain);;) + { + if ((res= backup(target.path, node, start, limit))) + break; + fil_node_t *next= UT_LIST_GET_NEXT(chain, node); + if (!next) + break; + const uint32_t size{node->size}; + start+= size; + if (limit >= size) + limit-= size; + else + limit= 0; + node= next; + } + } + else + { + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain);;) + { + if ((res= stream(sink.stream, node, start, limit))) + break; + fil_node_t *next= UT_LIST_GET_NEXT(chain, node); + if (!next) + break; + const uint32_t size{node->size}; + start+= size; + if (limit >= size) + limit-= size; + else + limit= 0; + node= next; + } + } +#else + int fd; + int (*method)(int, fil_node_t *, uint32_t, uint32_t); + if (sink.stream == sink.NO_STREAM) + { + fd= target.fd; + method= backup; + } + else + { + fd= sink.stream; + method= stream; + } + for (fil_node_t *node= UT_LIST_GET_FIRST(space->chain);;) + { +# ifdef HAVE_POSIX_FALLOCATE + if (limit & 3 && !UT_LIST_GET_NEXT(chain, node)) + { + const uint32_t page_size{space->physical_size()}; + if ((limit * page_size) & 4095) + /* os_file_set_size() extends ROW_FORMAT=COMPRESSED files to + multiples of 4096 bytes. There may be up to 3 pages + (of 1024 bytes) that have not been written out yet. + We must cap the limit to the actual file size. */ + limit= + std::min(limit, + uint32_t(os_file_get_size(node->handle) / page_size)); + } +# endif + if ((res= (*method)(fd, node, start, limit))) + break; + fil_node_t *next= UT_LIST_GET_NEXT(chain, node); + if (!next) + break; + const uint32_t size{node->size}; + start+= size; + if (limit >= size) + limit-= size; + else + limit= 0; + node= next; + } +#endif + space->release(); + if (res) + return res; + } + + size= std::min(size_t{std::numeric_limits::max()}, size); + return int(size); + } + + /** + Determine the logical time of the backup snapshot. + */ + void commit() noexcept + { + log_sys.latch.wr_lock(); + ut_ad(queue.empty()); + ut_ad(ctx); + ut_ad(ctx->last_lsn == LSN_MAX); + const lsn_t last_lsn{log_sys.get_lsn()}; + lsn_t lsn{log_sys.get_first_lsn()}; + if (logs.empty() || logs.back() != lsn) + { + /* Schedule the remaining log for copying */ + logs.emplace_back(lsn); + const lsn_t next_lsn{lsn + log_sys.capacity()}; + if (next_lsn < last_lsn) + logs.emplace_back(lsn= next_lsn); + } + ctx->max_first_lsn= lsn; + ctx->last_lsn= last_lsn; + ctx= nullptr; /* unsubscribe to checkpoint_complete() */ + log_sys.latch.wr_unlock(); + } + + /** + Finish copying or finalize the backup. + @param thd current session + @param phase backup phase + @param sink backup worker context + @return error code + @retval 0 on success + */ + int end(THD *thd, backup_phase phase, const backup_sink &sink) noexcept + { + context *const ctx{static_cast(sink.ha_data)}; + if (!ctx /* InnoDB_backup::init() must have failed */ || + ctx->cleaned_up /* aborting after phase=BACKUP_PHASE_NO_COMMIT */) + return 0; + ctx->cleaned_up= true; + if (phase == BACKUP_PHASE_ABORT) + ctx->last_hardlink.store(LSN_MAX, std::memory_order_relaxed); + log_sys.latch.wr_lock(); + ut_ad(!this->ctx || this->ctx == ctx); + this->ctx= nullptr; /* fini() will delete the object */ + ut_ad(!log_sys.resize_in_progress()); + ut_ad(log_sys.archive); + queue.clear(); + int fail{0}; + if (!old_size) + logs.clear(); + else + { + log_sys.latch.wr_unlock(); + fail= log_sys.backup_stop_archiving(thd); + log_sys.latch.wr_lock(); + delete_logs(); + logs.clear(); + } + + log_sys.backup_stop(old_size, thd); + return fail; + } + + /** + Clean up after end(). + @param target backup target + @param sink backup worker context + @return error code + @retval 0 on success + */ + int fini(const backup_target &target, const backup_sink &sink) noexcept + { + if (context *ctx{static_cast(sink.ha_data)}) + { + ut_ad(ctx != this->ctx); + int fail{ctx->cleanup(target, sink)}; + delete ctx; + return fail; + } + return 0; + } + + /** + Complete the first checkpoint in a new archive log file. + */ + void checkpoint_complete() noexcept + { + ut_ad(log_sys.latch_have_wr()); + if (ctx) + logs.emplace_back(log_sys.get_first_lsn() - log_sys.capacity()); + } + +private: + /** + Safely start backing up a tablespace file. + @param end array of fil_space_t::BACKUP_BATCH_SIZE block descriptors + @param space tablespace that is being backed up + @param end_page first page not to copy + @return pointer to the new end of the array, of write-fixed blocks + */ + static buf_page_t **backup_batch_start(buf_page_t **end, + fil_space_t *space, uint32_t end_page) + noexcept + { +#if 1 // FIXME: remove this + if (!end_page) + return end; +#endif + ut_ad(end_page); + /* buf_pool.mutex synchronizes with fil_space_t::backup_page_end() */ + mysql_mutex_lock(&buf_pool.mutex); + space->backup_start(end_page); + mysql_mutex_unlock(&buf_pool.mutex); + return innodb_backup_batch_wait(end, space->id, end_page - 1); + } + /** + Stop backing up a tablespace. + @param space tablespace + @param begin first write-fixed block descriptor + @param end end of write-fixed block descriptors + */ + static void backup_batch_stop(fil_space_t *space, + buf_page_t **begin, buf_page_t **end) noexcept + { + space->backup_stop(); + while (begin != end) + { + buf_page_t *b= *begin++; + b->write_unfix_try(); + b->unfix(); + } + } + + /** + Delete unnecessary logs that had been created for backup. + */ + void delete_logs() noexcept + { + ut_ad(log_sys.latch_have_wr()); + ut_ad(old_size); + const lsn_t first_lsn{log_sys.get_first_lsn()}; + for (const lsn_t lsn : logs) + if (lsn != first_lsn) + IF_WIN(DeleteFile,unlink)(log_sys.get_archive_path(lsn).c_str()); + } + + /** + Back up a persistent InnoDB data file. + @param target backup target directory + @param node InnoDB data file + @param start the page number at the start of the file + @param limit the size of the file at the start of backup + @return error code (non-positive) + @retval 0 on success + */ + static int backup(IF_WIN(const char *,int) target, fil_node_t *node, + uint32_t start, uint32_t limit) noexcept + { + for (bool tried_mkdir{false};;) + { +#ifdef _WIN32 + std::string path{target}; + path.push_back('/'); + path.append(node->name); + HANDLE f= CreateFile(path.c_str(), GENERIC_WRITE, 0, + my_win_file_secattr(), CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (f == INVALID_HANDLE_VALUE) + { + unsigned long err= GetLastError(); + if (err == ERROR_PATH_NOT_FOUND && !tried_mkdir && + node->space->id && !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + path.erase(path.rfind('/')); + if (CreateDirectory(path.c_str(), + my_dir_security_attributes.lpSecurityDescriptor + ? &my_dir_security_attributes : nullptr) || + (err= GetLastError()) == ERROR_ALREADY_EXISTS) + continue; + } + + my_osmaperr(err); + goto fail; + } +#else + int f; +# ifdef __APPLE__ + /* aio::synchronous() in another thread may concurrently invoke + pwrite(2) on node->handle. We assume that both pwrite(2) and + fclonefileat(2) are atomic with respect to each other. Should + this assumption be invalid, some data files in the backup may be + corrupted. This corruption can be fixed by either removing this + special handling, or by implementing file-level locking. */ + f= fclonefileat(node->handle, target, node->name, 0); + if (!f) + break; + switch (errno) { + case ENOENT: + goto try_mkdir; + case ENOTSUP: + break; + default: + goto fail; + } +# endif + f= openat(target, node->name, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (f < 0) + { + if (errno == ENOENT) + { +# ifdef __APPLE__ + try_mkdir: +# endif + if (!tried_mkdir && node->space->id && + !srv_is_undo_tablespace(node->space->id)) + { + tried_mkdir= true; + const char *sep= strchr(node->name, '/'); + ut_ad(sep); + sep= strchr(sep + 1, '/'); + ut_ad(sep); + std::string dir{node->name, size_t(sep - node->name)}; + if (!mkdirat(target, dir.c_str(), 0777) || errno == EEXIST) + continue; + } + } + goto fail; + } +#endif + const uint32_t page_size{node->space->physical_size()}; + int err{0}; + if (node->size < limit) + limit= node->size; + /* + For the system tablespace, there is a minimum size has been configured + which may be larger than the currently used size. Preserve the + original size. + + For other persistent data files, fil_node_t::read_page0() + expects at least 4 * innodb_page_size bytes. Small + ROW_FORMAT=COMPRESSED files may be zero-filled to this size. + */ + const uint64_t min_size= + std::max(uint64_t{FIL_IBD_FILE_INITIAL_SIZE} << srv_page_size_shift, + uint64_t{node->size} * page_size); + if (uint64_t{limit} * page_size < min_size) + { + /* Expand the target file to the minimum size. */ +#ifdef _WIN32 + LARGE_INTEGER li; + li.QuadPart= min_size; + err= !SetFilePointerEx(f, li, nullptr, FILE_BEGIN) || !SetEndOfFile(f); +#else + err= ftruncate(f, min_size); +#endif + if (err) + limit= 0; + } + + for (uint32_t page{0}; page < limit; ) + { + buf_page_t *blocks[fil_space_t::BACKUP_BATCH_SIZE], **end= blocks; + { + const uint32_t end_page{start + fil_space_t::BACKUP_BATCH_SIZE}; + end= backup_batch_start(end, node->space, end_page); + start= end_page; + } + uint32_t last{std::min(limit, page + fil_space_t::BACKUP_BATCH_SIZE)}; + /* TODO: avoid copying freed page ranges, or pages that were + allocated after the backup started */ + err= copy_file(node->handle, f, uint64_t{page} * page_size, + uint64_t{last} * page_size); + page= last; + backup_batch_stop(node->space, blocks, end); + if (err) + break; + } + + if (IF_WIN(!CloseHandle(f), close(f)) | err) + goto fail; + break; + } + return 0; + fail: + my_error(ER_CANT_CREATE_FILE, MYF(0), node->name, errno); + return -1; + } + + /** + Stream a persistent InnoDB data file. + @param stream backup target stream + @param node InnoDB data file + @param start the page number at the start of the file + @param limit the size of the file at the start of backup + @return error code (non-positive) + @retval 0 on success + */ + static int stream(IF_WIN(HANDLE,int) stream, fil_node_t *node, + uint32_t start, uint32_t limit) noexcept + { + const uint32_t page_size{node->space->physical_size()}, + file_size= std::max(std::max(limit, node->size), + (FIL_IBD_FILE_INITIAL_SIZE << srv_page_size_shift) / + page_size); + backup_chunk chunk[2]{ + {0, uint64_t{limit} * page_size}, + {uint64_t{file_size} * page_size, 0} + }; + if (file_size < limit) + { + limit= file_size; + chunk[0].length= chunk[1].offset; + } + int err= backup_stream_start(stream, node->name, 0644, + chunk[0].length, + chunk, (file_size > limit) * 2); + if (err) + limit= 0; + + for (uint32_t page{0}; page < limit; ) + { + buf_page_t *blocks[fil_space_t::BACKUP_BATCH_SIZE], **end= blocks; + { + const uint32_t end_page{start + fil_space_t::BACKUP_BATCH_SIZE}; + end= backup_batch_start(end, node->space, end_page); + start= end_page; + } + uint32_t last{std::min(limit, page + fil_space_t::BACKUP_BATCH_SIZE)}; + /* TODO: avoid copying freed page ranges, or pages that were + allocated after the backup started */ + err= backup_stream_append(node->handle, stream, + uint64_t{page} * page_size, + uint64_t{last} * page_size); + page= last; + backup_batch_stop(node->space, blocks, end); + if (err) + break; + } + + if (err) + my_error(ER_IO_WRITE_ERROR, MYF(0), errno, strerror(errno), + "BACKUP SERVER"); + return err; + } + +private: + /** + Initialize a checkpoint header buffer pointing to the start of the backup. + @param buf checkpoint buffer + @param c offset of the FILE_CHECKPOINT mini-transaction + */ + static void write_checkpoint_buf(uint64_t *buf, uint64_t c) noexcept + { + ut_ad(c >= log_sys.START_OFFSET); + if (log_sys.is_encrypted()) + log_crypt_write_header(reinterpret_cast(buf), true); + buf[4 * log_sys.is_encrypted()]= my_htobe64(c); + } + + /** Write a checkpoint header pointing to the start of the backup. + @param dst target file + @param buf checkpoint header + @return error code + @retval 0 on success */ + static int write_checkpoint(IF_WIN(HANDLE,int) dst, const void *buf) noexcept + { +#ifdef _WIN32 + using tpool::pwrite; +#endif + for (ssize_t o= 0, count= 64; count;) + { + ssize_t ret= + pwrite(dst, static_cast(buf) + o, count, o); + if (ret <= 0 || ret > count) + return -1; + o+= ret; + count-= ret; + } + return 0; + } + +public: + /** Maximum length of the configuration string */ + static constexpr size_t CONFIG_SIZE= + sizeof "[server]\n# checkpoint=" + + sizeof "innodb_log_recovery_start=" + + sizeof "innodb_log_recovery_target=\n" + 45 * 3; + + /** Write the configuration parameters for restoring the backup + @param config buffer for configuration string + @param ctx backup context + @return size of the configuration string */ + static size_t write_config_buf(char *config, const context &ctx) + noexcept + { + ut_ad(ctx.last_lsn != LSN_MAX); + return size_t(snprintf(config, CONFIG_SIZE, + "[server]\n# checkpoint=" LSN_PF "\n" + "innodb_log_recovery_start=" LSN_PF "\n" + "innodb_log_recovery_target=" LSN_PF "\n", + ctx.checkpoint, ctx.checkpoint_end_lsn, + ctx.last_lsn)); + } + + /** Write the configuration parameters for restoring the backup + @param target backup target + @param sink backup worker context + @param ctx backup context + @return error code (non-positive) + @retval 0 on success */ + static int write_config(const backup_target &target, + const backup_sink &sink) noexcept + { + char config[CONFIG_SIZE]; + const size_t size + {write_config_buf(config, *static_cast(sink.ha_data))}; + return sink.stream == sink.NO_STREAM + ? backup_config_append(IF_WIN(target.path, target.fd), config, size) + : backup_stream_config(sink.stream, config, size); + } + + /** + Hard-link (copy) or rename (move) or stream an archive log file. + @param lsn The first LSN in the file + @param target backup target + @param sink backup context + @param old lsn < log_sys.get_first_lsn() + @return error code + @retval 0 on success + */ + static int replicate(lsn_t lsn, + const backup_target &target, + const backup_sink &sink, bool old) noexcept + { + ut_ad(log_get_lsn() >= lsn); + const std::string p{log_sys.get_archive_path(lsn)}; + const char *const path= p.c_str(), *basename= strrchr(path, '/'); + if (!basename) + basename= path; + else + basename++; + context &ctx{*static_cast(sink.ha_data)}; + const bool move{old && !ctx.archived}; + uint64_t cp_buf[8]{}; +#ifdef _WIN32 + ut_ad(!target.path == (sink.stream != sink.NO_STREAM)); + std::string b; + const char *destname= nullptr; + if (!target.path) + goto send_file; + b= target.path; + b.push_back('/'); + b.append(basename); + destname= b.c_str(); + unsigned long err; + if (move) + { + if (!MoveFileEx(path, destname, MOVEFILE_COPY_ALLOWED)) + { + fail: + err= GetLastError(); + got_err: + my_osmaperr(err); + if (target.path) + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, + errno); + else + my_error(ER_IO_WRITE_ERROR, MYF(ME_ERROR_LOG), + errno, strerror(errno), "BACKUP SERVER"); + return -1; + } + + if (lsn < ctx.checkpoint) + { + if (!SetFileAttributes(destname, FILE_ATTRIBUTE_NORMAL)) + goto fail; + HANDLE dh= CreateFile(destname, GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dh == INVALID_HANDLE_VALUE) + goto fail; + if (os_file_set_sparse_win32(dh)) + std::ignore= + os_file_punch_hole(dh, 0, log_sys.START_OFFSET + + ((ctx.checkpoint - lsn) & ~4095ULL)); + write_checkpoint_buf(cp_buf, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + int fail= write_checkpoint(dh, cp_buf); + std::ignore= CloseHandle(dh); + if (fail) + goto fail; + } + return 0; + } + else if (CreateHardLink(destname, path, nullptr)) + { + ctx.note_hardlink(lsn); + return 0; + } + + if ((err= GetLastError()) != ERROR_NOT_SAME_DEVICE) + goto got_err; + /* Hard-linking failed. Try copying with the final name. */ + if (target.path) + { + b= target.path; + b.push_back('/'); + b.append(basename); + destname= b.c_str(); + + if (lsn >= ctx.checkpoint && lsn < ctx.max_first_lsn) + { + /* Copy a middle log file entirely. */ + if (CopyFileEx(path, basename, nullptr, nullptr, nullptr, + COPY_FILE_NO_BUFFERING)) + return 0; + goto fail; + } + } + + send_file: + HANDLE src; + for (;;) + { + src= CreateFile(path, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (src != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + goto fail; + } + HANDLE dst{sink.stream}; + if (dst == INVALID_HANDLE_VALUE) + { + dst= CreateFile(destname, GENERIC_WRITE, 0, my_win_file_secattr(), + CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr); + if (dst == INVALID_HANDLE_VALUE) + { + std::ignore= CloseHandle(src); + goto fail; + } + } +#else + if (sink.stream != sink.NO_STREAM); + else if (move + ? !renameat(AT_FDCWD, path, target.fd, basename) + : !linkat(AT_FDCWD, path, target.fd, basename, AT_SYMLINK_FOLLOW)) + { + if (!move) + ctx.note_hardlink(lsn); +# ifdef __linux__ + else if (lsn != ctx.first_lsn); + else if (off_t garbage= (ctx.checkpoint - lsn) & ~4095ULL) + /* Best effort to punch a hole to free up some garbage in + the first file. We do not care about failures. */ + if (!fchmodat(target.fd, basename, 0644, 0)) + { + int dst= openat(target.fd, basename, O_RDWR); + if (dst >= 0) + fallocate(dst, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + log_sys.START_OFFSET, garbage); + close(dst); + std::ignore= fchmodat(target.fd, basename, 0444, 0); + } +# endif + return 0; + } + else if (errno != EXDEV) + { + fail: + if (sink.stream == sink.NO_STREAM) + my_error(ER_ERROR_ON_RENAME, MYF(ME_ERROR_LOG), path, basename, errno); + else + my_error(ER_IO_WRITE_ERROR, MYF(ME_ERROR_LOG), errno, strerror(errno), + "BACKUP SERVER"); + return -1; + } + + const int src{open(path, O_RDONLY)}; + if (src < 0) + goto fail; + if (move && unlink(path)) + { + close_and_fail: + std::ignore= close(src); + goto fail; + } + int dst{sink.stream}; + if (dst < 0) + { + dst= openat(target.fd, basename, + O_CREAT | O_EXCL | O_TRUNC | O_WRONLY, 0666); + if (dst < 0) + goto close_and_fail; + } + int err; +#endif + backup_chunk chunks[3], *chunk{chunks}; + *chunk++= {log_sys.START_OFFSET, ctx.last_lsn - lsn}; + if (lsn < ctx.checkpoint) + { + /* Copy the necessary part of the first log file. */ + ut_ad(lsn == ctx.first_lsn); + write_checkpoint_buf(cp_buf, ctx.checkpoint_end_lsn - lsn + + log_sys.START_OFFSET); + chunk[-1]= {0, 512}; + const lsn_t end= + std::min(ctx.last_lsn, lsn + ctx.first_size - log_sys.START_OFFSET); + *chunk++= + {log_sys.START_OFFSET + ctx.checkpoint - lsn, end - ctx.checkpoint}; + chunk->offset= end - lsn; + goto pad_size; + } + else if (lsn < ctx.max_first_lsn) + { + /* Copy a middle log file entirely. */ +#ifdef _WIN32 + ut_ad(dst == sink.stream); + chunk->offset= os_file_get_size(src); +#else + if (dst != sink.stream) + { + err= copy_entire_file(src, dst); + goto close_dst; + } + chunk->offset= uint64_t(lseek(src, 0, SEEK_END)); +#endif + /* Omit the checkpoint header from the stream. */ + chunk[-1].length= chunk->offset - log_sys.START_OFFSET; + goto stream_file; + } + else + { + ut_ad(ctx.max_first_lsn == lsn); + ut_ad(ctx.last_lsn > lsn); + ut_ad(ctx.last_lsn != LSN_MAX); + ut_ad(chunk[-1].length == ctx.last_lsn - lsn); + chunk->offset= chunk[-1].length; + pad_size: + /* Set the logical size of the file. */ + chunk->offset= + std::max(log_sys.FILE_SIZE_MIN, + (chunk->offset + (log_sys.START_OFFSET + 4095)) & + ~4095ULL); + } + + if (dst == sink.stream) + { + stream_file: + chunk++->length= 0; + const backup_chunk &end{chunk[-2]}; + ut_ad(chunk - chunks == 2 || chunk - chunks == 3); + const size_t cp_size{(size_t(chunk - chunks) & 1) << 9}; + err= backup_stream_start(dst, basename, + 0444 | int{lsn == ctx.max_first_lsn} << 7, + end.length + cp_size, chunks, chunk - chunks); + if (!err && cp_size) + err= backup_stream_write(dst, cp_buf, sizeof cp_buf) || + backup_stream_write(dst, field_ref_zero, cp_size - sizeof cp_buf); + if (!err) + { + err= backup_stream_append_async(src, dst, end.offset, + end.offset + end.length); + if (err); + else if (size_t pad= size_t(end.length) & 511) + err= backup_stream_write(dst, field_ref_zero, 512 - pad); + } + } + else + { + /* First, extend the file to a valid size. */ +#ifdef _WIN32 + LARGE_INTEGER li; + li.QuadPart= chunk->offset; + err= !SetFilePointerEx(dst, li, nullptr, FILE_BEGIN) || + !SetEndOfFile(dst) || +#else + err= ftruncate(dst, chunk->offset) || +#endif + copy_file(src, dst, chunk[-1].offset, chunk[-1].offset + + chunk[-1].length) || + (lsn < ctx.checkpoint && write_checkpoint(dst, cp_buf)); +#ifdef _WIN32 + err|= !CloseHandle(dst); +#else + close_dst: + err|= close(dst); +#endif + } + + if (err | IF_WIN(!CloseHandle(src), close(src))) + goto fail; + + return 0; + } +}; + +/** The backup context; protected by log_sys.latch */ +static InnoDB_backup innodb_backup; +} + +bool log_t::backup_start(uint64_t *old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + ut_ad(!backup); + backup= true; + *old_size= 0; + if (archive) + return false; + const uint64_t old_file_size{file_size}; + latch.wr_unlock(); + const bool fail{set_archive(true, thd, true)}; + latch.wr_lock(); + if (!fail) + { + *old_size= old_file_size; + return false; + } + ut_ad(backup); + backup= false; + const uint64_t new_file_size{file_size}; + latch.wr_unlock(); + if (old_file_size != new_file_size && old_file_size && + resize_start(old_file_size, thd) == RESIZE_STARTED) + resize_finish(thd); + latch.wr_lock(); + return true; +} + +void log_t::backup_stop(uint64_t old_size, THD *thd) noexcept +{ + ut_ad(latch_have_wr()); + /* We will be invoked with old_size=0 after a failed backup_start(), + or if innodb_log_archive=ON held during a successful backup_start(). */ + ut_ad(!old_size || !resize_in_progress()); + ut_ad(!old_size || backup); + backup= false; + const uint64_t new_size{file_size}; + latch.wr_unlock(); + if (old_size && old_size != new_size && + resize_start(old_size, thd) == RESIZE_STARTED) + resize_finish(thd); +} + +void *innodb_backup_start(THD *thd, const backup_target *, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_PREPARE_START: + if (!fil_system.have_all_spaces) + { + /* To speed up startup, InnoDB does not normally open all + tablespace files that are pointed to by SYS_TABLES. + InnoDB_backup::init() assumes that the information of all + tablespaces is available, including files that had been created + before the server was started, and never opened in the course of + the current server execution. */ + dict_load_tablespaces(nullptr, true); + ut_ad(fil_system.have_all_spaces); + } + return 0; + case BACKUP_PHASE_START: + return innodb_backup.init(thd); + case BACKUP_PHASE_NO_COMMIT: + innodb_backup.commit(); + /* fall through */ + default: + return sink->ha_data; + } +} + +int innodb_backup_step(THD *, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_START: + case BACKUP_PHASE_NO_COMMIT: + case BACKUP_PHASE_FINISH: + return innodb_backup.step(*target, phase, *sink); + default: + return 0; + } +} + +int innodb_backup_end(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + default: + return 0; + case BACKUP_PHASE_FINISH: + return innodb_backup.fini(*target, *sink); + case BACKUP_PHASE_NO_COMMIT: + case BACKUP_PHASE_ABORT: + return innodb_backup.end(thd, phase, *sink); + } +} + +void innodb_backup_checkpoint() noexcept +{ + innodb_backup.checkpoint_complete(); +} diff --git a/storage/innobase/handler/backup_innodb.h b/storage/innobase/handler/backup_innodb.h new file mode 100644 index 0000000000000..e9393038e5a21 --- /dev/null +++ b/storage/innobase/handler/backup_innodb.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/** + Start of a BACKUP SERVER phase, + when no innodb_backup_step() or innodb_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to sink, or nullptr + @retval -1 on failure +*/ +void *innodb_backup_start(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) + noexcept; + +/** + Process a file that was collected in innodb_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @return number of files remaining, or negative on error + @retval 0 on completion +*/ +int innodb_backup_step(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current sesssion + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int innodb_backup_end(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Complete the first checkpoint in a new archive log file. +*/ +void innodb_backup_checkpoint() noexcept; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index d09250515aa27..e4f03b4e8521d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -152,6 +152,7 @@ MDL_ticket *get_mdl_ticket(TABLE *table) noexcept; #include "ha_innodb.h" #include "i_s.h" +#include "backup_innodb.h" #include #include @@ -4185,6 +4186,9 @@ static int innodb_init(void* p) = innodb_prepare_commit_versioned; innobase_hton->update_optimizer_costs= innobase_update_optimizer_costs; + innobase_hton->backup_start = innodb_backup_start; + innobase_hton->backup_step = innodb_backup_step; + innobase_hton->backup_end = innodb_backup_end; innobase_hton->binlog_init= innodb_binlog_init; innobase_hton->set_binlog_max_size= ibb_set_max_size; innobase_hton->binlog_write_direct_ordered= @@ -18885,39 +18889,7 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_CANT_CREATE_HANDLER_FILE); break; case log_t::RESIZE_STARTED: - for (timespec abstime;;) - { - if (thd_kill_level(thd)) - { - log_sys.resize_abort(thd); - break; - } - - set_timespec(abstime, 5); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - lsn_t resizing= log_sys.resize_in_progress(); - if (resizing > buf_pool.get_oldest_modification(0)) - { - buf_pool.page_cleaner_wakeup(true); - my_cond_timedwait(&buf_pool.done_flush_list, - &buf_pool.flush_list_mutex.m_mutex, &abstime); - resizing= log_sys.resize_in_progress(); - } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (!resizing || !log_sys.resize_running(thd)) - break; - log_sys.latch.wr_lock(); - while (resizing > log_sys.get_lsn()) - { - ut_ad(!log_sys.is_mmap()); - /* The server is almost idle. Write dummy FILE_CHECKPOINT records - to ensure that the log resizing will complete. */ - mtr_t mtr{nullptr}; - mtr.start(); - mtr.commit_files(log_sys.last_checkpoint_lsn); - } - log_sys.latch.wr_unlock(); - } + log_sys.resize_finish(thd); } } mysql_mutex_lock(&LOCK_global_system_variables); @@ -19774,7 +19746,9 @@ static void innodb_log_archive_update(THD *thd, st_mysql_sys_var*, { /* MDEV-36828 TODO: On failure, report which other setting conflicted with the request */ + mysql_mutex_unlock(&LOCK_global_system_variables); log_sys.set_archive(*static_cast(save), thd); + mysql_mutex_lock(&LOCK_global_system_variables); } static MYSQL_SYSVAR_BOOL(log_archive, log_sys.archive, @@ -19787,10 +19761,20 @@ static MYSQL_SYSVAR_UINT64_T(log_archive_start, innodb_log_archive_start, "initial value of innodb_lsn_archived; 0=auto-detect", nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); +static void innodb_log_recovery_start_update(THD *, st_mysql_sys_var*, + void *, const void *save) noexcept +{ + const lsn_t lsn{*static_cast(save)}; + recv_sys.recovery_start= lsn; + if (lsn && log_sys.archive) + log_sys.archived_checkpoint= lsn; +} + static MYSQL_SYSVAR_UINT64_T(log_recovery_start, recv_sys.recovery_start, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + PLUGIN_VAR_RQCMDARG, "LSN to start recovery from (0=automatic)", - nullptr, nullptr, 0, 0, std::numeric_limits::max(), 0); + nullptr, innodb_log_recovery_start_update, + 0, 0, std::numeric_limits::max(), 0); static MYSQL_SYSVAR_UINT64_T(log_recovery_target, recv_sys.rpo, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 3328319fd82b4..59da6711966f5 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -644,6 +644,14 @@ class buf_page_t /** @return whether this block is read fixed */ bool is_read_fixed() const noexcept { return is_read_fixed(state()); } + /** Try to write-fix a block. + @param s expected state() + @return new s (right before potentially setting the write-fix); + a write-fix was acquired if !is_freed(s) && !is_io_fixed(s) holds */ + inline uint32_t write_fix_try(uint32_t s) noexcept; + /** Try to undo a successful write_fix_try(). */ + inline void write_unfix_try() noexcept; + /** @return if this belongs to buf_pool.unzip_LRU */ bool belongs_to_unzip_LRU() const noexcept { return UNIV_LIKELY_NULL(zip.data) && frame; } @@ -671,14 +679,8 @@ class buf_page_t return old_state + (s - READ_FIX); } - void set_freed(uint32_t prev_state, uint32_t count= 0) noexcept - { - ut_ad(lock.is_write_locked()); - ut_ad(prev_state >= UNFIXED); - ut_ad(prev_state < READ_FIX); - ut_d(auto s=) zip.fix.fetch_sub((prev_state & LRU_MASK) - FREED - count); - ut_ad(!((prev_state ^ s) & LRU_MASK)); - } + /** Mark an X-latched block as freed in the tablespace. */ + void set_freed() noexcept; inline void set_state(uint32_t s) noexcept; inline void set_corrupt_id() noexcept; diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 67fa1bfd026f0..a00bf2c99e957 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -408,6 +408,9 @@ struct fil_space_t final /** Whether any corruption of this tablespace has been reported */ mutable std::atomic_flag is_corrupted= ATOMIC_FLAG_INIT; + /** first page number that is not yet being backed up, or 0 */ + Atomic_relaxed backup_end{0}; + public: /** mutex to protect freed_ranges and last_freed_lsn */ std::mutex freed_range_mutex; @@ -418,9 +421,11 @@ struct fil_space_t final /** LSN of freeing last page; protected by freed_range_mutex */ lsn_t last_freed_lsn= 0; - /** LSN of undo tablespace creation or 0; protected by latch */ - lsn_t create_lsn= 0; public: + /** LSN of tablespace creation or undo tablespace reinitialization; + protected by fil_system.mutex and (is_stopped() or log_sys.latch) */ + Atomic_relaxed create_lsn{0}; + /** @return whether this is the temporary tablespace */ bool is_temporary() const noexcept { return UNIV_UNLIKELY(id == SRV_TMP_SPACE_ID); } @@ -434,12 +439,6 @@ struct fil_space_t final /** @return whether a page has been freed */ inline bool is_freed(uint32_t page) noexcept; - /** Set create_lsn. */ - inline void set_create_lsn(lsn_t lsn) noexcept; - - /** @return the latest tablespace rebuild LSN, or 0 */ - lsn_t get_create_lsn() const noexcept { return create_lsn; } - /** Apply freed_ranges to the file. @param writable whether the file is writable @return number of pages written or hole-punched */ @@ -1058,6 +1057,17 @@ struct fil_space_t final VALIDATE_IMPORT }; + /** Note that we backing up some pages of the underlying files. + @param last_page the last page that is being backed up (0=stop backup) */ + void backup_start(uint32_t last_page) noexcept { backup_end= last_page; } + /** Note that we are not currently backing up the underlying files. */ + void backup_stop() noexcept { backup_start(0); } + /** @return the first page number that is not being backed up */ + inline uint32_t backup_page_end() const noexcept; + + /** The size of a backup copy_file() batch in pages */ + static constexpr uint32_t BACKUP_BATCH_SIZE{64}; + /** Update the data structures on write completion */ void complete_write() noexcept; @@ -1463,6 +1473,8 @@ struct fil_system_t my_bool buffered; /** whether fdatasync() is needed on data files */ Atomic_relaxed need_unflushed_spaces; + /** whether dict_load_tablespaces(nullptr, true) is unnecessary */ + Atomic_relaxed have_all_spaces; /** Try to enable or disable write-through of data files */ void set_write_through(bool write_through); diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 44a827dbf636d..6d07fa25e013d 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -221,6 +221,8 @@ struct log_t /** whether !archive log records may have been written with get_sequence_bit()==0 */ bool circular_recovery_from_sequence_bit_0:1; + /** whether we are between backup_start() and backup_stop() */ + bool backup:1; public: /** the default value of log_mmap */ static constexpr bool log_mmap_default= @@ -288,6 +290,8 @@ struct log_t Atomic_relaxed last_checkpoint_lsn; /** The log writer (protected by latch.wr_lock()) */ lsn_t (*writer)() noexcept; + /** the earliest available checkpoint; protected by latch.wr_lock() */ + lsn_t archived_checkpoint; /** end_lsn of the first available checkpoint, or 0; protected by latch.wr_lock() */ lsn_t archived_lsn; @@ -369,11 +373,24 @@ struct log_t RESIZE_NO_CHANGE, RESIZE_IN_PROGRESS, RESIZE_STARTED, RESIZE_FAILED }; +private: /** Start resizing the log and release the exclusive latch. + @param size requested new file_size + @param thd the current thread identifier + @param backup whether the caller is backup_start() or backup_stop() + @return whether the resizing was started successfully */ + resize_start_status resize_start(uint64_t size, void *thd, bool backup) + noexcept; +public: + /** Start resizing the log. @param size requested new file_size @param thd the current thread identifier @return whether the resizing was started successfully */ - resize_start_status resize_start(os_offset_t size, void *thd) noexcept; + resize_start_status resize_start(uint64_t size, void *thd) noexcept + { return resize_start(size, thd, false); } + + /** Wait for the completion of resize_start() == RESIZE_STARTED */ + void resize_finish(THD *thd) noexcept; /** Abort a resize_start() that we started. @param thd thread identifier that had been passed to resize_start() */ @@ -397,10 +414,37 @@ struct log_t resize_write_low(lsn, end, len, seq); } +private: + /** SET GLOBAL innodb_log_archive, or start/stop BACKUP SERVER + @param archive the new value of innodb_log_archive + @param thd SQL connection + @param backup whether the caller is backup_start() or backup_stop() + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd, bool backup) noexcept; +public: /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive - @param thd SQL connection */ - void set_archive(my_bool archive, THD *thd) noexcept; + @param thd SQL connection + @return whether the operation failed */ + bool set_archive(my_bool archive, THD *thd) noexcept + { return set_archive(archive, thd, false); } + + /** Start BACKUP SERVER. + @param old_size the old file_size, or 0 on failure or when + already running innodb_log_archive=ON + @param thd SQL connection + @return whether the operation failed */ + bool backup_start(uint64_t *old_size, THD *thd) noexcept; + /** Stop log archiving in BACKUP SERVER clean-up + @param thd SQL connection + @return whether the operation failed */ + bool backup_stop_archiving(THD *thd) noexcept + { return set_archive(false, thd, true); } + + /** Stop BACKUP SERVER. + @param old_size the value returned by backup_start() + @param thd SQL connection */ + void backup_stop(uint64_t old_size, THD *thd) noexcept; private: /** Replicate a write to the log. @@ -695,6 +739,18 @@ struct log_t /** @return the first LSN of the log file */ lsn_t get_first_lsn() const noexcept { return first_lsn; } + /** + Determine the latest checkpoint. + @param end LSN leading to the FILE_CHECKPOINT record + @return the latest checkpoint LSN + */ + lsn_t get_latest_checkpoint(lsn_t &end) const noexcept + { + ut_ad(latch_have_any()); + end= end_lsn; + return last_checkpoint_lsn; + } + /** Set the recovered checkpoint. @param lsn log sequence number of the checkpoint @param end_lsn LSN passed to write_checkpoint() diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index ee8d43a68e500..5d0d968bbf638 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -635,7 +635,7 @@ void log_t::set_buffered(bool buffered) noexcept } #endif - /** Try to enable or disable durable writes (update log_write_through) */ +/** Try to enable or disable durable writes (update log_write_through) */ void log_t::set_write_through(bool write_through) { if (is_mmap() || high_level_read_only || recv_sys.rpo) @@ -764,9 +764,12 @@ void log_t::header_rewrite(my_bool archive) noexcept /** SET GLOBAL innodb_log_archive @param archive the new value of innodb_log_archive -@param thd SQL connection */ -void log_t::set_archive(my_bool archive, THD *thd) noexcept +@param thd SQL connection +@param backup whether the caller is backup_start() or backup_stop() +@return whether the operation failed */ +bool log_t::set_archive(my_bool archive, THD *thd, bool backup) noexcept { + bool fail= false; thd_wait_begin(thd, THD_WAIT_DISKIO); tpool::tpool_wait_begin(); lsn_t wait_lsn; @@ -780,12 +783,20 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept my_printf_error(ER_WRONG_USAGE, "SET GLOBAL innodb_log_file_size is in progress", MYF(0)); + fail: + fail= true; + wait_lsn= 0; break; } if (archive == this->archive) break; - if (thd_kill_level(thd)) - break; + if ((!backup || archive) && thd_kill_level(thd)) + goto fail; + if (!backup && this->backup) + { + my_printf_error(ER_WRONG_USAGE, "BACKUP SERVER is in progress", MYF(0)); + goto fail; + } if (resize_log.is_opened()) { @@ -811,14 +822,15 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept if (archive) { - wait_lsn-= (wait_lsn - first_lsn) % capacity(); + const lsn_t limit{wait_lsn - (wait_lsn - first_lsn) % capacity()}; /* We are in innodb_log_archive=OFF. If the file has wrapped around between the checkpoint and the current position, we must wait for a log checkpoint not before the desired first_lsn of our innodb_log_archive=ON log file, because that format does not allow any wrap-around. */ - if (checkpoint < wait_lsn) + if (checkpoint < limit) goto retry_after_checkpoint; + wait_lsn= limit; } else if (circular_recovery_from_sequence_bit_0) { @@ -894,7 +906,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept if (!log.is_opened()) { my_error(ER_ERROR_ON_READ, MYF(0), old_name, errno); - break; + goto fail; } } #endif @@ -919,7 +931,7 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept { my_error(ER_ERROR_ON_RENAME, MYF(0), old_name, new_name, my_errno); first_lsn= old_first_lsn; - break; + goto fail; } if (archive) @@ -951,14 +963,16 @@ void log_t::set_archive(my_bool archive, THD *thd) noexcept thd_wait_end(thd); if (wait_lsn) mtr_flush_ahead(wait_lsn); + return fail; } -/** Start resizing the log and release the exclusive latch. -@param size requested new file_size -@param thd the current thread identifier +/** Start resizing the log. +@param size requested new file_size +@param thd the current thread identifier +@param backup whether the caller is backup_start() or backup_stop() @return whether the resizing was started successfully */ -log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) - noexcept +log_t::resize_start_status log_t::resize_start(uint64_t size, void *thd, + bool backup) noexcept { ut_ad(size >= 4U << 20); ut_ad(!(size & 4095)); @@ -989,6 +1003,9 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) resize_target= size; } } + else if (!backup && this->backup) + /* backup_start() or backup_stop() is running */ + status= RESIZE_FAILED; else { lsn_t start_lsn; @@ -1090,6 +1107,44 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) return status; } +/** Wait for the completion of resize_start() == RESIZE_STARTED */ +void log_t::resize_finish(THD *thd) noexcept +{ + for (timespec abstime;;) + { + if (thd_kill_level(thd)) + { + resize_abort(thd); + break; + } + + set_timespec(abstime, 5); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + lsn_t resizing= resize_in_progress(); + if (resizing > buf_pool.get_oldest_modification(0)) + { + buf_pool.page_cleaner_wakeup(true); + my_cond_timedwait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); + resizing= resize_in_progress(); + } + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + if (!resizing || !resize_running(thd)) + break; + latch.wr_lock(); + while (resizing > get_lsn()) + { + ut_ad(!is_mmap()); + /* The server is almost idle. Write dummy FILE_CHECKPOINT records + to ensure that the log resizing will complete. */ + mtr_t mtr{nullptr}; + mtr.start(); + mtr.commit_files(last_checkpoint_lsn); + } + latch.wr_unlock(); + } +} + /** Abort a resize_start() that we started. */ void log_t::resize_abort(void *thd) noexcept { diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 39b17bb30c649..46c2704bc5044 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2099,6 +2099,7 @@ dberr_t recv_sys_t::find_checkpoint() memset_aligned<4096>(const_cast(field_ref_zero), 0, 4096); /* Mark the redo log for upgrading. */ lsn= file_checkpoint= log_sys.last_checkpoint_lsn; + log_sys.archived_checkpoint= lsn; log_sys.set_recovered_lsn(lsn); if (rpo && rpo != lsn) { @@ -2177,7 +2178,8 @@ dberr_t recv_sys_t::find_checkpoint() log_sys.set_recovered_checkpoint(checkpoint_lsn, lsn= end_lsn, field == log_t::CHECKPOINT_1); } - if (!log_sys.last_checkpoint_lsn) + log_sys.archived_checkpoint= log_sys.last_checkpoint_lsn; + if (!log_sys.archived_checkpoint) goto got_no_checkpoint; else if (!log_sys.archived_lsn) log_sys.archived_lsn= lsn; @@ -4155,7 +4157,7 @@ static buf_block_t *recv_recover_page(buf_block_t *block, mtr_t &mtr, /* There have been no operations that modify the page. Any buffered changes will be merged in ibuf_upgrade(). */ ut_ad(!mtr.has_modifications()); - block->page.set_freed(block->page.state()); + block->page.set_freed(); } /* Make sure that committing mtr does not change the modification diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 21e4d1ae1c9ce..0cbcab1780e9b 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -313,7 +313,8 @@ void mtr_t::release_unlogged() buf_block_t *block= static_cast(slot.object); ut_d(const auto s=) block->page.unfix(); ut_ad(s >= buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* A fake "write fix" of InnoDB_backup may exist for a latched page */ + ut_ad(!buf_page_t::is_read_fixed(s)); if (slot.type & MTR_MEMO_MODIFY) { @@ -440,7 +441,8 @@ void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) noexcept ut_ad(b->page.id() < end_page_id); ut_d(const auto s= b->page.state()); ut_ad(s > buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* A fake "write fix" of InnoDB_backup may exist for a latched page */ + ut_ad(!buf_page_t::is_read_fixed(s)); ut_ad(mach_read_from_8(b->page.frame + FIL_PAGE_LSN) <= mtr->m_commit_lsn); mach_write_to_8(b->page.frame + FIL_PAGE_LSN, mtr->m_commit_lsn); @@ -492,7 +494,13 @@ void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) noexcept ut_ad(bpage->oldest_modification() < mtr->m_commit_lsn); ut_ad(bpage->id() < end_page_id); ut_ad(s >= buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* If a thread is executing between + InnoDB_backup::backup_batch_start() and + InnoDB_backup::backup_batch_stop() for this page, a fake + "write fix" may exist. We are free to modify the page in the + buffer pool, but buf_page_t::flush() will refuse to write it + to the file system. */ + ut_ad(!buf_page_t::is_read_fixed(s)); ut_ad(mach_read_from_8(bpage->frame + FIL_PAGE_LSN) <= mtr->m_commit_lsn); mach_write_to_8(bpage->frame + FIL_PAGE_LSN, mtr->m_commit_lsn); @@ -595,12 +603,34 @@ void mtr_t::rollback_to_savepoint(ulint begin, ulint end) m_memo.erase(m_memo.begin() + begin, m_memo.begin() + end); } -/** Set create_lsn. */ -inline void fil_space_t::set_create_lsn(lsn_t lsn) noexcept +/** Mark an X-latched block as freed in the tablespace. */ +void buf_page_t::set_freed() noexcept { - /* Concurrent log_checkpoint_low() must be impossible. */ - ut_ad(latch.have_wr()); - create_lsn= lsn; + /* + Ensure that lock.x_lock() is being held (hopefully, by us). + This blocks a concurrent flush(), protected by lock.u_lock(). + */ + ut_ad(lock.is_write_locked()); + uint32_t s{state()}; + do + { + ut_ad(s >= UNFIXED); + /* + InnoDB_backup::backup_batch_start() may set fix() and + write_fix_try() on any dirty block to prevent flush() from + writing changes back to the file system. We may safely mark such + blocks as FREED; write_unfix_try() will account for that. + */ + ut_ad(s > WRITE_FIX || s < READ_FIX); + } + /* + fetch_sub() is not safe here, because this may run concurrently + with write_fix_try() or write_unfix_try(). + */ + while (!zip.fix.compare_exchange_weak(s, FREED + (s & ~LRU_MASK), + std::memory_order_acquire, + std::memory_order_relaxed) && + !is_freed(s)); } /** Commit a mini-transaction that is shrinking a tablespace. @@ -635,7 +665,7 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) if (space.id == TRX_SYS_SPACE) srv_sys_space.set_last_file_size(file->size); else - space.set_create_lsn(m_commit_lsn); + space.create_lsn= m_commit_lsn; mysql_mutex_unlock(&fil_system.mutex); @@ -674,7 +704,8 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) const page_id_t id{b->page.id()}; const auto s= b->page.state(); ut_ad(s > buf_page_t::FREED); - ut_ad(s < buf_page_t::READ_FIX); + /* A fake "write fix" of InnoDB_backup may exist for a latched page */ + ut_ad(!buf_page_t::is_read_fixed(s)); ut_ad(b->page.frame); ut_ad(mach_read_from_8(b->page.frame + FIL_PAGE_LSN) <= m_commit_lsn); ut_ad(!b->page.zip.data); // we no not shrink ROW_FORMAT=COMPRESSED @@ -695,7 +726,7 @@ void mtr_t::commit_shrink(fil_space_t &space, uint32_t size) { ut_ad(id.space() == high.space()); if (s >= buf_page_t::UNFIXED) - b->page.set_freed(s); + b->page.set_freed(); if (b->page.oldest_modification() > 1) b->page.reset_oldest_modification(); slot.type= mtr_memo_type_t(slot.type & ~MTR_MEMO_MODIFY); @@ -1542,7 +1573,8 @@ buf_block_t *mtr_t::page_lock(buf_block_t *block, ulint rw_latch) noexcept case RW_SX_LATCH: fix_type= MTR_MEMO_PAGE_SX_FIX; block->page.lock.u_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); break; default: ut_ad(rw_latch == RW_X_LATCH); @@ -1553,7 +1585,8 @@ buf_block_t *mtr_t::page_lock(buf_block_t *block, ulint rw_latch) noexcept page_lock_upgrade(*block); return block; } - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); } done: @@ -1588,11 +1621,13 @@ void mtr_t::upgrade_buffer_fix(ulint savepoint, rw_lock_type_t rw_latch) break; case RW_SX_LATCH: block->page.lock.u_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); break; case RW_X_LATCH: block->page.lock.x_lock(); - ut_ad(!block->page.is_io_fixed()); + /* A fake "write fix" of InnoDB_backup may exist */ + ut_ad(!block->page.is_read_fixed()); } ut_ad(page_id_t(page_get_space_id(block->page.frame), @@ -1849,7 +1884,7 @@ void mtr_t::free(const fil_space_t &space, uint32_t offset) if (block->index) btr_search_drop_page_hash_index(block, nullptr); #endif /* BTR_CUR_HASH_ADAPT */ - block->page.set_freed(block->page.state()); + block->page.set_freed(); } } diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 6494c5e21b96e..4ef86095f78a3 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2017,7 +2017,15 @@ os_file_create_func( ); DWORD create_flag = OPEN_EXISTING; - DWORD share_mode = read_only + /* BACKUP SERVER may invoke CreateHardLink() on a log file that + may concurrently be written to. This is why we must allow + FILE_SHARE_WRITE. This has the side effect that multiple InnoDB + instances may be concurrently started on the same log file. + However, InnoDB will not write any log before it has successfully + opened data files. As long as the multiple instances are also + opening the same InnoDB data files (such as the system tablespace), + they should fail to start up concurrently. */ + DWORD share_mode = read_only || type == OS_LOG_FILE ? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE : FILE_SHARE_READ | FILE_SHARE_DELETE; diff --git a/storage/maria/CMakeLists.txt b/storage/maria/CMakeLists.txt index 9d528f980dc5b..974145ce8ea6c 100644 --- a/storage/maria/CMakeLists.txt +++ b/storage/maria/CMakeLists.txt @@ -45,6 +45,7 @@ SET(ARIA_SOURCES ma_init.c ma_open.c ma_extra.c ma_info.c ma_rkey.c ha_maria.h maria_def.h ma_recovery_util.c ma_servicethread.c ma_norec.c ma_crypt.c ma_backup.c + ma_backup_server.cc ma_backup_server.h ) IF(APPLE) diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 8f5e47daea728..802ad0f9fcc19 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -23,6 +23,7 @@ #include #include #include "ha_maria.h" +#include "ma_backup_server.h" #include "trnman_public.h" #include "trnman.h" @@ -3942,6 +3943,9 @@ static int ha_maria_init(void *p) maria_hton->prepare_for_backup= maria_prepare_for_backup; maria_hton->end_backup= maria_end_backup; maria_hton->update_optimizer_costs= aria_update_optimizer_costs; + maria_hton->backup_start= aria_backup_start; + //maria_hton->backup_step= aria_backup_step; + maria_hton->backup_end= aria_backup_end; /* TODO: decide if we support Maria being used for log tables */ maria_hton->flags= (HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES | diff --git a/storage/maria/ma_backup_server.cc b/storage/maria/ma_backup_server.cc new file mode 100644 index 0000000000000..e0775bfd9f213 --- /dev/null +++ b/storage/maria/ma_backup_server.cc @@ -0,0 +1,463 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "maria_def.h" +#include "ma_backup_server.h" +#include "mysqld_error.h" +#if 1 // tc_purge(), tdc_purge() +# include "sql_class.h" +# include "table_cache.h" +#endif +#include +#include +#include +#include +#include "span.h" + +/* + Implementation of functions declatred in ma_backup.h: + BACKUP SERVER support for Aria engine +*/ + +namespace +{ + /** Backup state; protected by log_sys.latch */ + class Aria_backup + { + public: + Aria_backup()= default; + ~Aria_backup() + { +#ifndef _WIN32 + if (datadir_fd >= 0) + std::ignore= close(datadir_fd); + if (logdir_fd >= 0) + std::ignore= close(logdir_fd); +#endif + if (translog_purge_disabled) + translog_enable_purge(); + } + + bool initialize() noexcept + { +#ifndef _WIN32 + /* Aria table files live under the server data directory + (mysql_real_data_home), while the transaction logs and control file + live under aria_log_dir_path (maria_data_root). These differ when + aria_log_dir_path is set, so open and scan them separately. */ + datadir_fd= open(mysql_real_data_home, O_DIRECTORY); + if (datadir_fd < 0) + { + my_error(ER_CANT_READ_DIR, MYF(0), mysql_real_data_home, errno); + return true; + } + logdir_fd= open(maria_data_root, O_DIRECTORY); + if (logdir_fd < 0) + { + my_error(ER_CANT_READ_DIR, MYF(0), maria_data_root, errno); + return true; + } +#endif // _WIN32 + assert(!translog_purge_disabled); + translog_purge_disabled= true; + translog_disable_purge(); + return false; + } + + int end(const backup_target &target, const backup_sink &sink) noexcept + { + int ret_val= perform_backup(target, sink); + assert(translog_purge_disabled); + translog_purge_disabled= false; + translog_enable_purge(); + return ret_val; + } + private: +#ifndef _WIN32 + /** The server data directory (Aria table files) */ + int datadir_fd{-1}; + /** The Aria log directory aria_log_dir_path (logs, control file) */ + int logdir_fd{-1}; +#endif + /** whether the Aria translog_disable_purge() is in effect */ + bool translog_purge_disabled{false}; + static constexpr const char zerobuf[511]{}; + using dir_contents = std::vector; + using database_dir = std::pair; + std::vector database_dirs; + std::vector log_files; + bool have_control_file = false; + + int perform_backup(const backup_target &target, const backup_sink &sink) + noexcept + { + return scan_datadir() || copy_databases(target, sink) || + copy_control_file(target, sink) || + translog_flush(translog_get_horizon()) || + copy_logs(target, sink); + } + + ATTRIBUTE_COLD ATTRIBUTE_NOINLINE + static int dir_error(const char *name) noexcept + { + my_error(ER_CANT_READ_DIR, MYF(0), name, my_errno); + return 1; + } + + int scan_datadir() noexcept + { + /* Scan the server data directory for Aria table files. */ + MY_DIR *data_dir= my_dir(mysql_real_data_home, MYF(MY_WANT_STAT)); + if (!data_dir) + return dir_error(mysql_real_data_home); + int fail= 0; + for (const fileinfo &fi : + st_::span{data_dir->dir_entry, + data_dir->number_of_files}) + if ((fi.mystat->st_mode & S_IFMT) == S_IFDIR) + if ((fail= scan_database_dir(fi.name)) != 0) + break; + my_dirend(data_dir); + if (fail) + return fail; + + /* Scan aria_log_dir_path for the transaction logs and control file. */ + MY_DIR *log_dir= my_dir(maria_data_root, MYF(MY_WANT_STAT)); + if (!log_dir) + return dir_error(maria_data_root); + for (const fileinfo &fi : + st_::span{log_dir->dir_entry, + log_dir->number_of_files}) + { + if (!strncmp(fi.name, C_STRING_WITH_LEN("aria_log."))) + log_files.emplace_back(fi.name); + else if (!strcmp(fi.name, "aria_log_control")) + have_control_file = true; + } + my_dirend(log_dir); + return 0; + } + + int scan_database_dir(const char* dir_name) noexcept + { + const std::string dir_path{make_path(mysql_real_data_home, dir_name)}; + MY_DIR *dir_info= my_dir(dir_path.c_str(), MYF(MY_WANT_STAT)); + if (!dir_info) + return dir_error(dir_path.c_str()); + std::vector files_to_backup; + for (const fileinfo &fi : + st_::span{dir_info->dir_entry, + dir_info->number_of_files}) + if (is_db_file(fi.name)) + files_to_backup.emplace_back(fi.name); + if (!files_to_backup.empty()) + database_dirs.emplace_back(dir_name, std::move(files_to_backup)); + my_dirend(dir_info); + return 0; + } + + int copy_databases(const backup_target &target, const backup_sink &sink) + noexcept + { + for (const database_dir &dir : database_dirs) + { + if (sink.stream != sink.NO_STREAM); + else if (int fail= ensure_target_subdir(target, dir.first.c_str())) + return fail; + if (int fail= copy_database(target, sink, dir)) + return fail; + } + return 0; + } + + /* + Create directory in the target directory if it does not exist. + Return 0 on success, non-0 on failure. Set errno in case of failure + */ + int ensure_target_subdir(const backup_target &target, const char *name) + noexcept + { +#ifdef _WIN32 + if (CreateDirectory(make_path(target.path, name).c_str(), nullptr)) + return 0; + DWORD err= GetLastError(); + if (err == ERROR_ALREADY_EXISTS) + return 0; + my_osmaperr(err); +#else + if (likely(!mkdirat(target.fd, name, 0777) || errno == EEXIST)) + return 0; +#endif + my_error(ER_CANT_CREATE_FILE, MYF(0), name, errno); + return 1; + } + + int copy_database(const backup_target &target, const backup_sink &sink, + const database_dir& dir) noexcept + { + std::string file_path; + for (const std::string &file : dir.second) + { + file_path= dir.first; + file_path.push_back('/'); + file_path.append(file); + if (int fail= copy_file(target, sink, file_path.c_str(), false)) + return fail; + } + return 0; + } + + int copy_control_file(const backup_target &target, const backup_sink &sink) + noexcept + { + if (!have_control_file) + return 0; + return copy_file(target, sink, "aria_log_control", true); + } + + int copy_logs(const backup_target &target, const backup_sink &sink) + noexcept + { + for (const std::string &file : log_files) + if (int fail= copy_file(target, sink, file.c_str(), true)) + return fail; + return 0; + } + + int copy_file(const backup_target &target, const backup_sink &sink, + const char *path, bool is_log) const noexcept + { +#ifndef _WIN32 + int ret_val{0}; + int src_fd{openat(is_log ? logdir_fd : datadir_fd, path, O_RDONLY)}; + if (src_fd < 0) + { + my_error(ER_CANT_OPEN_FILE, MYF(0), path, errno); + return 1; + } + int tgt_fd{sink.stream}; + if (tgt_fd == sink.NO_STREAM) + { + tgt_fd= openat(target.fd, path, + O_CREAT | O_EXCL | O_WRONLY, 0666); + if (tgt_fd < 0) + { + my_error(ER_CANT_CREATE_FILE, MYF(0), path, errno); + ret_val= 1; + } + else + { + ret_val= copy_entire_file(src_fd, tgt_fd); + if (ret_val | close(tgt_fd)) + { + write_error: + my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); + ret_val= 1; + } + } + } + else + { + uint64_t end= uint64_t(lseek(src_fd, 0, SEEK_END)); + if (backup_stream_start(tgt_fd, path, 0644, end, nullptr, 0) || + backup_stream_append(src_fd, tgt_fd, 0, end)) + goto write_error; + if (size_t pad= size_t(end) & 511) + if (backup_stream_write(tgt_fd, zerobuf, 512 - pad)) + goto write_error; + } + + close(src_fd); + return ret_val; +#else + const std::string src_path + {make_path(is_log ? maria_data_root : mysql_real_data_home, path)}; + + if (sink.stream == sink.NO_STREAM) + { + std::string dest_path{make_path(target.path, path)}; + if (!CopyFileEx(src_path.c_str(), dest_path.c_str(), + nullptr, nullptr, nullptr, COPY_FILE_NO_BUFFERING)) + { + my_osmaperr(GetLastError()); + my_error(ER_CANT_CREATE_FILE, MYF(0), dest_path.c_str(), errno); + return 1; + } + } + else + { + HANDLE src, dst{sink.stream}; + for (;;) + { + src= CreateFile(src_path.c_str(), GENERIC_READ, + FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, + my_win_file_secattr(), OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (src != INVALID_HANDLE_VALUE) + break; + switch (GetLastError()) { + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + + my_osmaperr(GetLastError()); + my_error(ER_FILE_NOT_FOUND, MYF(ME_ERROR_LOG), src_path.c_str(), + errno); + return -1; + } + + LARGE_INTEGER li; + if (!GetFileSizeEx(src, &li)) + { + write_error: + my_osmaperr(GetLastError()); + my_error(ER_ERROR_ON_WRITE, MYF(0), path, errno); + if (src != INVALID_HANDLE_VALUE) + CloseHandle(src); + return -1; + } + + if (backup_stream_start(dst, path, 0644, li.QuadPart, nullptr, 0) || + backup_stream_append_plain(src, dst, 0, li.QuadPart)) + goto write_error; + + if (size_t pad= size_t(li.LowPart) & 511) + if (backup_stream_write(dst, zerobuf, 512 - pad)) + goto write_error; + if (!CloseHandle(src)) + { + src= INVALID_HANDLE_VALUE; + goto write_error; + } + } + return 0; +#endif + } + + + static bool is_db_file(const char* file_name) noexcept + { + size_t len= strlen(file_name); + if (len < 4) + return false; + if (!memcmp(file_name, tmp_file_prefix, tmp_file_prefix_length)) + /* + As noted in MDEV-25854, file names that start with #sql + must be excluded from the backup. For example, a call to + MDL_context::upgrade_shared_lock() in + mysql_inplace_alter_table() could time out, resulting in + cleanup_table_after_inplace_alter() deleting a + #sql-alter*.frm file before we get a chance to copy it. + */ + return false; + uint32_t suffix; + memcpy(&suffix, file_name + len - 4, 4); + switch (suffix) { + default: + return len == 6 && !memcmp(file_name, C_STRING_WITH_LEN("db.opt")); +#ifdef WORDS_BIGENDIAN + case 0x2e41524d: /* .ARM ENGINE=ARCHIVE metadata */ + case 0x2e41525a: /* .ARZ ENGINE=ARCHIVE compressed data */ + case 0x2e43534d: /* .CSM ENGINE=CSV metadata */ + case 0x2e435356: /* .CSV ENGINE=CSV data ("comma separated values") */ + case 0x2e4d4144: /* .MAD ENGINE=Aria data heap */ + case 0x2e4d4149: /* .MAI ENGINE=Aria indexes */ + case 0x2e4d5247: /* .MRG ENGINE=MRG_MyISAM */ + case 0x2e4d5944: /* .MYD ENGINE=MyISAM data heap */ + case 0x2e4d5949: /* .MYI ENGINE=MyISAM indexes */ + case 0x2e66726d: /* .frm form (SHOW CREATE TABLE) */ + case 0x2e706172: /* .par PARTITION metadata */ +#else + case 0x4d52412e: /* .ARM ENGINE=ARCHIVE metadata */ + case 0x5a52412e: /* .ARZ ENGINE=ARCHIVE compressed data */ + case 0x4d53432e: /* .CSM ENGINE=CSV metadata */ + case 0x5653432e: /* .CSV ENGINE=CSV data ("comma separated values") */ + case 0x44414d2e: /* .MAD ENGINE=Aria data heap */ + case 0x49414d2e: /* .MAI ENGINE=Aria indexes */ + case 0x47524d2e: /* .MRG ENGINE=MRG_MyISAM */ + case 0x44594d2e: /* .MYD ENGINE=MyISAM data heap */ + case 0x49594d2e: /* .MYI ENGINE=MyISAM indexes */ + case 0x6d72662e: /* .frm form (SHOW CREATE TABLE) */ + case 0x7261702e: /* .par PARTITION metadata */ +#endif + return true; + } + } + + /** + Construct a file path. + @param dir directory name + @param name file name + @return dir/name + */ + static std::string make_path(const char *dir, const char *name) + { + std::string path{dir}; + path.push_back('/'); + path.append(name); + return path; + } + }; +} + +void *aria_backup_start(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept +{ + switch (phase) { + case BACKUP_PHASE_PREPARE_START: + return 0; + default: + return sink->ha_data; + case BACKUP_PHASE_NO_COMMIT: + assert(!sink->ha_data); + Aria_backup *aria_backup{new Aria_backup}; + if (aria_backup->initialize()) + { + delete aria_backup; + return reinterpret_cast(-1); + } + return aria_backup; + } +} + +#if 0 // FIXME: implement the actual copying here +int aria_backup_step(THD*, const backup_target*, backup_phase, + const backup_sink*) noexcept +{ + return 0; +} +#endif + +int aria_backup_end(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink) noexcept +{ + Aria_backup *aria_backup= static_cast(sink->ha_data); + switch (phase) { + case BACKUP_PHASE_NO_COMMIT: + assert(aria_backup); +#if 1 // FIXME: invoke these only for Aria, MyISAM, CSV but not others + tc_purge(); + tdc_purge(true); +#endif + return aria_backup->end(*target, *sink); + case BACKUP_PHASE_FINISH: + delete aria_backup; + /* fall through */ + default: + return 0; + } +} diff --git a/storage/maria/ma_backup_server.h b/storage/maria/ma_backup_server.h new file mode 100644 index 0000000000000..6c72bb46eb60f --- /dev/null +++ b/storage/maria/ma_backup_server.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2026, MariaDB plc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#pragma once + +/* BACKUP SERVER support for Aria engine. */ + +#include +#include + +/** + Start of a BACKUP SERVER phase, + when no aria_backup_step() or aria_backup_end() is pending. + @param thd current session + @param target backup target + @param phase BACKUP_PHASE_START, ... (not BACKUP_PHASE_ABORT) + @param sink worker context + @return backup context object to be attached to backup_target, or nullptr + @retval -1 on failure +*/ +void *aria_backup_start(THD *thd, const backup_target *target, + backup_phase phase, const backup_sink *sink) noexcept; + +/** + Process a file that was collected in aria_backup_start(). + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked + @param sink worker context + @retval 0 on completion +*/ +int aria_backup_step(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink) noexcept; + +/** + Finish a phase, once all calls for the current phase are completed. + @param thd current session + @param target backup target + @param phase last phase on which backup_start() was successfully invoked, + or BACKUP_PHASE_ABORT or BACKUP_PHASE_FINISH + @param sink worker context + @return error code + @retval 0 on success +*/ +int aria_backup_end(THD *thd, const backup_target *target, backup_phase phase, + const backup_sink *sink) noexcept;