From 3148997538dd2015557649f07682ac9283af152c Mon Sep 17 00:00:00 2001 From: Eric Yan Date: Wed, 11 Nov 2026 16:12:42 +0000 Subject: [PATCH 1/4] move-tables: replace private design references in comments Remove internal task numbers, section markers, and private design links from move-tables comments and point to doc/move-tables-design.md instead. --- go/base/context.go | 2 +- go/logic/applier.go | 15 +++--- go/logic/applier_test.go | 16 +++---- go/logic/hooks.go | 2 +- go/logic/migrator.go | 47 +++++++++---------- go/logic/migrator_move_tables_cutover_test.go | 31 +++++------- .../move-tables/multiple-three/create.sql | 2 +- .../move-tables/multiple-two/create.sql | 2 +- 8 files changed, 52 insertions(+), 65 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index fcf9e6a17..bce0feee4 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -633,7 +633,7 @@ func (mctx *MigrationContext) MoveTableDelName(tableName string) string { // or a given table name. func (mctx *MigrationContext) GetChangelogTableName() string { if mctx.IsMoveTablesMode() { - panic("GetChangelogTableName() must not be called in move-tables mode; there is no changelog table (§1.2)") + panic("GetChangelogTableName() must not be called in move-tables mode; there is no changelog table") } if mctx.ForceTmpTableName != "" { return getSafeTableName(mctx.ForceTmpTableName, "ghc") diff --git a/go/logic/applier.go b/go/logic/applier.go index 318694f45..e6d2682ba 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -476,10 +476,11 @@ func (apl *Applier) prepareQueries() (err error) { } // Move-tables mode: build one set of query builders per migrated table. DML is - // routed to the right set at apply time by source table name (§2.1). There is - // no representative/primary table: every table is handled identically through - // its own builders, and the checkpoint uses a table-agnostic schema written by - // WriteMoveTableCheckpoints (no checkpointInsertQueryBuilder). + // routed to the right set at apply time by source table name (see + // doc/move-tables-design.md). There is no representative/primary table: every + // table is handled identically through its own builders, and the checkpoint + // uses a table-agnostic schema written by WriteMoveTableCheckpoints (no + // checkpointInsertQueryBuilder). apl.moveTablesBuilders = make(map[string]*moveTableBuilders, len(apl.migrationContext.MoveTables.TableNames)) for _, mt := range apl.migrationContext.OrderedMoveTables() { if mt.UniqueKey == nil { @@ -1226,7 +1227,7 @@ func (apl *Applier) DropGhostTable() error { // WriteChangelog writes a value to the changelog table. // It returns the hint (or an empty string in move-tables mode), for convenience func (apl *Applier) WriteChangelog(hint, value string) (string, error) { - // In move-tables mode, there is no changelog table (§1.2). All changelog + // In move-tables mode, there is no changelog table. All changelog // writes are no-ops. This is a single chokepoint rather than per-caller // guards to prevent drift when new callers are added. if apl.migrationContext.IsMoveTablesMode() { @@ -1527,7 +1528,7 @@ func (apl *Applier) ReadMoveTablesCutOverCheckpoint() (*Checkpoint, error) { // InitiateHeartbeat creates a heartbeat cycle, writing to the changelog table. // Apl is done asynchronously func (apl *Applier) InitiateHeartbeat() { - // In move-tables mode, there is no heartbeat table (§1.2). + // In move-tables mode, there is no heartbeat table. if apl.migrationContext.IsMoveTablesMode() { return } @@ -2578,7 +2579,7 @@ func (apl *Applier) updateModifiesUniqueKeyColumns(dmlEvent *binlog.BinlogDMLEve func (apl *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) []*dmlBuildResult { // Resolve the query builders + schema for the table this event targets. In // move-tables mode the set is selected by source table name (one binlog - // stream feeds every table; routing happens here, §2.1). In standard mode + // stream feeds every table; routing happens here). In standard mode // there is a single set on the applier. deleteBuilder := apl.dmlDeleteQueryBuilder insertBuilder := apl.dmlInsertQueryBuilder diff --git a/go/logic/applier_test.go b/go/logic/applier_test.go index 4391827be..da4d9b8ec 100644 --- a/go/logic/applier_test.go +++ b/go/logic/applier_test.go @@ -460,7 +460,6 @@ func (suite *ApplierTestSuite) TestInitiateApplierMoveTablesMode_NoGhostOrChange err = applier.InitDBConnections() suite.Require().NoError(err) - // #8206 [Task] [1.2] Skip ghost/changelog tables, heartbeat in gh-ost move-tables mode // In move-tables mode, no ghost or changelog table should exist. // InitDBConnections() should succeed without them. suite.Require().False(applier.tableExists("_testing_gho"), "ghost table should not exist in move-tables mode") @@ -593,7 +592,7 @@ func (suite *ApplierTestSuite) TestFinalCleanupMoveTablesMode_SkipsDrops() { // initiateStreaming() requires a binlog-capable MySQL connection to call directly. // This test verifies IsMoveTablesMode() and that no changelog table is referenced -// in move-tables mode (§1.2): no `_ghc` table exists on the source or target +// in move-tables mode: no `_ghc` table exists on the source or target // database. A new streamer always starts with zero listeners; the real proof that // no changelog listener is registered comes from the full run not failing on a // nonexistent _ghc table. @@ -629,7 +628,7 @@ func (suite *ApplierTestSuite) TestInitiateStreamingMoveTablesMode_NoChangelogLi // // A stronger test would instrument InitiateHeartbeat() (e.g., via a callback or // channel) to assert the goroutine is never started. That requires test-infrastructure -// changes to the Applier and is beyond #8206's scope. +// changes to the Applier. func (suite *ApplierTestSuite) TestNoHeartbeatInMoveTablesMode() { migrationContext := newTestMigrationContext() migrationContext.MoveTables.TableNames = []string{testMysqlTableName} @@ -760,7 +759,6 @@ func (suite *ApplierTestSuite) TestWriteChangelogNoOpInMoveTablesMode() { err = applier.InitDBConnections() suite.Require().NoError(err) - // #8206 [Task] [1.2] Skip ghost/changelog tables, heartbeat in gh-ost move-tables mode // WriteChangelog should be a no-op in move-tables mode. // No changelog table exists, so if it tried to execute, it would fail. hint, err := applier.WriteChangelog("heartbeat", "2026-06-05T00:00:00Z") @@ -1047,9 +1045,9 @@ func (suite *ApplierTestSuite) TestAnalyzeGhostTable() { suite.Require().ErrorContains(applier.AnalyzeGhostTable(), "failed; refusing cut-over") } -// TestCreateTargetTable_HappyPath exercises #8207 AC #1: -// "A move table run targeting a clean target schema creates the migrated table -// with a SHOW CREATE TABLE output equivalent to the source's." +// TestCreateTargetTable_HappyPath verifies that a move-tables run targeting a +// clean target schema creates the migrated table with a SHOW CREATE TABLE +// output equivalent to the source's. // // It calls CreateTargetTable (not just IsMoveTablesMode()), asserts the table // exists on the target database, verifies schema equivalence via SHOW CREATE TABLE, @@ -1106,8 +1104,8 @@ func (suite *ApplierTestSuite) TestCreateTargetTable_HappyPath() { suite.Require().Equal(1, count, "target table must exist exactly once on the target database") } -// TestCreateTargetTable_AbortsIfExists exercises #8207 AC #2: -// "A move table run aborts before any data is copied if the target table already exists." +// TestCreateTargetTable_AbortsIfExists verifies that a move-tables run aborts +// before any data is copied if the target table already exists. // // It pre-creates the target table, then calls CreateTargetTable and asserts it // returns a descriptive error (not just MySQL's raw ERROR 1050). diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 28cde0ca6..73a736c43 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -279,7 +279,7 @@ func (he *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) []s env = append(env, fmt.Sprintf("GH_OST_REVERT=%t", he.migrationContext.Revert)) env = append(env, fmt.Sprintf("GH_OST_MOVE_TABLES=%t", he.migrationContext.IsMoveTablesMode())) if he.migrationContext.IsMoveTablesMode() { - // Comma-joined list of all migrated tables (§2.4). + // Comma-joined list of all migrated tables. env = append(env, fmt.Sprintf("GH_OST_TABLES=%s", strings.Join(he.migrationContext.MoveTables.TableNames, ","))) } env = append(env, fmt.Sprintf("GH_OST_TARGET_DATABASE_NAME=%s", he.migrationContext.GetTargetDatabaseName())) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index f2816bc87..2e72149e6 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -31,7 +31,7 @@ var ( checkpointTimeout = 2 * time.Second // moveTablesCutOverDrainPollInterval is the per-iteration sleep in T3's - // drain poll. 100ms per move_table_mode.md §1.5. + // drain poll. moveTablesCutOverDrainPollInterval = 100 * time.Millisecond ) @@ -856,10 +856,10 @@ func prepareMoveTableColumnMetadata(inspector *Inspector, databaseName, tableNam } // prepareMoveTablesCopyState initializes per-table runtime state for row copy in -// move-tables mode (§2.1). Each migrated table is inspected and validated -// independently into its own container (schema, unique key, row estimate, CREATE -// statement). There is no representative table: a single-entry --move-tables is -// simply an array of one, handled by the same per-table loop. +// move-tables mode (see doc/move-tables-design.md). Each migrated table is inspected +// and validated independently into its own container (schema, unique key, row +// estimate, CREATE statement). There is no representative table: a single-entry +// --move-tables is simply an array of one, handled by the same per-table loop. func (mgtr *Migrator) prepareMoveTablesCopyState() error { mgtr.migrationContext.InitMoveTableContainers() @@ -1368,8 +1368,8 @@ func (mgtr *Migrator) MoveTables() (err error) { if err := mgtr.countTableRows(); err != nil { return err } - // Read each migrated table's full row-copy range into its per-table container - // (§2.3). Ranges are read from the source via the inspector connection. + // Read each migrated table's full row-copy range into its per-table container. + // Ranges are read from the source via the inspector connection. for _, mt := range mgtr.migrationContext.OrderedMoveTables() { if err := mgtr.applier.ReadMoveTableMigrationRangeValues(mgtr.inspector.db, mt); err != nil { return fmt.Errorf("failed to read migration range for %s.%s: %w", @@ -1429,7 +1429,7 @@ func (mgtr *Migrator) MoveTables() (err error) { // moveTablesCutOver orchestrates the cooperative cutover protocol for move-tables // mode. It implements the T0-T6 transitions described in -// docs/learning/design-refs/coop_cutover.md §1.3. +// doc/move-tables-design.md. // // NOT the standard cutOver() path: every internal call from cutOver() (throttle, // atomicCutOver, waitForEventsUpToLock, heartbeat-lag) was built on a @@ -1444,11 +1444,11 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { // ----- Postpone gate (precedes T0) ----- // Mirrors standard cutOver()'s sleepWhileTrue postpone structure but DROPS the - // heartbeat-lag branch: move-tables mode disables _ghc heartbeat writes (#8206), + // heartbeat-lag branch: move-tables mode disables _ghc heartbeat writes, // so TimeSinceLastHeartbeatOnChangelog() returns ~58 years (time.Since(zero)) // and would deadlock the gate forever. KEEPS the postpone-flag-file + - // unpostpone-socket gate because per coop_cutover.md §1.1 P4, operator-removes- - // postpone is the trigger for the entire cutover phase. + // unpostpone-socket gate because removing the postpone gate is the operator + // trigger for the cutover phase. mgtr.migrationContext.Log.Debugf("checking for cut-over postpone") if err := mgtr.sleepWhileTrue("cut_over_postpone", func() (bool, error) { if mgtr.migrationContext.PostponeCutOverFlagFile == "" { @@ -1505,7 +1505,7 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { sourceDB := mgtr.migrationContext.DatabaseName // Build a single atomic multi-table RENAME covering every table in - // --move-tables order (§2.4): `RENAME TABLE db.t1 TO db._t1_del, db.t2 TO + // --move-tables order: `RENAME TABLE db.t1 TO db._t1_del, db.t2 TO // db._t2_del, ...`. MySQL executes this as one event group with one GTID, so // the existing single-drain-GTID mechanism covers the whole move set. renameClauses := make([]string, 0, len(mgtr.migrationContext.MoveTables.TableNames)) @@ -1522,8 +1522,8 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { len(renameClauses), renameAndCaptureQuery) // MySQL exposes @@global.gtid_executed, while MariaDB exposes - // @@global.gtid_binlog_pos. - // Design: https://github.com/github/gh-ost-tablemove-poc/blob/9dc6df75c4c88ff473906a497836c7518f5614ec/design/coop_cutover.md#32-correctness-verification-for-p4 + // @@global.gtid_binlog_pos. See doc/move-tables-design.md for the cutover + // drain protocol. drainGTIDStr, err := func() (string, error) { rows, err := mgtr.sourcePrimaryDB.QueryContext(cutOverCtx, renameAndCaptureQuery) if err != nil { @@ -1599,12 +1599,9 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) { mgtr.migrationContext.NewFailPoint("move-tables-panic-before-on-success-hook", base.WithFailPointWait(2*time.Second)) // ----- T5: on-success hook ----- - // Hook unlocks user_rw@target via db-user-management and flips the - // write_cutover? feature flag. Standard env vars only — GH_OST_DRAIN_GTID + - // GH_OST_TARGET_* are #8211 (1.7), not this PR. The pre-protocol placeholder - // OnSuccess call that used to live in MoveTables() (after finalCleanup) has - // been removed so the hook fires in the order coop_cutover.md §3.2 step 6 - // requires (T5 between T4 and T6, BEFORE finalCleanup). + // Runs after the completed-drain checkpoint marker is persisted (T4) and + // before finalCleanup (T6), with GH_OST_DRAIN_GTID set to the drained + // source GTID position. See doc/move-tables-design.md. mgtr.migrationContext.MoveTables.DrainGTID = drainGTID if err := mgtr.hooksExecutor.OnSuccess(false); err != nil { return fmt.Errorf("on-success hook failed: %w", err) @@ -2279,7 +2276,7 @@ func (mgtr *Migrator) printMigrationStatusHint(writers ...io.Writer) { w := io.MultiWriter(writers...) if mgtr.migrationContext.IsMoveTablesMode() { // In move-tables mode there may be several migrated tables; list each - // source -> target mapping rather than a single primary table (§2.3). + // source -> target mapping rather than a single primary table. // Table names match on source and target; only the database may differ. sourceDatabaseName := mgtr.migrationContext.DatabaseName targetDatabaseName := mgtr.migrationContext.GetTargetDatabaseName() @@ -2535,7 +2532,7 @@ func (mgtr *Migrator) printStatus(rule PrintStatusRule, snap migrationProgressSn fmt.Fprintln(w, status) // In move-tables mode, surface per-table row-copy progress so all migrated - // tables are visibly advancing concurrently (§2.3). + // tables are visibly advancing concurrently. if mgtr.migrationContext.IsMoveTablesMode() { for _, mt := range mgtr.migrationContext.OrderedMoveTables() { copied := atomic.LoadInt64(&mt.RowsCopied) @@ -2639,7 +2636,7 @@ func (mgtr *Migrator) initiateStreaming() error { // addDMLEventsListener begins listening for binlog events on the migrated // table(s), and creates & enqueues a write task per such event. In move-tables // mode it registers one listener per migrated table on the shared events -// streamer (§2.2); all listeners feed the same apply queue, parameterized only +// streamer; all listeners feed the same apply queue, parameterized only // by table name. The streamer already dispatches per (database, table), and the // applier routes DML to the right per-table query builders by table name. func (mgtr *Migrator) addDMLEventsListener() error { @@ -2718,7 +2715,7 @@ func (mgtr *Migrator) initiateApplier() error { } } // Create every migrated table on the target from its captured CREATE - // statement (§2.1). Containers were populated by prepareMoveTablesCopyState. + // statement. Containers were populated by prepareMoveTablesCopyState. for _, mt := range mgtr.migrationContext.OrderedMoveTables() { createTableStatement := mt.CreateTableStatement if createTableStatement == "" { @@ -2900,7 +2897,7 @@ func (mgtr *Migrator) iterateChunks() error { } } -// iterateChunksMoveTables drives the interleaved, multi-table row copy (§2.3). +// iterateChunksMoveTables drives the interleaved, multi-table row copy. // It round-robins over the migrated tables in --move-tables order, enqueuing one // chunk-copy task per not-yet-complete table per cycle so all tables make // progress concurrently through the single shared apply pipeline. Each task diff --git a/go/logic/migrator_move_tables_cutover_test.go b/go/logic/migrator_move_tables_cutover_test.go index 7cdc68f21..d45c0d708 100644 --- a/go/logic/migrator_move_tables_cutover_test.go +++ b/go/logic/migrator_move_tables_cutover_test.go @@ -31,9 +31,9 @@ import ( // initialized error instead of the earlier error it asserts on. // ----------------------------------------------------------------------------- -// TestMoveTablesCutOver_NoopShortCircuits maps to the Noop semantics decision -// in #8209-implement-protocol (Noop returns immediately, no postpone gate, no -// hooks, no RENAME). +// TestMoveTablesCutOver_NoopShortCircuits verifies that Noop mode in +// moveTablesCutOver returns immediately without waiting on the postpone gate, +// running hooks, or renaming source tables. func TestMoveTablesCutOver_NoopShortCircuits(t *testing.T) { var calls []string fakeHooks := &recordingHooks{name: "fake", calls: &calls} @@ -54,8 +54,8 @@ func TestMoveTablesCutOver_NoopShortCircuits(t *testing.T) { require.Empty(t, calls, "post-state: Noop must not fire any hook") } -// TestMoveTablesCutOver_OnBeforeCutOverHookAbortsBeforeRename maps to T0 in -// coop_cutover.md section 1.3 ("non-zero return code aborts cutover"). The "aborts +// TestMoveTablesCutOver_OnBeforeCutOverHookAbortsBeforeRename verifies T0 in +// moveTablesCutOver ("non-zero return code aborts cutover"). The "aborts // BEFORE source DDL" assertion is enforced as a proxy: m.sourcePrimaryDB is nil, // so if T1 RENAME executed it would fail with the source-primary-not-initialized // error rather than the asserted hook error. @@ -129,11 +129,10 @@ func (h *onValidatedCheckHooks) OnValidated() error { return h.onValidatedCheck() } -// TestMoveTablesCutOver_PostponeGateFiresOnBeginPostponedOnce maps to the -// postpone-gate decision in #8209-implement-protocol (keep OnBeginPostponed -// firing logic with the same once-per-cutover semantics as standard cutOver). -// Also exercises Edge Case Test Quality #5 by asserting both pre-state and -// post-state of IsPostponingCutOver. +// TestMoveTablesCutOver_PostponeGateFiresOnBeginPostponedOnce verifies that +// moveTablesCutOver fires OnBeginPostponed with the same once-per-cutover +// semantics as standard cutOver, and asserts both pre-state and post-state of +// IsPostponingCutOver. func TestMoveTablesCutOver_PostponeGateFiresOnBeginPostponedOnce(t *testing.T) { flagPath := filepath.Join(t.TempDir(), "postpone.flag") require.NoError(t, os.WriteFile(flagPath, nil, 0o644)) @@ -244,13 +243,7 @@ func TestValidateMoveTablesSourceReadHost(t *testing.T) { // // These live under a dedicated suite (NOT MigratorTestSuite) so the parent // function name TestMoveTablesCutOver matches the `-run MoveTablesCutOver` -// verifier alongside the pure unit tests above. SetupSuite duplicates the -// testcontainer setup pattern used by MigratorTestSuite intentionally to -// keep this commit's diff strictly additive (no edits to existing tests or -// shared helpers, per the commit-3 prompt). -// -// Known-environmental flakes when Docker/testcontainers is unavailable mirror -// the same class as #8206; they are not regressions of #8209. +// verifier alongside the pure unit tests above. // ----------------------------------------------------------------------------- type MoveTablesCutOverSuite struct { @@ -659,8 +652,6 @@ func (s *MoveTablesCutOverSuite) TestAssertConnectionWritableRejectsReadOnly() { // TestHappyPath drives the full T0-T6 protocol against the test container. // Asserts hook ordering (T0 then T5), T4 flag set, and the source-side rename. -// Maps to acceptance criterion #8209 "RENAME executes; drain completes; -// CutOverCompleteFlag set; on-success hook fires". func (s *MoveTablesCutOverSuite) TestHappyPath() { ctx := context.Background() _, err := s.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE %s (id INT PRIMARY KEY)", getTestTableName())) @@ -680,7 +671,7 @@ func (s *MoveTablesCutOverSuite) TestHappyPath() { s.Require().Equal([]string{"fake:OnBeforeCutOver", "fake:OnSuccess"}, calls, "post-state: T0 hook precedes T5 hook") - // Source-side post-state (Edge Case Test Quality #5: assert both sides). + // Source-side post-state: assert both sides. var renamed string s.Require().NoError(s.db.QueryRow(fmt.Sprintf("SHOW TABLES IN %s LIKE '_%s_del'", testMysqlDatabase, testMysqlTableName)).Scan(&renamed)) diff --git a/localtests/move-tables/multiple-three/create.sql b/localtests/move-tables/multiple-three/create.sql index 631653315..f92cef32e 100644 --- a/localtests/move-tables/multiple-three/create.sql +++ b/localtests/move-tables/multiple-three/create.sql @@ -1,5 +1,5 @@ -- Three tables with distinct schemas, primary-key types, and row counts. This --- exercises the multi-table move-tables path (§2.1-2.4) at its widest: per-table +-- exercises the multi-table move-tables path at its widest: per-table -- runtime state, per-table query builders, interleaved row copy where the tables -- finish at different times, and a single atomic multi-table RENAME at cutover. -- diff --git a/localtests/move-tables/multiple-two/create.sql b/localtests/move-tables/multiple-two/create.sql index 3d4a5d70b..ba9d85f52 100644 --- a/localtests/move-tables/multiple-two/create.sql +++ b/localtests/move-tables/multiple-two/create.sql @@ -1,5 +1,5 @@ -- Two tables with different schemas, primary-key types, and row counts. This --- exercises the multi-table move-tables path (§2.1-2.4): per-table runtime +-- exercises the multi-table move-tables path: per-table runtime -- state, per-table query builders, interleaved row copy where the tables finish -- at different times, and a single atomic multi-table RENAME at cutover. From 09c7d5e0b5fbc09297e8160146c00478a9fa2c47 Mon Sep 17 00:00:00 2001 From: Eric Yan Date: Wed, 11 Nov 2026 16:12:42 +0000 Subject: [PATCH 2/4] move-tables: include target context in sample hooks Log target host, database, tables, and drain GTID in the before-cut-over and success sample hooks when GH_OST_MOVE_TABLES is set. --- resources/hooks-sample/gh-ost-on-before-cut-over-hook | 7 ++++++- resources/hooks-sample/gh-ost-on-success-hook | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/resources/hooks-sample/gh-ost-on-before-cut-over-hook b/resources/hooks-sample/gh-ost-on-before-cut-over-hook index 41bf2d051..77e100319 100755 --- a/resources/hooks-sample/gh-ost-on-before-cut-over-hook +++ b/resources/hooks-sample/gh-ost-on-before-cut-over-hook @@ -2,4 +2,9 @@ # Sample hook file for gh-ost-on-before-cut-over -echo "$(date) gh-ost-on-before-cut-over $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log +message="$GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" +if [ "$GH_OST_MOVE_TABLES" = "true" ]; then + message="$GH_OST_DATABASE_NAME ($GH_OST_TABLES) -> $GH_OST_TARGET_HOST/$GH_OST_TARGET_DATABASE_NAME" +fi + +echo "$(date) gh-ost-on-before-cut-over $message" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-success-hook b/resources/hooks-sample/gh-ost-on-success-hook index aa4aedc22..52828f174 100755 --- a/resources/hooks-sample/gh-ost-on-success-hook +++ b/resources/hooks-sample/gh-ost-on-success-hook @@ -2,4 +2,9 @@ # Sample hook file for gh-ost-on-success -echo "$(date) gh-ost-on-success $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log +message="$GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" +if [ "$GH_OST_MOVE_TABLES" = "true" ]; then + message="$GH_OST_DATABASE_NAME ($GH_OST_TABLES) -> $GH_OST_TARGET_HOST/$GH_OST_TARGET_DATABASE_NAME (drain_gtid: $GH_OST_DRAIN_GTID)" +fi + +echo "$(date) gh-ost-on-success $message" >> /tmp/gh-ost.log From 9c3b25389e76b6303ef6324ca3a16f11fc10f960 Mon Sep 17 00:00:00 2001 From: Eric Yan Date: Wed, 11 Nov 2026 16:12:42 +0000 Subject: [PATCH 3/4] docs: document --move-tables mode Add the move-tables operator guide and technical reference, and update the README, flag reference, hook docs, resume docs, throttling docs, and output docs. --- README.md | 6 +- doc/command-line-flags.md | 72 ++++++++++---- doc/hooks.md | 30 ++++-- doc/move-tables-design.md | 101 +++++++++++++++++++ doc/move-tables.md | 192 ++++++++++++++++++++++++++++++++++++ doc/resume.md | 25 ++++- doc/throttle.md | 14 ++- doc/understanding-output.md | 14 +++ 8 files changed, 424 insertions(+), 30 deletions(-) create mode 100644 doc/move-tables-design.md create mode 100644 doc/move-tables.md diff --git a/README.md b/README.md index 898b2d2f5..101535970 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Please refer to the [docs](doc) for more information. No, really, read the [docs ## Usage -The [cheatsheet](doc/cheatsheet.md) has it all. You may be interested in invoking `gh-ost` in various modes: +For online schema changes, see the [cheatsheet](doc/cheatsheet.md). You may be interested in invoking `gh-ost` in various modes: - a _noop_ migration (merely testing that the migration is valid and good to go) - a real migration, utilizing a replica (the migration runs on the master; `gh-ost` figures out identities of servers involved. Required mode if your master uses Statement Based Replication) @@ -45,7 +45,9 @@ The [cheatsheet](doc/cheatsheet.md) has it all. You may be interested in invokin - a real migration on a replica (master untouched) - a test migration on a replica, the way for you to build trust with `gh-ost`'s operation. -Our tips: +`gh-ost` also supports **moving tables across clusters** without a schema change (`--move-tables`); you switch application traffic yourself after cut-over. See the [user guide](doc/move-tables.md) and [design doc](doc/move-tables-design.md). + +Our tips for online schema changes: - [Testing above all](doc/testing-on-replica.md), try out `--test-on-replica` first few times. Better yet, make it continuous. We have multiple replicas where we iterate our entire fleet of production tables, migrating them one by one, checksumming the results, verifying migration is good. - For each master migration, first issue a _noop_ diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index ff781b250..87d7b877b 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -16,7 +16,7 @@ Allows the user to make schema changes that include a zero date or zero in date ### alter -Mandatory unless using [`--revert`](#revert). The schema change to apply to the migrated table. You can pass just the alter options, for example `--alter="ADD COLUMN created_at timestamp NULL"`, together with [`--database`](#database) and [`--table`](#table). You can also pass a full `ALTER TABLE [database.]table ...` statement; an explicit database or table in `--alter` can be used instead of the corresponding flag. +Mandatory unless using [`--revert`](#revert) or [`--move-tables`](#move-tables). The schema change to apply to the migrated table. You can pass just the alter options, for example `--alter="ADD COLUMN created_at timestamp NULL"`, together with [`--database`](#database) and [`--table`](#table). You can also pass a full `ALTER TABLE [database.]table ...` statement; an explicit database or table in `--alter` can be used instead of the corresponding flag. Cannot be used with [`--move-tables`](#move-tables). ### azure @@ -36,6 +36,10 @@ By default, `gh-ost` would like you to connect to a replica, from where it figur If, for some reason, you do not wish `gh-ost` to connect to a replica, you may connect it directly to the master and approve this via `--allow-on-master`. +### allow-on-source-primary + +Allows [`--move-tables`](#move-tables) to use the source primary as `--host`, which is refused by default. Prefer a source replica to keep copy load off the primary. + ### allow-setup-metadata-lock-instruments `--allow-setup-metadata-lock-instruments` allows gh-ost to enable the [`metadata_locks`](https://dev.mysql.com/doc/refman/8.0/en/performance-schema-metadata-locks-table.html) table in `performance_schema`, if it is not already enabled. This is used for a safety check before cut-over. @@ -94,7 +98,10 @@ Example: `gh-ost --check-flag --cut-over-lock-timeout-seconds 1 --nice-ratio 0` ### checkpoint `--checkpoint` enables periodic checkpoints of the gh-ost's state so that gh-ost can resume a migration from the checkpoint with `--resume`. Checkpoints are written to a separate table named `_${original_table_name}_ghk`. It is recommended to use with `--gtid` for checkpoints. -See also: [`resuming-migrations`](resume.md) + +In [`--move-tables`](#move-tables) CLI runs, checkpoints are enabled automatically and written on the target cluster. + +See also: [`resuming-migrations`](resume.md) and [`move-tables`](move-tables.md) ### checkpoint-seconds @@ -124,6 +131,8 @@ Comma delimited status-name=threshold, same format as [`--max-load`](#max-load). `--critical-load` defines a threshold that, when met, `gh-ost` panics and bails out. The default behavior is to bail out immediately when meeting this threshold. +In [`--move-tables`](#move-tables) mode, these metrics are evaluated on the target primary, as with `--max-load`. + This may sometimes lead to migrations bailing out on a very short spike, that, while in itself is impacting production and is worth investigating, isn't reason enough to kill a 10-hour migration. ### critical-load-hibernate-seconds @@ -140,7 +149,7 @@ This is somewhat similar to a Nagios `n`-times test, where `n` in our case is al ### cut-over -Optional. Default is `atomic`. Accepted values are `atomic`, `default`, and `two-step`; `default` is a legacy alias for `atomic`. `atomic` performs an atomic table swap where pending connections briefly block and either all see the old table or all see the new table. `two-step` uses a non-atomic rename sequence where the original table is renamed away before the ghost table takes its place, leaving a brief interval where the table does not exist. See more discussion in [`cut-over`](cut-over.md) +Optional. Default is `atomic`. Accepted values are `atomic`, `default`, and `two-step`; `default` is a legacy alias for `atomic`. `atomic` performs an atomic table swap where pending connections briefly block and either all see the old table or all see the new table. `two-step` uses a non-atomic rename sequence where the original table is renamed away before the ghost table takes its place, leaving a brief interval where the table does not exist. Not used in [`--move-tables`](#move-tables) mode. See more discussion in [`cut-over`](cut-over.md). ### cut-over-exponential-backoff @@ -148,11 +157,12 @@ Wait exponentially longer intervals between failed cut-over attempts. The maximu ### cut-over-lock-timeout-seconds -Default `3`. Max number of seconds to hold locks on tables while attempting to cut-over (retry attempted when lock exceeds timeout). +- **Standard OSC:** default `3`. Maximum seconds to hold locks while attempting cut-over; a lock timeout causes a retry. +- **[`--move-tables`](#move-tables):** default `60`. Maximum seconds to drain the source stream and target applier through the source cut-over GTID after renaming the source tables. This is not a limit on rename lock waits or total application outage. A failed drain leaves the source tables renamed; it does not automatically retry the rename or roll back. ### database -The database containing the table to migrate. Required unless the database is specified in a full [`--alter`](#alter) statement. +The database containing the table to migrate (or the source database in [`--move-tables`](#move-tables) mode). Required unless the database is specified in a full [`--alter`](#alter) statement. ### debug @@ -222,7 +232,7 @@ Add this flag when executing on a 1st generation Google Cloud Platform (GCP). ### gtid -Add this flag to enable support for [MySQL replication GTIDs](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html) for replication positioning. On MySQL this requires `gtid_mode` and `enforce_gtid_consistency` to be set to `ON`. +Add this flag to enable support for [MySQL replication GTIDs](https://dev.mysql.com/doc/refman/5.7/en/replication-gtids-concepts.html) for replication positioning. On MySQL this requires `gtid_mode` and `enforce_gtid_consistency` to be set to `ON`. Automatically enabled in [`--move-tables`](#move-tables) CLI runs. [MariaDB GTIDs](https://mariadb.com/kb/en/gtid/) are also supported: gh-ost detects the server flavor automatically and uses the appropriate GTID dialect. MariaDB has no `gtid_mode`/`enforce_gtid_consistency` settings — GTIDs are always recorded when binary logging is enabled, so no extra configuration is required beyond `--gtid`. @@ -298,15 +308,21 @@ When [`--throttle-control-replicas`](#throttle-control-replicas) is provided, th When using on master or when `--allow-on-master` is provided, `max-lag-millis` is also considered a threshold for starting the cutover stage of the migration. If the row copy is complete and the heartbeat lag is less than `max-lag-millis` cutover phase of the migration will start. +In [`--move-tables`](#move-tables) mode, applies only to target replicas listed in [`--throttle-control-replicas`](#throttle-control-replicas); the source replica is not monitored automatically. See [move-tables throttling](throttle.md#move-tables-mode). + See also: [Sub-second replication lag throttling](subsecond-lag.md) ### max-load -List of metrics and threshold values; topping the threshold of any will cause throttler to kick in. See also: [`throttling`](throttle.md#status-thresholds) +List of metrics and threshold values; topping the threshold of any will cause throttler to kick in. Evaluated on the target primary in [`--move-tables`](#move-tables) mode. See also: [`throttling`](throttle.md#status-thresholds) ### migrate-on-replica -Typically `gh-ost` is used to migrate tables on a master. If you wish to only perform the migration in full on a replica, connect `gh-ost` to said replica and pass `--migrate-on-replica`. `gh-ost` will briefly connect to the master but otherwise will make no changes on the master. Migration will be fully executed on the replica, while making sure to maintain a small replication lag. +Typically `gh-ost` is used to migrate tables on a master. If you wish to only perform the migration in full on a replica, connect `gh-ost` to said replica and pass `--migrate-on-replica`. `gh-ost` will briefly connect to the master but otherwise will make no changes on the master. Migration will be fully executed on the replica, while making sure to maintain a small replication lag. Cannot be used with [`--move-tables`](#move-tables). + +### move-tables + +Comma-delimited list of one or more tables to move from the source cluster to the target cluster without a schema change, for example `--move-tables="orders,order_items"`. Requires [`--target-host`](#target-host) and [`--postpone-cut-over-flag-file`](#postpone-cut-over-flag-file), and automatically enables [`--checkpoint`](#checkpoint) and [`--gtid`](#gtid) in the CLI. Mutually exclusive with [`--alter`](#alter), [`--table`](#table), [`--test-on-replica`](#test-on-replica), [`--migrate-on-replica`](#migrate-on-replica), [`--revert`](#revert), and [`--timestamp-old-table`](#timestamp-old-table). See [`move-tables`](move-tables.md). ### mysql-timeout @@ -318,7 +334,9 @@ Makes row-copy operations sleep after each chunk, proportional to the time spent ### ok-to-drop-table -Allows `gh-ost` to drop the old table at the end of a successful migration. This is disabled by default because dropping a table can be a long locking operation. +Allows `gh-ost` to drop the old table (and checkpoint table) at the end of a successful migration. This is disabled by default because dropping a table can be a long locking operation. + +In [`--move-tables`](#move-tables) mode, drops the `__del` tables on the source primary and the checkpoint table on the target primary, removing the source rollback handles. ### old-table @@ -344,7 +362,7 @@ MySQL port to connect to. Preferably this is a replica, not the master. Default ### postpone-cut-over-flag-file -Indicate a file name, such that the final [cut-over](cut-over.md) step does not take place as long as the file exists. +Indicate a file name, such that the final [cut-over](cut-over.md) step does not take place as long as the file exists. Mandatory when using [`--move-tables`](#move-tables). When this flag is set, `gh-ost` expects the file to exist on startup, or else tries to create it. `gh-ost` exits with error if the file does not exist and `gh-ost` is unable to create it. With this flag set, the migration will cut-over upon deletion of the file or upon `cut-over` [interactive command](interactive-commands.md). @@ -371,11 +389,11 @@ Deprecated. `gh-ost` uses an internal subsecond-resolution lag query instead. ### resume `--resume` attempts to resume a migration that was previously interrupted from the last checkpoint. The first `gh-ost` invocation must run with `--checkpoint` and have successfully written a checkpoint in order for `--resume` to work. -See also: [`resuming-migrations`](resume.md) +See also: [`resuming-migrations`](resume.md) and [move-tables recovery](move-tables.md#cleanup-and-recovery) ### revert -Attempts to revert a completed migration. Must be used with [`--old-table`](#old-table). `--revert` cannot be used together with [`--resume`](#resume), and ignores migration options such as `--alter`, `--attempt-instant-ddl`, `--include-triggers`, and `--discard-foreign-keys`. +Attempts to revert a completed migration. Must be used with [`--old-table`](#old-table). `--revert` cannot be used together with [`--resume`](#resume) or [`--move-tables`](#move-tables), and ignores migration options such as `--alter`, `--attempt-instant-ddl`, `--include-triggers`, and `--discard-foreign-keys`. ### runtime-metrics-interval @@ -465,11 +483,31 @@ Allows `gh-ost` to automatically switch the replica's binary log format to `ROW` ### table -The table name to migrate, without the database name. Required unless the table is specified in a full [`--alter`](#alter) statement. See also [`--database`](#database). +The table name to migrate, without the database name. Required unless the table is specified in a full [`--alter`](#alter) statement or [`--move-tables`](#move-tables) is used. Cannot be used with [`--move-tables`](#move-tables). See also [`--database`](#database). + +### target-database + +Target MySQL database name for [`--move-tables`](#move-tables) mode. If not provided, defaults to [`--database`](#database). The database must already exist on the target cluster. + +### target-host + +Target MySQL primary hostname for [`--move-tables`](#move-tables) mode. Required when `--move-tables` is specified, and must point to a writable primary (`read_only=OFF`). + +### target-password + +Target MySQL password for [`--move-tables`](#move-tables) mode. If not provided, defaults to the source connection password. + +### target-port + +Target MySQL port for [`--move-tables`](#move-tables) mode. Defaults to `3306`. + +### target-user + +Target MySQL username for [`--move-tables`](#move-tables) mode. If not provided, defaults to the source connection user. ### test-on-replica -Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md) +Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. Cannot be used with [`--move-tables`](#move-tables). See [`testing-on-replica`](testing-on-replica.md) ### test-on-replica-skip-replica-stop @@ -485,11 +523,11 @@ A flag file that pauses operation while it exists. Prefer a file name specific t ### throttle-query -A custom query issued every second to decide whether the migration should throttle. The query runs on the migrated server and should return `0` for no throttle and a value greater than `0` to throttle. Keep this query lightweight. +A custom query issued every second to decide whether the migration should throttle. The query runs on the migrated server (or the target primary in [`--move-tables`](#move-tables) mode) and should return `0` for no throttle and a value greater than `0` to throttle. Keep this query lightweight. ### throttle-control-replicas -Provide a command delimited list of replicas; `gh-ost` will throttle when any of the given replicas lag beyond [`--max-lag-millis`](#max-lag-millis). The list can be queried and updated dynamically via [interactive commands](interactive-commands.md) +Provide a command delimited list of replicas; `gh-ost` will throttle when any of the given replicas lag beyond [`--max-lag-millis`](#max-lag-millis). In [`--move-tables`](#move-tables) mode, these replicas are in the target cluster and are checked using the target credentials. The list can be queried and updated dynamically via [interactive commands](interactive-commands.md) ### throttle-http @@ -505,7 +543,7 @@ Defaults to 1000 (1 second). Configures the HTTP throttler check timeout in mill ### timestamp-old-table -Makes the _old_ table include a timestamp value. The _old_ table is what the original table is renamed to at the end of a successful migration. For example, if the table is `gh_ost_test`, then the _old_ table would normally be `_gh_ost_test_del`. With `--timestamp-old-table` it would be, for example, `_gh_ost_test_20170221103147_del`. +Makes the _old_ table include a timestamp value. The _old_ table is what the original table is renamed to at the end of a successful migration. For example, if the table is `gh_ost_test`, then the _old_ table would normally be `_gh_ost_test_del`. With `--timestamp-old-table` it would be, for example, `_gh_ost_test_20170221103147_del`. Not supported with [`--move-tables`](#move-tables). ### trigger-suffix diff --git a/doc/hooks.md b/doc/hooks.md index 47ee59bd9..c39adce11 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -54,10 +54,6 @@ The full list of supported hooks is best found in code: [hooks.go](https://githu For CLI runs, `gh-ost-on-failure` runs when the migration returns an error, even if `on-startup` did not run or `on-success` may already have enabled target writes. -> [!IMPORTANT] -> In move-tables mode, inspect source and target state before recovery; failure -> hooks must not blindly roll back. - ### Context `gh-ost` will set environment variables per hook invocation. Hooks are then able to read those variables, indicating schema name, table name, `alter` statement, migrated host name etc. Some variables are available on all hooks, and some are available on relevant hooks. @@ -75,29 +71,49 @@ The following variables are available on all hooks: - `GH_OST_COPIED_ROWS` - number of rows copied by `gh-ost` - `GH_OST_INSPECTED_LAG` - lag in seconds (floating point) of inspected server - `GH_OST_HEARTBEAT_LAG` - lag in seconds (floating point) of heartbeat +- `GH_OST_BINLOG_WRITER_LAG_SECONDS` - writer lag in seconds (floating point) in `--move-tables` mode (`0` in standard mode) - `GH_OST_PROGRESS` - progress pct ([0..100], floating point) of migration - `GH_OST_ETA_SECONDS` - estimated duration until migration finishes in seconds - `GH_OST_MIGRATED_HOST` - `GH_OST_INSPECTED_HOST` - `GH_OST_EXECUTING_HOST` -- `GH_OST_TARGET_HOST` - the target/applier hostname (in `--move-tables`, this is the target cluster host) +- `GH_OST_TARGET_HOST` - target/applier hostname - `GH_OST_HOOKS_HINT` - copy of `--hooks-hint` value - `GH_OST_HOOKS_HINT_OWNER` - copy of `--hooks-hint-owner` value - `GH_OST_HOOKS_HINT_TOKEN` - copy of `--hooks-hint-token` value - `GH_OST_DRY_RUN` - whether or not the `gh-ost` run is a dry run - `GH_OST_REVERT` - whether or not `gh-ost` is running in revert mode - `GH_OST_MOVE_TABLES` - whether or not `gh-ost` is running in `--move-tables` mode -- `GH_OST_TARGET_DATABASE_NAME` - operation target database name (in `--move-tables`, this is the explicit target database) +- `GH_OST_TARGET_DATABASE_NAME` - operation target database name - `GH_OST_TARGET_TABLE_NAME` - operation target table name (mode-dependent) The following variables are available on particular hooks: - `GH_OST_INSTANT_DDL` is only available in `gh-ost-on-success`. The value is `true` if instant DDL was successful, and `false` if it was not. -- `GH_OST_DRAIN_GTID` is only available in `gh-ost-on-success` and only in `--move-tables` mode. It contains the source `@@gtid_executed` captured immediately after the source `RENAME TABLE` (cutover) and represents the drain target the applier waits for. - `GH_OST_COMMAND` is only available in `gh-ost-on-interactive-command` - `GH_OST_STATUS` is only available in `gh-ost-on-status` - `GH_OST_LAST_BATCH_COPY_ERROR` is only available in `gh-ost-on-batch-copy-retry` +#### `--move-tables` mode + +When [`--move-tables`](move-tables.md) is enabled (`GH_OST_MOVE_TABLES=true`), these variables are added or change meaning: + +- `GH_OST_TABLES` - comma-joined list of moved table names in `--move-tables` order (available on all hooks) +- `GH_OST_TABLE_NAME`, `GH_OST_GHOST_TABLE_NAME`, `GH_OST_TARGET_TABLE_NAME` - comma-joined list of moved table names +- `GH_OST_OLD_TABLE_NAME` - comma-joined list of source deletion table names (`_
_del`) +- `GH_OST_TARGET_DATABASE_NAME` - target database name (`--target-database`, defaulting to `--database`) +- `GH_OST_TARGET_HOST` - target cluster hostname (available on all hooks; `GH_OST_MIGRATED_HOST` also matches it once the applier connection is initialized) +- `GH_OST_BINLOG_WRITER_LAG_SECONDS` and `GH_OST_HEARTBEAT_LAG` - writer lag in seconds derived from applied binlog event timestamps +- `GH_OST_DRAIN_GTID` (only in `gh-ost-on-success`) - source GTID position captured immediately after the cut-over `RENAME TABLE` (`@@global.gtid_executed` on MySQL, `@@global.gtid_binlog_pos` on MariaDB), representing the drain target the applier waits for (not the target cluster's replication position) + +Review existing OSC hooks before reusing them. Hooks may run more than once across initial and resumed attempts and should be idempotent. + +- On `--resume`, `gh-ost-on-startup` runs again; a pre-drain cut-over resume skips row-copy hooks and `gh-ost-on-before-cut-over`. +- Keep write-enabling and routing in `gh-ost-on-success` synchronous; do not return success while those operations run in the background. +- Do not blindly roll back in `gh-ost-on-failure`; target writes may already be enabled. + +See [cut-over](move-tables.md#cut-over) and [recovery](move-tables.md#cleanup-and-recovery). + ### Examples See [sample hooks](https://github.com/github/gh-ost/tree/master/resources/hooks-sample), as `bash` implementation samples. diff --git a/doc/move-tables-design.md b/doc/move-tables-design.md new file mode 100644 index 000000000..1f641356e --- /dev/null +++ b/doc/move-tables-design.md @@ -0,0 +1,101 @@ +# Move-Tables Design Doc + +For setup and recovery procedures, see the [user guide](move-tables.md). + +## Architecture + +| Aspect | Standard `gh-ost` | Move-tables (`--move-tables`) | +| --- | --- | --- | +| **Scope** | Single table altered in place via `_
_gho` | One or more tables copied directly to `--target-host` | +| **Bookkeeping** | `_
_ghc` changelog/heartbeat + optional `_
_ghk` on source | No `_ghc` table; required checkpoint table on target | +| **Data path** | `INSERT IGNORE ... SELECT` on the source primary | `SELECT` chunks + binlog stream from `--host`, applied to `--target-host` | +| **Cut-over** | Two-session lock + atomic swap with `_
_gho` | Source `RENAME TABLE` to `_
_del` + drain to the post-rename source GTID | + +Schema inspection, row counts, chunk reads, and binlog streaming run on `--host`, normally a source replica. The writable source primary handles the advisory lock, table-type/resume checks, cut-over rename and GTID capture, and deletion-table cleanup. + +Tables have independent schema metadata, unique keys, and chunk ranges, but share one binlog streamer and target writer loop. Chunk copies run round-robin; binlog DML takes priority. The target checkpoint table stores one row per table per checkpoint, combining per-table key ranges with run-wide GTID coordinates and cut-over markers. + +Source replication to `--host`, gh-ost copy/replay to the target primary, and native target replication to its replicas advance independently. Target throttling does not establish source-replica readiness, and the drain GTID (`GH_OST_DRAIN_GTID`) is a **source** stream coordinate rather than a target replication position. + +`GET_LOCK('gh-ost-move-tables', 0)` serializes moves per source primary. It does not lock the target, coordinate with standard OSC, protect overlapping source/target roles, or fence primary failover and external sessions. + +## Lifecycle and cut-over (`T0`–`T6`) + +```mermaid +sequenceDiagram + actor Op as Operator / App + participant G as gh-ost + participant SP as Source Primary + participant SR as Source Replica + participant TP as Target Primary + + Note over Op,TP: Precondition: no app writes to target tables + G->>SP: Validate, acquire GET_LOCK + G->>TP: Create target and checkpoint tables + loop Copy and replay + G->>SR: Read chunks and binlog events + G->>TP: Apply rows, DML, and checkpoints + end + Note over G: All tables copied: postpone cut-over + Op->>G: Release postpone gate + G->>Op: [T0] on-before-cut-over hook + Note over G,SP: Single round trip + G->>SP: [T1] RENAME to _del + G->>SP: [T2] Capture drain GTID + Note over Op,SP: Original table names now unavailable + G->>TP: Pre-drain checkpoint + SP-->>SR: Replicate RENAME + G->>TP: [T3] Apply binlog from SR through drain GTID + G->>TP: [T4] Completed-drain checkpoint + G->>Op: [T5] on-success hook (GH_OST_DRAIN_GTID) + Op->>TP: Enable target writes, switch traffic + G->>G: [T6] Cleanup +``` + +Cut-over disables throttling. The `T0`–`T6` labels match code comments, tests, and logs. + +| Phase | Behavior | +| --- | --- | +| `T0`: Before-cut-over hook | `gh-ost-on-before-cut-over` runs synchronously; a non-zero exit aborts before source DDL. | +| `T1`: Rename | `RENAME TABLE` moves all source tables to `_
_del` in one statement; original names become unavailable. Never retried. | +| `T2`: Capture drain GTID | In the same round trip, reads `@@global.gtid_executed` (MySQL) or `@@global.gtid_binlog_pos` (MariaDB), so the boundary covers every write before the rename. A target checkpoint then records `cutover_started=1`, `is_cutover=0`, and the drain GTID. The round trip is not atomic: if `T1` commits but `T2` or this checkpoint fails, resume refuses and recovery is manual. | +| `T3`: Drain | Polls every `100ms`, up to `--cut-over-lock-timeout-seconds` (default `60s`), until queues are empty, no apply handler is active, and the applier reaches the `T2` GTID. For non-row tail events, streamer progress must hold for two consecutive polls. | +| `T4`: Record completion | Persists `gh_ost_is_cutover=1` and lets the streamer exit. A failed write aborts before `T5`. | +| `T5`: Success hook | `gh-ost-on-success` runs with `GH_OST_DRAIN_GTID`; automation can enable target writes and switch traffic. | +| `T6`: Cleanup | Closes the streamer; keeps or drops source `_
_del` tables and the checkpoint table per `--ok-to-drop-table`. | + +The sampled drain checks are not a synchronized transaction barrier. The drain timeout excludes rename lock waits, hooks, and routing; it does not cap total outage. + +## Correctness and boundaries + +### Assumptions + +- Target application writes stay disabled until `T4` persists `gh_ost_is_cutover=1`; enable them through or after `gh-ost-on-success`. +- No external DDL alters moved or `_
_del` tables during the move or recovery. +- Prior `gh-ost` and hook processes have stopped before `--resume`. +- All source and target hosts use the same database flavor (checked at startup). + +### Replay and transaction visibility + +Checkpoints record the applier's last applied source position. Replay starts at the earliest per-table coordinate; `INSERT IGNORE` chunks and idempotent DML (`INSERT IGNORE`, `UPDATE`, `DELETE`) allow convergence. + +This tracking does **not** preserve source transaction boundaries for target readers. Asynchronous batches can expose partially applied multi-row or cross-table transactions during copying and postponed cut-over. + +### Cut-over safety + +1. `RENAME TABLE` waits for transactions holding metadata locks on the moved tables. After rename, statements resolving the original names fail unless external DDL recreates them. +2. The post-rename GTID boundary includes preceding writes to every moved table. +3. `T3` checks source progress and observed target apply work before declaring the drain complete. +4. Recording completion before the success hook prevents resume from replaying source changes over application writes the hook may enable. + +Because `gh-ost` neither routes traffic nor fences target writes, this is not an atomic cross-cluster handoff. Rename crash atomicity depends on server version and storage engines ([MySQL](https://docs.oracle.com/cd/E17952_01/mysql-8.0-en/atomic-ddl.html), [MariaDB](https://mariadb.com/docs/server/reference/sql-statements/data-definition/rename-table)) and is not validated by `gh-ost`; partial renames require manual recovery. + +## Failure boundaries and resume + +`--resume` preflight validates target cut-over checkpoints and source primary table state before running `gh-ost-on-startup` or starting binlog streaming: + +- **Copy checkpoint (`cutover_started=0`, `is_cutover=0`) + original source `BASE TABLE`s intact:** resumes chunk copying from per-table key ranges and the earliest per-table checkpoint coordinate. +- **Pre-drain checkpoint (`cutover_started=1`, `is_cutover=0`, non-empty drain GTID) + only `_
_del` `BASE TABLE`s on source:** resumes replay/drain without repeating the rename, postpone gate, or `gh-ost-on-before-cut-over`, then persists `T4` and runs `gh-ost-on-success`. +- **Fail-closed states:** refuses resume if `gh_ost_is_cutover=1` (target writes may already be enabled), if cut-over coordinates are missing, or if source table state disagrees with checkpoint state (including interruptions between `T1` and the pre-drain checkpoint, or partial renames). + +`gh_ost_is_cutover=1` does not prove traffic switched, cleanup errors do not undo cut-over, and `_
_del` receives no target-side writes. For operator inspection queries and recovery steps, see the [user guide](move-tables.md#cleanup-and-recovery). diff --git a/doc/move-tables.md b/doc/move-tables.md new file mode 100644 index 000000000..e8a518c5c --- /dev/null +++ b/doc/move-tables.md @@ -0,0 +1,192 @@ +# Moving Tables Across Clusters + +`gh-ost`'s `--move-tables` mode copies tables to another cluster without a schema change, replays ongoing source writes, and waits for you to initiate cut-over. + +This guide assumes familiarity with normal online schema changes (OSC). See the [design doc](move-tables-design.md) for the architecture and correctness argument, and the [flag reference](command-line-flags.md#move-tables) for individual options. + +## What changes from normal OSC? + +| Normal OSC | Move-tables | +| --- | --- | +| One table, with a schema change | One or more tables, without a schema change; use `--move-tables` instead of `--table` and `--alter` | +| Rows are copied by `INSERT ... SELECT` on the source primary | Rows are read in chunks from `--host`, preferably a source replica, and written to `--target-host` | +| Writes go to a ghost table on the same cluster | Writes go directly to the same table names on `--target-host`, optionally in a different database | +| Cut-over swaps the original and ghost tables | Cut-over renames the source tables to `_
_del`, then drains remaining changes to the target | +| Application connections continue using the same cluster | You must switch application traffic and enable target writes | +| `--test-on-replica` and `--revert` are available | Neither is supported; test on disposable tables and plan recovery before starting | + +> [!IMPORTANT] +> **`gh-ost` does not route application traffic, fence application writes, checksum data, or roll back automatically.** Keep target application writes disabled until cut-over completes. Unlike an in-place OSC swap, source table names become unavailable during cut-over and remain unavailable if the move fails after renaming them. + +## Before you start + +- Use the same database flavor across the source primary, source read host, and target primary: MySQL/Percona 8.0+ or MariaDB 10.5+. On MariaDB < 10.6, a crash during multi-table rename can leave a partial rename. +- Point `--host` at a source replica with binary logging enabled, `binlog_format=ROW`, `binlog_row_image=FULL`, and `log_slave_updates=ON`. Enable GTIDs on MySQL (`gtid_mode=ON`, `enforce_gtid_consistency=ON`); MariaDB records GTIDs when binary logging is enabled. +- The source tables must exist with identical schemas on the source primary and read replica. Only `BASE TABLE` is supported. The usual [key, foreign-key, and trigger limitations](requirements-and-limitations.md#limitations) still apply. +- Point `--target-host` at a writable target primary. The target database must already exist, with none of the moved table names present. Source `_
_del` names must also be available for cut-over. +- Keep target application writes disabled, avoid external source DDL during the move, and pause automated cleanup of source `_
_del` tables and target checkpoint tables. +- Ensure the machine running `gh-ost` can reach all three roles below. Allow only one move at a time per source primary; the source lock does not coordinate operations on the target. + +### Connections and credentials + +| Role | Credentials and required access | +| --- | --- | +| Source read host (`--host`) | Source credentials from `--user` / `--password`, `--ask-pass`, or `--conf`. Requires the [standard gh-ost grants](requirements-and-limitations.md#requirements). | +| Source primary | Discovered by traversing the source replication topology, using source credentials by default. It must allow metadata inspection, the advisory lock, cut-over `RENAME TABLE`, GTID capture, and optional deletion-table cleanup. | +| Target primary (`--target-host`) | Defaults to source credentials; override with `--target-user` and `--target-password`. Requires schema inspection and `CREATE`, `DROP`, `SELECT`, `INSERT`, `UPDATE`, and `DELETE` on the target database. | + +On MariaDB, ensure source credentials can read replica status; this requires `SLAVE MONITOR` or `SUPER` on current releases. + +Use [`--assume-master-host`](command-line-flags.md#assume-master-host) to specify the source primary explicitly; [`--master-user`](command-line-flags.md#master-user) and [`--master-password`](command-line-flags.md#master-password) can then supply different credentials for it. Target-replica lag checks use the target credentials and need access to replication status on those replicas. + +For a lab without a source replica, point `--host` at the source primary and add [`--allow-on-source-primary`](command-line-flags.md#allow-on-source-primary), not `--allow-on-master`. This puts the full copy load on the primary; prefer a replica for production. + +## Start a move + +### 1. Run a noop check + +Verify `ROW` / `FULL` binlogging on the source replica, then run without `--execute`: + +```shell +gh-ost \ + --move-tables="orders" \ + --host=source-replica.example.com --database="app_db" \ + --user="gh-ost" --ask-pass --assume-rbr \ + --target-host=target-primary.example.com \ + --postpone-cut-over-flag-file=/tmp/gh-ost.move-tables.postpone.flag \ + --verbose +``` + +`gh-ost` validates connections and schemas, creates the target and `_gho__ghk` checkpoint tables, then drops them and exits 0. A noop is **not read-only**, and it skips row copy, cut-over, and `gh-ost-on-success`; other hooks may run. A failed noop may leave target tables behind; see [abandoning before the source rename](#abandon-before-the-source-rename). + +`--assume-rbr` skips restarting source replication to apply binlog settings; it does not configure them. + +### 2. Copy and replay + +Rerun the same command with `--execute`, and keep it running. `gh-ost` creates the postpone flag file, copies rows, and replays ongoing source changes to the target. + +Copying is done when the status shows `State: postponing cut-over` with every table `complete` (see [status output](understanding-output.md#move-tables-progress)). Changes keep replaying until you start cut-over. To spot-check replay, update a row on the source primary and read it on the target primary. When ready, continue to [cut-over](#cut-over). + +### Options + +| Flag | Use | +| --- | --- | +| `--move-tables="orders,order_items"` | Move several tables together; cut-over waits for all of them to finish copying. | +| `--target-database` | Write to a different existing database on the target. | +| `--max-load`, `--throttle-query` | Throttle on the **target primary**'s load. | +| `--critical-load` | Abort on the **target primary**'s load. | +| `--throttle-control-replicas`, `--max-lag-millis` | Throttle on **target replica** lag. | +| `--hooks-path` | Integrate the application, for example to switch traffic in `gh-ost-on-success`. Review existing OSC hooks first; see [move-tables hooks](hooks.md#--move-tables-mode). | +| `--checkpoint-seconds` | Checkpoint interval (default `300`); resume needs at least one checkpoint. | + +Throttling protects the target only. Monitor source load and replication separately; see [throttling](throttle.md#move-tables-mode). + +While cut-over is postponed, you may route reads to the target if the application tolerates stale reads and partially applied transactions. Keep writes on the source. + +## Cut-over + +### Before releasing the postpone gate + +Keep the postpone gate closed until you are ready. Removing its file during copying allows cut-over as soon as copying finishes; `unpostpone` is accepted only while actively postponing cut-over. + +1. Verify the source replica is caught up and healthy; `WriterLag: 0` and an empty backlog do not prove this. +2. Wait for `State: postponing cut-over` with all tables marked `complete`, and check that `WriterLag` and `Backlog` are near zero. +3. Ensure gh-ost is not throttled, keep target application writes disabled, and prepare routing for the source outage. +4. If reads will use target replicas, plan to verify they have replayed the target primary's writes. `GH_OST_DRAIN_GTID` is a **source** stream boundary, not the target's replication position. + +The drain timeout ([`--cut-over-lock-timeout-seconds`](command-line-flags.md#cut-over-lock-timeout-seconds), default `60s`) is **not an upper bound on application outage**: rename lock waits, hooks, and routing add time, and a failed drain leaves the source tables renamed. + +### Release the gate + +Delete the flag file, or send `unpostpone` through the [interactive interface](interactive-commands.md): + +```shell +rm /tmp/gh-ost.move-tables.postpone.flag +``` + +`gh-ost` runs `gh-ost-on-before-cut-over`, renames the source tables to `_
_del`, drains remaining source events to the target primary, records drain completion, runs `gh-ost-on-success` with `GH_OST_DRAIN_GTID` set, and performs final cleanup. + +### Switch application traffic + +**Enable target writes and switch traffic in `gh-ost-on-success`, or manually after the `--execute` run exits cleanly.** When switching manually, omit `--ok-to-drop-table` and clean up later so table drops do not extend the outage. + +Confirm the source primary has `_orders_del`, the target serves `orders`, and application writes to the target succeed. + +### If cut-over does not exit cleanly + +- **Drain timeout:** the source tables stay renamed. See [recovery](#cleanup-and-recovery). +- **`move-tables cutover completed but cleanup failed`:** the drain finished, and `gh-ost-on-success` may already have enabled target writes. Do not rerun or resume to get a clean exit; confirm state via [recovery](#cleanup-and-recovery) before finishing any traffic switch. + +## Cleanup and recovery + +By default, `gh-ost` retains source `_
_del` tables and the target `_gho__ghk` checkpoint table, and logs their `DROP TABLE` commands. [`--ok-to-drop-table`](command-line-flags.md#ok-to-drop-table) drops both after `gh-ost-on-success`; omit it while rollback remains an option. + +> [!IMPORTANT] +> **Rollback is unsafe once target writes begin**, even with `_
_del` retained: target writes are not replicated back to the source. A non-zero exit does not prove cut-over did not happen, and a log line alone is not grounds to enable writes. + +Before any recovery action, **stop the old `gh-ost` process and all hooks**, then inspect both clusters and hook logs. If target-write history is unknown, assess recovery manually rather than resuming or rolling back. + +### Inspect source and checkpoint state + +On the **source primary** (not a possibly lagging replica), check each moved table and its `_
_del` pair: + +```sql +SELECT TABLE_NAME, TABLE_TYPE +FROM information_schema.tables +WHERE TABLE_SCHEMA = 'app_db' + AND TABLE_NAME IN ('orders', '_orders_del'); +``` + +Before rename, only the original name should exist; afterward, only `_
_del`. Both, neither, or a mix across tables is inconsistent. + +On the **target primary**, check whether the moved tables and `_gho__ghk` exist; `` is the run ID in the first log line (`Moving tables [...] (run )`). If the checkpoint table exists, read its most significant row; cut-over markers outrank later copy checkpoints: + +```sql +SELECT gh_ost_chk_id, gh_ost_chk_table_name, gh_ost_chk_coords, + gh_ost_move_tables_cutover_started, gh_ost_is_cutover, + gh_ost_move_tables_drain_gtid +FROM `app_db`.`_gho__ghk` +ORDER BY gh_ost_is_cutover DESC, + gh_ost_move_tables_cutover_started DESC, + gh_ost_chk_id DESC +LIMIT 1; +``` + +Markers are run-wide values alongside per-table copy state: + +- `gh_ost_move_tables_cutover_started=0`, `gh_ost_is_cutover=0`: copy checkpoint. Compare source table state; rename may have happened without a cut-over checkpoint. +- `gh_ost_move_tables_cutover_started=1`, `gh_ost_is_cutover=0`, non-empty `gh_ost_move_tables_drain_gtid`: pre-drain checkpoint. +- `gh_ost_is_cutover=1`: drain completed; hook/routing status unknown. +- No table or rows: no recorded resume state. **Absence of a marker does not prove cut-over never happened**; `--ok-to-drop-table` cleanup can remove the table. + +These queries do not replace resume preflight or hook/application checks. **Do not edit markers to force a resume.** + +### Decide whether to resume + +Use the first matching row. Resume requires usable checkpoint coordinates and retained source binlogs. + +| Observed state | Next action | +| --- | --- | +| Target writes were enabled | Do not resume or rename source tables back. Recovery must account for target-side writes. | +| Completed-drain marker set, whether or not the success hook finished | `--resume` refuses. Confirm source/hook/application state, then finish pending traffic switching or assess manual recovery. | +| Rename outcome unknown, partial rename, or source/checkpoint mismatch | Recover manually; do not retry the rename blindly. | +| All source tables renamed; usable pre-drain checkpoint and target tables exist | Resume draining, or roll back manually. Prepare routing first: resume skips the postpone gate and `gh-ost-on-before-cut-over`, then runs `gh-ost-on-success` after draining. | +| Original source names intact; usable copy checkpoint and target tables exist | Resume copying, or abandon the move. | +| No usable checkpoint or required binlogs unavailable | Resume is unavailable. Abandon if original source names are intact; otherwise assess manual recovery. | + +Rerun the original command with `--resume --execute`, preserving the table set, source database, and target host/database. Preflight validates source tables and target cut-over checkpoints before startup hooks or streaming. See [resuming migrations](resume.md#resuming---move-tables-runs). + +### Abandon before the source rename + +If the original source names are intact and target writes were never enabled, route any target reads back to the source. Then drop **only** the target tables this attempt created, never pre-existing ones. + +### Roll back after the source rename + +**Only if target writes were never enabled:** rename the `_
_del` tables back on the source primary, let source replicas catch up, route any target reads back to the source, and drop this attempt's target tables: + +```sql +-- Source primary; for multiple tables, rename them all in one statement +RENAME TABLE `app_db`.`_orders_del` TO `app_db`.`orders`; +-- Target primary; substitute the run ID for +DROP TABLE `app_db`.`orders`, `app_db`.`_gho__ghk`; +``` diff --git a/doc/resume.md b/doc/resume.md index 6f12ab7a3..f91f42fb8 100644 --- a/doc/resume.md +++ b/doc/resume.md @@ -19,7 +19,7 @@ gh-ost \ --host=replica1.company.com \ --database="mydb" \ --table="mytable" \ ---alter="add column mycol varchar(20)" +--alter="add column mycol varchar(20)" \ --gtid \ --checkpoint \ --checkpoint-seconds=60 \ @@ -30,14 +30,33 @@ In this example `gh-ost` writes a checkpoint to a table `_mytable_ghk` every 60 ```shell # resume migration gh-ost \ ---chunk-size=100 +--chunk-size=100 \ --host=replica1.company.com \ --database="mydb" \ --table="mytable" \ ---alter="add column mycol varchar(20)" +--alter="add column mycol varchar(20)" \ --gtid \ --resume \ --execute ``` `gh-ost` then reconnects at the binlog coordinates of the last checkpoint and resumes copying rows at the chunk specified by the checkpoint. The data integrity of the ghost table is preserved because `gh-ost` applies row DMLs and copies row in an idempotent way. + +## Resuming `--move-tables` runs + +[`--move-tables`](move-tables.md) enables `--checkpoint` and `--gtid` automatically in the CLI, storing checkpoints in `_gho__ghk` in the **target** database. + +Before rerunning the original command with `--resume --execute`, confirm: + +- The previous `gh-ost` process and all hooks have stopped. +- Target writes were never enabled. +- Target tables, the checkpoint table, and required source binlogs remain intact. +- The moved table set, source database, and target host/database are unchanged. + +Preflight validates source tables and target cut-over checkpoints before startup hooks or streaming: + +- An interrupted row copy resumes copying. +- A pre-drain cut-over resumes draining, then runs `gh-ost-on-success`, without the postpone gate or `gh-ost-on-before-cut-over`. Prepare routing first. +- A completed drain or source/checkpoint mismatch causes resume to fail closed. + +See [move-tables recovery](move-tables.md#decide-whether-to-resume) for the decision table and alternatives. diff --git a/doc/throttle.md b/doc/throttle.md index 7b923b663..8a8e15e23 100644 --- a/doc/throttle.md +++ b/doc/throttle.md @@ -12,13 +12,15 @@ Since `gh-ost` is not based on triggers, but of reading binary logs, it controls `gh-ost` supports various means for controlling throttling behavior; it is operations friendly in that it allows the user greater, dynamic control of throttler behavior. +The changelog/heartbeat and source-master descriptions above apply to standard OSC. Cross-cluster moves monitor the target instead; see [move-tables mode](#move-tables-mode). + ### Throttling parameters and factors Throttling is controlled via the following explicit and implicit factors: #### Replication-lag -The recommended way of running `gh-ost` is by connecting it to a replica. It will figure out the master by traversing the topology. It is by design that `gh-ost` is throttle aware: it generates its own _heartbeat_ mechanism; while it is running the migration, it is self-checking the replica to which it is connected for replication lag. +For standard OSC, the recommended way of running `gh-ost` is by connecting it to a replica. It will figure out the master by traversing the topology. It is by design that `gh-ost` is throttle aware: it generates its own _heartbeat_ mechanism; while it is running the migration, it is self-checking the replica to which it is connected for replication lag. Otherwise you may specify your own list of replica servers you wish it to observe. @@ -106,3 +108,13 @@ Having said that, throttling for so long is far fetching, in that the `gh-ost` p It is worth mentioning that some deployments have external scheduled scripts that purge binary logs, regardless of the `expire_logs_days` configuration. Please verify your own deployment configuration. To clarify, you only need to keep binary logs on the single server `gh-ost` connects to. + +### Move-tables mode + +In [`--move-tables`](move-tables.md) mode, `gh-ost` writes to the target primary without a `_ghc` heartbeat table: + +- `--max-load`, `--critical-load`, and `--throttle-query` run on the **target primary**. +- `--throttle-control-replicas` checks **target-cluster replicas** via `SHOW SLAVE STATUS` / `SHOW REPLICA STATUS` (1s resolution, target credentials). +- The source replica is **not monitored automatically**. Monitor source load and replication health separately, particularly before cut-over. +- `WriterLag` is a status metric, not an automatic throttle threshold. A zero value and an empty backlog do not prove source-replica catch-up. +- Releasing the postpone gate suspends throttling for cut-over; prepare before releasing it. diff --git a/doc/understanding-output.md b/doc/understanding-output.md index e83da7d7d..b3323c91d 100644 --- a/doc/understanding-output.md +++ b/doc/understanding-output.md @@ -105,6 +105,20 @@ Notes: - `Copy: 31291200/43138418`, `Copy: 31389700/43138432`: this migration executed with `--exact-rowcount`. `gh-ost` continuously heuristically updates the total number of expected row copies as migration proceeds, hence the change from `43138418` to `43138432` - `streamer: mysql-bin.006793:179473435` tells us which binary log entry is `gh-ost` processing at this time. +### Move-tables progress + +In [`--move-tables`](move-tables.md) mode, `Lag` and `HeartbeatLag` are replaced by `WriterLag`, and each status line is followed by per-table progress. For example: + +```text +Copy: 1500/2000 75.0%; Applied: 42; Backlog: 0/1000; Time: 30s(total), 25s(copy); streamer: source-uuid:1-123; WriterLag: 0.00s, State: migrating; ETA: 8s + - `app_db`.`orders`: Copy 1000/1000 100.0%; iteration 1; complete + - `app_db`.`order_items`: Copy 500/1000 50.0%; iteration 1; copying +``` + +- The top line totals all moved tables; the indented lines show each table. +- `complete` means that table's initial row copy finished, not that cut-over finished; `gh-ost` continues replaying binlog changes and postpones cut-over once all tables finish copying. +- `WriterLag` is the idle-adjusted age of the last applied binlog event on the target (reset while no moved-table events arrive), not source- or target-replica catch-up; see [cut-over checks](move-tables.md#cut-over) and [move-tables throttling](throttle.md#move-tables-mode). + ### Status hint In addition, once every `10` minutes, a friendly reminder is printed, in the following form: From cdb5b22254ed0ff96e6ea97fa3b47c074fa8d8c5 Mon Sep 17 00:00:00 2001 From: Eric Yan Date: Thu, 12 Nov 2026 11:54:56 +0000 Subject: [PATCH 4/4] docs: address move-tables review feedback - Document cut-over-lock-timeout-seconds ranges and the fallback to 3s - Warn that --resume can skip part of a transaction (#1781) - Include the checkpoint table when abandoning a move --- doc/command-line-flags.md | 7 +++++-- doc/move-tables.md | 2 +- doc/resume.md | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 87d7b877b..65dbdf9b8 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -157,8 +157,11 @@ Wait exponentially longer intervals between failed cut-over attempts. The maximu ### cut-over-lock-timeout-seconds -- **Standard OSC:** default `3`. Maximum seconds to hold locks while attempting cut-over; a lock timeout causes a retry. -- **[`--move-tables`](#move-tables):** default `60`. Maximum seconds to drain the source stream and target applier through the source cut-over GTID after renaming the source tables. This is not a limit on rename lock waits or total application outage. A failed drain leaves the source tables renamed; it does not automatically retry the rename or roll back. +- **Standard OSC:** default `3`, range `1`–`10`. Maximum seconds to hold locks while attempting cut-over; a lock timeout causes a retry. +- **[`--move-tables`](#move-tables):** default `60`, range `1`–`60`. Maximum seconds to drain the source stream and target applier through the source cut-over GTID after renaming the source tables. This is not a limit on rename lock waits or total application outage. A failed drain leaves the source tables renamed; it does not automatically retry the rename or roll back. + +> [!WARNING] +> An out-of-range value logs an error and falls back to `3`, including in `--move-tables` mode. ### database diff --git a/doc/move-tables.md b/doc/move-tables.md index e8a518c5c..1ad2113d6 100644 --- a/doc/move-tables.md +++ b/doc/move-tables.md @@ -178,7 +178,7 @@ Rerun the original command with `--resume --execute`, preserving the table set, ### Abandon before the source rename -If the original source names are intact and target writes were never enabled, route any target reads back to the source. Then drop **only** the target tables this attempt created, never pre-existing ones. +If the original source names are intact and target writes were never enabled, route any target reads back to the source. Then drop **only** the target tables this attempt created, including the `_gho__ghk` checkpoint table (also after a failed noop), never pre-existing ones. A leftover checkpoint table blocks the next attempt for the same table set. ### Roll back after the source rename diff --git a/doc/resume.md b/doc/resume.md index f91f42fb8..a6a199335 100644 --- a/doc/resume.md +++ b/doc/resume.md @@ -8,7 +8,7 @@ To resume, invoke `gh-ost` again with the same arguments with the `--resume` flag. -> [!WARNING] +> [!NOTE] > It is recommended use `--checkpoint` with `--gtid` enabled so that checkpoint binlog coordinates store GTID sets rather than file positions. In that case, `gh-ost` can resume using a different replica than it originally attached to. ## Example @@ -42,6 +42,9 @@ gh-ost \ `gh-ost` then reconnects at the binlog coordinates of the last checkpoint and resumes copying rows at the chunk specified by the checkpoint. The data integrity of the ghost table is preserved because `gh-ost` applies row DMLs and copies row in an idempotent way. +> [!WARNING] +> A successful resume does not prove data completeness. A checkpoint can be written while a source transaction is only partially applied, so `--resume` may skip the rest of that transaction ([#1781](https://github.com/github/gh-ost/issues/1781)). If the previous run may have stopped mid-transaction, verify the affected rows before cut-over. + ## Resuming `--move-tables` runs [`--move-tables`](move-tables.md) enables `--checkpoint` and `--gtid` automatically in the CLI, storing checkpoints in `_gho__ghk` in the **target** database.