From 8132ca26f16a0f564b5630c72543e8e20c8f4579 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Fri, 4 Sep 2026 20:15:10 +0700 Subject: [PATCH] feat(sidebar): cross-engine copy, per-table row filters and server-side copies Claude-Session: https://claude.ai/code/session_011EqgjCjCAU6tiiVmnMpF86 --- CHANGELOG.md | 6 + .../CrossEngine/CanonicalColumnType.swift | 95 +++++ .../CrossEngineConversionNote.swift | 54 +++ .../CrossEngine/CrossEngineDefaultValue.swift | 142 +++++++ .../CrossEngineIndexTranslator.swift | 191 ++++++++++ .../CrossEngineStructureTranslator.swift | 277 ++++++++++++++ .../CrossEngine/CrossEngineValueCoercer.swift | 207 ++++++++++ TablePro/Core/CrossEngine/SQLTypeFamily.swift | 62 +++ .../CrossEngine/SQLTypeParser+Families.swift | 252 ++++++++++++ TablePro/Core/CrossEngine/SQLTypeParser.swift | 179 +++++++++ .../SQLTypeRenderer+BundledFamilies.swift | 289 ++++++++++++++ .../SQLTypeRenderer+RegistryFamilies.swift | 359 ++++++++++++++++++ .../Core/CrossEngine/SQLTypeRenderer.swift | 123 ++++++ .../ObjectCopy/ObjectCopyEligibility.swift | 68 +++- TablePro/Core/ObjectCopy/ObjectCopyPlan.swift | 106 +++++- .../Core/ObjectCopy/ObjectCopyPlanner.swift | 280 +++++--------- .../Core/ObjectCopy/ObjectCopyRequest.swift | 21 +- .../Core/ObjectCopy/ObjectCopyRowCopier.swift | 38 +- .../ObjectCopyServerSideInsert.swift | 124 ++++++ .../Core/ObjectCopy/ObjectCopySession.swift | 83 +++- .../ObjectCopy/ObjectCopyTableDraft.swift | 276 ++++++++++++++ .../Views/Export/ExportRowScopeEditor.swift | 1 + .../ObjectCopy/CopyObjectsListView.swift | 105 ++++- .../ObjectCopy/CopyObjectsReviewView.swift | 47 +++ .../CrossEngineStructureTranslatorTests.swift | 320 ++++++++++++++++ .../CrossEngineValueCoercerTests.swift | 167 ++++++++ .../Core/CrossEngine/SQLTypeParserTests.swift | 121 ++++++ .../CrossEngine/SQLTypeRendererTests.swift | 168 ++++++++ .../ObjectCopyEligibilityTests.swift | 59 ++- .../ObjectCopy/ObjectCopyRowCopierTests.swift | 70 +++- .../ObjectCopySelectQueryTests.swift | 68 ++++ .../ObjectCopyServerSideInsertTests.swift | 222 +++++++++++ .../ObjectCopy/ObjectCopySessionTests.swift | 63 ++- TableProUITests/CopyObjectsUITests.swift | 85 +++++ docs/features/copy-objects.mdx | 91 ++++- 35 files changed, 4580 insertions(+), 239 deletions(-) create mode 100644 TablePro/Core/CrossEngine/CanonicalColumnType.swift create mode 100644 TablePro/Core/CrossEngine/CrossEngineConversionNote.swift create mode 100644 TablePro/Core/CrossEngine/CrossEngineDefaultValue.swift create mode 100644 TablePro/Core/CrossEngine/CrossEngineIndexTranslator.swift create mode 100644 TablePro/Core/CrossEngine/CrossEngineStructureTranslator.swift create mode 100644 TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift create mode 100644 TablePro/Core/CrossEngine/SQLTypeFamily.swift create mode 100644 TablePro/Core/CrossEngine/SQLTypeParser+Families.swift create mode 100644 TablePro/Core/CrossEngine/SQLTypeParser.swift create mode 100644 TablePro/Core/CrossEngine/SQLTypeRenderer+BundledFamilies.swift create mode 100644 TablePro/Core/CrossEngine/SQLTypeRenderer+RegistryFamilies.swift create mode 100644 TablePro/Core/CrossEngine/SQLTypeRenderer.swift create mode 100644 TablePro/Core/ObjectCopy/ObjectCopyServerSideInsert.swift create mode 100644 TablePro/Core/ObjectCopy/ObjectCopyTableDraft.swift create mode 100644 TableProTests/Core/CrossEngine/CrossEngineStructureTranslatorTests.swift create mode 100644 TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift create mode 100644 TableProTests/Core/CrossEngine/SQLTypeParserTests.swift create mode 100644 TableProTests/Core/CrossEngine/SQLTypeRendererTests.swift create mode 100644 TableProTests/Core/ObjectCopy/ObjectCopyServerSideInsertTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index e2080ad68..bc18cca8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Copy To across database engines, with every type approximation listed before the copy runs. (#1491) +- Per-table `WHERE` and row limit in Copy To. (#1491) +- Server-side `INSERT … SELECT` when a copy's two sides are one connection. (#1491) + ### Fixed - SQL `IN`, `AND`, `OR`, `NOT`, `BY` and `ON` in the editor's plain text colour. (#2634) diff --git a/TablePro/Core/CrossEngine/CanonicalColumnType.swift b/TablePro/Core/CrossEngine/CanonicalColumnType.swift new file mode 100644 index 000000000..2de91db67 --- /dev/null +++ b/TablePro/Core/CrossEngine/CanonicalColumnType.swift @@ -0,0 +1,95 @@ +// +// CanonicalColumnType.swift +// TablePro +// +// One vocabulary every engine's column types can be said in. +// +// `PluginColumnKind` has five cases and `ColumnType` is documented as +// display-only, and neither keeps a length, a precision or a time zone. Both +// are enough to decide how to draw a cell and neither is enough to write a +// `CREATE TABLE` for a different engine, which is why a copy across engines +// was refused rather than approximated. This is the third vocabulary and the +// only one that carries what DDL needs. +// +// It is deliberately not a superset of every engine's type system. A type no +// family here can express arrives as `.unsupported` carrying the source's own +// spelling, so the renderer can say what it could not translate instead of +// guessing a shape for it. +// + +import Foundation + +/// What a column holds, said without naming an engine. +internal enum CanonicalTypeKind: Hashable, Sendable { + case boolean + /// Width in bytes, which is what decides the target spelling: 1 is a MySQL `TINYINT` and a + /// PostgreSQL `SMALLINT`, and no engine has a type for every width. + case integer(bytes: Int) + case decimal(precision: Int?, scale: Int?) + case floatingPoint(bits: Int) + case text(length: Int?, isFixed: Bool) + case binary(length: Int?, isFixed: Bool) + case date + case time(precision: Int?, hasTimeZone: Bool) + case timestamp(precision: Int?, hasTimeZone: Bool) + case interval + case uuid + case json + case xml + /// Carries its labels because every engine that lacks `ENUM` needs them to size the text + /// column that replaces it, and MySQL needs them to write the type back out. + case enumeration(values: [String]) + case bitString(length: Int?) + case money + case spatial + indirect case array(element: CanonicalTypeKind) + case unsupported +} + +internal struct CanonicalColumnType: Hashable, Sendable { + internal let kind: CanonicalTypeKind + /// MySQL's own modifier. Kept beside the kind rather than inside it because it doubles the + /// integer cases for one engine, and because it is the fact that forces a widening on every + /// other engine rather than a different type. + internal let isUnsigned: Bool + /// Exactly what the source called it, so a refusal can quote the user's own type name. + internal let sourceSpelling: String + + internal init(kind: CanonicalTypeKind, isUnsigned: Bool = false, sourceSpelling: String) { + self.kind = kind + self.isUnsigned = isUnsigned + self.sourceSpelling = sourceSpelling + } +} + +/// How close a rendered spelling is to what the source held. +/// +/// Ordered so a table's worst column decides what the review step says about the table. +internal enum CanonicalTypeFidelity: Int, Comparable, Sendable { + /// The target has the same type. Nothing to tell the user. + case exact + /// The target has no type this narrow, so a wider one is used. Every source value still fits. + case widened + /// The target has nothing equivalent and a different shape is used instead. Values survive as + /// text, constraints and semantics do not. + case approximated + /// Nothing sensible to write. The column is left out. + case unsupported + + internal static func < (lhs: CanonicalTypeFidelity, rhs: CanonicalTypeFidelity) -> Bool { + lhs.rawValue < rhs.rawValue + } +} + +internal struct RenderedColumnType: Sendable { + internal let spelling: String + internal let fidelity: CanonicalTypeFidelity + /// Why it is not exact, in the user's words. Nil when it is. + internal let reason: String? + + internal init(spelling: String, fidelity: CanonicalTypeFidelity = .exact, reason: String? = nil) { + self.spelling = spelling + self.fidelity = fidelity + self.reason = reason + } +} diff --git a/TablePro/Core/CrossEngine/CrossEngineConversionNote.swift b/TablePro/Core/CrossEngine/CrossEngineConversionNote.swift new file mode 100644 index 000000000..a559bc867 --- /dev/null +++ b/TablePro/Core/CrossEngine/CrossEngineConversionNote.swift @@ -0,0 +1,54 @@ +// +// CrossEngineConversionNote.swift +// TablePro +// +// One thing the copy could not carry across unchanged. +// +// Every note is produced before anything runs and read out in the review step, +// because a conversion the user finds out about afterwards is indistinguishable +// from a bug. A copy that says "TIMESTAMPTZ became DATETIME and the offset was +// dropped" is a decision; the same copy in silence is data loss. +// + +import Foundation + +internal struct CrossEngineConversionNote: Identifiable, Hashable, Sendable { + internal let table: String + /// The column or index this is about. Empty for something about the table itself. + internal let subject: String + /// What changed, in the form the review step lists: `created_at: TIMESTAMPTZ → DATETIME`. + internal let summary: String + internal let reason: String + internal let fidelity: CanonicalTypeFidelity + + internal init( + table: String, + subject: String, + summary: String, + reason: String, + fidelity: CanonicalTypeFidelity + ) { + self.table = table + self.subject = subject + self.summary = summary + self.reason = reason + self.fidelity = fidelity + } + + internal var id: String { "\(table)\u{1F}\(subject)\u{1F}\(summary)" } + + /// Whether the note is about something the user may want to change their mind over, rather + /// than a widening that keeps every value. + internal var isLossy: Bool { fidelity >= .approximated } +} + +internal extension Array where Element == CrossEngineConversionNote { + /// Lossy first, then by table and subject, so the review step opens on the ones that matter. + var orderedForReview: [CrossEngineConversionNote] { + sorted { lhs, rhs in + guard lhs.fidelity == rhs.fidelity else { return lhs.fidelity > rhs.fidelity } + guard lhs.table == rhs.table else { return lhs.table < rhs.table } + return lhs.subject < rhs.subject + } + } +} diff --git a/TablePro/Core/CrossEngine/CrossEngineDefaultValue.swift b/TablePro/Core/CrossEngine/CrossEngineDefaultValue.swift new file mode 100644 index 000000000..f5f7a819c --- /dev/null +++ b/TablePro/Core/CrossEngine/CrossEngineDefaultValue.swift @@ -0,0 +1,142 @@ +// +// CrossEngineDefaultValue.swift +// TablePro +// +// What a column's default becomes on the other engine. +// +// A default is the source's own SQL text and the target's DDL writer quotes +// whatever it does not recognise, so an untranslated `now()` does not fail: it +// becomes the literal string `'now()'` in every row, which is worse than +// failing because nothing reports it. Everything here is therefore either +// translated to the target's own spelling or dropped with a reason. +// +// `nextval(...)` is the one that is not a default at all. PostgreSQL renders a +// `SERIAL` as an ordinary integer whose default calls its sequence, and the +// sequence is a separate object written in PostgreSQL's DDL. Copied across +// engines the sequence cannot come with it, so the default becomes the target's +// own auto-increment instead, which is what a `SERIAL` meant in the first place. +// + +import Foundation + +internal enum CrossEngineDefaultValue { + internal enum Outcome: Equatable, Sendable { + case none + case keep(String) + /// The column carries the target's own generated-key attribute and no default text. + case autoIncrement + case drop(reason: String) + } + + internal static func translate( + _ value: String?, + kind: CanonicalTypeKind, + to target: SQLTypeFamily + ) -> Outcome { + guard let raw = value?.trimmingCharacters(in: .whitespacesAndNewlines), !raw.isEmpty else { + return .none + } + let stripped = strippingCast(raw) + let upper = stripped.uppercased() + + if upper.hasPrefix("NEXTVAL(") { return .autoIncrement } + if upper == "NULL" { return .keep("NULL") } + if let now = currentTimestamp(upper, target: target) { return .keep(now) } + if let today = currentDate(upper, target: target) { return .keep(today) } + if let boolean = booleanLiteral(upper, kind: kind, target: target) { return .keep(boolean) } + if isNumericLiteral(stripped) { return .keep(stripped) } + if let literal = stringLiteral(stripped) { return .keep(literal) } + if isBareWord(stripped) { return .keep(stripped) } + + return .drop(reason: String( + format: String(localized: "The default %@ is not written in a form this engine shares."), + raw + )) + } + + // MARK: - Shapes + + /// PostgreSQL writes a default's type onto it: `'active'::character varying`, `0::numeric`, + /// `'{}'::jsonb`. The cast is PostgreSQL's own syntax and every other engine rejects it, while + /// the value in front of it is portable. + internal static func strippingCast(_ value: String) -> String { + guard let range = value.range(of: "::") else { return value } + /// Only a cast that ends the expression. A `::` inside quotes is part of the value, and one + /// in the middle of a larger expression means the expression is not a bare literal anyway. + let head = String(value[value.startIndex.. String? { + let names = [ + "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP()", "NOW()", "GETDATE()", "GETUTCDATE()", + "SYSDATETIME()", "SYSDATE", "SYSTIMESTAMP", "LOCALTIMESTAMP", "LOCALTIMESTAMP()", + "STATEMENT_TIMESTAMP()", "TRANSACTION_TIMESTAMP()", "CLOCK_TIMESTAMP()" + ] + let matches = names.contains(upper) || upper.hasPrefix("CURRENT_TIMESTAMP(") + guard matches else { return nil } + return target == .clickhouse ? "now()" : "CURRENT_TIMESTAMP" + } + + private static func currentDate(_ upper: String, target: SQLTypeFamily) -> String? { + guard ["CURRENT_DATE", "CURRENT_DATE()", "CURDATE()", "TODAY()"].contains(upper) else { return nil } + return target == .clickhouse ? "today()" : "CURRENT_DATE" + } + + /// Every engine spells a boolean default differently and half of them have no boolean type at + /// all, so the literal follows the target rather than the source. PostgreSQL reports a `false` + /// default on a `boolean` column as `false`, and MySQL stores it in a `TINYINT(1)` as 0. + private static func booleanLiteral( + _ upper: String, + kind: CanonicalTypeKind, + target: SQLTypeFamily + ) -> String? { + guard case .boolean = kind else { return nil } + let isTrue = ["TRUE", "'T'", "T", "1", "'1'", "YES", "'YES'", "B'1'"].contains(upper) + let isFalse = ["FALSE", "'F'", "F", "0", "'0'", "NO", "'NO'", "B'0'"].contains(upper) + guard isTrue || isFalse else { return nil } + switch target { + case .postgres, .duckdb: + return isTrue ? "TRUE" : "FALSE" + case .clickhouse: + return isTrue ? "true" : "false" + case .mysql, .sqlite, .mssql, .oracle, .generic: + return isTrue ? "1" : "0" + } + } + + private static func isNumericLiteral(_ value: String) -> Bool { + guard !value.isEmpty else { return false } + return Int64(value) != nil || Double(value) != nil + } + + /// A quoted literal, which every engine here writes the same way. A doubled quote inside is the + /// escape all of them share; a backslash escape is MySQL's alone and is left to be dropped. + private static func stringLiteral(_ value: String) -> String? { + guard value.hasPrefix("'"), value.hasSuffix("'"), value.count >= 2 else { return nil } + guard !value.contains("\\") else { return nil } + let inner = value.dropFirst().dropLast() + var index = inner.startIndex + while index < inner.endIndex { + guard inner[index] == "'" else { + index = inner.index(after: index) + continue + } + let next = inner.index(after: index) + guard next < inner.endIndex, inner[next] == "'" else { return nil } + index = inner.index(after: next) + } + return value + } + + /// MySQL reports a string default without its quotes, so `active` on a `VARCHAR` arrives as a + /// bare word. Every target's DDL writer quotes an unrecognised word, which is the right answer + /// for it; what must not reach them is a word with a bracket or a space in it, because that is + /// an expression and quoting one turns it into a string. + private static func isBareWord(_ value: String) -> Bool { + guard !value.isEmpty, value.count <= 128 else { return false } + return value.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" || $0 == "-" || $0 == "." } + } +} diff --git a/TablePro/Core/CrossEngine/CrossEngineIndexTranslator.swift b/TablePro/Core/CrossEngine/CrossEngineIndexTranslator.swift new file mode 100644 index 000000000..141f477aa --- /dev/null +++ b/TablePro/Core/CrossEngine/CrossEngineIndexTranslator.swift @@ -0,0 +1,191 @@ +// +// CrossEngineIndexTranslator.swift +// TablePro +// +// Which of a table's indexes survive the crossing, and in what shape. +// +// An index is the part of a copy that fails loudest. A `GIN` index arrives at +// MySQL as a plain `INDEX` over a column that is now `JSON`, and MySQL refuses +// the whole `CREATE TABLE` for it; an index over a column that became +// `LONGTEXT` is refused with "used in key specification without a key length". +// Both take the table down with them, so an index that cannot be written is +// dropped here and named in the review instead. +// + +import Foundation +import TableProPluginKit + +internal enum CrossEngineIndexTranslator { + internal struct Result: Sendable { + internal let indexes: [EditableIndexDefinition] + internal let notes: [CrossEngineConversionNote] + } + + /// The length a text key is cut to where the engine needs one. 255 is what MySQL's own + /// tooling uses and it fits inside the 3072-byte limit of a four-byte character set. + private static let textKeyPrefix = 255 + + internal static func translate( + _ indexes: [EditableIndexDefinition], + table: String, + to family: SQLTypeFamily, + unboundedColumns: Set + ) -> Result { + var kept: [EditableIndexDefinition] = [] + var notes: [CrossEngineConversionNote] = [] + + for index in indexes { + /// The primary key is not an index the target creates separately: it comes out of + /// `primaryKeyColumns` inside the `CREATE TABLE`, and the column translation has + /// already bounded whatever it needed to. + guard !index.isPrimary else { + kept.append(index) + continue + } + guard let translated = translate( + index, table: table, to: family, unboundedColumns: unboundedColumns, notes: ¬es + ) else { continue } + kept.append(translated) + } + return Result(indexes: kept, notes: notes) + } + + private static func translate( + _ index: EditableIndexDefinition, + table: String, + to family: SQLTypeFamily, + unboundedColumns: Set, + notes: inout [CrossEngineConversionNote] + ) -> EditableIndexDefinition? { + guard supportsSecondaryIndexes(family) else { + notes.append(dropped(index, table: table, reason: String( + localized: "This engine does not take a secondary index in a CREATE TABLE." + ))) + return nil + } + + guard let type = translatedType(index.type, family: family) else { + notes.append(dropped(index, table: table, reason: String( + format: String(localized: "A %@ index has no equivalent on this engine."), + index.type.rawValue + ))) + return nil + } + + let unbounded = index.columns.filter { unboundedColumns.contains($0.lowercased()) } + if !unbounded.isEmpty, !supportsKeyPrefixes(family) { + notes.append(dropped(index, table: table, reason: String( + format: String( + localized: "%@ is unbounded text or binary here, which this engine cannot index." + ), + unbounded.joined(separator: ", ") + ))) + return nil + } + + var translated = index + translated.type = type + translated.columnPrefixes = keyPrefixes( + index, family: family, unboundedColumns: unboundedColumns + ) + /// A prefix on a unique index is not the same constraint. Two rows differing only after the + /// prefix collide, so the copy fails part way through the data phase on a table the source + /// considered valid, and where the rows do fit the target enforces less than the source did. + if !unbounded.isEmpty, index.isUnique { + notes.append(CrossEngineConversionNote( + table: table, + subject: index.name, + summary: String( + format: String(localized: "%1$@ becomes unique on the first %2$lld characters"), + index.name, textKeyPrefix + ), + reason: String( + format: String( + localized: "%@ is unbounded text here, which this engine indexes only by a prefix. Rows differing only past that are refused as duplicates." + ), + unbounded.joined(separator: ", ") + ), + fidelity: .approximated + )) + } + /// A partial index is PostgreSQL's, SQLite's and DuckDB's syntax. Elsewhere the clause is + /// dropped and the index becomes a full one, which indexes more rather than less. + if index.whereClause?.nilIfEmpty != nil, !supportsPartialIndexes(family) { + translated.whereClause = nil + notes.append(CrossEngineConversionNote( + table: table, + subject: index.name, + summary: String( + format: String(localized: "%@ stops being a partial index"), index.name + ), + reason: String( + localized: "Its WHERE clause is not supported here, so it covers every row." + ), + fidelity: .approximated + )) + } + return translated + } + + private static func dropped( + _ index: EditableIndexDefinition, + table: String, + reason: String + ) -> CrossEngineConversionNote { + CrossEngineConversionNote( + table: table, + subject: index.name, + summary: String(format: String(localized: "The index %@ is left out"), index.name), + reason: reason, + fidelity: .approximated + ) + } + + // MARK: - Family rules + + /// ClickHouse's `CREATE TABLE` takes a sorting key and data-skipping indexes, neither of which + /// is what a b-tree index from another engine means, and its driver writes neither. + private static func supportsSecondaryIndexes(_ family: SQLTypeFamily) -> Bool { + family != .clickhouse + } + + private static func supportsKeyPrefixes(_ family: SQLTypeFamily) -> Bool { + family == .mysql + } + + private static func supportsPartialIndexes(_ family: SQLTypeFamily) -> Bool { + family == .postgres || family == .sqlite || family == .duckdb + } + + private static func translatedType( + _ type: EditableIndexDefinition.IndexType, + family: SQLTypeFamily + ) -> EditableIndexDefinition.IndexType? { + switch type { + case .btree: + return .btree + case .hash: + return family == .mysql || family == .postgres ? .hash : .btree + case .fulltext, .spatial: + return family == .mysql ? type : nil + case .gin, .gist, .brin: + return family == .postgres ? type : nil + } + } + + /// A prefix is kept only where the engine has them, and one is added for a column the crossing + /// made unbounded. Carried to an engine without them the number is ignored by the driver, but + /// carrying a MySQL prefix into a MySQL copy of a bounded column is still what the source meant. + private static func keyPrefixes( + _ index: EditableIndexDefinition, + family: SQLTypeFamily, + unboundedColumns: Set + ) -> [String: Int] { + guard supportsKeyPrefixes(family) else { return [:] } + var prefixes = index.columnPrefixes + for column in index.columns where unboundedColumns.contains(column.lowercased()) { + prefixes[column] = min(prefixes[column] ?? textKeyPrefix, textKeyPrefix) + } + return prefixes + } +} diff --git a/TablePro/Core/CrossEngine/CrossEngineStructureTranslator.swift b/TablePro/Core/CrossEngine/CrossEngineStructureTranslator.swift new file mode 100644 index 000000000..4088c0864 --- /dev/null +++ b/TablePro/Core/CrossEngine/CrossEngineStructureTranslator.swift @@ -0,0 +1,277 @@ +// +// CrossEngineStructureTranslator.swift +// TablePro +// +// Says one engine's table in another engine's terms. +// +// It rewrites nothing the target driver already owns. Quoting, the primary +// key clause, index and foreign key syntax, `SERIAL` versus `AUTO_INCREMENT` +// versus `IDENTITY` are all decided by `generateCreateTableSQL` on the target +// side, and were already correct. What was source-native and reached the +// target unchanged is the list here: the type spelling, the default +// expression, the character set, the generation expression and the index +// kind. Translating exactly those is what turns a refusal into a copy. +// +// Same-family pairs return the snapshot they were given, byte for byte. A +// MySQL to MariaDB copy runs the path it always ran, which is the only way a +// change this wide can be trusted not to move what already worked. +// + +import Foundation +import TableProPluginKit + +internal enum CrossEngineStructureTranslator { + internal struct Result: Sendable { + internal let snapshot: TableStructureSnapshot + internal let notes: [CrossEngineConversionNote] + /// What each column held on the source, keyed by the column's name. + /// + /// The coercer needs both sides. Which reshaping a value needs is not answerable from + /// either alone: a boolean has to be recognised on the source, because `t` is boolean only + /// where the source said so, while a time zone has to be dropped according to the target, + /// because only the target knows whether it has one. + internal let sourceKinds: [String: CanonicalTypeKind] + /// What each written column holds on the target, keyed by the same name. + /// + /// Read back out of the spelling the renderer produced rather than carried over from the + /// source. They are not the same question: a PostgreSQL `timestamptz` is rendered as MySQL + /// `DATETIME`, which has no zone, and recording the source's answer told the coercer the + /// target still had one, so the offset it exists to strip was left on every value. + internal let targetKinds: [String: CanonicalTypeKind] + /// True when the source's own defaults could not come across as written, so the plan must + /// not also copy the sequences those defaults named. + internal let translated: Bool + } + + internal static func translate( + _ snapshot: TableStructureSnapshot, + from source: DatabaseType, + to target: DatabaseType + ) -> Result { + let targetFamily = SQLTypeFamily.of(target) + guard SQLTypeFamily.needsTranslation(from: source, to: target) else { + let kinds = kinds(of: snapshot, family: targetFamily) + return Result( + snapshot: snapshot, + notes: [], + sourceKinds: kinds, + targetKinds: kinds, + translated: false + ) + } + + let sourceFamily = SQLTypeFamily.of(source) + var notes: [CrossEngineConversionNote] = [] + var sourceKindsByColumn: [String: CanonicalTypeKind] = [:] + var kindsByColumn: [String: CanonicalTypeKind] = [:] + let keyColumns = Set(snapshot.primaryKeyColumns.map { $0.lowercased() }) + let indexed = Set( + snapshot.indexes.flatMap(\.columns).map { $0.lowercased() } + ).union(keyColumns) + + var columns: [EditableColumnDefinition] = [] + for column in snapshot.columns { + let outcome = translate( + column, + table: snapshot.name, + from: sourceFamily, + to: targetFamily, + isKeyColumn: keyColumns.contains(column.name.lowercased()) + ) + columns.append(outcome.column) + sourceKindsByColumn[outcome.column.name] = outcome.sourceKind + kindsByColumn[outcome.column.name] = outcome.targetKind + notes += outcome.notes + } + + let unboundedIndexColumns = Set( + columns + .filter { indexed.contains($0.name.lowercased()) && isUnbounded(kindsByColumn[$0.name]) } + .map { $0.name.lowercased() } + ) + let indexOutcome = CrossEngineIndexTranslator.translate( + snapshot.indexes, + table: snapshot.name, + to: targetFamily, + unboundedColumns: unboundedIndexColumns + ) + notes += indexOutcome.notes + + let translated = TableStructureSnapshot( + name: snapshot.name, + schema: snapshot.schema, + columns: columns, + indexes: indexOutcome.indexes, + foreignKeys: snapshot.foreignKeys, + /// `ENGINE=InnoDB`, a MySQL character set and a MySQL collation are all rejected + /// outright by every other engine's `CREATE TABLE`. + engine: nil, + charset: nil, + collation: nil + ) + return Result( + snapshot: translated, + notes: notes, + sourceKinds: sourceKindsByColumn, + targetKinds: kindsByColumn, + translated: true + ) + } + + /// What the columns of a table already on the target hold, for a copy that appends into it + /// rather than creating it. The same answer the translation produces, read from the other side. + internal static func kinds( + of snapshot: TableStructureSnapshot, + family: SQLTypeFamily + ) -> [String: CanonicalTypeKind] { + var kinds: [String: CanonicalTypeKind] = [:] + for column in snapshot.columns { + kinds[column.name] = SQLTypeParser.parse(column.dataType, family: family).kind + } + return kinds + } + + // MARK: - Columns + + private struct ColumnOutcome { + let column: EditableColumnDefinition + let sourceKind: CanonicalTypeKind + let targetKind: CanonicalTypeKind + let notes: [CrossEngineConversionNote] + } + + private static func translate( + _ column: EditableColumnDefinition, + table: String, + from sourceFamily: SQLTypeFamily, + to targetFamily: SQLTypeFamily, + isKeyColumn: Bool + ) -> ColumnOutcome { + let canonical = SQLTypeParser.parse(column.dataType, family: sourceFamily) + var rendered = SQLTypeRenderer.render(canonical, family: targetFamily) + var notes: [CrossEngineConversionNote] = [] + + if isKeyColumn, let bounded = boundedKeyType(rendered, kind: canonical.kind, family: targetFamily) { + rendered = bounded + } + + var translated = column + translated.dataType = rendered.spelling + translated.unsigned = targetFamily == .mysql && canonical.isUnsigned + /// Both name a source-side object. A `utf8mb4_0900_ai_ci` collation does not exist anywhere + /// but MySQL 8, and a character set clause is MySQL syntax outright. + translated.charset = nil + translated.collation = nil + translated.extra = nil + /// `ON UPDATE CURRENT_TIMESTAMP` is MySQL's alone; no other engine has a column-level one. + translated.onUpdate = targetFamily == .mysql ? column.onUpdate : nil + + if rendered.fidelity != .exact, let reason = rendered.reason { + notes.append(CrossEngineConversionNote( + table: table, + subject: column.name, + summary: "\(column.name): \(column.dataType) → \(rendered.spelling)", + reason: reason, + fidelity: rendered.fidelity + )) + } + + if let expression = column.generationExpression?.nilIfEmpty { + translated.generationExpression = nil + translated.generationKind = nil + notes.append(CrossEngineConversionNote( + table: table, + subject: column.name, + summary: String( + format: String(localized: "%@ stops being a computed column"), column.name + ), + reason: String( + format: String( + localized: "Its expression %@ is written in the source's own dialect, so the column is created as an ordinary one and its values are copied." + ), + expression + ), + fidelity: .approximated + )) + } + + let defaultOutcome = CrossEngineDefaultValue.translate( + column.defaultValue, kind: canonical.kind, to: targetFamily + ) + switch defaultOutcome { + case .none: + translated.defaultValue = nil + case .keep(let value): + translated.defaultValue = value + case .autoIncrement: + translated.defaultValue = nil + translated.autoIncrement = true + case .drop(let reason): + translated.defaultValue = nil + notes.append(CrossEngineConversionNote( + table: table, + subject: column.name, + summary: String(format: String(localized: "%@ loses its default"), column.name), + reason: reason, + fidelity: .approximated + )) + } + + /// Read back out of what was actually written, not carried over from what was read. The + /// renderer is the only thing that knows what it chose, and a spelling that lost a time + /// zone or turned an array into JSON has to say so or the coercer works from the wrong + /// side of the crossing. + let targetKind = SQLTypeParser.parse(rendered.spelling, family: targetFamily).kind + return ColumnOutcome( + column: translated, sourceKind: canonical.kind, targetKind: targetKind, notes: notes + ) + } + + // MARK: - Keys + + /// A key column cannot be unbounded text on the engines whose index entries are size-limited. + /// MySQL refuses `PRIMARY KEY` on a `LONGTEXT` outright, SQL Server caps a key at 900 bytes and + /// Oracle cannot index a `CLOB` at all, so the `CREATE TABLE` fails rather than the copy losing + /// anything. A bounded spelling is used for those columns instead, which is why it is a note. + private static func boundedKeyType( + _ rendered: RenderedColumnType, + kind: CanonicalTypeKind, + family: SQLTypeFamily + ) -> RenderedColumnType? { + guard isUnbounded(kind) else { return nil } + let spelling: String + switch family { + case .mysql: + spelling = isBinary(kind) ? "VARBINARY(255)" : "VARCHAR(255)" + case .mssql: + spelling = isBinary(kind) ? "VARBINARY(450)" : "NVARCHAR(450)" + case .oracle: + spelling = isBinary(kind) ? "RAW(2000)" : "VARCHAR2(2000)" + case .postgres, .sqlite, .clickhouse, .duckdb, .generic: + return nil + } + return RenderedColumnType( + spelling: spelling, + fidelity: .approximated, + reason: String( + format: String( + localized: "A key column cannot be unbounded here, so it is created as %@." + ), + spelling + ) + ) + } + + private static func isUnbounded(_ kind: CanonicalTypeKind?) -> Bool { + switch kind { + case .text(let length, _), .binary(let length, _): return length == nil + case .json, .xml, .spatial, .array: return true + default: return false + } + } + + private static func isBinary(_ kind: CanonicalTypeKind) -> Bool { + guard case .binary = kind else { return false } + return true + } +} diff --git a/TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift b/TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift new file mode 100644 index 000000000..4f02cb11b --- /dev/null +++ b/TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift @@ -0,0 +1,207 @@ +// +// CrossEngineValueCoercer.swift +// TablePro +// +// Reshapes the values a row carries so the other engine accepts them. +// +// Values cross as `PluginCellValue`, which is text, bytes or null, and each +// one is bound as a parameter rather than written into SQL. That is already +// enough for almost everything: a number, a string and a blob all mean the +// same thing to both sides. Three things do not, and each of them fails in a +// way that is hard to read from the far end of a long copy. +// +// PostgreSQL renders a boolean as `t` and `f`. Bound into a MySQL +// `TINYINT(1)` that is 0 in both cases outside strict mode, so every `true` in +// the table silently becomes `false`, and inside strict mode it is an error on +// the first row. MySQL renders a missing date as `0000-00-00`, which no other +// engine will accept at all. And a `TIMESTAMPTZ` arrives carrying `+07`, which +// a target column with no time zone rejects. +// +// Nothing here guesses at a value's meaning: each coercion is chosen from the +// target column's own canonical kind, so a `t` in a text column stays `t`. +// + +import Foundation +import TableProPluginKit + +internal struct CrossEngineValueCoercer: Sendable { + /// One written column, said on both sides of the crossing. + /// + /// Neither side answers on its own. `t` is a boolean because the *source* column was one, and + /// nothing about a MySQL `TINYINT(1)` says the value arriving is not the literal letter t. A + /// time zone is dropped because the *target* has none, and nothing about a PostgreSQL + /// `timestamptz` says where it is going. + internal struct ColumnPair: Sendable { + internal let source: CanonicalTypeKind? + internal let target: CanonicalTypeKind? + } + + private let pairs: [ColumnPair] + private let sourceFamily: SQLTypeFamily + private let positions: [Int] + + internal init(pairs: [ColumnPair], from sourceFamily: SQLTypeFamily) { + self.pairs = pairs + self.sourceFamily = sourceFamily + self.positions = pairs.enumerated() + .filter { Self.needsCoercion($0.element) } + .map(\.offset) + } + + /// Whether any column in this table needs looking at, so a table of numbers and strings runs + /// the same loop it ran before this existed. + internal var isNeeded: Bool { !positions.isEmpty } + + internal func coerce(_ row: [PluginCellValue]) -> [PluginCellValue] { + guard isNeeded else { return row } + var values = row + for index in positions where index < values.count { + values[index] = coerce(values[index], pair: pairs[index]) + } + return values + } + + // MARK: - One value + + private func coerce(_ value: PluginCellValue, pair: ColumnPair) -> PluginCellValue { + if Self.carriesBoolean(pair) { return boolean(value) } + switch pair.target { + case .date, .time, .timestamp: + return temporal(value, kind: pair.target) + case .json: + return json(value) + default: + return value + } + } + + /// A boolean the target will store as a number or a boolean. + /// + /// Not one going into text: a `boolean` column copied into a `VARCHAR` keeps whatever the + /// source wrote, because `t` is the value there rather than a spelling of one, and rewriting it + /// to `1` would change the data rather than carry it. + private static func carriesBoolean(_ pair: ColumnPair) -> Bool { + guard case .boolean = pair.source else { return false } + switch pair.target { + case .boolean, .integer, .decimal, .bitString, nil: return true + default: return false + } + } + + /// Every engine here reads `1` and `0` in a boolean column, and none of them reads all of + /// `t`, `true`, `yes` and `on`. A value that is none of those is left alone rather than + /// guessed at, so a column the source did not really use as a boolean fails visibly. + private func boolean(_ value: PluginCellValue) -> PluginCellValue { + switch value { + case .null: + return value + case .bytes(let data): + guard data.count == 1, let byte = data.first else { return value } + return .text(byte == 0 ? "0" : "1") + case .text(let text): + switch PluginSQLLiteral.booleanSynonym(for: text) { + case .isTrue: return .text("1") + case .isFalse: return .text("0") + case nil: return Self.singleLetterBoolean(text) ?? value + /// `PluginBooleanSynonym` is published by a resilient library, so it can gain a case a + /// build compiled against today's PluginKit has never seen. One it cannot name is one + /// it cannot spell for the target either, so the value is left as it came. + @unknown default: return value + } + } + } + + /// PostgreSQL's own rendering of a boolean, which `PluginSQLLiteral` does not cover because + /// `t` and `f` are ordinary text everywhere else. + private static func singleLetterBoolean(_ text: String) -> PluginCellValue? { + switch text.lowercased() { + case "t": return .text("1") + case "f": return .text("0") + default: return nil + } + } + + private func temporal(_ value: PluginCellValue, kind: CanonicalTypeKind?) -> PluginCellValue { + guard case .text(let text) = value else { return value } + if sourceFamily == .mysql, Self.isZeroDate(text) { return .null } + guard !hasTimeZone(kind) else { return value } + guard let stripped = Self.strippingTimeZoneOffset(text) else { return value } + return .text(stripped) + } + + /// A PostgreSQL array arrives as `{1,2,3}`, which a JSON column on the target rejects. The + /// elements are the same; only the brackets and the quoting differ. + private func json(_ value: PluginCellValue) -> PluginCellValue { + guard sourceFamily == .postgres, case .text(let text) = value else { return value } + guard text.hasPrefix("{"), text.hasSuffix("}") else { return value } + guard let elements = PostgresArrayLiteralCodec.parse(text) else { return value } + var items: [String] = [] + items.reserveCapacity(elements.count) + for element in elements { + switch element { + case .null: + items.append("null") + case .value(let raw): + items.append(Self.jsonScalar(raw)) + /// A case this build has never seen cannot be rendered as JSON, and a wrong guess would + /// be written into every row. The value is left as the array literal it arrived as. + @unknown default: + return value + } + } + return .text("[\(items.joined(separator: ","))]") + } + + private func hasTimeZone(_ kind: CanonicalTypeKind?) -> Bool { + switch kind { + case .time(_, let hasTimeZone), .timestamp(_, let hasTimeZone): return hasTimeZone + default: return false + } + } + + // MARK: - Shapes + + private static func needsCoercion(_ pair: ColumnPair) -> Bool { + if carriesBoolean(pair) { return true } + switch pair.target { + case .date, .time, .timestamp, .json: return true + default: return false + } + } + + internal static func isZeroDate(_ text: String) -> Bool { + text.hasPrefix("0000-00-00") + } + + /// The trailing `+07`, `+07:00`, `-0330` or `Z` a zone-aware value carries. Removed only when + /// what is left still looks like a date, so a plain `2024-01-01` and a negative number are both + /// left alone. + internal static func strippingTimeZoneOffset(_ text: String) -> String? { + guard let expression = zonedTimestamp else { return nil } + let subject = text as NSString + let whole = NSRange(location: 0, length: subject.length) + guard let match = expression.firstMatch(in: text, options: [], range: whole), + match.range == whole, match.numberOfRanges > 1 else { + return nil + } + return subject.substring(with: match.range(at: 1)) + } + + private static let zonedTimestamp: NSRegularExpression? = { + let pattern = "^(\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}(?::\\d{2}(?:\\.\\d+)?)?)" + + "\\s*(?:Z|[+-]\\d{2}(?::?\\d{2})?)$" + return try? NSRegularExpression(pattern: pattern, options: [.caseInsensitive]) + }() + + private static func jsonScalar(_ raw: String) -> String { + if raw == "true" || raw == "false" { return raw } + if Int64(raw) != nil || Double(raw) != nil { return raw } + let escaped = raw + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") + .replacingOccurrences(of: "\n", with: "\\n") + .replacingOccurrences(of: "\r", with: "\\r") + .replacingOccurrences(of: "\t", with: "\\t") + return "\"\(escaped)\"" + } +} diff --git a/TablePro/Core/CrossEngine/SQLTypeFamily.swift b/TablePro/Core/CrossEngine/SQLTypeFamily.swift new file mode 100644 index 000000000..d4f1a4433 --- /dev/null +++ b/TablePro/Core/CrossEngine/SQLTypeFamily.swift @@ -0,0 +1,62 @@ +// +// SQLTypeFamily.swift +// TablePro +// +// Which engines spell their column types the same way. +// +// Not the same grouping as `SqlDialect`, which exists to lex a script and puts +// DuckDB with SQLite because both take its string literals. Their type systems +// are not related at all: DuckDB has `HUGEINT`, `STRUCT`, `LIST` and a real +// `TIMESTAMPTZ`, and SQLite has five storage classes and accepts any spelling +// at all. Reusing that grouping would have rendered a DuckDB target as SQLite. +// +// Curated by name, like `SqlDialect.from(databaseTypeId:)` already is, because +// a type spelling is a fact about an engine and no capability the plugin +// registry publishes implies it. `DatabaseType` is open, so anything not named +// here is `.generic` and renders portable ANSI SQL rather than being refused. +// + +import Foundation + +internal enum SQLTypeFamily: String, Hashable, Sendable, CaseIterable { + case mysql + case postgres + case sqlite + case mssql + case oracle + case clickhouse + case duckdb + case generic + + internal static func of(_ type: DatabaseType) -> SQLTypeFamily { + familiesByTypeId[type.rawValue] ?? .generic + } + + /// Whether a copy between these two needs its types translated at all. + internal static func needsTranslation(from source: DatabaseType, to target: DatabaseType) -> Bool { + guard source != target else { return false } + let sourceFamily = of(source) + return sourceFamily != of(target) || sourceFamily == .generic + } + + private static let familiesByTypeId: [String: SQLTypeFamily] = [ + "MySQL": .mysql, + "MariaDB": .mysql, + "PostgreSQL": .postgres, + "Redshift": .postgres, + "CockroachDB": .postgres, + "PGlite": .postgres, + "AlloyDB": .postgres, + "Citus": .postgres, + "Greenplum": .postgres, + "SQLite": .sqlite, + "libSQL": .sqlite, + "Turso": .sqlite, + "Cloudflare D1": .sqlite, + "SQL Server": .mssql, + "Oracle": .oracle, + "Dameng": .oracle, + "ClickHouse": .clickhouse, + "DuckDB": .duckdb + ] +} diff --git a/TablePro/Core/CrossEngine/SQLTypeParser+Families.swift b/TablePro/Core/CrossEngine/SQLTypeParser+Families.swift new file mode 100644 index 000000000..6b3db3e4a --- /dev/null +++ b/TablePro/Core/CrossEngine/SQLTypeParser+Families.swift @@ -0,0 +1,252 @@ +// +// SQLTypeParser+Families.swift +// TablePro +// +// One reading per family, kept apart so a word can mean two things. +// + +import Foundation + +internal extension SQLTypeParser { + static func mysqlKind(base: String, params: String?) -> CanonicalTypeKind { + switch base { + case "BOOL", "BOOLEAN": return .boolean + /// `TINYINT(1)` is how MySQL stores a boolean, and how every MySQL ORM writes one. Read as + /// a one-byte integer it becomes a `SMALLINT` on the target and every `true` arrives as 1. + case "TINYINT": return integers(in: params).first == 1 ? .boolean : .integer(bytes: 1) + case "SMALLINT": return .integer(bytes: 2) + case "MEDIUMINT": return .integer(bytes: 3) + case "INT", "INTEGER": return .integer(bytes: 4) + case "BIGINT": return .integer(bytes: 8) + case "YEAR": return .integer(bytes: 2) + case "BIT": return .bitString(length: length(params)) + case "DECIMAL", "NUMERIC", "DEC", "FIXED": return decimalKind(params) + case "FLOAT": return .floatingPoint(bits: 32) + case "DOUBLE", "DOUBLE PRECISION", "REAL": return .floatingPoint(bits: 64) + case "CHAR", "NCHAR": return .text(length: length(params), isFixed: true) + case "VARCHAR", "NVARCHAR", "CHARACTER VARYING": return .text(length: length(params), isFixed: false) + case "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT": return .text(length: nil, isFixed: false) + case "BINARY": return .binary(length: length(params), isFixed: true) + case "VARBINARY": return .binary(length: length(params), isFixed: false) + case "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB": return .binary(length: nil, isFixed: false) + case "DATE": return .date + case "TIME": return .time(precision: length(params), hasTimeZone: false) + /// Both are wall-clock as far as DDL is concerned. `TIMESTAMP` is stored as UTC and read + /// back in the session's zone, but it declares no zone and its values arrive without one. + case "DATETIME", "TIMESTAMP": return .timestamp(precision: length(params), hasTimeZone: false) + case "ENUM": return .enumeration(values: labels(in: params)) + case "JSON": return .json + case "GEOMETRY", "POINT", "LINESTRING", "POLYGON", "MULTIPOINT", + "MULTILINESTRING", "MULTIPOLYGON", "GEOMETRYCOLLECTION": return .spatial + default: return .unsupported + } + } + + static func postgresKind(base: String, params: String?) -> CanonicalTypeKind { + switch base { + case "BOOL", "BOOLEAN": return .boolean + case "INT2", "SMALLINT", "SMALLSERIAL", "SERIAL2": return .integer(bytes: 2) + case "INT4", "INT", "INTEGER", "SERIAL", "SERIAL4": return .integer(bytes: 4) + case "INT8", "BIGINT", "BIGSERIAL", "SERIAL8": return .integer(bytes: 8) + case "NUMERIC", "DECIMAL": return decimalKind(params) + case "FLOAT4", "REAL": return .floatingPoint(bits: 32) + case "FLOAT8", "DOUBLE PRECISION": return .floatingPoint(bits: 64) + case "MONEY": return .money + case "CHAR", "BPCHAR", "CHARACTER": return .text(length: length(params), isFixed: true) + case "VARCHAR", "CHARACTER VARYING": return .text(length: length(params), isFixed: false) + case "TEXT", "NAME", "CITEXT": return .text(length: nil, isFixed: false) + case "BYTEA": return .binary(length: nil, isFixed: false) + case "DATE": return .date + case "TIME", "TIME WITHOUT TIME ZONE": return .time(precision: length(params), hasTimeZone: false) + case "TIMETZ", "TIME WITH TIME ZONE": return .time(precision: length(params), hasTimeZone: true) + case "TIMESTAMP", "TIMESTAMP WITHOUT TIME ZONE": + return .timestamp(precision: length(params), hasTimeZone: false) + case "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE": + return .timestamp(precision: length(params), hasTimeZone: true) + case "INTERVAL": return .interval + case "UUID": return .uuid + case "JSON", "JSONB": return .json + case "XML": return .xml + case "BIT", "VARBIT", "BIT VARYING": return .bitString(length: length(params)) + case "GEOMETRY", "GEOGRAPHY", "BOX", "CIRCLE", "LINE", "LSEG", "PATH", "POINT", "POLYGON": + return .spatial + default: return .unsupported + } + } + + /// SQLite declares an affinity, not a type, and stores whatever spelling the `CREATE TABLE` + /// used, so a table written by another tool carries that tool's words. The five affinity names + /// are read first and anything else falls through to the ANSI reading rather than to + /// `.unsupported`, which is what keeps a `VARCHAR(255)` in a SQLite file a `VARCHAR(255)`. + static func sqliteKind(base: String, params: String?) -> CanonicalTypeKind { + switch base { + case "INTEGER", "INT": return .integer(bytes: 8) + case "REAL": return .floatingPoint(bits: 64) + case "TEXT": return .text(length: nil, isFixed: false) + case "BLOB": return .binary(length: nil, isFixed: false) + case "NUMERIC": return decimalKind(params) + default: return ansiKind(base: base, params: params) + } + } + + static func mssqlKind(base: String, params: String?) -> CanonicalTypeKind { + switch base { + case "BIT": return .boolean + case "TINYINT": return .integer(bytes: 1) + case "SMALLINT": return .integer(bytes: 2) + case "INT", "INTEGER": return .integer(bytes: 4) + case "BIGINT": return .integer(bytes: 8) + case "DECIMAL", "NUMERIC": return decimalKind(params) + case "REAL": return .floatingPoint(bits: 32) + /// `FLOAT(n)` is 32 bits up to a mantissa of 24 and 64 above it, which is why the parameter + /// is read rather than assumed. `FLOAT` with none is 53, so 64. + case "FLOAT": return .floatingPoint(bits: (length(params) ?? 53) <= 24 ? 32 : 64) + case "MONEY", "SMALLMONEY": return .money + case "CHAR", "NCHAR": return .text(length: length(params), isFixed: true) + /// `varchar(max)` reaches here with no integer parameter, and so does a `varchar` with none + /// at all; both are unbounded as far as the target is concerned. + case "VARCHAR", "NVARCHAR": return .text(length: length(params), isFixed: false) + case "TEXT", "NTEXT": return .text(length: nil, isFixed: false) + case "BINARY": return .binary(length: length(params), isFixed: true) + case "VARBINARY", "IMAGE": return .binary(length: length(params), isFixed: false) + case "DATE": return .date + case "TIME": return .time(precision: length(params), hasTimeZone: false) + case "DATETIME", "DATETIME2", "SMALLDATETIME": + return .timestamp(precision: length(params), hasTimeZone: false) + case "DATETIMEOFFSET": return .timestamp(precision: length(params), hasTimeZone: true) + case "UNIQUEIDENTIFIER": return .uuid + case "XML": return .xml + case "GEOMETRY", "GEOGRAPHY": return .spatial + default: return .unsupported + } + } + + /// Oracle has one numeric type, so a whole-number column is a `NUMBER(p, 0)` and its precision + /// is the only thing that says how wide an integer the target needs. Read as a decimal, an + /// Oracle primary key arrives on MySQL as `DECIMAL(10,0)` and stops being an integer. + static func oracleKind(base: String, params: String?) -> CanonicalTypeKind { + switch base { + case "NUMBER", "DECIMAL", "NUMERIC", "DEC": + let numbers = integers(in: params) + guard numbers.count > 1, numbers[1] == 0, let precision = numbers.first else { + return decimalKind(params) + } + return .integer(bytes: integerWidth(forDecimalDigits: precision)) + case "INT", "INTEGER", "SMALLINT": return .integer(bytes: 4) + case "FLOAT", "BINARY_DOUBLE", "DOUBLE PRECISION": return .floatingPoint(bits: 64) + case "BINARY_FLOAT": return .floatingPoint(bits: 32) + case "CHAR", "NCHAR": return .text(length: length(params), isFixed: true) + case "VARCHAR", "VARCHAR2", "NVARCHAR2": return .text(length: length(params), isFixed: false) + case "CLOB", "NCLOB", "LONG": return .text(length: nil, isFixed: false) + case "ROWID", "UROWID": return .text(length: 18, isFixed: false) + case "BLOB", "LONG RAW", "BFILE": return .binary(length: nil, isFixed: false) + case "RAW": return .binary(length: length(params), isFixed: false) + /// Oracle's `DATE` carries hours, minutes and seconds. Copied to a `DATE` on any other + /// engine it loses the time of day silently. + case "DATE": return .timestamp(precision: 0, hasTimeZone: false) + case "TIMESTAMP": return .timestamp(precision: length(params), hasTimeZone: false) + case "TIMESTAMP WITH TIME ZONE", "TIMESTAMP WITH LOCAL TIME ZONE": + return .timestamp(precision: length(params), hasTimeZone: true) + case "XMLTYPE": return .xml + case "SDO_GEOMETRY": return .spatial + default: + return base.hasPrefix("INTERVAL") ? .interval : .unsupported + } + } + + static func clickHouseKind(base: String, params: String?) -> CanonicalTypeKind { + switch base { + case "BOOL", "BOOLEAN": return .boolean + case "INT8", "UINT8": return .integer(bytes: 1) + case "INT16", "UINT16": return .integer(bytes: 2) + case "INT32", "UINT32": return .integer(bytes: 4) + case "INT64", "UINT64": return .integer(bytes: 8) + case "INT128", "UINT128", "INT256", "UINT256": return .integer(bytes: 16) + case "FLOAT32": return .floatingPoint(bits: 32) + case "FLOAT64": return .floatingPoint(bits: 64) + case "DECIMAL", "DECIMAL32", "DECIMAL64", "DECIMAL128", "DECIMAL256": return decimalKind(params) + case "STRING": return .text(length: nil, isFixed: false) + case "FIXEDSTRING": return .text(length: length(params), isFixed: true) + case "DATE", "DATE32": return .date + case "DATETIME", "DATETIME64": return .timestamp(precision: length(params), hasTimeZone: false) + case "UUID": return .uuid + case "JSON", "OBJECT": return .json + case "ENUM", "ENUM8", "ENUM16": return .enumeration(values: labels(in: params)) + case "IPV4", "IPV6": return .text(length: 45, isFixed: false) + default: return .unsupported + } + } + + static func duckDBKind(base: String, params: String?) -> CanonicalTypeKind { + switch base { + case "BOOL", "BOOLEAN", "LOGICAL": return .boolean + case "TINYINT", "INT1", "UTINYINT": return .integer(bytes: 1) + case "SMALLINT", "INT2", "SHORT", "USMALLINT": return .integer(bytes: 2) + case "INTEGER", "INT", "INT4", "SIGNED", "UINTEGER": return .integer(bytes: 4) + case "BIGINT", "INT8", "LONG", "UBIGINT": return .integer(bytes: 8) + case "HUGEINT", "UHUGEINT": return .integer(bytes: 16) + case "DECIMAL", "NUMERIC": return decimalKind(params) + case "REAL", "FLOAT4", "FLOAT": return .floatingPoint(bits: 32) + case "DOUBLE", "FLOAT8": return .floatingPoint(bits: 64) + case "VARCHAR", "CHAR", "BPCHAR", "TEXT", "STRING": return .text(length: length(params), isFixed: false) + case "BLOB", "BYTEA", "BINARY", "VARBINARY": return .binary(length: nil, isFixed: false) + case "DATE": return .date + case "TIME": return .time(precision: length(params), hasTimeZone: false) + case "TIMETZ", "TIME WITH TIME ZONE": return .time(precision: length(params), hasTimeZone: true) + case "TIMESTAMP", "DATETIME": return .timestamp(precision: length(params), hasTimeZone: false) + case "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE": + return .timestamp(precision: length(params), hasTimeZone: true) + case "INTERVAL": return .interval + case "UUID": return .uuid + case "JSON": return .json + case "BIT", "BITSTRING": return .bitString(length: length(params)) + default: return .unsupported + } + } + + /// The words the SQL standard defines, for an engine no family here names. Nothing engine + /// specific belongs in it: a plugin added later reads its own types through this, and a guess + /// borrowed from one engine would be wrong for the next. + static func ansiKind(base: String, params: String?) -> CanonicalTypeKind { + switch base { + case "BOOL", "BOOLEAN": return .boolean + case "TINYINT", "BYTEINT": return .integer(bytes: 1) + case "SMALLINT", "INT2": return .integer(bytes: 2) + case "INT", "INTEGER", "INT4": return .integer(bytes: 4) + case "BIGINT", "INT8", "LONG": return .integer(bytes: 8) + case "DECIMAL", "NUMERIC", "DEC", "NUMBER": return decimalKind(params) + case "REAL", "FLOAT4": return .floatingPoint(bits: 32) + case "FLOAT", "DOUBLE", "DOUBLE PRECISION", "FLOAT8": return .floatingPoint(bits: 64) + case "CHAR", "CHARACTER", "NCHAR": return .text(length: length(params), isFixed: true) + case "VARCHAR", "CHARACTER VARYING", "NVARCHAR", "VARCHAR2", "STRING": + return .text(length: length(params), isFixed: false) + case "TEXT", "CLOB", "NCLOB": return .text(length: nil, isFixed: false) + case "BINARY", "VARBINARY", "BLOB", "BYTEA", "BYTES": + return .binary(length: length(params), isFixed: base == "BINARY") + case "DATE": return .date + case "TIME": return .time(precision: length(params), hasTimeZone: false) + case "TIMETZ", "TIME WITH TIME ZONE": return .time(precision: length(params), hasTimeZone: true) + case "TIMESTAMP", "DATETIME", "TIMESTAMP WITHOUT TIME ZONE": + return .timestamp(precision: length(params), hasTimeZone: false) + case "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE": + return .timestamp(precision: length(params), hasTimeZone: true) + case "INTERVAL": return .interval + case "UUID": return .uuid + case "JSON", "JSONB", "VARIANT": return .json + case "XML": return .xml + default: return .unsupported + } + } + + /// The narrowest integer that holds every value of that many decimal digits. Oracle and + /// Teradata both declare integers this way and nothing else says how wide the column is. + static func integerWidth(forDecimalDigits digits: Int) -> Int { + switch digits { + case ..<3: return 1 + case ..<5: return 2 + case ..<10: return 4 + case ..<19: return 8 + default: return 16 + } + } +} diff --git a/TablePro/Core/CrossEngine/SQLTypeParser.swift b/TablePro/Core/CrossEngine/SQLTypeParser.swift new file mode 100644 index 000000000..547321379 --- /dev/null +++ b/TablePro/Core/CrossEngine/SQLTypeParser.swift @@ -0,0 +1,179 @@ +// +// SQLTypeParser.swift +// TablePro +// +// Reads one engine's spelling of a column type into the canonical vocabulary. +// +// Every reading is family-specific even where the word is shared, because the +// same word means different things: Oracle's `DATE` carries a time and +// PostgreSQL's does not, MySQL's `TINYINT(1)` is a boolean and SQL Server's +// `TINYINT` is an unsigned byte, and `REAL` is 32 bits on PostgreSQL and 64 on +// SQLite. A shared table keyed on the word alone would get each of those +// backwards for one of the two engines. +// +// A word no family here knows becomes `.unsupported` carrying the source's own +// spelling. That is not a failure: the renderer turns it into the target's +// widest text type and the review step names it, which is what lets a copy +// carry a PostGIS geometry or a ClickHouse tuple across as text rather than +// refusing the whole table. +// + +import Foundation + +internal enum SQLTypeParser { + internal static func parse(_ spelling: String, family: SQLTypeFamily) -> CanonicalColumnType { + let trimmed = spelling.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { + return CanonicalColumnType(kind: .unsupported, sourceSpelling: spelling) + } + + var body = stripWrappers(trimmed, family: family) + let unsigned = hasUnsignedModifier(body) + if unsigned { body = removingModifiers(body) } + + if let element = arrayElement(of: body, family: family) { + let inner = parse(element, family: family) + return CanonicalColumnType( + kind: .array(element: inner.kind), isUnsigned: inner.isUnsigned, sourceSpelling: spelling + ) + } + + let (base, params) = split(body) + let kind = kind(base: base.uppercased(), params: params, family: family) + return CanonicalColumnType( + kind: kind, isUnsigned: unsigned || impliesUnsigned(base: base.uppercased(), family: family), + sourceSpelling: spelling + ) + } + + // MARK: - Shape + + /// ClickHouse writes nullability and its dictionary encoding into the type itself, and both + /// wrap whatever they qualify. Read without stripping them, every ClickHouse column is one + /// unknown type called `Nullable`. + private static func stripWrappers(_ value: String, family: SQLTypeFamily) -> String { + guard family == .clickhouse else { return value } + for prefix in ["Nullable(", "LowCardinality("] where value.hasPrefix(prefix) && value.hasSuffix(")") { + let inner = value.dropFirst(prefix.count).dropLast() + return stripWrappers(String(inner), family: family) + } + return value + } + + private static func hasUnsignedModifier(_ value: String) -> Bool { + value.range(of: "\\bUNSIGNED\\b", options: [.regularExpression, .caseInsensitive]) != nil + } + + /// MySQL hangs `UNSIGNED`, `ZEROFILL` and a character set off the type itself, and SQL Server + /// and Oracle hang a collation off theirs. None of them is part of the type name, and a base + /// left holding one matches nothing in the family table. + private static func removingModifiers(_ value: String) -> String { + let pattern = "\\s+(UNSIGNED|ZEROFILL)\\b" + return value + .replacingOccurrences( + of: pattern, with: "", options: [.regularExpression, .caseInsensitive] + ) + .trimmingCharacters(in: .whitespaces) + } + + private static func arrayElement(of value: String, family: SQLTypeFamily) -> String? { + if value.hasSuffix("[]") { + let element = String(value.dropLast(2)).trimmingCharacters(in: .whitespaces) + return element.isEmpty ? nil : element + } + guard family == .clickhouse || family == .duckdb else { return nil } + for prefix in ["Array(", "ARRAY("] where value.uppercased().hasPrefix(prefix.uppercased()) + && value.hasSuffix(")") { + let element = String(value.dropFirst(prefix.count).dropLast()).trimmingCharacters(in: .whitespaces) + return element.isEmpty ? nil : element + } + return nil + } + + private static func split(_ value: String) -> (base: String, params: String?) { + guard let open = value.firstIndex(of: "("), let close = value.lastIndex(of: ")"), open < close else { + return (value.trimmingCharacters(in: .whitespaces), nil) + } + let base = String(value[value.startIndex.. [Int] { + guard let params else { return [] } + return params.split(separator: ",").compactMap { + Int($0.trimmingCharacters(in: .whitespaces)) + } + } + + /// The labels of a MySQL `ENUM` or a ClickHouse `Enum8`, whose parameter list is quoted text + /// rather than numbers. A ClickHouse label carries an `= 1` the value list does not need. + internal static func labels(in params: String?) -> [String] { + guard let params, !params.isEmpty else { return [] } + var labels: [String] = [] + var current = "" + var quote: Character? + var isEscaped = false + for character in params { + if isEscaped { + current.append(character) + isEscaped = false + continue + } + switch character { + case "\\" where quote != nil: + isEscaped = true + case "'", "\"": + if quote == character { + quote = nil + labels.append(current) + current = "" + } else if quote == nil { + quote = character + current = "" + } else { + current.append(character) + } + default: + if quote != nil { current.append(character) } + } + } + return labels + } + + // MARK: - Family readings + + private static func kind(base: String, params: String?, family: SQLTypeFamily) -> CanonicalTypeKind { + switch family { + case .mysql: return mysqlKind(base: base, params: params) + case .postgres: return postgresKind(base: base, params: params) + case .sqlite: return sqliteKind(base: base, params: params) + case .mssql: return mssqlKind(base: base, params: params) + case .oracle: return oracleKind(base: base, params: params) + case .clickhouse: return clickHouseKind(base: base, params: params) + case .duckdb: return duckDBKind(base: base, params: params) + case .generic: return ansiKind(base: base, params: params) + } + } + + private static func impliesUnsigned(base: String, family: SQLTypeFamily) -> Bool { + switch family { + case .mssql: return base == "TINYINT" + case .clickhouse, .duckdb: return base.uppercased().hasPrefix("U") && base.uppercased() != "UUID" + default: return false + } + } + + internal static func length(_ params: String?) -> Int? { integers(in: params).first } + + internal static func decimalKind(_ params: String?) -> CanonicalTypeKind { + let numbers = integers(in: params) + return .decimal(precision: numbers.first, scale: numbers.count > 1 ? numbers[1] : nil) + } +} diff --git a/TablePro/Core/CrossEngine/SQLTypeRenderer+BundledFamilies.swift b/TablePro/Core/CrossEngine/SQLTypeRenderer+BundledFamilies.swift new file mode 100644 index 000000000..8be0499f1 --- /dev/null +++ b/TablePro/Core/CrossEngine/SQLTypeRenderer+BundledFamilies.swift @@ -0,0 +1,289 @@ +// +// SQLTypeRenderer+BundledFamilies.swift +// TablePro +// +// MySQL, PostgreSQL and SQLite as copy targets. +// + +import Foundation + +internal extension SQLTypeRenderer { + // MARK: - MySQL + + /// `UNSIGNED` is never spelled here. It is a column attribute rather than part of the type name + /// as far as `PluginColumnDefinition` is concerned, and `mysqlColumnAttributesSQL` writes it + /// from `unsigned`, so putting it in the type as well produced `INT UNSIGNED UNSIGNED`. + static func mysql(_ type: CanonicalColumnType) -> RenderedColumnType { + switch type.kind { + case .boolean: + return RenderedColumnType(spelling: "TINYINT(1)") + case .integer(let bytes): + return mysqlInteger(bytes: bytes, isUnsigned: type.isUnsigned) + case .decimal(let precision, let scale): + return decimalSpelling( + "DECIMAL", precision: precision, scale: scale, precisionCeiling: 65 + ) + case .floatingPoint(let bits): + return RenderedColumnType(spelling: bits <= 32 ? "FLOAT" : "DOUBLE") + case .text(let length, let isFixed): + return mysqlText(length: length, isFixed: isFixed) + case .binary(let length, let isFixed): + return mysqlBinary(length: length, isFixed: isFixed) + case .date: + return RenderedColumnType(spelling: "DATE") + case .time(let precision, let hasTimeZone): + return RenderedColumnType( + spelling: "TIME" + precisionSuffix(precision), + fidelity: hasTimeZone ? .approximated : .exact, + reason: hasTimeZone ? timeZoneDropped : nil + ) + case .timestamp(let precision, let hasTimeZone): + return RenderedColumnType( + spelling: "DATETIME" + precisionSuffix(precision), + fidelity: hasTimeZone ? .approximated : .exact, + reason: hasTimeZone ? timeZoneDropped : nil + ) + case .interval: + return RenderedColumnType( + spelling: "VARCHAR(64)", fidelity: .approximated, + reason: noEquivalent("INTERVAL", as: "VARCHAR(64)") + ) + case .uuid: + return RenderedColumnType( + spelling: "CHAR(36)", fidelity: .widened, reason: widenedTo("CHAR(36)") + ) + case .json: + return RenderedColumnType(spelling: "JSON") + case .xml: + return RenderedColumnType( + spelling: "LONGTEXT", fidelity: .approximated, reason: noEquivalent("XML", as: "LONGTEXT") + ) + case .enumeration(let values): + return mysqlEnum(values) + case .bitString(let length): + let bits = length ?? 1 + guard bits <= 64 else { + return RenderedColumnType( + spelling: "VARBINARY(\(max(1, (bits + 7) / 8)))", fidelity: .widened, + reason: widenedTo("VARBINARY") + ) + } + return RenderedColumnType(spelling: "BIT(\(bits))") + case .money: + return RenderedColumnType( + spelling: "DECIMAL(19, 4)", fidelity: .widened, reason: widenedTo("DECIMAL(19, 4)") + ) + case .spatial: + return RenderedColumnType( + spelling: "LONGTEXT", fidelity: .approximated, + reason: noEquivalent(type.sourceSpelling, as: String(localized: "text")) + ) + case .array: + return RenderedColumnType( + spelling: "JSON", fidelity: .approximated, + reason: noEquivalent(type.sourceSpelling, as: "JSON") + ) + case .unsupported: + return RenderedColumnType( + spelling: "LONGTEXT", fidelity: .approximated, + reason: noEquivalent(type.sourceSpelling, as: String(localized: "text")) + ) + } + } + + private static func mysqlInteger(bytes: Int, isUnsigned: Bool) -> RenderedColumnType { + switch bytes { + case ...1: return RenderedColumnType(spelling: "TINYINT") + case 2: return RenderedColumnType(spelling: "SMALLINT") + case 3: return RenderedColumnType(spelling: "MEDIUMINT") + case 4: return RenderedColumnType(spelling: "INT") + case 8: return RenderedColumnType(spelling: "BIGINT") + default: + let digits = decimalDigits(forIntegerBytes: bytes, isUnsigned: isUnsigned, ceiling: 65) + return RenderedColumnType( + spelling: "DECIMAL(\(digits), 0)", fidelity: .widened, + reason: widenedTo("DECIMAL(\(digits), 0)") + ) + } + } + + /// `LONGTEXT` rather than `TEXT` for anything unbounded, because `TEXT` holds 64 KB and a + /// PostgreSQL `text` column holds a gigabyte. Truncation would be silent outside strict mode. + private static func mysqlText(length: Int?, isFixed: Bool) -> RenderedColumnType { + if isFixed, let length, length <= 255 { + return RenderedColumnType(spelling: "CHAR(\(length))") + } + guard let length, length <= 4_000 else { + return RenderedColumnType( + spelling: "LONGTEXT", + fidelity: length == nil ? .exact : .widened, + reason: length == nil ? nil : widenedTo("LONGTEXT") + ) + } + return RenderedColumnType(spelling: "VARCHAR(\(length))") + } + + private static func mysqlBinary(length: Int?, isFixed: Bool) -> RenderedColumnType { + if isFixed, let length, length <= 255 { + return RenderedColumnType(spelling: "BINARY(\(length))") + } + guard let length, length <= 4_000 else { + return RenderedColumnType( + spelling: "LONGBLOB", + fidelity: length == nil ? .exact : .widened, + reason: length == nil ? nil : widenedTo("LONGBLOB") + ) + } + return RenderedColumnType(spelling: "VARBINARY(\(length))") + } + + private static func mysqlEnum(_ values: [String]) -> RenderedColumnType { + guard !values.isEmpty else { + return RenderedColumnType(spelling: "VARCHAR(255)", fidelity: .approximated, reason: enumBecomesText) + } + let labels = values + .map { "'\($0.replacingOccurrences(of: "'", with: "''"))'" } + .joined(separator: ", ") + return RenderedColumnType(spelling: "ENUM(\(labels))") + } + + // MARK: - PostgreSQL + + static func postgres(_ type: CanonicalColumnType) -> RenderedColumnType { + switch type.kind { + case .boolean: + return RenderedColumnType(spelling: "BOOLEAN") + case .integer(let bytes): + return postgresInteger(bytes: bytes, isUnsigned: type.isUnsigned) + case .decimal(let precision, let scale): + return decimalSpelling( + "NUMERIC", precision: precision, scale: scale, precisionCeiling: 1_000 + ) + case .floatingPoint(let bits): + return RenderedColumnType(spelling: bits <= 32 ? "REAL" : "DOUBLE PRECISION") + case .text(let length, let isFixed): + guard let length else { return RenderedColumnType(spelling: "TEXT") } + return RenderedColumnType(spelling: isFixed ? "CHAR(\(length))" : "VARCHAR(\(length))") + case .binary(let length, _): + return RenderedColumnType( + spelling: "BYTEA", + fidelity: length == nil ? .exact : .widened, + reason: length == nil ? nil : widenedTo("BYTEA") + ) + case .date: + return RenderedColumnType(spelling: "DATE") + case .time(let precision, let hasTimeZone): + let base = hasTimeZone ? "TIMETZ" : "TIME" + return RenderedColumnType(spelling: base + precisionSuffix(precision)) + case .timestamp(let precision, let hasTimeZone): + let base = hasTimeZone ? "TIMESTAMPTZ" : "TIMESTAMP" + return RenderedColumnType(spelling: base + precisionSuffix(precision)) + case .interval: + return RenderedColumnType(spelling: "INTERVAL") + case .uuid: + return RenderedColumnType(spelling: "UUID") + case .json: + return RenderedColumnType(spelling: "JSONB") + case .xml: + return RenderedColumnType(spelling: "XML") + case .enumeration(let values): + return RenderedColumnType( + spelling: "VARCHAR(\(longestLabel(in: values)))", + fidelity: .approximated, reason: enumBecomesText + ) + case .bitString(let length): + guard let length else { return RenderedColumnType(spelling: "BIT VARYING") } + return RenderedColumnType(spelling: "BIT(\(length))") + /// Not `MONEY`, whose text form and rounding follow the server's `lc_monetary`, so the same + /// copy produces different values on two servers. + case .money: + return RenderedColumnType( + spelling: "NUMERIC(19, 4)", fidelity: .widened, reason: widenedTo("NUMERIC(19, 4)") + ) + case .spatial: + return RenderedColumnType( + spelling: "TEXT", fidelity: .approximated, + reason: noEquivalent(type.sourceSpelling, as: String(localized: "text")) + ) + case .array(let element): + let inner = postgres(CanonicalColumnType( + kind: element, isUnsigned: type.isUnsigned, sourceSpelling: type.sourceSpelling + )) + return RenderedColumnType(spelling: "\(inner.spelling)[]", fidelity: inner.fidelity, reason: inner.reason) + case .unsupported: + return RenderedColumnType( + spelling: "TEXT", fidelity: .approximated, + reason: noEquivalent(type.sourceSpelling, as: String(localized: "text")) + ) + } + } + + /// PostgreSQL has no unsigned integers, so an unsigned source widens by one step and the widest + /// becomes a `NUMERIC`. Kept as the same width, a `BIGINT UNSIGNED` above 2^63 fails on the row + /// that first exceeds it, which on a long copy is minutes in. + private static func postgresInteger(bytes: Int, isUnsigned: Bool) -> RenderedColumnType { + let effective = isUnsigned ? bytes * 2 : bytes + switch effective { + case ...2: return RenderedColumnType(spelling: "SMALLINT") + case 3...4: return RenderedColumnType(spelling: "INTEGER") + case 5...8: return RenderedColumnType(spelling: "BIGINT") + default: + let digits = decimalDigits(forIntegerBytes: bytes, isUnsigned: isUnsigned, ceiling: 1_000) + return RenderedColumnType( + spelling: "NUMERIC(\(digits), 0)", fidelity: .widened, + reason: widenedTo("NUMERIC(\(digits), 0)") + ) + } + } + + // MARK: - SQLite + + /// SQLite declares an affinity rather than a type and stores the spelling verbatim, so the + /// spellings here are chosen for what TablePro and every other reader make of them rather than + /// for what the engine enforces, which is nothing. + static func sqlite(_ type: CanonicalColumnType) -> RenderedColumnType { + switch type.kind { + case .boolean: + return RenderedColumnType(spelling: "BOOLEAN") + case .integer: + return RenderedColumnType(spelling: "INTEGER") + case .decimal(let precision, let scale): + return decimalSpelling( + "NUMERIC", precision: precision, scale: scale, precisionCeiling: 38 + ) + case .floatingPoint: + return RenderedColumnType(spelling: "REAL") + case .text(let length, _): + guard let length else { return RenderedColumnType(spelling: "TEXT") } + return RenderedColumnType(spelling: "VARCHAR(\(length))", fidelity: .widened, reason: lengthNotEnforced) + case .binary: + return RenderedColumnType(spelling: "BLOB") + case .date: + return RenderedColumnType(spelling: "DATE") + case .time(_, let hasTimeZone): + return RenderedColumnType( + spelling: "TIME", fidelity: hasTimeZone ? .approximated : .exact, + reason: hasTimeZone ? timeZoneDropped : nil + ) + case .timestamp(_, let hasTimeZone): + return RenderedColumnType( + spelling: "DATETIME", fidelity: hasTimeZone ? .approximated : .exact, + reason: hasTimeZone ? timeZoneDropped : nil + ) + case .money: + return RenderedColumnType(spelling: "NUMERIC(19, 4)") + case .uuid, .json, .xml, .interval, .bitString, .enumeration, .spatial, .array, .unsupported: + return sqliteText(type) + } + } + + private static func sqliteText(_ type: CanonicalColumnType) -> RenderedColumnType { + if case .enumeration = type.kind { + return RenderedColumnType(spelling: "TEXT", fidelity: .approximated, reason: enumBecomesText) + } + return RenderedColumnType( + spelling: "TEXT", fidelity: .approximated, + reason: noEquivalent(type.sourceSpelling, as: "TEXT") + ) + } +} diff --git a/TablePro/Core/CrossEngine/SQLTypeRenderer+RegistryFamilies.swift b/TablePro/Core/CrossEngine/SQLTypeRenderer+RegistryFamilies.swift new file mode 100644 index 000000000..14e183041 --- /dev/null +++ b/TablePro/Core/CrossEngine/SQLTypeRenderer+RegistryFamilies.swift @@ -0,0 +1,359 @@ +// +// SQLTypeRenderer+RegistryFamilies.swift +// TablePro +// +// SQL Server, Oracle, ClickHouse and DuckDB as copy targets, and the ANSI +// spellings used for an engine no family names. +// + +import Foundation + +internal extension SQLTypeRenderer { + // MARK: - SQL Server + + static func mssql(_ type: CanonicalColumnType) -> RenderedColumnType { + switch type.kind { + case .boolean: + return RenderedColumnType(spelling: "BIT") + case .integer(let bytes): + return mssqlInteger(bytes: bytes, isUnsigned: type.isUnsigned) + case .decimal(let precision, let scale): + return decimalSpelling( + "DECIMAL", precision: precision, scale: scale, precisionCeiling: 38 + ) + case .floatingPoint(let bits): + return RenderedColumnType(spelling: bits <= 32 ? "REAL" : "FLOAT") + case .text(let length, let isFixed): + return mssqlText(length: length, isFixed: isFixed) + case .binary(let length, let isFixed): + guard let length, length <= 8_000 else { + return RenderedColumnType(spelling: "VARBINARY(MAX)") + } + return RenderedColumnType(spelling: isFixed ? "BINARY(\(length))" : "VARBINARY(\(length))") + case .date: + return RenderedColumnType(spelling: "DATE") + case .time(let precision, let hasTimeZone): + return RenderedColumnType( + spelling: "TIME" + precisionSuffix(precision), + fidelity: hasTimeZone ? .approximated : .exact, + reason: hasTimeZone ? timeZoneDropped : nil + ) + case .timestamp(let precision, let hasTimeZone): + let base = hasTimeZone ? "DATETIMEOFFSET" : "DATETIME2" + return RenderedColumnType(spelling: base + precisionSuffix(precision)) + case .interval: + return RenderedColumnType( + spelling: "NVARCHAR(64)", fidelity: .approximated, + reason: noEquivalent("INTERVAL", as: "NVARCHAR(64)") + ) + case .uuid: + return RenderedColumnType(spelling: "UNIQUEIDENTIFIER") + case .json: + return RenderedColumnType( + spelling: "NVARCHAR(MAX)", fidelity: .widened, reason: widenedTo("NVARCHAR(MAX)") + ) + case .xml: + return RenderedColumnType(spelling: "XML") + case .enumeration(let values): + return RenderedColumnType( + spelling: "NVARCHAR(\(longestLabel(in: values)))", + fidelity: .approximated, reason: enumBecomesText + ) + case .bitString(let length): + return RenderedColumnType( + spelling: "VARBINARY(\(max(1, ((length ?? 1) + 7) / 8)))", + fidelity: .widened, reason: widenedTo("VARBINARY") + ) + case .money: + return RenderedColumnType(spelling: "MONEY") + case .spatial, .array, .unsupported: + return RenderedColumnType( + spelling: "NVARCHAR(MAX)", fidelity: .approximated, + reason: noEquivalent(type.sourceSpelling, as: "NVARCHAR(MAX)") + ) + } + } + + /// `TINYINT` on SQL Server is unsigned and holds 0 to 255, so a signed one-byte source widens + /// to `SMALLINT`. Left as `TINYINT`, every negative value in the column fails on insert. + private static func mssqlInteger(bytes: Int, isUnsigned: Bool) -> RenderedColumnType { + if bytes <= 1, isUnsigned { return RenderedColumnType(spelling: "TINYINT") } + let effective = isUnsigned ? bytes * 2 : max(bytes, 2) + switch effective { + case ...2: return RenderedColumnType(spelling: "SMALLINT") + case 3...4: return RenderedColumnType(spelling: "INT") + case 5...8: return RenderedColumnType(spelling: "BIGINT") + default: + let digits = decimalDigits(forIntegerBytes: bytes, isUnsigned: isUnsigned, ceiling: 38) + return RenderedColumnType( + spelling: "DECIMAL(\(digits), 0)", fidelity: .widened, + reason: widenedTo("DECIMAL(\(digits), 0)") + ) + } + } + + /// The `N` spellings throughout, so text that crossed from a UTF-8 engine is not narrowed to + /// the server's own code page. A non-`N` `VARCHAR` on a Latin-1 collation drops every + /// character outside it, and it drops them silently. + private static func mssqlText(length: Int?, isFixed: Bool) -> RenderedColumnType { + guard let length, length <= 4_000 else { + return RenderedColumnType( + spelling: "NVARCHAR(MAX)", + fidelity: length == nil ? .exact : .widened, + reason: length == nil ? nil : widenedTo("NVARCHAR(MAX)") + ) + } + return RenderedColumnType(spelling: isFixed ? "NCHAR(\(length))" : "NVARCHAR(\(length))") + } + + // MARK: - Oracle + + static func oracle(_ type: CanonicalColumnType) -> RenderedColumnType { + switch type.kind { + /// Oracle had no `BOOLEAN` in SQL before 23ai, and `NUMBER(1)` is what every Oracle schema + /// uses for one. The value coercer writes 1 and 0 into it. + case .boolean: + return RenderedColumnType( + spelling: "NUMBER(1)", fidelity: .approximated, + reason: noEquivalent("BOOLEAN", as: "NUMBER(1)") + ) + case .integer(let bytes): + let digits = decimalDigits(forIntegerBytes: bytes, isUnsigned: type.isUnsigned, ceiling: 38) + return RenderedColumnType(spelling: "NUMBER(\(digits))") + case .decimal(let precision, let scale): + return decimalSpelling( + "NUMBER", precision: precision, scale: scale, precisionCeiling: 38 + ) + case .floatingPoint(let bits): + return RenderedColumnType(spelling: bits <= 32 ? "BINARY_FLOAT" : "BINARY_DOUBLE") + case .text(let length, let isFixed): + return oracleText(length: length, isFixed: isFixed) + case .binary(let length, _): + guard let length, length <= 2_000 else { return RenderedColumnType(spelling: "BLOB") } + return RenderedColumnType(spelling: "RAW(\(length))") + case .date: + return RenderedColumnType(spelling: "DATE") + case .time(let precision, _): + return RenderedColumnType( + spelling: "INTERVAL DAY(0) TO SECOND\(precisionSuffix(precision))", + fidelity: .approximated, reason: noEquivalent("TIME", as: "INTERVAL DAY TO SECOND") + ) + case .timestamp(let precision, let hasTimeZone): + let suffix = hasTimeZone ? " WITH TIME ZONE" : "" + return RenderedColumnType(spelling: "TIMESTAMP" + precisionSuffix(precision) + suffix) + case .interval: + return RenderedColumnType(spelling: "INTERVAL DAY TO SECOND") + case .uuid: + return RenderedColumnType( + spelling: "VARCHAR2(36)", fidelity: .widened, reason: widenedTo("VARCHAR2(36)") + ) + case .json: + return RenderedColumnType(spelling: "CLOB", fidelity: .widened, reason: widenedTo("CLOB")) + case .xml: + return RenderedColumnType(spelling: "XMLTYPE") + case .enumeration(let values): + return RenderedColumnType( + spelling: "VARCHAR2(\(longestLabel(in: values)))", + fidelity: .approximated, reason: enumBecomesText + ) + case .money: + return RenderedColumnType(spelling: "NUMBER(19, 4)") + case .bitString, .spatial, .array, .unsupported: + return RenderedColumnType( + spelling: "CLOB", fidelity: .approximated, + reason: noEquivalent(type.sourceSpelling, as: "CLOB") + ) + } + } + + private static func oracleText(length: Int?, isFixed: Bool) -> RenderedColumnType { + if isFixed, let length, length <= 2_000 { + return RenderedColumnType(spelling: "CHAR(\(length))") + } + guard let length, length <= 4_000 else { + return RenderedColumnType( + spelling: "CLOB", + fidelity: length == nil ? .exact : .widened, + reason: length == nil ? nil : widenedTo("CLOB") + ) + } + return RenderedColumnType(spelling: "VARCHAR2(\(length))") + } + + // MARK: - ClickHouse + + static func clickHouse(_ type: CanonicalColumnType) -> RenderedColumnType { + switch type.kind { + case .boolean: + return RenderedColumnType(spelling: "Bool") + case .integer(let bytes): + let prefix = type.isUnsigned ? "UInt" : "Int" + let bits: Int + switch bytes { + case ...1: bits = 8 + case 2: bits = 16 + case 3, 4: bits = 32 + case 8: bits = 64 + default: bits = 128 + } + return RenderedColumnType(spelling: "\(prefix)\(bits)") + case .decimal(let precision, let scale): + return decimalSpelling( + "Decimal", precision: precision, scale: scale, precisionCeiling: 76 + ) + case .floatingPoint(let bits): + return RenderedColumnType(spelling: bits <= 32 ? "Float32" : "Float64") + case .text(let length, let isFixed): + guard isFixed, let length else { return RenderedColumnType(spelling: "String") } + return RenderedColumnType(spelling: "FixedString(\(length))") + case .binary: + return RenderedColumnType(spelling: "String", fidelity: .widened, reason: widenedTo("String")) + case .date: + return RenderedColumnType(spelling: "Date32") + case .timestamp(let precision, let hasTimeZone): + let spelling = (precision ?? 0) > 0 ? "DateTime64(\(precision ?? 3))" : "DateTime64(3)" + return RenderedColumnType( + spelling: spelling, fidelity: hasTimeZone ? .approximated : .exact, + reason: hasTimeZone ? timeZoneDropped : nil + ) + case .uuid: + return RenderedColumnType(spelling: "UUID") + case .enumeration(let values): + return clickHouseEnum(values) + case .array(let element): + let inner = clickHouse(CanonicalColumnType( + kind: element, isUnsigned: type.isUnsigned, sourceSpelling: type.sourceSpelling + )) + return RenderedColumnType( + spelling: "Array(\(inner.spelling))", fidelity: inner.fidelity, reason: inner.reason + ) + case .time, .interval, .json, .xml, .bitString, .money, .spatial, .unsupported: + return RenderedColumnType( + spelling: "String", fidelity: .approximated, + reason: noEquivalent(type.sourceSpelling, as: "String") + ) + } + } + + private static func clickHouseEnum(_ values: [String]) -> RenderedColumnType { + guard !values.isEmpty, values.count <= 255 else { + return RenderedColumnType(spelling: "String", fidelity: .approximated, reason: enumBecomesText) + } + let labels = values.enumerated() + .map { "'\($0.element.replacingOccurrences(of: "'", with: "\\'"))' = \($0.offset + 1)" } + .joined(separator: ", ") + return RenderedColumnType(spelling: "Enum8(\(labels))") + } + + // MARK: - DuckDB + + static func duckDB(_ type: CanonicalColumnType) -> RenderedColumnType { + switch type.kind { + case .boolean: + return RenderedColumnType(spelling: "BOOLEAN") + case .integer(let bytes): + let prefix = type.isUnsigned ? "U" : "" + switch bytes { + case ...1: return RenderedColumnType(spelling: prefix + "TINYINT") + case 2: return RenderedColumnType(spelling: prefix + "SMALLINT") + case 3, 4: return RenderedColumnType(spelling: prefix + "INTEGER") + case 8: return RenderedColumnType(spelling: prefix + "BIGINT") + default: return RenderedColumnType(spelling: prefix + "HUGEINT") + } + case .decimal(let precision, let scale): + return decimalSpelling( + "DECIMAL", precision: precision, scale: scale, precisionCeiling: 38 + ) + case .floatingPoint(let bits): + return RenderedColumnType(spelling: bits <= 32 ? "FLOAT" : "DOUBLE") + case .text(let length, _): + guard let length else { return RenderedColumnType(spelling: "VARCHAR") } + return RenderedColumnType(spelling: "VARCHAR(\(length))") + case .binary: + return RenderedColumnType(spelling: "BLOB") + case .date: + return RenderedColumnType(spelling: "DATE") + case .time(_, let hasTimeZone): + return RenderedColumnType(spelling: hasTimeZone ? "TIMETZ" : "TIME") + case .timestamp(_, let hasTimeZone): + return RenderedColumnType(spelling: hasTimeZone ? "TIMESTAMPTZ" : "TIMESTAMP") + case .interval: + return RenderedColumnType(spelling: "INTERVAL") + case .uuid: + return RenderedColumnType(spelling: "UUID") + case .json: + return RenderedColumnType(spelling: "JSON") + case .bitString: + return RenderedColumnType(spelling: "BIT") + case .money: + return RenderedColumnType(spelling: "DECIMAL(19, 4)") + case .array(let element): + let inner = duckDB(CanonicalColumnType( + kind: element, isUnsigned: type.isUnsigned, sourceSpelling: type.sourceSpelling + )) + return RenderedColumnType( + spelling: "\(inner.spelling)[]", fidelity: inner.fidelity, reason: inner.reason + ) + case .enumeration: + return RenderedColumnType(spelling: "VARCHAR", fidelity: .approximated, reason: enumBecomesText) + case .xml, .spatial, .unsupported: + return RenderedColumnType( + spelling: "VARCHAR", fidelity: .approximated, + reason: noEquivalent(type.sourceSpelling, as: "VARCHAR") + ) + } + } + + // MARK: - ANSI + + /// What an engine no family names is given. Only words the SQL standard defines, because the + /// next plugin to be copied into is not known here and a borrowed spelling would be a guess. + static func ansi(_ type: CanonicalColumnType) -> RenderedColumnType { + switch type.kind { + case .boolean: + return RenderedColumnType(spelling: "BOOLEAN") + case .integer(let bytes): + let effective = type.isUnsigned ? bytes * 2 : bytes + switch effective { + case ...2: return RenderedColumnType(spelling: "SMALLINT") + case 3...4: return RenderedColumnType(spelling: "INTEGER") + case 5...8: return RenderedColumnType(spelling: "BIGINT") + default: + let digits = decimalDigits(forIntegerBytes: bytes, isUnsigned: type.isUnsigned, ceiling: 38) + return RenderedColumnType( + spelling: "DECIMAL(\(digits), 0)", fidelity: .widened, + reason: widenedTo("DECIMAL(\(digits), 0)") + ) + } + case .decimal(let precision, let scale): + return decimalSpelling( + "DECIMAL", precision: precision, scale: scale, precisionCeiling: 38 + ) + case .floatingPoint(let bits): + return RenderedColumnType(spelling: bits <= 32 ? "REAL" : "DOUBLE PRECISION") + case .text(let length, let isFixed): + guard let length else { return RenderedColumnType(spelling: "TEXT") } + return RenderedColumnType(spelling: isFixed ? "CHAR(\(length))" : "VARCHAR(\(length))") + case .binary(let length, _): + guard let length else { return RenderedColumnType(spelling: "BLOB") } + return RenderedColumnType(spelling: "VARBINARY(\(length))") + case .date: + return RenderedColumnType(spelling: "DATE") + case .time(let precision, let hasTimeZone): + let suffix = hasTimeZone ? " WITH TIME ZONE" : "" + return RenderedColumnType(spelling: "TIME" + precisionSuffix(precision) + suffix) + case .timestamp(let precision, let hasTimeZone): + let suffix = hasTimeZone ? " WITH TIME ZONE" : "" + return RenderedColumnType(spelling: "TIMESTAMP" + precisionSuffix(precision) + suffix) + case .money: + return RenderedColumnType(spelling: "DECIMAL(19, 4)") + case .uuid: + return RenderedColumnType(spelling: "CHAR(36)", fidelity: .widened, reason: widenedTo("CHAR(36)")) + case .interval, .json, .xml, .enumeration, .bitString, .spatial, .array, .unsupported: + return RenderedColumnType( + spelling: "TEXT", fidelity: .approximated, + reason: noEquivalent(type.sourceSpelling, as: "TEXT") + ) + } + } +} diff --git a/TablePro/Core/CrossEngine/SQLTypeRenderer.swift b/TablePro/Core/CrossEngine/SQLTypeRenderer.swift new file mode 100644 index 000000000..c5a97e6dc --- /dev/null +++ b/TablePro/Core/CrossEngine/SQLTypeRenderer.swift @@ -0,0 +1,123 @@ +// +// SQLTypeRenderer.swift +// TablePro +// +// Writes a canonical type in the target engine's own words. +// +// Every family answers every kind. There is no "this engine cannot hold that" +// arm, because a copy that drops a column is worse than a copy that says the +// column arrived as text: the row is written either way, and only one of the +// two tells the user what happened. So a kind with no equivalent is rendered +// as the family's widest text type and carries the reason with it, which is +// what the review step reads out before anything runs. +// +// Widening is always toward the larger type. A MySQL `BIGINT UNSIGNED` becomes +// a PostgreSQL `NUMERIC(20,0)` rather than a `BIGINT`, because half of its +// range does not fit in one and the failure would be per row, at the end of a +// long copy, on whichever row first exceeded it. +// + +import Foundation + +internal enum SQLTypeRenderer { + internal static func render(_ type: CanonicalColumnType, family: SQLTypeFamily) -> RenderedColumnType { + switch family { + case .mysql: return mysql(type) + case .postgres: return postgres(type) + case .sqlite: return sqlite(type) + case .mssql: return mssql(type) + case .oracle: return oracle(type) + case .clickhouse: return clickHouse(type) + case .duckdb: return duckDB(type) + case .generic: return ansi(type) + } + } + + // MARK: - Shared reasons + + internal static func noEquivalent(_ spelling: String, as substitute: String) -> String { + String( + format: String(localized: "%1$@ has no equivalent here, so the values arrive as %2$@."), + spelling, substitute + ) + } + + internal static var timeZoneDropped: String { + String(localized: "The target has no time zone on this type, so the offset is dropped.") + } + + internal static func widenedTo(_ substitute: String) -> String { + String(format: String(localized: "Widened to %@, which holds every source value."), substitute) + } + + internal static var enumBecomesText: String { + String(localized: "The list of allowed values is not carried over.") + } + + internal static var lengthNotEnforced: String { + String(localized: "The declared length is kept for reference but is not enforced.") + } + + // MARK: - Shared shapes + + /// A `DECIMAL` wide enough for that many bytes of integer, for a target with no integer type + /// that wide. 16 bytes needs 39 digits, which is past what most engines allow, so the caller + /// passes the ceiling its own engine accepts. + internal static func decimalDigits(forIntegerBytes bytes: Int, isUnsigned: Bool, ceiling: Int) -> Int { + let digits: Int + switch bytes { + case ...1: digits = isUnsigned ? 3 : 3 + case 2: digits = 5 + case 3: digits = 8 + case 4: digits = 10 + case 8: digits = isUnsigned ? 20 : 19 + default: digits = 39 + } + return min(digits, ceiling) + } + + /// A cut precision is a narrowing and says so. + /// + /// MySQL allows 65 digits and SQL Server, Oracle and DuckDB allow 38, so a `DECIMAL(65, 30)` + /// crossing to any of them loses 27 of them. Reported as exact, the review step said nothing + /// and the copy failed part way through the data phase on the first row that needed the digits + /// the target no longer had. + internal static func decimalSpelling( + _ name: String, + precision: Int?, + scale: Int?, + precisionCeiling: Int + ) -> RenderedColumnType { + let requested = precision ?? 38 + let resolved = min(requested, precisionCeiling) + let spelling: String + if let scale { + spelling = "\(name)(\(resolved), \(min(scale, resolved)))" + } else { + spelling = "\(name)(\(resolved))" + } + guard resolved < requested else { return RenderedColumnType(spelling: spelling) } + return RenderedColumnType( + spelling: spelling, + fidelity: .approximated, + reason: String( + format: String( + localized: "This engine holds %1$lld digits, not %2$lld, so the extra ones are lost." + ), + resolved, requested + ) + ) + } + + /// A parenthesised precision only where the engine accepts one and the source had a real + /// value. Rendering `TIMESTAMP(0)` where the source said nothing changes a column that would + /// have kept fractional seconds into one that truncates them. + internal static func precisionSuffix(_ precision: Int?) -> String { + guard let precision, precision > 0 else { return "" } + return "(\(precision))" + } + + internal static func longestLabel(in values: [String]) -> Int { + max(1, values.map { ($0 as NSString).length }.max() ?? 0) + } +} diff --git a/TablePro/Core/ObjectCopy/ObjectCopyEligibility.swift b/TablePro/Core/ObjectCopy/ObjectCopyEligibility.swift index 79cab20ed..b4ee6a59a 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopyEligibility.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopyEligibility.swift @@ -47,17 +47,67 @@ internal enum ObjectCopyEligibility { target.ineligibleAsTargetReason } - /// A copy stays inside one engine, whatever half of an object it carries. + /// A copy stays inside SQL, and inside SQL it may cross engines. /// - /// Structure cannot cross because column data types are driver-native strings. Data cannot - /// cross either, and the first version let it: the row writer emits `INSERT … VALUES` and a - /// MongoDB or Elasticsearch target parses neither, while a SQL Server `dbo` source handed a - /// MySQL target a schema that engine does not have. Comparing the two remains available in - /// Compare & Sync, which reads rather than writes. - internal static func engineRefusal(from source: DatabaseType, to target: DatabaseType) -> String? { - guard !CompareSyncEngineFamily.canGenerateStructureScript(from: source, to: target) else { return nil } + /// It could not before, because a column's data type is the source driver's own string and + /// handing it to another engine's `CREATE TABLE` produced DDL that engine rejects. + /// `CrossEngineStructureTranslator` is what removed that reason: the types, defaults and + /// indexes are said in the target's own words before the target driver ever sees them, and + /// every approximation is listed in the review step. + /// + /// What has not changed is the floor. The row writer emits `INSERT … VALUES` and the structure + /// writer emits `CREATE TABLE`, so an engine whose query language is not SQL parses neither. + /// Comparing the two remains available in Compare & Sync, which reads rather than writes. + internal static func engineRefusal( + from source: DatabaseType, + to target: DatabaseType, + sourceLanguage: EditorLanguage, + targetLanguage: EditorLanguage + ) -> String? { + guard supportsCopying(editorLanguage: sourceLanguage), + supportsCopying(editorLanguage: targetLanguage) else { + return String( + format: String(localized: "%1$@ cannot be copied to %2$@. Choose a target that speaks SQL."), + source.rawValue, target.rawValue + ) + } + return crossEngineRefusal(from: source, to: target) + } + + /// The second half of the gate, and the one the editor language cannot answer. + /// + /// `.sql` is what DynamoDB declares for PartiQL and Cassandra for CQL, so the language alone + /// lets a MySQL to DynamoDB copy through to a planner that can only fail. A crossing is offered + /// where both engines have a type system `SQLTypeFamily` names, which is the same set the + /// translation was written and tested against. Within one engine nothing is translated, so an + /// engine no family names still copies to itself. + private static func crossEngineRefusal(from source: DatabaseType, to target: DatabaseType) -> String? { + guard SQLTypeFamily.needsTranslation(from: source, to: target) else { return nil } + let unnamed = [source, target].filter { SQLTypeFamily.of($0) == .generic } + guard let first = unnamed.first else { return nil } + return String( + format: String( + localized: "A copy to another engine needs a type system TablePro can translate, and %@ has none it knows. Copy to a target of the same type." + ), + first.rawValue + ) + } + + /// Why a view, routine or trigger cannot cross to another engine. + /// + /// Its definition is the source's own SQL text and nothing here parses it, so a MySQL view's + /// backtick quoting, a PostgreSQL function's `$$` body and a SQL Server trigger's `inserted` + /// pseudo-table all arrive verbatim at an engine that has none of them. A table has a + /// structure the translator can restate; a definition has only text. + internal static func definitionEngineRefusal( + from source: DatabaseType, + to target: DatabaseType + ) -> String? { + guard SQLTypeFamily.needsTranslation(from: source, to: target) else { return nil } return String( - format: String(localized: "%1$@ cannot be copied to %2$@. Choose a target of the same type."), + format: String( + localized: "Its definition is written in %1$@'s own SQL, which %2$@ does not parse." + ), source.rawValue, target.rawValue ) } diff --git a/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift b/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift index 1c7ec8d18..20afcd6e0 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift @@ -47,10 +47,59 @@ internal struct ObjectCopyTableStep: Identifiable, Sendable { internal let copiesData: Bool /// True when a column this step writes is one the server may insist on generating itself. internal let copiesIdentityColumn: Bool + /// What the crossing changed about this table's structure, listed before anything runs. Empty + /// within one type family, and empty for a step that writes no structure. + internal let conversionNotes: [CrossEngineConversionNote] + /// Reshapes the values whose spelling the two engines disagree about. Nil where they agree. + internal let coercer: CrossEngineValueCoercer? + /// The one statement that copies this table's rows without them leaving the server. Set only + /// when both sides are the same connection; nil means the rows are streamed through the app. + internal let serverSideInsert: SyncStatement? /// Set when this table is in the plan but part of it cannot run, so the sheet can say why /// before the user presses Copy. internal let note: String? + /// Spelled out rather than left to the memberwise init so the three fields a same-engine copy + /// never sets can default. Every one of them describes something only a crossing or a + /// same-connection copy produces, and a caller that has neither should not have to say so. + internal init( + selection: ObjectCopySelection, + dropStatements: [SyncStatement], + sequenceStatements: [SyncStatement], + createStatements: [SyncStatement], + truncateStatements: [SyncStatement], + columns: [String], + primaryKeyColumns: [String], + sourceQuery: String, + targetTable: String, + targetSchema: String?, + estimatedRows: Int?, + copiesData: Bool, + copiesIdentityColumn: Bool, + conversionNotes: [CrossEngineConversionNote] = [], + coercer: CrossEngineValueCoercer? = nil, + serverSideInsert: SyncStatement? = nil, + note: String? + ) { + self.selection = selection + self.dropStatements = dropStatements + self.sequenceStatements = sequenceStatements + self.createStatements = createStatements + self.truncateStatements = truncateStatements + self.columns = columns + self.primaryKeyColumns = primaryKeyColumns + self.sourceQuery = sourceQuery + self.targetTable = targetTable + self.targetSchema = targetSchema + self.estimatedRows = estimatedRows + self.copiesData = copiesData + self.copiesIdentityColumn = copiesIdentityColumn + self.conversionNotes = conversionNotes + self.coercer = coercer + self.serverSideInsert = serverSideInsert + self.note = note + } + internal var id: String { selection.id } /// What the DDL phase runs for this table. The truncate is deliberately absent: it belongs to @@ -133,14 +182,54 @@ internal struct ObjectCopyPlan: Sendable { self.skipped = skipped } - /// Shown above the script. The engine cannot differ any more, so the one caveat left is the - /// one the copy cannot do anything about: a key column the server insists on generating - /// refuses the value the source holds, and the table's own error is the first the user sees. + /// Shown above the script, for the caveats the copy cannot do anything about: a key column the + /// server insists on generating refuses the value the source holds, and the table's own error + /// is the first the user sees. + /// + /// A crossing between engines is named here as well as itemised below, because the itemised + /// list is per column and the fact that the types were rewritten at all is per copy. internal var warnings: [String] { - guard dataSteps.contains(where: \.copiesIdentityColumn) else { return [] } - return [String( + var warnings: [String] = [] + /// Asked of the steps rather than of `conversionNotes`, which flattens and sorts every + /// note in the plan. This is read from a view body on every redraw, and a copy of a + /// hundred tables has thousands of them. + if tableSteps.contains(where: { !$0.conversionNotes.isEmpty }) { + warnings.append(String( + format: String( + localized: "%1$@ and %2$@ do not share a type system, so the structure below was rewritten. Check it before copying." + ), + request.source.databaseType.rawValue, request.target.databaseType.rawValue + )) + } + if dataSteps.contains(where: { $0.serverSideInsert != nil }) { + warnings.append(String( + localized: "Both sides are one connection, so the server copies the rows itself. There is no row-by-row progress, and Stop cannot interrupt it." + )) + } + guard dataSteps.contains(where: \.copiesIdentityColumn) else { return warnings } + warnings.append(String( localized: "Identity and auto-increment values are written as they are. A column the server generates always may refuse them." - )] + )) + return warnings + } + + /// Every type, default and index the crossing changed, worst first. + internal var conversionNotes: [CrossEngineConversionNote] { + tableSteps.flatMap(\.conversionNotes).orderedForReview + } + + /// What the review step lists, and how many it could not. + /// + /// A whole-database crossing produces a note per converted column, which on a hundred tables is + /// thousands of them. The list is inside a `ScrollView` rather than a `List`, so SwiftUI builds + /// every row it is given whether or not any of them is on screen. The worst are first, so a cap + /// keeps exactly the ones worth reading. + internal func reviewedConversionNotes( + limit: Int = 200 + ) -> (shown: [CrossEngineConversionNote], hidden: Int) { + let ordered = conversionNotes + guard ordered.count > limit else { return (ordered, 0) } + return (Array(ordered.prefix(limit)), ordered.count - limit) } /// Emptiness is about work, not about steps. A data-only copy into a table the target does not @@ -240,7 +329,10 @@ internal struct ObjectCopyPlan: Sendable { for step in dataSteps { lines.append("") lines.append(String(format: String(localized: "-- Copy rows into %@"), step.qualifiedTargetName)) - lines.append(step.sourceQuery + ";") + /// The statement itself where the server runs it, because that one is real SQL the + /// user is about to approve. The streamed path has no statement to show: its INSERTs + /// do not exist yet and never all exist at once, so the query it walks stands in. + lines.append(step.serverSideInsert?.sql ?? step.sourceQuery + ";") } guard !afterDataStatements.isEmpty else { return lines.joined(separator: "\n") } lines.append("") diff --git a/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift b/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift index 2d586328b..a6d714494 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift @@ -198,7 +198,10 @@ internal struct ObjectCopyPlanner { throw ObjectCopyError.refused(reason) } if let reason = ObjectCopyEligibility.engineRefusal( - from: request.source.databaseType, to: request.target.databaseType + from: request.source.databaseType, + to: request.target.databaseType, + sourceLanguage: PluginManager.shared.editorLanguage(for: request.source.databaseType), + targetLanguage: PluginManager.shared.editorLanguage(for: request.target.databaseType) ) { throw ObjectCopyError.refused(reason) } @@ -335,6 +338,9 @@ internal struct ObjectCopyPlanner { sourceNamespace: sourceNamespace, targetNamespace: targetNamespace ) + let serverSide = try await buildServerSideInserts( + drafts, request: request, sourceEndpoint: sourceEndpoint, targetEndpoint: targetEndpoint + ) return drafts.map { draft in let parts = sourceParts[draft.selection.id] let statements = ddl[draft.selection.id] ?? ObjectCopyTableDDL() @@ -342,7 +348,7 @@ internal struct ObjectCopyPlanner { selection: draft.selection, dropStatements: statements.drop, sequenceStatements: Self.sequenceStatements( - parts?.sequences ?? [], table: draft.targetTable + draft.isCrossEngine ? [] : (parts?.sequences ?? []), table: draft.targetTable ), createStatements: statements.create, truncateStatements: statements.truncate, @@ -354,6 +360,9 @@ internal struct ObjectCopyPlanner { estimatedRows: parts?.estimatedRows, copiesData: draft.copiesData, copiesIdentityColumn: draft.copiesIdentityColumn, + conversionNotes: draft.conversionNotes, + coercer: draft.coercer, + serverSideInsert: draft.copiesData ? serverSide[draft.selection.id] : nil, note: draft.note ) } @@ -365,6 +374,62 @@ internal struct ObjectCopyPlanner { let sequences: [String] } + /// One `INSERT … SELECT` per table the server can copy on its own. + /// + /// Built with the target driver's quoting, which is also the source's: the fast path only + /// exists when both endpoints are the same connection. Nothing is opened at all unless a table + /// qualifies, so a copy between two connections pays nothing for this. + private func buildServerSideInserts( + _ drafts: [ObjectCopyTableDraft], + request: ObjectCopyRequest, + sourceEndpoint: DatabaseEndpoint, + targetEndpoint: DatabaseEndpoint + ) async throws -> [String: SyncStatement] { + guard ObjectCopyServerSideInsert.isEligible(source: sourceEndpoint, target: targetEndpoint) else { + return [:] + } + let inputs = drafts.filter(\.copiesData).map { draft in + ( + id: draft.selection.id, + table: draft.targetTable, + input: ObjectCopyServerSideInsert.Input( + source: sourceEndpoint, + target: targetEndpoint, + sourceTable: draft.snapshot.name, + sourceSchema: draft.sourceSchema, + targetTable: draft.targetTable, + targetSchema: draft.targetSchema, + sourceColumns: draft.sourceColumns, + targetColumns: draft.targetColumns, + scope: draft.rowScope + ) + ) + } + guard !inputs.isEmpty else { return [:] } + + return try await manager.withMetadataDriver( + scope: targetScope(request, endpoint: targetEndpoint) + ) { driver in + guard let plugin = CompareMetadataService.pluginDriver(from: driver) else { + throw ObjectCopyError.refused(Self.noTargetDriver) + } + var statements: [String: SyncStatement] = [:] + for input in inputs { + guard let sql = ObjectCopyServerSideInsert.statement(input.input, driver: plugin) else { + continue + } + statements[input.id] = SyncStatement( + sql: sql, + objectName: input.table, + summary: String( + format: String(localized: "Copy rows into %@ on the server"), input.table + ) + ) + } + return statements + } + } + /// One scoped call for every table, because each `withMetadataDriver` either leases a pooled /// connection or takes the session gate. private func readSourceParts( @@ -381,7 +446,11 @@ internal struct ObjectCopyPlanner { schema: $0.sourceSchema, columns: $0.sourceColumns, copiesData: $0.copiesData, - writesStructure: $0.writesStructure + /// A sequence is read only where its `CREATE SEQUENCE` could run. Across engines it + /// is the source's own DDL, and the column that defaulted from it already carries + /// the target's own generated-key attribute instead. + writesStructure: $0.writesStructure && !$0.isCrossEngine, + scope: $0.rowScope ) } guard inputs.contains(where: { $0.copiesData || $0.writesStructure }) else { return [:] } @@ -402,11 +471,19 @@ internal struct ObjectCopyPlanner { /// structure-only copy scan every table it named. if input.copiesData { query = ObjectCopySelectQuery.build( - columns: input.columns, table: input.table, schema: input.schema, driver: plugin + columns: input.columns, table: input.table, schema: input.schema, + driver: plugin, scope: input.scope ) - estimatedRows = (try? await plugin.fetchApproximateRowCount( + let counted = (try? await plugin.fetchApproximateRowCount( table: input.table, schema: input.schema )) ?? nil + /// The driver counts the whole table, so a filtered step would show a bar + /// running to a total it can never reach. A limit is a ceiling the copy will + /// not pass; a `WHERE` has no knowable count without running it, so the + /// estimate is withheld and the review says Unknown. + estimatedRows = Self.estimate( + counted, scope: input.scope + ) } var sequences: [String] = [] if input.writesStructure { @@ -426,6 +503,16 @@ internal struct ObjectCopyPlanner { } } + /// What the progress bar counts this table against. Nil for a filtered table, because the only + /// count the driver has is of every row, and a bar that can never fill reads as a stall. + nonisolated internal static func estimate(_ counted: Int?, scope: PluginExportRowScope?) -> Int? { + guard let scope else { return counted } + guard scope.sanitizedFilter.isEmpty else { return scope.rowLimit } + guard let limit = scope.rowLimit else { return counted } + guard let counted else { return limit } + return min(counted, limit) + } + /// A copied table's default names its sequence, so the sequence has to be there before the /// `CREATE TABLE` runs. /// @@ -456,8 +543,10 @@ internal struct ObjectCopyPlanner { let inputs = drafts.map { ObjectCopyDDLInput( id: $0.selection.id, + /// The translated structure, so the `CREATE TABLE` the target driver writes names + /// types that engine has. Identical to the source's within one type family. snapshot: Self.retargeted( - $0.snapshot, from: sourceNamespace, to: targetNamespace, schema: $0.targetSchema + $0.targetStructure, from: sourceNamespace, to: targetNamespace, schema: $0.targetSchema ), targetSchema: $0.targetSchema, writesStructure: $0.writesStructure, @@ -634,6 +723,16 @@ internal struct ObjectCopyPlanner { let sourceNamespace = ObjectCopyNamespace.name(for: sourceEndpoint) let targetNamespace = ObjectCopyNamespace.name(for: targetEndpoint) + /// Asked before the namespace question and for the same reason: it depends on the two + /// engines alone, so one skip per object here costs nothing while reading every body first + /// and discarding all of them costs a round trip per object. + if let reason = ObjectCopyEligibility.definitionEngineRefusal( + from: request.source.databaseType, to: request.target.databaseType + ) { + skipped += selections.map { ObjectCopySkip(selection: $0, reason: reason) } + return [] + } + /// Asked once for the whole scope, and before anything is read. It depends only on the two /// namespaces, so a cross-namespace copy rejected every object anyway: asking per object /// first fetched every view body, routine body and trigger body from the source and then @@ -898,172 +997,3 @@ internal struct ObjectCopyPlanner { ) nonisolated private static let noSourceDriver = String(localized: "The source driver cannot be read.") } - -// MARK: - Drafts - -/// One table's decisions, made before any driver is opened so the two scoped calls that follow can -/// each run over the whole list. -private struct ObjectCopyTableDraft { - let selection: ObjectCopySelection - let snapshot: TableStructureSnapshot - let sourceSchema: String? - let targetSchema: String? - let targetTable: String - /// Read with these, written with those. A case-insensitive match pairs two spellings of one - /// column, and each side has to be quoted the way its own server spells it. - let sourceColumns: [String] - let targetColumns: [String] - let writesStructure: Bool - let dropsFirst: Bool - let emptiesFirst: Bool - let copiesData: Bool - let copiesIdentityColumn: Bool - let note: String? - - init( - selection: ObjectCopySelection, - read: TableStructureRead, - snapshot: TableStructureSnapshot, - targetSnapshot: TableStructureSnapshot?, - existsInTarget: Bool, - sourceSchema: String?, - targetSchema: String?, - request: ObjectCopyRequest - ) { - self.selection = selection - self.snapshot = snapshot - self.sourceSchema = sourceSchema - /// Never the source's. A target endpoint that names no schema means the target driver's - /// own current scope, and inheriting the source's put a SQL Server `dbo` into a MySQL - /// INSERT, naming a database that engine does not have. - self.targetSchema = targetSchema - - let keepsTargetStructure = existsInTarget && request.existingPolicy != .replace - let writesStructure = request.content.includesStructure && !keepsTargetStructure - self.writesStructure = writesStructure - self.dropsFirst = writesStructure && existsInTarget - - /// The target's own name when it already has the table, because a case-insensitive match - /// pairs `Orders` with `orders` and the INSERT has to quote the one that exists. - self.targetTable = (writesStructure ? nil : targetSnapshot?.name) ?? snapshot.name - - /// Read from the driver's own columns rather than from the snapshot. SQL Server computed - /// columns and ClickHouse ALIAS columns set `isGenerated` with no expression, and - /// PostgreSQL reports identity through `identityKind`; the snapshot conversion keeps - /// neither, so those columns looked ordinary and writable. - let pairs = Self.writableColumnPairs( - columns: read.columns, - snapshot: snapshot, - targetSnapshot: writesStructure ? nil : targetSnapshot - ) - self.sourceColumns = pairs.map(\.source) - self.targetColumns = pairs.map(\.target) - let copiesData = request.content.includesData && !pairs.isEmpty && (writesStructure || existsInTarget) - self.copiesData = copiesData - - /// A data-only replace has no DROP and CREATE to clear the table, so it is emptied instead, - /// and only where rows are going back into it. Emptying without that condition deleted - /// every row of a table whose columns the target does not share, and then wrote nothing: - /// the step was dropped from the data phase for having no writable column while its DELETE - /// stayed in the clear phase, and the review said only that the two sides shared no column. - self.emptiesFirst = copiesData - && existsInTarget - && request.existingPolicy == .replace - && !writesStructure - - let written = Set(pairs.map { $0.source.lowercased() }) - self.copiesIdentityColumn = request.content.includesData && read.columns.contains { - written.contains($0.name.lowercased()) && ($0.isIdentity || $0.extra?.lowercased().contains("auto_increment") == true) - } - - if request.content.includesData, !writesStructure, !existsInTarget { - self.note = String( - localized: "The target has no table of this name, so the rows have nowhere to go." - ) - } else if request.content.includesData, pairs.isEmpty { - self.note = String(localized: "The source and the target share no writable column.") - } else { - self.note = nil - } - } - - /// The columns the copy writes, paired source spelling to target spelling. - /// - /// The source's own order, without the ones the server computes: an `INSERT` into a generated - /// column is rejected by every engine that has them. When the target's structure is not being - /// written the answer narrows to what both sides have, matched without regard to case, because - /// a column the target lacks cannot be written to and one it has that the source lacks keeps - /// its default. - static func writableColumnPairs( - columns: [PluginColumnInfo], - snapshot: TableStructureSnapshot, - targetSnapshot: TableStructureSnapshot? - ) -> [(source: String, target: String)] { - let generated = Set(columns.filter(\.isGenerated).map { $0.name.lowercased() }) - let sourceColumns = snapshot.columns - .filter { $0.generationExpression == nil && !generated.contains($0.name.lowercased()) } - .map(\.name) - guard let targetSnapshot else { return sourceColumns.map { ($0, $0) } } - - /// Exact spellings first. PostgreSQL allows quoted `Orders` and `orders` in one schema, so - /// folding case unconditionally resolved either to whichever row came back first. - var exact: [String: String] = [:] - var folded: [String: [String]] = [:] - for column in targetSnapshot.columns where column.generationExpression == nil { - exact[column.name] = column.name - folded[column.name.lowercased(), default: []].append(column.name) - } - return sourceColumns.compactMap { name in - if let target = exact[name] { return (name, target) } - guard let candidates = folded[name.lowercased()], candidates.count == 1 else { return nil } - return (name, candidates[0]) - } - } -} - -private struct ObjectCopyDDLInput: Sendable { - let id: String - let snapshot: TableStructureSnapshot - let targetSchema: String? - let writesStructure: Bool - let dropsFirst: Bool - let emptiesFirst: Bool - let clearsWithDelete: Bool -} - -private struct ObjectCopyTableDDL: Sendable { - var drop: [SyncStatement] = [] - var create: [SyncStatement] = [] - var truncate: [SyncStatement] = [] -} - -internal enum ObjectCopyError: LocalizedError { - case refused(String) - - internal var errorDescription: String? { - switch self { - case .refused(let message): return message - } - } -} - -/// The read side of a table copy: the exact columns that will be written, in the order they will be -/// written, so the stream and the INSERT cannot drift apart. -internal enum ObjectCopySelectQuery { - internal static func build( - columns: [String], - table: String, - schema: String?, - driver: any PluginDatabaseDriver - ) -> String { - let list = columns.isEmpty - ? "*" - : columns.map { driver.quoteIdentifier($0) }.joined(separator: ", ") - return "SELECT \(list) FROM \(qualified(table, schema, driver))" - } - - private static func qualified(_ table: String, _ schema: String?, _ driver: any PluginDatabaseDriver) -> String { - guard let schema, !schema.isEmpty else { return driver.quoteIdentifier(table) } - return "\(driver.quoteIdentifier(schema)).\(driver.quoteIdentifier(table))" - } -} diff --git a/TablePro/Core/ObjectCopy/ObjectCopyRequest.swift b/TablePro/Core/ObjectCopy/ObjectCopyRequest.swift index 7187a81c7..88ffb941c 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopyRequest.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopyRequest.swift @@ -136,6 +136,16 @@ internal struct ObjectCopyRequest: Sendable { internal let existingPolicy: ObjectCopyExistingPolicy internal let errorHandling: ImportErrorHandling internal let wrapEachTableInTransaction: Bool + /// A `WHERE` and a row limit per table, keyed by `ObjectCopySelection.id`. + /// + /// The same type the export tree narrows a table with, so the rule that a filter is one + /// expression is written once. That rule is not cosmetic: the text is spliced into a `SELECT`, + /// and `sanitizedFilter` is what stops a second statement riding in with it. + /// + /// It narrows the rows only. A copy that also wrote the structure would otherwise create a + /// table whose columns and whose `INSERT` disagreed, so `PluginExportRowScope.columns` is + /// deliberately not carried. + internal let rowScopes: [String: PluginExportRowScope] internal init( source: DatabaseEndpoint, @@ -144,7 +154,8 @@ internal struct ObjectCopyRequest: Sendable { content: ObjectCopyContent, existingPolicy: ObjectCopyExistingPolicy, errorHandling: ImportErrorHandling = .stopAndRollback, - wrapEachTableInTransaction: Bool = true + wrapEachTableInTransaction: Bool = true, + rowScopes: [String: PluginExportRowScope] = [:] ) { self.source = source self.destination = destination @@ -153,6 +164,14 @@ internal struct ObjectCopyRequest: Sendable { self.existingPolicy = existingPolicy self.errorHandling = errorHandling self.wrapEachTableInTransaction = wrapEachTableInTransaction + self.rowScopes = rowScopes + } + + /// The rows this table contributes, with the column subset dropped so the scope can only ever + /// narrow rows. + internal func rowScope(for selection: ObjectCopySelection) -> PluginExportRowScope? { + guard let scope = rowScopes[selection.id], !scope.isUnrestricted else { return nil } + return PluginExportRowScope(filter: scope.filter, rowLimit: scope.rowLimit) } internal var target: DatabaseEndpoint { destination.endpoint } diff --git a/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift b/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift index a428cbef7..b3f0e34c9 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift @@ -59,6 +59,9 @@ internal struct ObjectCopyRowCopier: Sendable { to targetDriver: any PluginDatabaseDriver, onProgress: @Sendable (Int) -> Void ) async throws -> Outcome { + if let statement = step.serverSideInsert { + return try await copyOnServer(statement, using: targetDriver, onProgress: onProgress) + } let generator = try makeGenerator(targetDriver: targetDriver) let batchSize = Self.batchSize(columnCount: step.columns.count, generator: generator) var stream = sourceDriver.streamRows(query: step.sourceQuery).makeAsyncIterator() @@ -90,6 +93,34 @@ internal struct ObjectCopyRowCopier: Sendable { return Outcome(inserted: inserted, cancelled: Task.isCancelled) } + // MARK: - Server side + + /// One statement, run on the target driver, which for this path is also the source's. + /// + /// Cancellation is checked before it is sent and not after: a statement the server has already + /// been given runs to completion whatever the app does next, and reporting it as stopped would + /// leave the sheet claiming nothing was written over rows that were. The plan warns about that + /// before the user presses Copy. + private func copyOnServer( + _ statement: SyncStatement, + using driver: any PluginDatabaseDriver, + onProgress: @Sendable (Int) -> Void + ) async throws -> Outcome { + if Task.isCancelled { return Outcome(inserted: 0, cancelled: true) } + let result = try await driver.execute(query: statement.sql) + /// The server's own count, never the plan's estimate. `rowsAffected` is not optional, so a + /// driver that reports nothing and a table that held nothing both come back as zero, and + /// standing the estimate in for that told the user a copy of an empty table had written + /// however many rows a stale table statistic guessed at. Under-reporting a count is a + /// worse-looking number; over-reporting one is a false claim about their data. + let inserted = result.rowsAffected + onProgress(inserted) + /// `committed` is left at zero, as the streamed path leaves it: whether these rows survive + /// is the caller's transaction to decide, and it is the caller that fills it in when there + /// is no transaction to roll back. + return Outcome(inserted: inserted, cancelled: false) + } + // MARK: - Statements private func makeGenerator(targetDriver: any PluginDatabaseDriver) throws -> SQLStatementGenerator { @@ -131,6 +162,10 @@ internal struct ObjectCopyRowCopier: Sendable { /// The SELECT names its columns, so a row that arrives with a different width means the source /// answered a different question from the one the plan asked. Writing it would put values in /// the wrong columns, so it stops the table instead. + /// + /// The width is checked before the coercion rather than after, because the coercion is + /// positional: a short row would have every value past the gap reshaped against the wrong + /// column's type. private func aligned(_ row: [PluginCellValue]) throws -> [PluginCellValue] { guard row.count == step.columns.count else { throw ObjectCopyError.refused(String( @@ -138,7 +173,8 @@ internal struct ObjectCopyRowCopier: Sendable { step.selection.qualifiedName, row.count, step.columns.count )) } - return row + guard let coercer = step.coercer else { return row } + return coercer.coerce(row) } /// Every value in the batch is one bind parameter, and each engine has its own ceiling on how diff --git a/TablePro/Core/ObjectCopy/ObjectCopyServerSideInsert.swift b/TablePro/Core/ObjectCopy/ObjectCopyServerSideInsert.swift new file mode 100644 index 000000000..2ed52e54e --- /dev/null +++ b/TablePro/Core/ObjectCopy/ObjectCopyServerSideInsert.swift @@ -0,0 +1,124 @@ +// +// ObjectCopyServerSideInsert.swift +// TablePro +// +// The copy that never leaves the server. +// +// Streaming a table through the app costs a read, a decode, a re-encode and a +// write per row, and every one of those bytes crosses the network twice. When +// both sides of a copy are the same connection, the server can do the whole +// thing itself with one `INSERT INTO … SELECT`, which is the difference +// between minutes and seconds on a large table. +// +// Two things are given up for that, and both are why this is not the only path. +// There is no per-row progress, because the statement reports nothing until it +// finishes; and Stop cannot land inside it, because a statement already sent is +// the server's to finish. The plan says so and the review shows the statement +// itself, so neither is a surprise. +// +// The gate is narrow on purpose. Naming the other side of a copy needs a +// reference the engine resolves the same way from where the statement runs, and +// the engines disagree about whether one database can name another at all: +// MySQL and ClickHouse write `db.table`, SQL Server writes `db.schema.table`, +// and PostgreSQL cannot do it in a single statement under any spelling. Getting +// that wrong does not corrupt anything, because the statement is rejected +// rather than resolved to the wrong table, but a copy that fails at its last +// step is worse than one that took the slow path. +// + +import Foundation +import TableProPluginKit + +internal enum ObjectCopyServerSideInsert { + /// Whether the server can be asked to copy between these two on its own. + /// + /// One connection, so one session and one transaction, and the same engine follows from that. + /// Beyond that it is only a question of whether the target's scope can name the source's. + internal static func isEligible(source: DatabaseEndpoint, target: DatabaseEndpoint) -> Bool { + guard source.connectionId == target.connectionId else { return false } + guard !crossesDatabases(SQLTypeFamily.of(target.databaseType)) else { return true } + return source.database == target.database + } + + internal static func statement( + _ input: Input, + driver: any PluginDatabaseDriver + ) -> String? { + guard !input.sourceColumns.isEmpty, + input.sourceColumns.count == input.targetColumns.count else { return nil } + guard let from = reference( + database: input.source.database, + schema: input.sourceSchema, + table: input.sourceTable, + family: SQLTypeFamily.of(input.target.databaseType), + driver: driver + ) else { return nil } + + let into = ObjectCopySelectQuery.qualified(input.targetTable, input.targetSchema, driver) + let targetList = input.targetColumns.map { driver.quoteIdentifier($0) }.joined(separator: ", ") + let sourceList = input.sourceColumns.map { driver.quoteIdentifier($0) }.joined(separator: ", ") + var select = "SELECT \(sourceList) FROM \(from)" + if let filter = input.scope?.sanitizedFilter, !filter.isEmpty { + select += " WHERE \(filter)" + } + if let rowLimit = input.scope?.rowLimit { + select = driver.injectRowLimit(select, limit: rowLimit) ?? "\(select) LIMIT \(rowLimit)" + } + return "INSERT INTO \(into) (\(targetList)) \(select);" + } + + internal struct Input: Sendable { + internal let source: DatabaseEndpoint + internal let target: DatabaseEndpoint + internal let sourceTable: String + internal let sourceSchema: String? + internal let targetTable: String + internal let targetSchema: String? + internal let sourceColumns: [String] + internal let targetColumns: [String] + internal let scope: PluginExportRowScope? + } + + // MARK: - Naming the other side + + /// Spelled as the engine spells it, and only where the engine has a spelling. + /// + /// An engine that can name a second database is given the fully qualified name even when the + /// two sides share one, because it is unambiguous either way and comparing the two database + /// strings is not. `Shop` and `shop` are one database on a case-insensitive MySQL server and + /// two on a case-sensitive one; treating them as one and dropping the qualifier would resolve + /// the SELECT against the database the statement runs in, which is the target's, so the copy + /// would read the table it was about to write. + /// + /// The others get the ordinary qualified name and are only reached when the databases match + /// exactly. PostgreSQL has no spelling at all: a second database needs `dblink` or a foreign + /// data wrapper, neither of which a copy may install on the user's server. + private static func reference( + database: String, + schema: String?, + table: String, + family: SQLTypeFamily, + driver: any PluginDatabaseDriver + ) -> String? { + guard crossesDatabases(family), !database.isEmpty else { + return ObjectCopySelectQuery.qualified(table, schema, driver) + } + switch family { + case .mysql, .clickhouse: + return "\(driver.quoteIdentifier(database)).\(driver.quoteIdentifier(table))" + case .mssql: + let resolved = schema?.nilIfEmpty ?? "dbo" + return "\(driver.quoteIdentifier(database)).\(driver.quoteIdentifier(resolved))" + + ".\(driver.quoteIdentifier(table))" + case .postgres, .sqlite, .oracle, .duckdb, .generic: + return nil + } + } + + private static func crossesDatabases(_ family: SQLTypeFamily) -> Bool { + switch family { + case .mysql, .clickhouse, .mssql: return true + case .postgres, .sqlite, .oracle, .duckdb, .generic: return false + } + } +} diff --git a/TablePro/Core/ObjectCopy/ObjectCopySession.swift b/TablePro/Core/ObjectCopy/ObjectCopySession.swift index 873865dcd..1f211e5ce 100644 --- a/TablePro/Core/ObjectCopy/ObjectCopySession.swift +++ b/TablePro/Core/ObjectCopy/ObjectCopySession.swift @@ -65,6 +65,14 @@ internal final class ObjectCopySession { internal var existingPolicy: ObjectCopyExistingPolicy = .skip internal var errorHandling: ImportErrorHandling = .stopAndRollback internal var searchText = "" + /// A `WHERE` and a row limit per table, keyed by the selection's own id so two overloads or two + /// same-named triggers cannot share one. Absent means every row, which is what a copy did + /// before this existed. + internal var rowScopes: [String: PluginExportRowScope] = [:] + /// The table whose filter popover is open, if any. Held here rather than in the row, because a + /// row is rebuilt whenever the search text changes and a popover anchored to `@State` inside + /// one closes as soon as the list diffs under a keystroke. + internal var rowFilterObjectId: String? // MARK: - Catalog @@ -158,6 +166,12 @@ internal final class ObjectCopySession { if selectedObjectIds.isEmpty { return String(localized: "Choose at least one object to copy.") } + if let object = rejectedFilterObject { + return String( + format: String(localized: "The filter on %@ is not one expression. Remove the semicolon."), + object.displayName + ) + } switch mode { case .copyTo: guard let target else { return String(localized: "Choose where to copy to.") } @@ -166,7 +180,10 @@ internal final class ObjectCopySession { return reason } if let reason = ObjectCopyEligibility.engineRefusal( - from: source.databaseType, to: target.databaseType + from: source.databaseType, + to: target.databaseType, + sourceLanguage: PluginManager.shared.editorLanguage(for: source.databaseType), + targetLanguage: PluginManager.shared.editorLanguage(for: target.databaseType) ) { return reason } @@ -218,10 +235,72 @@ internal final class ObjectCopySession { content: content, existingPolicy: existingPolicy, errorHandling: errorHandling, - wrapEachTableInTransaction: true + wrapEachTableInTransaction: true, + rowScopes: activeRowScopes ) } + // MARK: - Row filters + + /// A filter narrows the rows a table contributes, so it means nothing for an object that has + /// none and nothing for a copy that is not carrying rows. + internal func allowsRowFilter(for object: ObjectCopySelection) -> Bool { + object.kind.carriesRows && content.includesData + } + + internal func rowScope(for object: ObjectCopySelection) -> PluginExportRowScope { + rowScopes[object.id] ?? .unrestricted + } + + /// Kept when it narrows something, and kept when it is refused. + /// + /// `isUnrestricted` answers yes to a filter the sanitizer threw out, because `sanitizedFilter` + /// is empty for text holding a second statement. Dropping the scope on that answer discarded + /// the very thing `rejectedFilterObject` exists to catch, so the copy went ahead over every row + /// of a table the user had tried to narrow. + internal func setRowScope(_ scope: PluginExportRowScope, for object: ObjectCopySelection) { + guard isNarrowing(scope) else { + rowScopes.removeValue(forKey: object.id) + return + } + rowScopes[object.id] = scope + } + + /// Whether this table's rows are narrowed, or an attempt at narrowing them is outstanding. + /// What the funnel is drawn from, so a refused filter is still findable. + internal func hasRowFilter(for object: ObjectCopySelection) -> Bool { + isNarrowing(rowScope(for: object)) + } + + private func isNarrowing(_ scope: PluginExportRowScope) -> Bool { + !scope.isUnrestricted || scope.hasRejectedFilter + } + + /// Only the filters that belong to an object still selected, and only while rows are being + /// copied. A filter left behind by a deselected table would otherwise reach the planner, which + /// keys them by id and would silently narrow a different copy if that table were ticked again. + private var activeRowScopes: [String: PluginExportRowScope] { + guard content.includesData else { return [:] } + var active: [String: PluginExportRowScope] = [:] + for object in selectedObjects where object.kind.carriesRows { + guard let scope = rowScopes[object.id], !scope.isUnrestricted else { continue } + active[object.id] = scope + } + return active + } + + /// A filter the sanitizer refused, which is the one case the copy must not run: the text holds + /// a second statement, and dropping it silently would copy every row of a table the user meant + /// to narrow. + /// + /// Only while rows are being copied. A filter left behind by a switch to structure only reaches + /// no query, so holding Continue over it would be a dead button with no way to clear it: the + /// funnel that set it is not offered in that mode either. + internal var rejectedFilterObject: ObjectCopySelection? { + guard content.includesData else { return nil } + return selectedObjects.first { rowScopes[$0.id]?.hasRejectedFilter == true } + } + // MARK: - Catalog internal func loadObjects(catalog: ObjectCopyCatalog = ObjectCopyCatalog()) async { diff --git a/TablePro/Core/ObjectCopy/ObjectCopyTableDraft.swift b/TablePro/Core/ObjectCopy/ObjectCopyTableDraft.swift new file mode 100644 index 000000000..c5da4a1a1 --- /dev/null +++ b/TablePro/Core/ObjectCopy/ObjectCopyTableDraft.swift @@ -0,0 +1,276 @@ +// +// ObjectCopyTableDraft.swift +// TablePro +// +// One table's decisions, and the two statements they produce. +// +// Split out of `ObjectCopyPlanner`, which reached its length limit. The +// planner resolves and reads; everything here is what one table resolved to, +// so the two scoped calls that follow can each run over the whole list. +// + +import Foundation +import TableProPluginKit + +/// One table's decisions, made before any driver is opened so the two scoped calls that follow can +/// each run over the whole list. +internal struct ObjectCopyTableDraft { + let selection: ObjectCopySelection + let snapshot: TableStructureSnapshot + /// The same table said in the target's own types, which is what the target driver is handed. + /// Identical to `snapshot` whenever the two engines share a type family. + let targetStructure: TableStructureSnapshot + let sourceSchema: String? + let targetSchema: String? + let targetTable: String + /// Read with these, written with those. A case-insensitive match pairs two spellings of one + /// column, and each side has to be quoted the way its own server spells it. + let sourceColumns: [String] + let targetColumns: [String] + let writesStructure: Bool + let dropsFirst: Bool + let emptiesFirst: Bool + let copiesData: Bool + let copiesIdentityColumn: Bool + /// True when the two engines do not share a type family, so the structure was rewritten and + /// the values need reshaping on the way in. + let isCrossEngine: Bool + let conversionNotes: [CrossEngineConversionNote] + let coercer: CrossEngineValueCoercer? + /// The `WHERE` and row limit the user set on this table, or nil for every row. + let rowScope: PluginExportRowScope? + let note: String? + + init( + selection: ObjectCopySelection, + read: TableStructureRead, + snapshot: TableStructureSnapshot, + targetSnapshot: TableStructureSnapshot?, + existsInTarget: Bool, + sourceSchema: String?, + targetSchema: String?, + request: ObjectCopyRequest + ) { + self.selection = selection + self.snapshot = snapshot + self.rowScope = request.rowScope(for: selection) + self.sourceSchema = sourceSchema + /// Never the source's. A target endpoint that names no schema means the target driver's + /// own current scope, and inheriting the source's put a SQL Server `dbo` into a MySQL + /// INSERT, naming a database that engine does not have. + self.targetSchema = targetSchema + + let keepsTargetStructure = existsInTarget && request.existingPolicy != .replace + let writesStructure = request.content.includesStructure && !keepsTargetStructure + self.writesStructure = writesStructure + self.dropsFirst = writesStructure && existsInTarget + + /// The target's own name when it already has the table, because a case-insensitive match + /// pairs `Orders` with `orders` and the INSERT has to quote the one that exists. + self.targetTable = (writesStructure ? nil : targetSnapshot?.name) ?? snapshot.name + + let translation = CrossEngineStructureTranslator.translate( + snapshot, from: request.source.databaseType, to: request.target.databaseType + ) + self.targetStructure = translation.snapshot + self.isCrossEngine = translation.translated + /// Only a run that writes the structure has anything to report about it. Appending into a + /// table the target already has changes no type, so a list of conversions would describe + /// DDL that is not going to run. + self.conversionNotes = writesStructure ? translation.notes : [] + + /// Read from the driver's own columns rather than from the snapshot. SQL Server computed + /// columns and ClickHouse ALIAS columns set `isGenerated` with no expression, and + /// PostgreSQL reports identity through `identityKind`; the snapshot conversion keeps + /// neither, so those columns looked ordinary and writable. + /// + /// A computed column is the exception once the engines differ: its expression cannot come + /// across, so the copy creates it as an ordinary column, and a column the target will never + /// compute has to be written or it arrives empty. + let carriesGeneratedColumns = translation.translated && writesStructure + let pairs = Self.writableColumnPairs( + columns: read.columns, + snapshot: carriesGeneratedColumns ? translation.snapshot : snapshot, + targetSnapshot: writesStructure ? nil : targetSnapshot, + includesGenerated: carriesGeneratedColumns + ) + self.sourceColumns = pairs.map(\.source) + self.targetColumns = pairs.map(\.target) + let copiesData = request.content.includesData && !pairs.isEmpty && (writesStructure || existsInTarget) + self.copiesData = copiesData + + /// A data-only replace has no DROP and CREATE to clear the table, so it is emptied instead, + /// and only where rows are going back into it. Emptying without that condition deleted + /// every row of a table whose columns the target does not share, and then wrote nothing: + /// the step was dropped from the data phase for having no writable column while its DELETE + /// stayed in the clear phase, and the review said only that the two sides shared no column. + self.emptiesFirst = copiesData + && existsInTarget + && request.existingPolicy == .replace + && !writesStructure + + let written = Set(pairs.map { $0.source.lowercased() }) + self.copiesIdentityColumn = request.content.includesData && read.columns.contains { + written.contains($0.name.lowercased()) && ($0.isIdentity || $0.extra?.lowercased().contains("auto_increment") == true) + } + + /// Built from whichever side decides the target's types: the translation when the copy + /// creates the table, and the target's own structure when it appends into one that is + /// already there and whose columns the user may have declared differently. + self.coercer = Self.coercer( + for: pairs, + translation: translation, + targetSnapshot: writesStructure ? nil : targetSnapshot, + request: request + ) + + if request.content.includesData, !writesStructure, !existsInTarget { + self.note = String( + localized: "The target has no table of this name, so the rows have nowhere to go." + ) + } else if request.content.includesData, pairs.isEmpty { + self.note = String(localized: "The source and the target share no writable column.") + } else { + self.note = nil + } + } + + /// The columns the copy writes, paired source spelling to target spelling. + /// + /// The source's own order, without the ones the server computes: an `INSERT` into a generated + /// column is rejected by every engine that has them. When the target's structure is not being + /// written the answer narrows to what both sides have, matched without regard to case, because + /// a column the target lacks cannot be written to and one it has that the source lacks keeps + /// its default. + static func writableColumnPairs( + columns: [PluginColumnInfo], + snapshot: TableStructureSnapshot, + targetSnapshot: TableStructureSnapshot?, + includesGenerated: Bool = false + ) -> [(source: String, target: String)] { + let generated = includesGenerated + ? Set() + : Set(columns.filter(\.isGenerated).map { $0.name.lowercased() }) + let sourceColumns = snapshot.columns + .filter { $0.generationExpression == nil && !generated.contains($0.name.lowercased()) } + .map(\.name) + guard let targetSnapshot else { return sourceColumns.map { ($0, $0) } } + + /// Exact spellings first. PostgreSQL allows quoted `Orders` and `orders` in one schema, so + /// folding case unconditionally resolved either to whichever row came back first. + var exact: [String: String] = [:] + var folded: [String: [String]] = [:] + for column in targetSnapshot.columns where column.generationExpression == nil { + exact[column.name] = column.name + folded[column.name.lowercased(), default: []].append(column.name) + } + return sourceColumns.compactMap { name in + if let target = exact[name] { return (name, target) } + guard let candidates = folded[name.lowercased()], candidates.count == 1 else { return nil } + return (name, candidates[0]) + } + } + + /// Nil where the two engines share a type family, and nil where the columns that are written + /// hold nothing whose spelling differs between them. A copy that needs no reshaping runs the + /// loop it ran before this existed. + static func coercer( + for pairs: [(source: String, target: String)], + translation: CrossEngineStructureTranslator.Result, + targetSnapshot: TableStructureSnapshot?, + request: ObjectCopyRequest + ) -> CrossEngineValueCoercer? { + guard translation.translated, !pairs.isEmpty else { return nil } + let source = SQLTypeFamily.of(request.source.databaseType) + /// The target's own columns when the copy appends into a table that is already there, and + /// the translation's when it creates one. The user may have declared an existing table's + /// columns differently from anything this copy would have chosen. + let declared = targetSnapshot.map { + CrossEngineStructureTranslator.kinds( + of: $0, family: SQLTypeFamily.of(request.target.databaseType) + ) + } ?? translation.targetKinds + let sourceKinds = folded(translation.sourceKinds) + let targetKinds = folded(declared) + let columnPairs = pairs.map { + CrossEngineValueCoercer.ColumnPair( + source: sourceKinds[$0.source.lowercased()], + target: targetKinds[$0.target.lowercased()] + ) + } + let coercer = CrossEngineValueCoercer(pairs: columnPairs, from: source) + return coercer.isNeeded ? coercer : nil + } + + /// Matched without regard to case, the way the column pairing itself is: PostgreSQL allows + /// quoted `Orders` and `orders` in one schema and the two sides need not spell one the same. + private static func folded( + _ kinds: [String: CanonicalTypeKind] + ) -> [String: CanonicalTypeKind] { + var folded: [String: CanonicalTypeKind] = [:] + for (name, kind) in kinds { folded[name.lowercased()] = kind } + return folded + } +} + +internal struct ObjectCopyDDLInput: Sendable { + let id: String + let snapshot: TableStructureSnapshot + let targetSchema: String? + let writesStructure: Bool + let dropsFirst: Bool + let emptiesFirst: Bool + let clearsWithDelete: Bool +} + +internal struct ObjectCopyTableDDL: Sendable { + var drop: [SyncStatement] = [] + var create: [SyncStatement] = [] + var truncate: [SyncStatement] = [] +} + +internal enum ObjectCopyError: LocalizedError { + case refused(String) + + internal var errorDescription: String? { + switch self { + case .refused(let message): return message + } + } +} + +/// The read side of a table copy: the exact columns that will be written, in the order they will be +/// written, so the stream and the INSERT cannot drift apart. +internal enum ObjectCopySelectQuery { + internal static func build( + columns: [String], + table: String, + schema: String?, + driver: any PluginDatabaseDriver, + scope: PluginExportRowScope? = nil + ) -> String { + let list = columns.isEmpty + ? "*" + : columns.map { driver.quoteIdentifier($0) }.joined(separator: ", ") + var query = "SELECT \(list) FROM \(qualified(table, schema, driver))" + /// `sanitizedFilter` rather than `filter`. The text is the user's own SQL against their own + /// connection, but it is spliced into this statement, and the sanitizer is what keeps it to + /// the single expression the field is for. + if let filter = scope?.sanitizedFilter, !filter.isEmpty { + query += " WHERE \(filter)" + } + guard let rowLimit = scope?.rowLimit else { return query } + /// Through the driver's own injection, because `LIMIT` is not the spelling on SQL Server or + /// on Oracle before 12c. + return driver.injectRowLimit(query, limit: rowLimit) ?? "\(query) LIMIT \(rowLimit)" + } + + internal static func qualified( + _ table: String, + _ schema: String?, + _ driver: any PluginDatabaseDriver + ) -> String { + guard let schema, !schema.isEmpty else { return driver.quoteIdentifier(table) } + return "\(driver.quoteIdentifier(schema)).\(driver.quoteIdentifier(table))" + } +} diff --git a/TablePro/Views/Export/ExportRowScopeEditor.swift b/TablePro/Views/Export/ExportRowScopeEditor.swift index 0da8285b8..49aec994f 100644 --- a/TablePro/Views/Export/ExportRowScopeEditor.swift +++ b/TablePro/Views/Export/ExportRowScopeEditor.swift @@ -40,6 +40,7 @@ internal struct ExportRowScopeEditor: View { .textFieldStyle(.roundedBorder) .lineLimit(2 ... 4) .font(ThemeEngine.shared.valueFontSwiftUI) + .accessibilityIdentifier("row-scope-filter") if hasRejectedFilter { Text("A filter is one expression. Remove the semicolon.") .font(.caption) diff --git a/TablePro/Views/ObjectCopy/CopyObjectsListView.swift b/TablePro/Views/ObjectCopy/CopyObjectsListView.swift index d89ddcbaf..4c405c8e7 100644 --- a/TablePro/Views/ObjectCopy/CopyObjectsListView.swift +++ b/TablePro/Views/ObjectCopy/CopyObjectsListView.swift @@ -6,6 +6,7 @@ // import SwiftUI +import TableProPluginKit internal struct CopyObjectsListView: View { @Bindable internal var session: ObjectCopySession @@ -94,28 +95,94 @@ internal struct CopyObjectsListView: View { } } + /// The tick and the filter are siblings rather than one inside the other: a button inside a + /// `Toggle`'s label is still part of the toggle's hit area, so opening the filter would flip + /// the object out of the copy on the way. private func row(_ object: ObjectCopySelection) -> some View { - /// The value SwiftUI hands the setter, never an unconditional flip. A binding that inverts - /// whatever it is written turns any write of the value it already holds into a change, so - /// a re-render, an accessibility `setValue`, or a second delivery while the list diffs - /// under a search keystroke silently took an object out of the copy or put one in. - Toggle(isOn: Binding( - get: { session.selectedObjectIds.contains(object.id) }, - set: { session.setSelected(object, $0) } - )) { - HStack(spacing: 6) { - /// The signature or the owning table, not just the name: two overloads and two - /// same-named triggers are two rows and have to read as two. - Text(object.displayName) - .lineLimit(1) - .truncationMode(.middle) - Spacer(minLength: 8) - Text(object.kind.displayName) - .font(.caption) - .foregroundStyle(.secondary) + HStack(spacing: 6) { + /// The value SwiftUI hands the setter, never an unconditional flip. A binding that + /// inverts whatever it is written turns any write of the value it already holds into a + /// change, so a re-render, an accessibility `setValue`, or a second delivery while the + /// list diffs under a search keystroke silently took an object out of the copy or put + /// one in. + Toggle(isOn: Binding( + get: { session.selectedObjectIds.contains(object.id) }, + set: { session.setSelected(object, $0) } + )) { + HStack(spacing: 6) { + /// The signature or the owning table, not just the name: two overloads and two + /// same-named triggers are two rows and have to read as two. + Text(object.displayName) + .lineLimit(1) + .truncationMode(.middle) + Spacer(minLength: 8) + Text(object.kind.displayName) + .font(.caption) + .foregroundStyle(.secondary) + } + } + .toggleStyle(.checkbox) + /// Outside the toggle, beside the control that sets it. A `Toggle` folds its whole + /// label into one accessibility element, so a summary placed in there is spoken as part + /// of the checkbox and cannot be read on its own. + scopeSummary(object) + if session.allowsRowFilter(for: object) { + filterButton(object) } } - .toggleStyle(.checkbox) + } + + /// Shown only where the filter would act. A copy switched to structure only carries no rows for + /// a `WHERE` to narrow, and a row still advertising one would misreport what is about to happen. + @ViewBuilder + private func scopeSummary(_ object: ObjectCopySelection) -> some View { + let scope = session.rowScope(for: object) + if session.allowsRowFilter(for: object), !scope.isUnrestricted, !scope.summary.isEmpty { + Text(scope.summary) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.middle) + .accessibilityIdentifier("copy-objects-row-scope-\(object.qualifiedName)") + } + } + + /// The same editor the export tree narrows a table with, minus its column list: a copy that + /// also writes the structure would otherwise build a table whose columns and whose `INSERT` + /// disagree. + private func filterButton(_ object: ObjectCopySelection) -> some View { + /// Filled for a refused filter as well as a working one. A filter the sanitizer threw out + /// is the one the user most needs to find again, and it is also the one holding Continue. + let isFiltered = session.hasRowFilter(for: object) + return Button { + session.rowFilterObjectId = object.id + } label: { + Image(systemName: isFiltered + ? "line.3.horizontal.decrease.circle.fill" + : "line.3.horizontal.decrease.circle") + } + .buttonStyle(.borderless) + .foregroundStyle(isFiltered ? AnyShapeStyle(Color.accentColor) : AnyShapeStyle(.secondary)) + .help(String(localized: "Choose which rows of this table are copied")) + .accessibilityLabel(String(localized: "Filter rows")) + .accessibilityIdentifier("copy-objects-row-filter-\(object.qualifiedName)") + .popover( + isPresented: Binding( + get: { session.rowFilterObjectId == object.id }, + set: { if !$0, session.rowFilterObjectId == object.id { session.rowFilterObjectId = nil } } + ), + arrowEdge: .trailing + ) { + ExportRowScopeEditor( + objectName: object.displayName, + availableColumns: [], + scope: Binding( + get: { session.rowScope(for: object) }, + set: { session.setRowScope($0, for: object) } + ), + dismiss: { session.rowFilterObjectId = nil } + ) + } } private func centred(@ViewBuilder content: () -> some View) -> some View { diff --git a/TablePro/Views/ObjectCopy/CopyObjectsReviewView.swift b/TablePro/Views/ObjectCopy/CopyObjectsReviewView.swift index 9ccaf3e12..f2490cf08 100644 --- a/TablePro/Views/ObjectCopy/CopyObjectsReviewView.swift +++ b/TablePro/Views/ObjectCopy/CopyObjectsReviewView.swift @@ -47,6 +47,7 @@ internal struct CopyObjectsReviewView: View { .fixedSize(horizontal: false, vertical: true) } rowPlan(plan) + conversions(plan) notes(plan) skipped(plan) } @@ -106,6 +107,52 @@ internal struct CopyObjectsReviewView: View { return String(format: template, rows.formatted(.number.grouping(.automatic))) } + /// What the crossing between two engines changed, one row per column or index. + /// + /// Listed rather than summarised, because "some types were converted" is not something a user + /// can act on and "amount: DECIMAL(19,4) → NUMBER(19,4)" is. A row that loses something carries + /// the same warning triangle the copy's own caveats use; a widening reads as ordinary text, + /// because every value still fits and there is nothing to decide. + @ViewBuilder + private func conversions(_ plan: ObjectCopyPlan) -> some View { + let notes = plan.reviewedConversionNotes() + if !notes.shown.isEmpty { + VStack(alignment: .leading, spacing: 6) { + Text("Type changes") + .font(.subheadline.weight(.medium)) + ForEach(notes.shown) { note in + conversion(note) + } + if notes.hidden > 0 { + Text(String( + format: String(localized: "%@ more, in the script beside this"), + notes.hidden.formatted(.number.grouping(.automatic)) + )) + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + } + + private func conversion(_ note: CrossEngineConversionNote) -> some View { + VStack(alignment: .leading, spacing: 2) { + Label { + Text(note.summary) + } icon: { + Image(systemName: note.isLossy ? "exclamationmark.triangle" : "arrow.right.circle") + } + .font(.callout) + .foregroundStyle(note.isLossy ? AnyShapeStyle(.orange) : AnyShapeStyle(.secondary)) + Text(note.reason) + .font(.caption) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + .frame(maxWidth: .infinity, alignment: .leading) + .accessibilityElement(children: .combine) + } + @ViewBuilder private func notes(_ plan: ObjectCopyPlan) -> some View { let noted = plan.tableSteps.compactMap { step in step.note.map { (step.id, step.selection, $0) } } diff --git a/TableProTests/Core/CrossEngine/CrossEngineStructureTranslatorTests.swift b/TableProTests/Core/CrossEngine/CrossEngineStructureTranslatorTests.swift new file mode 100644 index 000000000..852b9a504 --- /dev/null +++ b/TableProTests/Core/CrossEngine/CrossEngineStructureTranslatorTests.swift @@ -0,0 +1,320 @@ +// +// CrossEngineStructureTranslatorTests.swift +// TableProTests +// + +import TableProPluginKit +import XCTest +@testable import TablePro + +final class CrossEngineStructureTranslatorTests: XCTestCase { + private func column( + _ name: String, + _ type: String, + nullable: Bool = true, + defaultValue: String? = nil, + autoIncrement: Bool = false, + unsigned: Bool = false, + charset: String? = nil, + collation: String? = nil, + onUpdate: String? = nil, + extra: String? = nil, + generation: String? = nil, + isPrimaryKey: Bool = false + ) -> EditableColumnDefinition { + EditableColumnDefinition( + id: UUID(), + name: name, + dataType: type, + isNullable: nullable, + defaultValue: defaultValue, + autoIncrement: autoIncrement, + unsigned: unsigned, + comment: nil, + collation: collation, + onUpdate: onUpdate, + charset: charset, + extra: extra, + generationExpression: generation, + generationKind: generation == nil ? nil : .stored, + isPrimaryKey: isPrimaryKey + ) + } + + private func index( + _ name: String, + _ columns: [String], + type: EditableIndexDefinition.IndexType = .btree, + unique: Bool = false, + primary: Bool = false, + whereClause: String? = nil + ) -> EditableIndexDefinition { + EditableIndexDefinition( + id: UUID(), + name: name, + columns: columns, + type: type, + isUnique: unique, + isPrimary: primary, + comment: nil, + columnPrefixes: [:], + whereClause: whereClause + ) + } + + private func snapshot( + columns: [EditableColumnDefinition], + indexes: [EditableIndexDefinition] = [] + ) -> TableStructureSnapshot { + TableStructureSnapshot( + name: "orders", + schema: "public", + columns: columns, + indexes: indexes, + engine: "InnoDB", + charset: "utf8mb4", + collation: "utf8mb4_0900_ai_ci" + ) + } + + // MARK: - Same family + + /// The one guarantee that lets a change this wide be trusted: a copy that already worked runs + /// the path it always ran, byte for byte. + func testASameEngineCopyIsUntouched() { + let source = snapshot(columns: [column("id", "INT", autoIncrement: true, isPrimaryKey: true)]) + let result = CrossEngineStructureTranslator.translate(source, from: .mysql, to: .mysql) + XCTAssertFalse(result.translated) + XCTAssertTrue(result.notes.isEmpty) + XCTAssertEqual(result.snapshot.columns.map(\.dataType), ["INT"]) + XCTAssertEqual(result.snapshot.engine, "InnoDB") + XCTAssertEqual(result.snapshot.charset, "utf8mb4") + } + + func testTwoEnginesOfOneFamilyAreUntouched() { + let source = snapshot(columns: [column("id", "INT")]) + XCTAssertFalse( + CrossEngineStructureTranslator.translate(source, from: .mysql, to: .mariadb).translated + ) + } + + // MARK: - Types + + func testTypesAreSaidInTheTargetsOwnWords() { + let source = snapshot(columns: [ + column("id", "INT", autoIncrement: true, isPrimaryKey: true), + column("flag", "TINYINT(1)"), + column("body", "LONGTEXT"), + column("made", "DATETIME(6)") + ]) + let result = CrossEngineStructureTranslator.translate(source, from: .mysql, to: .postgresql) + XCTAssertTrue(result.translated) + XCTAssertEqual( + result.snapshot.columns.map(\.dataType), + ["INTEGER", "BOOLEAN", "TEXT", "TIMESTAMP(6)"] + ) + } + + /// All three name something on the source's side and are rejected outright by every other + /// engine's `CREATE TABLE`. + func testMySQLOnlyAttributesAreDropped() { + let source = snapshot(columns: [ + column("made", "TIMESTAMP", onUpdate: "CURRENT_TIMESTAMP", extra: "on update CURRENT_TIMESTAMP"), + column("name", "VARCHAR(20)", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci") + ]) + let result = CrossEngineStructureTranslator.translate(source, from: .mysql, to: .postgresql) + XCTAssertNil(result.snapshot.columns[0].onUpdate) + XCTAssertNil(result.snapshot.columns[0].extra) + XCTAssertNil(result.snapshot.columns[1].charset) + XCTAssertNil(result.snapshot.columns[1].collation) + XCTAssertNil(result.snapshot.engine) + XCTAssertNil(result.snapshot.charset) + XCTAssertNil(result.snapshot.collation) + } + + /// `UNSIGNED` is written by the MySQL driver from the attribute, so it survives into a MySQL + /// target and is dropped for every other one, where it would be a syntax error. + func testUnsignedFollowsTheTarget() { + let source = snapshot(columns: [column("count", "INT UNSIGNED", unsigned: true)]) + let toPostgres = CrossEngineStructureTranslator.translate(source, from: .mysql, to: .postgresql) + XCTAssertFalse(toPostgres.snapshot.columns[0].unsigned) + XCTAssertEqual(toPostgres.snapshot.columns[0].dataType, "BIGINT") + + let fromPostgres = snapshot(columns: [column("count", "bigint")]) + let toMySQL = CrossEngineStructureTranslator.translate(fromPostgres, from: .postgresql, to: .mysql) + XCTAssertFalse(toMySQL.snapshot.columns[0].unsigned) + } + + // MARK: - Defaults + + /// A `SERIAL` is an integer whose default calls a sequence written in PostgreSQL's own DDL. + /// Carried over as text the target either rejects it or stores the literal string; carried as + /// the target's own generated key it means what the source meant. + func testASequenceDefaultBecomesAutoIncrement() { + let source = snapshot(columns: [ + column("id", "integer", defaultValue: "nextval('orders_id_seq'::regclass)", isPrimaryKey: true) + ]) + let result = CrossEngineStructureTranslator.translate(source, from: .postgresql, to: .mysql) + XCTAssertTrue(result.snapshot.columns[0].autoIncrement) + XCTAssertNil(result.snapshot.columns[0].defaultValue) + } + + func testTheCurrentTimestampDefaultSurvives() { + let source = snapshot(columns: [column("made", "timestamp", defaultValue: "now()")]) + let result = CrossEngineStructureTranslator.translate(source, from: .postgresql, to: .mysql) + XCTAssertEqual(result.snapshot.columns[0].defaultValue, "CURRENT_TIMESTAMP") + } + + /// The target's DDL writer quotes whatever it does not recognise, so a default it cannot read + /// would become the literal string `'uuid_generate_v4()'` in every row. + func testAnUntranslatableDefaultIsDroppedWithAReason() { + let source = snapshot(columns: [column("id", "uuid", defaultValue: "uuid_generate_v4()")]) + let result = CrossEngineStructureTranslator.translate(source, from: .postgresql, to: .mysql) + XCTAssertNil(result.snapshot.columns[0].defaultValue) + XCTAssertTrue(result.notes.contains { $0.subject == "id" && $0.isLossy }) + } + + func testAPostgresCastIsStrippedFromALiteralDefault() { + let source = snapshot(columns: [column("state", "character varying(10)", defaultValue: "'new'::character varying")]) + let result = CrossEngineStructureTranslator.translate(source, from: .postgresql, to: .mysql) + XCTAssertEqual(result.snapshot.columns[0].defaultValue, "'new'") + } + + func testABooleanDefaultFollowsTheTargetsSpelling() { + let source = snapshot(columns: [column("live", "boolean", defaultValue: "false")]) + XCTAssertEqual( + CrossEngineStructureTranslator.translate(source, from: .postgresql, to: .mysql) + .snapshot.columns[0].defaultValue, + "0" + ) + let mysqlSource = snapshot(columns: [column("live", "TINYINT(1)", defaultValue: "1")]) + XCTAssertEqual( + CrossEngineStructureTranslator.translate(mysqlSource, from: .mysql, to: .postgresql) + .snapshot.columns[0].defaultValue, + "TRUE" + ) + } + + // MARK: - Generated columns + + /// The expression is the source's own SQL and nothing parses it, so the column is created as + /// an ordinary one. That is what lets its values be copied instead of arriving empty. + func testAComputedColumnBecomesAnOrdinaryOne() { + let source = snapshot(columns: [ + column("total", "DECIMAL(10,2)", generation: "price * quantity") + ]) + let result = CrossEngineStructureTranslator.translate(source, from: .mysql, to: .postgresql) + XCTAssertNil(result.snapshot.columns[0].generationExpression) + XCTAssertNil(result.snapshot.columns[0].generationKind) + XCTAssertTrue(result.notes.contains { $0.subject == "total" }) + } + + // MARK: - Keys and indexes + + /// MySQL refuses a `PRIMARY KEY` on a `LONGTEXT` outright, and the whole `CREATE TABLE` fails + /// with it rather than the copy losing anything. + func testAKeyColumnIsBoundedWhereTheEngineNeedsIt() { + let source = snapshot(columns: [column("code", "text", isPrimaryKey: true)]) + let result = CrossEngineStructureTranslator.translate(source, from: .postgresql, to: .mysql) + XCTAssertEqual(result.snapshot.columns[0].dataType, "VARCHAR(255)") + XCTAssertTrue(result.notes.contains { $0.subject == "code" }) + } + + func testAKeyColumnIsLeftUnboundedWhereTheEngineAllowsIt() { + let source = snapshot(columns: [column("code", "LONGTEXT", isPrimaryKey: true)]) + let result = CrossEngineStructureTranslator.translate(source, from: .mysql, to: .postgresql) + XCTAssertEqual(result.snapshot.columns[0].dataType, "TEXT") + } + + /// A `GIN` index reaches MySQL as a plain `INDEX` over a column that is now `JSON`, and MySQL + /// refuses the whole table for it. + func testAnIndexTheTargetCannotBuildIsLeftOut() { + let source = snapshot( + columns: [column("doc", "jsonb")], + indexes: [index("doc_gin", ["doc"], type: .gin)] + ) + let result = CrossEngineStructureTranslator.translate(source, from: .postgresql, to: .mysql) + XCTAssertTrue(result.snapshot.indexes.isEmpty) + XCTAssertTrue(result.notes.contains { $0.subject == "doc_gin" }) + } + + /// MySQL indexes an unbounded text column only with a key length, and refuses it without one. + func testAnIndexOnUnboundedTextGetsAKeyPrefixOnMySQL() { + let source = snapshot( + columns: [column("body", "text")], + indexes: [index("body_idx", ["body"])] + ) + let result = CrossEngineStructureTranslator.translate(source, from: .postgresql, to: .mysql) + XCTAssertEqual(result.snapshot.indexes.first?.columnPrefixes["body"], 255) + } + + func testAPartialIndexLosesItsClauseWhereTheEngineHasNone() { + let source = snapshot( + columns: [column("state", "text")], + indexes: [index("live_idx", ["state"], whereClause: "state = 'live'")] + ) + let result = CrossEngineStructureTranslator.translate(source, from: .postgresql, to: .mysql) + XCTAssertNil(result.snapshot.indexes.first?.whereClause) + XCTAssertTrue(result.notes.contains { $0.subject == "live_idx" }) + } + + func testThePrimaryKeyIndexIsKeptForTheTargetToWrite() { + let source = snapshot( + columns: [column("id", "INT", isPrimaryKey: true)], + indexes: [index("PRIMARY", ["id"], unique: true, primary: true)] + ) + let result = CrossEngineStructureTranslator.translate(source, from: .mysql, to: .postgresql) + XCTAssertEqual(result.snapshot.primaryKeyColumns, ["id"]) + } + + // MARK: - Value kinds + + func testTheTargetKindsDescribeEveryColumn() { + let source = snapshot(columns: [column("flag", "TINYINT(1)"), column("made", "DATETIME")]) + let result = CrossEngineStructureTranslator.translate(source, from: .mysql, to: .postgresql) + XCTAssertEqual(result.targetKinds["flag"], .boolean) + XCTAssertEqual(result.targetKinds["made"], .timestamp(precision: nil, hasTimeZone: false)) + } + + /// The kinds describe what was written, not what was read. Carrying the source's answer over + /// told the coercer a MySQL `DATETIME` still had the time zone its PostgreSQL source declared, + /// so the offset it exists to strip was left on every value, and an array bound for a `JSON` + /// column was not recognised as needing conversion at all. + func testTheTargetKindsFollowWhatTheRendererWroteRatherThanTheSource() { + let source = snapshot(columns: [ + column("made", "timestamptz"), + column("tags", "integer[]"), + column("live", "boolean") + ]) + let result = CrossEngineStructureTranslator.translate(source, from: .postgresql, to: .mysql) + + XCTAssertEqual(result.snapshot.columns.map(\.dataType), ["DATETIME", "JSON", "TINYINT(1)"]) + XCTAssertEqual(result.targetKinds["made"], .timestamp(precision: nil, hasTimeZone: false)) + XCTAssertEqual(result.targetKinds["tags"], .json) + XCTAssertEqual(result.targetKinds["live"], .boolean) + + XCTAssertEqual(result.sourceKinds["made"], .timestamp(precision: nil, hasTimeZone: true)) + XCTAssertEqual(result.sourceKinds["tags"], .array(element: .integer(bytes: 4))) + XCTAssertEqual(result.sourceKinds["live"], .boolean) + } + + /// A cut precision loses digits, so it is a conversion the review step has to name. + func testAClampedDecimalPrecisionIsReported() { + let source = snapshot(columns: [column("amount", "DECIMAL(65,30)")]) + let result = CrossEngineStructureTranslator.translate(source, from: .mysql, to: .mssql) + XCTAssertEqual(result.snapshot.columns[0].dataType, "DECIMAL(38, 30)") + XCTAssertTrue(result.notes.contains { $0.subject == "amount" && $0.isLossy }) + } + + /// A prefix on a unique index is a weaker constraint than the source had, and the copy fails on + /// the first pair of rows that agree for 255 characters. + func testAUniqueIndexCutToAPrefixIsReported() { + let source = snapshot( + columns: [column("body", "text")], + indexes: [index("body_unique", ["body"], unique: true)] + ) + let result = CrossEngineStructureTranslator.translate(source, from: .postgresql, to: .mysql) + XCTAssertEqual(result.snapshot.indexes.first?.columnPrefixes["body"], 255) + XCTAssertTrue(result.notes.contains { $0.subject == "body_unique" && $0.isLossy }) + } +} diff --git a/TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift b/TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift new file mode 100644 index 000000000..c35bdaf77 --- /dev/null +++ b/TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift @@ -0,0 +1,167 @@ +// +// CrossEngineValueCoercerTests.swift +// TableProTests +// + +import TableProPluginKit +import XCTest +@testable import TablePro + +final class CrossEngineValueCoercerTests: XCTestCase { + /// The target's kinds, with the source's taken to match unless a test says otherwise. Most + /// coercions are decided by the target; the boolean one is the exception and names both. + private func coercer( + _ targets: [CanonicalTypeKind?], + sources: [CanonicalTypeKind?]? = nil, + from source: SQLTypeFamily = .postgres + ) -> CrossEngineValueCoercer { + let sourceKinds = sources ?? targets + let pairs = targets.enumerated().map { index, target in + CrossEngineValueCoercer.ColumnPair( + source: index < sourceKinds.count ? sourceKinds[index] : nil, target: target + ) + } + return CrossEngineValueCoercer(pairs: pairs, from: source) + } + + // MARK: - When it runs at all + + /// A table of numbers and strings runs the loop it ran before this existed. + func testATableWithNothingToReshapeNeedsNoCoercion() { + let plain = coercer([.integer(bytes: 4), .text(length: nil, isFixed: false)]) + XCTAssertFalse(plain.isNeeded) + XCTAssertTrue(coercer([.boolean]).isNeeded) + XCTAssertTrue(coercer([.timestamp(precision: nil, hasTimeZone: false)]).isNeeded) + } + + // MARK: - Booleans + + /// PostgreSQL renders a boolean as `t` and `f`. Bound into a MySQL `TINYINT(1)` that is 0 in + /// both cases outside strict mode, so every `true` in the table silently becomes `false`. + func testPostgresBooleansBecomeOneAndZero() { + let row = coercer([.boolean, .boolean]).coerce([.text("t"), .text("f")]) + XCTAssertEqual(row, [.text("1"), .text("0")]) + } + + func testTheOtherBooleanSpellingsAreNormalised() { + let subject = coercer([.boolean]) + XCTAssertEqual(subject.coerce([.text("true")]), [.text("1")]) + XCTAssertEqual(subject.coerce([.text("YES")]), [.text("1")]) + XCTAssertEqual(subject.coerce([.text("off")]), [.text("0")]) + XCTAssertEqual(subject.coerce([.text("0")]), [.text("0")]) + } + + /// A MySQL `BIT(1)` arrives as one byte rather than as text. + func testASingleByteBooleanIsRead() { + let subject = coercer([.boolean], from: .mysql) + XCTAssertEqual(subject.coerce([.bytes(Data([1]))]), [.text("1")]) + XCTAssertEqual(subject.coerce([.bytes(Data([0]))]), [.text("0")]) + } + + /// A value that is not a boolean spelling is left alone rather than guessed at, so a column the + /// source did not really use as one fails visibly instead of arriving wrong. + func testAnUnrecognisedBooleanIsLeftAlone() { + XCTAssertEqual(coercer([.boolean]).coerce([.text("maybe")]), [.text("maybe")]) + XCTAssertEqual(coercer([.boolean]).coerce([.null]), [.null]) + } + + /// A `t` that was text on the source is text on the target too. + func testATextColumnIsNotTouched() { + XCTAssertEqual( + coercer([.text(length: nil, isFixed: false)]).coerce([.text("t")]), [.text("t")] + ) + } + + /// A boolean copied into a text column keeps what the source wrote. `t` is the value there + /// rather than a spelling of one, and rewriting it to `1` would change the data. + func testABooleanCopiedIntoTextKeepsItsOwnSpelling() { + let subject = coercer([.text(length: nil, isFixed: false)], sources: [.boolean]) + XCTAssertEqual(subject.coerce([.text("t")]), [.text("t")]) + } + + /// Oracle has no boolean and takes `NUMBER(1)`, so the target kind is a decimal while the + /// values are still boolean-shaped. Deciding from the target alone left `t` bound into a number. + func testABooleanIntoANumericTargetIsStillNormalised() { + let subject = coercer([.decimal(precision: 1, scale: nil)], sources: [.boolean]) + XCTAssertEqual(subject.coerce([.text("t")]), [.text("1")]) + } + + // MARK: - Time zones + + func testAZoneOffsetIsStrippedForATargetWithoutOne() { + let subject = coercer([.timestamp(precision: nil, hasTimeZone: false)]) + XCTAssertEqual( + subject.coerce([.text("2024-01-01 10:00:00+07")]), [.text("2024-01-01 10:00:00")] + ) + XCTAssertEqual( + subject.coerce([.text("2024-01-01T10:00:00.123456Z")]), [.text("2024-01-01T10:00:00.123456")] + ) + XCTAssertEqual( + subject.coerce([.text("2024-01-01 10:00:00-03:30")]), [.text("2024-01-01 10:00:00")] + ) + } + + /// The crossing that motivates the whole coercion: PostgreSQL `timestamptz` to MySQL + /// `DATETIME`. The source has a zone and the target does not, so the decision has to come from + /// the target's side. + func testAZonedSourceIsStrippedForAnUnzonedTarget() { + let subject = coercer( + [.timestamp(precision: nil, hasTimeZone: false)], + sources: [.timestamp(precision: nil, hasTimeZone: true)] + ) + XCTAssertEqual( + subject.coerce([.text("2024-01-01 10:00:00+07")]), [.text("2024-01-01 10:00:00")] + ) + } + + func testAZoneAwareTargetKeepsTheOffset() { + let subject = coercer([.timestamp(precision: nil, hasTimeZone: true)]) + XCTAssertEqual( + subject.coerce([.text("2024-01-01 10:00:00+07")]), [.text("2024-01-01 10:00:00+07")] + ) + } + + func testAPlainTimestampIsUnchanged() { + let subject = coercer([.timestamp(precision: nil, hasTimeZone: false)]) + XCTAssertEqual(subject.coerce([.text("2024-01-01 10:00:00")]), [.text("2024-01-01 10:00:00")]) + XCTAssertEqual(subject.coerce([.text("2024-01-01")]), [.text("2024-01-01")]) + } + + // MARK: - Zero dates + + /// No engine but MySQL accepts `0000-00-00`, and a copy that sends it fails on that row. + func testMySQLZeroDatesBecomeNull() { + let subject = coercer([.date, .timestamp(precision: nil, hasTimeZone: false)], from: .mysql) + XCTAssertEqual( + subject.coerce([.text("0000-00-00"), .text("0000-00-00 00:00:00")]), [.null, .null] + ) + } + + func testAZeroDateFromAnotherEngineIsLeftAlone() { + XCTAssertEqual(coercer([.date], from: .postgres).coerce([.text("0000-00-00")]), [.text("0000-00-00")]) + } + + // MARK: - Arrays into JSON + + func testAPostgresArrayBecomesJson() { + let subject = coercer([.json], from: .postgres) + XCTAssertEqual(subject.coerce([.text("{1,2,3}")]), [.text("[1,2,3]")]) + XCTAssertEqual(subject.coerce([.text("{a,b}")]), [.text("[\"a\",\"b\"]")]) + XCTAssertEqual(subject.coerce([.text("{}")]), [.text("[]")]) + } + + /// A JSON object arrives with the same brackets an array literal uses, and it is already JSON. + func testAJsonObjectIsNotMistakenForAnArray() { + let subject = coercer([.json], from: .postgres) + XCTAssertEqual(subject.coerce([.text("{\"a\": 1}")]), [.text("{\"a\": 1}")]) + } + + // MARK: - Shape + + /// The width is checked before the coercion, so a short row can never reach it. Guarded here + /// too, because a positional reshape against the wrong column's type is silent. + func testAShortRowIsNotReshapedPastItsEnd() { + let subject = coercer([.boolean, .boolean, .boolean]) + XCTAssertEqual(subject.coerce([.text("t")]), [.text("1")]) + } +} diff --git a/TableProTests/Core/CrossEngine/SQLTypeParserTests.swift b/TableProTests/Core/CrossEngine/SQLTypeParserTests.swift new file mode 100644 index 000000000..aebff9c67 --- /dev/null +++ b/TableProTests/Core/CrossEngine/SQLTypeParserTests.swift @@ -0,0 +1,121 @@ +// +// SQLTypeParserTests.swift +// TableProTests +// + +import XCTest +@testable import TablePro + +final class SQLTypeParserTests: XCTestCase { + private func kind(_ spelling: String, _ family: SQLTypeFamily) -> CanonicalTypeKind { + SQLTypeParser.parse(spelling, family: family).kind + } + + // MARK: - The same word means two things + + /// The reason the parser is keyed by family rather than by word. Each of these is read + /// correctly for one engine and wrongly for the other by any shared table. + func testTheSameWordIsReadDifferentlyPerEngine() { + XCTAssertEqual(kind("TINYINT(1)", .mysql), .boolean) + XCTAssertEqual(kind("TINYINT", .mssql), .integer(bytes: 1)) + XCTAssertEqual(kind("DATE", .postgres), .date) + XCTAssertEqual(kind("DATE", .oracle), .timestamp(precision: 0, hasTimeZone: false)) + XCTAssertEqual(kind("REAL", .postgres), .floatingPoint(bits: 32)) + XCTAssertEqual(kind("REAL", .sqlite), .floatingPoint(bits: 64)) + } + + /// SQL Server's `TINYINT` holds 0 to 255, so it is unsigned however it is spelled. + func testSQLServerTinyIntIsUnsigned() { + XCTAssertTrue(SQLTypeParser.parse("TINYINT", family: .mssql).isUnsigned) + XCTAssertFalse(SQLTypeParser.parse("TINYINT", family: .mysql).isUnsigned) + } + + // MARK: - Modifiers + + func testUnsignedIsReadAsAModifierRatherThanAsPartOfTheName() { + let parsed = SQLTypeParser.parse("BIGINT UNSIGNED", family: .mysql) + XCTAssertEqual(parsed.kind, .integer(bytes: 8)) + XCTAssertTrue(parsed.isUnsigned) + } + + func testZerofillDoesNotHideTheTypeName() { + XCTAssertEqual(kind("INT UNSIGNED ZEROFILL", .mysql), .integer(bytes: 4)) + } + + /// Without stripping these every ClickHouse column is one unknown type called `Nullable`. + func testClickHouseWrappersAreStripped() { + XCTAssertEqual(kind("Nullable(Int32)", .clickhouse), .integer(bytes: 4)) + XCTAssertEqual(kind("LowCardinality(Nullable(String))", .clickhouse), .text(length: nil, isFixed: false)) + } + + // MARK: - Parameters + + func testLengthAndPrecisionAreKept() { + XCTAssertEqual(kind("VARCHAR(255)", .mysql), .text(length: 255, isFixed: false)) + XCTAssertEqual(kind("CHAR(2)", .postgres), .text(length: 2, isFixed: true)) + XCTAssertEqual(kind("NUMERIC(19,4)", .postgres), .decimal(precision: 19, scale: 4)) + XCTAssertEqual(kind("DECIMAL(10)", .mysql), .decimal(precision: 10, scale: nil)) + } + + /// The parenthesised part sits in the middle of these, so a parser that stops at the first + /// bracket reads `timestamp(3) with time zone` as an ordinary `timestamp`. + func testASuffixAfterTheParametersIsPartOfTheName() { + XCTAssertEqual(kind("timestamp(3) with time zone", .postgres), .timestamp(precision: 3, hasTimeZone: true)) + XCTAssertEqual(kind("time(6) without time zone", .postgres), .time(precision: 6, hasTimeZone: false)) + } + + func testEnumLabelsAreRead() { + XCTAssertEqual(kind("enum('small','large')", .mysql), .enumeration(values: ["small", "large"])) + XCTAssertEqual( + kind("Enum8('a' = 1, 'b' = 2)", .clickhouse), .enumeration(values: ["a", "b"]) + ) + } + + func testAnEmptyParameterListIsNotALength() { + XCTAssertEqual(kind("VARCHAR", .mysql), .text(length: nil, isFixed: false)) + XCTAssertEqual(kind("NVARCHAR(MAX)", .mssql), .text(length: nil, isFixed: false)) + } + + // MARK: - Arrays + + func testArraysAreReadOnBothSpellings() { + XCTAssertEqual(kind("integer[]", .postgres), .array(element: .integer(bytes: 4))) + XCTAssertEqual(kind("Array(String)", .clickhouse), .array(element: .text(length: nil, isFixed: false))) + } + + // MARK: - Oracle numbers + + /// Oracle has one numeric type, so a whole-number column is `NUMBER(p, 0)` and only its + /// precision says how wide an integer the target needs. + /// The width is the narrowest integer that holds every value of that many digits, so the + /// boundaries are where the digit count outgrows the type: 10 digits do not fit in four bytes + /// and 19 do not fit in eight. + func testOracleWholeNumbersBecomeIntegers() { + XCTAssertEqual(kind("NUMBER(2,0)", .oracle), .integer(bytes: 1)) + XCTAssertEqual(kind("NUMBER(4,0)", .oracle), .integer(bytes: 2)) + XCTAssertEqual(kind("NUMBER(9,0)", .oracle), .integer(bytes: 4)) + XCTAssertEqual(kind("NUMBER(10,0)", .oracle), .integer(bytes: 8)) + XCTAssertEqual(kind("NUMBER(18,0)", .oracle), .integer(bytes: 8)) + XCTAssertEqual(kind("NUMBER(38,0)", .oracle), .integer(bytes: 16)) + XCTAssertEqual(kind("NUMBER(10,2)", .oracle), .decimal(precision: 10, scale: 2)) + XCTAssertEqual(kind("NUMBER", .oracle), .decimal(precision: nil, scale: nil)) + } + + // MARK: - Unknowns + + /// An unknown word carries its own spelling so the renderer can name it, rather than being + /// guessed into a shape it does not have. + func testAnUnknownTypeKeepsItsSpelling() { + let parsed = SQLTypeParser.parse("tsvector", family: .postgres) + XCTAssertEqual(parsed.kind, .unsupported) + XCTAssertEqual(parsed.sourceSpelling, "tsvector") + } + + /// SQLite stores whatever spelling the `CREATE TABLE` used, so a file written by another tool + /// is full of other engines' words and none of them may fall through to unsupported. + func testSQLiteFallsBackToTheAnsiReading() { + XCTAssertEqual(kind("VARCHAR(80)", .sqlite), .text(length: 80, isFixed: false)) + XCTAssertEqual(kind("BOOLEAN", .sqlite), .boolean) + XCTAssertEqual(kind("DATETIME", .sqlite), .timestamp(precision: nil, hasTimeZone: false)) + } +} diff --git a/TableProTests/Core/CrossEngine/SQLTypeRendererTests.swift b/TableProTests/Core/CrossEngine/SQLTypeRendererTests.swift new file mode 100644 index 000000000..58d1a5f2a --- /dev/null +++ b/TableProTests/Core/CrossEngine/SQLTypeRendererTests.swift @@ -0,0 +1,168 @@ +// +// SQLTypeRendererTests.swift +// TableProTests +// + +import XCTest +@testable import TablePro + +final class SQLTypeRendererTests: XCTestCase { + private func rendered( + _ spelling: String, + from source: SQLTypeFamily, + to target: SQLTypeFamily + ) -> RenderedColumnType { + SQLTypeRenderer.render(SQLTypeParser.parse(spelling, family: source), family: target) + } + + private func spelling( + _ value: String, + from source: SQLTypeFamily, + to target: SQLTypeFamily + ) -> String { + rendered(value, from: source, to: target).spelling + } + + // MARK: - The pairs users actually copy + + func testMySQLToPostgres() { + XCTAssertEqual(spelling("TINYINT(1)", from: .mysql, to: .postgres), "BOOLEAN") + XCTAssertEqual(spelling("INT", from: .mysql, to: .postgres), "INTEGER") + XCTAssertEqual(spelling("MEDIUMINT", from: .mysql, to: .postgres), "INTEGER") + XCTAssertEqual(spelling("VARCHAR(255)", from: .mysql, to: .postgres), "VARCHAR(255)") + XCTAssertEqual(spelling("LONGTEXT", from: .mysql, to: .postgres), "TEXT") + XCTAssertEqual(spelling("DATETIME(6)", from: .mysql, to: .postgres), "TIMESTAMP(6)") + XCTAssertEqual(spelling("LONGBLOB", from: .mysql, to: .postgres), "BYTEA") + XCTAssertEqual(spelling("JSON", from: .mysql, to: .postgres), "JSONB") + XCTAssertEqual(spelling("DOUBLE", from: .mysql, to: .postgres), "DOUBLE PRECISION") + } + + func testPostgresToMySQL() { + XCTAssertEqual(spelling("boolean", from: .postgres, to: .mysql), "TINYINT(1)") + XCTAssertEqual(spelling("int4", from: .postgres, to: .mysql), "INT") + XCTAssertEqual(spelling("text", from: .postgres, to: .mysql), "LONGTEXT") + XCTAssertEqual(spelling("bytea", from: .postgres, to: .mysql), "LONGBLOB") + XCTAssertEqual(spelling("uuid", from: .postgres, to: .mysql), "CHAR(36)") + XCTAssertEqual(spelling("jsonb", from: .postgres, to: .mysql), "JSON") + XCTAssertEqual(spelling("numeric(19,4)", from: .postgres, to: .mysql), "DECIMAL(19, 4)") + } + + /// `text` on PostgreSQL holds a gigabyte and MySQL's `TEXT` holds 64 KB, so the unbounded case + /// has to reach `LONGTEXT` or a long value is truncated with no error outside strict mode. + func testUnboundedTextReachesTheWidestMySQLType() { + XCTAssertEqual(spelling("text", from: .postgres, to: .mysql), "LONGTEXT") + XCTAssertEqual(spelling("CLOB", from: .oracle, to: .mysql), "LONGTEXT") + XCTAssertEqual(spelling("NVARCHAR(MAX)", from: .mssql, to: .mysql), "LONGTEXT") + } + + // MARK: - Unsigned + + /// PostgreSQL has no unsigned integers. Kept at the same width, half of a `BIGINT UNSIGNED`'s + /// range fails on whichever row first exceeds it, minutes into a copy. + func testUnsignedIntegersWidenRatherThanOverflow() { + XCTAssertEqual(spelling("INT UNSIGNED", from: .mysql, to: .postgres), "BIGINT") + XCTAssertEqual(spelling("SMALLINT UNSIGNED", from: .mysql, to: .postgres), "INTEGER") + XCTAssertEqual(spelling("BIGINT UNSIGNED", from: .mysql, to: .postgres), "NUMERIC(20, 0)") + XCTAssertEqual(spelling("BIGINT UNSIGNED", from: .mysql, to: .mssql), "DECIMAL(20, 0)") + } + + /// `UNSIGNED` is a column attribute on the MySQL side, and the driver writes it from that + /// attribute. Spelling it into the type as well produced `INT UNSIGNED UNSIGNED`. + func testTheMySQLRendererNeverSpellsUnsigned() { + let type = CanonicalColumnType(kind: .integer(bytes: 4), isUnsigned: true, sourceSpelling: "int") + XCTAssertEqual(SQLTypeRenderer.render(type, family: .mysql).spelling, "INT") + } + + /// A signed byte does not fit in SQL Server's `TINYINT`, which holds 0 to 255. + func testSQLServerWidensASignedByte() { + XCTAssertEqual(spelling("TINYINT", from: .mysql, to: .mssql), "SMALLINT") + let unsigned = CanonicalColumnType(kind: .integer(bytes: 1), isUnsigned: true, sourceSpelling: "tinyint") + XCTAssertEqual(SQLTypeRenderer.render(unsigned, family: .mssql).spelling, "TINYINT") + } + + // MARK: - Fidelity + + func testAnExactMappingReportsNothing() { + let result = rendered("INT", from: .mysql, to: .postgres) + XCTAssertEqual(result.fidelity, .exact) + XCTAssertNil(result.reason) + } + + func testALossyMappingCarriesItsReason() { + let result = rendered("timestamptz", from: .postgres, to: .mysql) + XCTAssertEqual(result.fidelity, .approximated) + XCTAssertNotNil(result.reason) + XCTAssertEqual(result.spelling, "DATETIME") + } + + func testAWideningCarriesItsReasonAndKeepsEveryValue() { + let result = rendered("uuid", from: .postgres, to: .mysql) + XCTAssertEqual(result.fidelity, .widened) + XCTAssertNotNil(result.reason) + } + + /// Nothing is ever dropped. A type no family can express becomes the target's widest text and + /// says so, because a copy that writes the value as text is better than one missing a column. + func testAnUnknownTypeBecomesTextRatherThanDisappearing() { + for family in SQLTypeFamily.allCases where family != .postgres { + let result = rendered("tsvector", from: .postgres, to: family) + XCTAssertFalse(result.spelling.isEmpty, "\(family) produced no spelling") + XCTAssertEqual(result.fidelity, .approximated, "\(family)") + XCTAssertNotNil(result.reason, "\(family)") + } + } + + /// Every family answers every kind, so no copy can be refused for a type the source happened + /// to use. + func testEveryFamilyAnswersEveryKind() { + let kinds: [CanonicalTypeKind] = [ + .boolean, .integer(bytes: 4), .decimal(precision: 10, scale: 2), .floatingPoint(bits: 64), + .text(length: nil, isFixed: false), .binary(length: nil, isFixed: false), .date, + .time(precision: nil, hasTimeZone: true), .timestamp(precision: 3, hasTimeZone: true), + .interval, .uuid, .json, .xml, .enumeration(values: ["a"]), .bitString(length: 8), + .money, .spatial, .array(element: .integer(bytes: 4)), .unsupported + ] + for family in SQLTypeFamily.allCases { + for kind in kinds { + let type = CanonicalColumnType(kind: kind, sourceSpelling: "source_type") + let spelling = SQLTypeRenderer.render(type, family: family).spelling + XCTAssertFalse(spelling.isEmpty, "\(family) had no spelling for \(kind)") + } + } + } + + // MARK: - Enums + + func testAMySQLTargetKeepsAnEnum() { + XCTAssertEqual( + SQLTypeRenderer.render( + SQLTypeParser.parse("enum('a','b')", family: .mysql), family: .mysql + ).spelling, + "ENUM('a', 'b')" + ) + } + + /// A label with a quote in it has to survive being written back out. + func testAnEnumLabelIsEscaped() { + let type = CanonicalColumnType(kind: .enumeration(values: ["it's"]), sourceSpelling: "enum") + XCTAssertEqual(SQLTypeRenderer.render(type, family: .mysql).spelling, "ENUM('it''s')") + } + + /// PostgreSQL enums are a separate `CREATE TYPE` with a dependency of its own, so the column + /// becomes text wide enough for the longest label and the review says so. + func testAPostgresTargetSizesTextToTheLongestLabel() { + let type = CanonicalColumnType(kind: .enumeration(values: ["a", "medium"]), sourceSpelling: "enum") + XCTAssertEqual(SQLTypeRenderer.render(type, family: .postgres).spelling, "VARCHAR(6)") + } + + // MARK: - Arrays + + func testAPostgresTargetKeepsAnArray() { + XCTAssertEqual(spelling("integer[]", from: .postgres, to: .postgres), "INTEGER[]") + XCTAssertEqual(spelling("Array(Int32)", from: .clickhouse, to: .postgres), "INTEGER[]") + } + + func testAnArrayBecomesJsonOnMySQL() { + XCTAssertEqual(spelling("integer[]", from: .postgres, to: .mysql), "JSON") + } +} diff --git a/TableProTests/Core/ObjectCopy/ObjectCopyEligibilityTests.swift b/TableProTests/Core/ObjectCopy/ObjectCopyEligibilityTests.swift index 2225072be..cf6473488 100644 --- a/TableProTests/Core/ObjectCopy/ObjectCopyEligibilityTests.swift +++ b/TableProTests/Core/ObjectCopy/ObjectCopyEligibilityTests.swift @@ -31,14 +31,57 @@ final class ObjectCopyEligibilityTests: XCTestCase { XCTAssertNil(ObjectCopyEligibility.targetRefusal(endpoint("app"))) } - /// Both halves of a copy are refused across engines, not only structure. The row writer emits - /// `INSERT … VALUES`, which a MongoDB target cannot parse, and a SQL Server `dbo` source hands - /// a MySQL target a schema that engine does not have. - func testACopyStaysInsideOneEngine() { - XCTAssertNotNil(ObjectCopyEligibility.engineRefusal(from: .mysql, to: .postgresql)) - XCTAssertNotNil(ObjectCopyEligibility.engineRefusal(from: .mssql, to: .mysql)) - XCTAssertNil(ObjectCopyEligibility.engineRefusal(from: .mysql, to: .mysql)) - XCTAssertNil(ObjectCopyEligibility.engineRefusal(from: .mysql, to: .mariadb)) + /// A copy stays inside SQL and may cross engines within it. The types are translated before + /// the target driver writes any DDL, so the refusal that is left is the one nothing can + /// translate: an engine whose query language is not SQL parses neither `CREATE TABLE` nor + /// `INSERT … VALUES`. + func testACopyCrossesEnginesButNotLanguages() { + XCTAssertNil(ObjectCopyEligibility.engineRefusal( + from: .mysql, to: .postgresql, sourceLanguage: .sql, targetLanguage: .sql + )) + XCTAssertNil(ObjectCopyEligibility.engineRefusal( + from: .mssql, to: .mysql, sourceLanguage: .sql, targetLanguage: .sql + )) + XCTAssertNil(ObjectCopyEligibility.engineRefusal( + from: .mysql, to: .mariadb, sourceLanguage: .sql, targetLanguage: .sql + )) + XCTAssertNotNil(ObjectCopyEligibility.engineRefusal( + from: .mysql, to: .mongodb, sourceLanguage: .sql, targetLanguage: .javascript + )) + XCTAssertNotNil(ObjectCopyEligibility.engineRefusal( + from: .mongodb, to: .mysql, sourceLanguage: .javascript, targetLanguage: .sql + )) + } + + /// DynamoDB declares `.sql` for PartiQL and Cassandra for CQL, so the editor language alone + /// let a MySQL to DynamoDB copy through to a planner that could only fail. A crossing needs a + /// type system the translator knows on both sides. + func testACrossingNeedsATypeSystemOnBothSides() { + XCTAssertNotNil(ObjectCopyEligibility.engineRefusal( + from: .mysql, to: .dynamodb, sourceLanguage: .sql, targetLanguage: .sql + )) + XCTAssertNotNil(ObjectCopyEligibility.engineRefusal( + from: .cassandra, to: .postgresql, sourceLanguage: .sql, targetLanguage: .sql + )) + } + + /// Nothing is translated within one engine, so an engine no family names still copies to + /// itself. That is the path every registry-only driver already used. + func testAnEngineWithNoNamedFamilyStillCopiesToItself() { + XCTAssertNil(ObjectCopyEligibility.engineRefusal( + from: .dynamodb, to: .dynamodb, sourceLanguage: .sql, targetLanguage: .sql + )) + XCTAssertNil(ObjectCopyEligibility.engineRefusal( + from: .cassandra, to: .cassandra, sourceLanguage: .sql, targetLanguage: .sql + )) + } + + /// A view, routine or trigger is its definition, and the definition is the source engine's own + /// SQL text. Nothing here parses it, so it cannot cross even where the tables beside it can. + func testADefinitionDoesNotCrossEngines() { + XCTAssertNotNil(ObjectCopyEligibility.definitionEngineRefusal(from: .mysql, to: .postgresql)) + XCTAssertNil(ObjectCopyEligibility.definitionEngineRefusal(from: .mysql, to: .mariadb)) + XCTAssertNil(ObjectCopyEligibility.definitionEngineRefusal(from: .postgresql, to: .postgresql)) } /// Copying a database onto itself either drops the rows it is about to read or doubles them. diff --git a/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift b/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift index fc9cae702..20014daea 100644 --- a/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift +++ b/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift @@ -75,7 +75,10 @@ private final class CopyDriver: PluginDatabaseDriver, @unchecked Sendable { final class ObjectCopyRowCopierTests: XCTestCase { private func step( columns: [String] = ["id", "name"], - schema: String? = "public" + schema: String? = "public", + coercer: CrossEngineValueCoercer? = nil, + serverSideInsert: SyncStatement? = nil, + estimatedRows: Int? = nil ) -> ObjectCopyTableStep { ObjectCopyTableStep( selection: ObjectCopySelection(kind: .table, name: "orders", schema: schema), @@ -88,9 +91,11 @@ final class ObjectCopyRowCopierTests: XCTestCase { sourceQuery: "SELECT `id`, `name` FROM `public`.`orders`", targetTable: "orders", targetSchema: schema, - estimatedRows: nil, + estimatedRows: estimatedRows, copiesData: true, copiesIdentityColumn: false, + coercer: coercer, + serverSideInsert: serverSideInsert, note: nil ) } @@ -251,6 +256,67 @@ final class ObjectCopyRowCopierTests: XCTestCase { XCTAssertEqual(ObjectCopyRowCopier.batchSize(columnCount: 5_000, generator: mssql), 1) } + + // MARK: - Crossing engines + + /// A `t` from PostgreSQL bound into a MySQL `TINYINT(1)` is 0 outside strict mode, so every + /// `true` in the table would arrive as `false` with nothing reported. + func testValuesAreReshapedForTheTarget() async throws { + let source = CopyDriver() + source.streamed = [[.text("1"), .text("t")]] + let target = CopyDriver() + let coercer = CrossEngineValueCoercer( + pairs: [ + .init(source: .integer(bytes: 4), target: .integer(bytes: 4)), + .init(source: .boolean, target: .boolean) + ], + from: .postgres + ) + + _ = try await ObjectCopyRowCopier( + step: step(columns: ["id", "live"], coercer: coercer), targetDatabaseType: .mysql + ).copy(from: source, to: target) { _ in } + + XCTAssertEqual(target.executedParameters.first?.map(\.sortKey), ["1", "1"]) + } + + // MARK: - The server-side path + + /// One statement instead of a stream, and the source driver is never read from at all. + func testAServerSideCopyRunsOneStatementAndReadsNothing() async throws { + let source = CopyDriver() + source.streamed = rows(5) + let target = CopyDriver() + let statement = SyncStatement( + sql: "INSERT INTO `orders` (`id`) SELECT `id` FROM `shop`.`orders`;", + objectName: "orders", + summary: "server side" + ) + + let outcome = try await ObjectCopyRowCopier( + step: step(serverSideInsert: statement, estimatedRows: 5), targetDatabaseType: .mysql + ).copy(from: source, to: target) { _ in } + + XCTAssertEqual(target.executedQueries, [statement.sql]) + XCTAssertTrue(target.executedParameters.isEmpty) + XCTAssertFalse(outcome.cancelled) + } + + /// The server's own count and never the plan's estimate. `rowsAffected` cannot tell "the driver + /// reported nothing" from "the statement matched nothing", so standing the estimate in for zero + /// told the user a copy of an empty table had written however many rows a stale table statistic + /// had guessed at. + func testTheServerCountIsReportedRatherThanTheEstimate() async throws { + let outcome = try await ObjectCopyRowCopier( + step: step( + serverSideInsert: SyncStatement(sql: "INSERT INTO x SELECT 1;", objectName: "x", summary: ""), + estimatedRows: 42 + ), + targetDatabaseType: .mysql + ).copy(from: CopyDriver(), to: CopyDriver()) { _ in } + + XCTAssertEqual(outcome.inserted, 0) + } } private final class Reported: @unchecked Sendable { diff --git a/TableProTests/Core/ObjectCopy/ObjectCopySelectQueryTests.swift b/TableProTests/Core/ObjectCopy/ObjectCopySelectQueryTests.swift index 6f3581719..a1ae02542 100644 --- a/TableProTests/Core/ObjectCopy/ObjectCopySelectQueryTests.swift +++ b/TableProTests/Core/ObjectCopy/ObjectCopySelectQueryTests.swift @@ -17,6 +17,7 @@ private final class QuotingDriver: PluginDatabaseDriver, @unchecked Sendable { PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) } func quoteIdentifier(_ name: String) -> String { "\"\(name)\"" } + func injectRowLimit(_ query: String, limit: Int) -> String? { "\(query) LIMIT \(limit)" } func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } @@ -66,4 +67,71 @@ final class ObjectCopySelectQueryTests: XCTestCase { "SELECT * FROM \"orders\"" ) } + + // MARK: - Row scope + + func testAFilterBecomesAWhereClause() { + XCTAssertEqual( + ObjectCopySelectQuery.build( + columns: ["id"], table: "orders", schema: nil, driver: driver, + scope: PluginExportRowScope(filter: "total > 10") + ), + "SELECT \"id\" FROM \"orders\" WHERE total > 10" + ) + } + + /// Through the driver's own injection, because `LIMIT` is not the spelling on SQL Server or on + /// Oracle before 12c. + func testARowLimitGoesThroughTheDriver() { + XCTAssertEqual( + ObjectCopySelectQuery.build( + columns: ["id"], table: "orders", schema: nil, driver: driver, + scope: PluginExportRowScope(filter: "total > 10", rowLimit: 50) + ), + "SELECT \"id\" FROM \"orders\" WHERE total > 10 LIMIT 50" + ) + } + + /// The text is spliced into this statement, so the rule that a filter is one expression is what + /// stops a second statement riding in with it. + func testAFilterCarryingASecondStatementIsRefused() { + XCTAssertEqual( + ObjectCopySelectQuery.build( + columns: ["id"], table: "orders", schema: nil, driver: driver, + scope: PluginExportRowScope(filter: "1=1; DROP TABLE orders") + ), + "SELECT \"id\" FROM \"orders\"" + ) + } + + func testATrailingSemicolonIsATypingHabitRatherThanARefusal() { + XCTAssertEqual( + ObjectCopySelectQuery.build( + columns: ["id"], table: "orders", schema: nil, driver: driver, + scope: PluginExportRowScope(filter: "total > 10;") + ), + "SELECT \"id\" FROM \"orders\" WHERE total > 10" + ) + } + + // MARK: - Estimates + + /// The driver counts the whole table, so a filtered step would show a bar running to a total it + /// can never reach. + func testAFilteredTableReportsNoEstimate() { + XCTAssertNil(ObjectCopyPlanner.estimate( + 5_000, scope: PluginExportRowScope(filter: "total > 10") + )) + XCTAssertEqual( + ObjectCopyPlanner.estimate(5_000, scope: PluginExportRowScope(filter: "total > 10", rowLimit: 20)), + 20 + ) + } + + func testARowLimitIsACeilingOnTheEstimate() { + XCTAssertEqual(ObjectCopyPlanner.estimate(5_000, scope: PluginExportRowScope(rowLimit: 20)), 20) + XCTAssertEqual(ObjectCopyPlanner.estimate(10, scope: PluginExportRowScope(rowLimit: 20)), 10) + XCTAssertEqual(ObjectCopyPlanner.estimate(nil, scope: PluginExportRowScope(rowLimit: 20)), 20) + XCTAssertEqual(ObjectCopyPlanner.estimate(5_000, scope: nil), 5_000) + } } diff --git a/TableProTests/Core/ObjectCopy/ObjectCopyServerSideInsertTests.swift b/TableProTests/Core/ObjectCopy/ObjectCopyServerSideInsertTests.swift new file mode 100644 index 000000000..c3b5abd53 --- /dev/null +++ b/TableProTests/Core/ObjectCopy/ObjectCopyServerSideInsertTests.swift @@ -0,0 +1,222 @@ +// +// ObjectCopyServerSideInsertTests.swift +// TableProTests +// + +import TableProPluginKit +import XCTest +@testable import TablePro + +/// Quotes with backticks and takes the MySQL spelling of a row limit, which is enough to read the +/// statements back. The rules under test are about which name is used, not about how it is quoted. +private final class ServerSideInsertDriver: PluginDatabaseDriver, @unchecked Sendable { + func connect() async throws {} + func disconnect() {} + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { + "`\(identifier.replacingOccurrences(of: "`", with: "``"))`" + } + + func injectRowLimit(_ query: String, limit: Int) -> String? { + "\(query) LIMIT \(limit)" + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} + +final class ObjectCopyServerSideInsertTests: XCTestCase { + private let connectionId = UUID() + + private func endpoint( + _ database: String, + schema: String? = nil, + type: DatabaseType = .mysql, + connectionId: UUID? = nil + ) -> DatabaseEndpoint { + DatabaseEndpoint( + scope: DatabaseScope( + connectionId: connectionId ?? self.connectionId, database: database, schema: schema + ), + connectionName: "server", + databaseType: type, + safeModeLevel: .silent, + color: .blue + ) + } + + private func input( + source: DatabaseEndpoint, + target: DatabaseEndpoint, + sourceSchema: String? = nil, + targetSchema: String? = nil, + scope: PluginExportRowScope? = nil + ) -> ObjectCopyServerSideInsert.Input { + ObjectCopyServerSideInsert.Input( + source: source, + target: target, + sourceTable: "orders", + sourceSchema: sourceSchema, + targetTable: "orders", + targetSchema: targetSchema, + sourceColumns: ["id", "name"], + targetColumns: ["id", "name"], + scope: scope + ) + } + + // MARK: - Eligibility + + /// The whole point of the fast path is that one session can see both sides. Two connections + /// cannot, whatever engine they run. + func testTwoConnectionsAreNeverEligible() { + XCTAssertFalse(ObjectCopyServerSideInsert.isEligible( + source: endpoint("shop"), + target: endpoint("shop_copy", connectionId: UUID()) + )) + } + + func testOneConnectionAndOneDatabaseIsAlwaysEligible() { + for type in [DatabaseType.mysql, .postgresql, .sqlite, .oracle, .duckdb] { + XCTAssertTrue( + ObjectCopyServerSideInsert.isEligible( + source: endpoint("shop", schema: "a", type: type), + target: endpoint("shop", schema: "b", type: type) + ), + "\(type.rawValue)" + ) + } + } + + /// PostgreSQL cannot name a second database in one statement under any spelling, so a copy + /// between two of them streams through the app as it always did. + func testOnlyTheEnginesThatCanNameASecondDatabaseCrossOne() { + XCTAssertTrue(ObjectCopyServerSideInsert.isEligible( + source: endpoint("shop", type: .mysql), target: endpoint("shop_copy", type: .mysql) + )) + XCTAssertTrue(ObjectCopyServerSideInsert.isEligible( + source: endpoint("shop", type: .mssql), target: endpoint("shop_copy", type: .mssql) + )) + XCTAssertFalse(ObjectCopyServerSideInsert.isEligible( + source: endpoint("shop", type: .postgresql), target: endpoint("shop_copy", type: .postgresql) + )) + XCTAssertFalse(ObjectCopyServerSideInsert.isEligible( + source: endpoint("shop", type: .duckdb), target: endpoint("shop_copy", type: .duckdb) + )) + } + + // MARK: - The statement + + func testAWithinDatabaseCopyNamesTheSchema() { + let sql = ObjectCopyServerSideInsert.statement( + input( + source: endpoint("shop", schema: "old", type: .postgresql), + target: endpoint("shop", schema: "new", type: .postgresql), + sourceSchema: "old", + targetSchema: "new" + ), + driver: ServerSideInsertDriver() + ) + XCTAssertEqual( + sql, + "INSERT INTO `new`.`orders` (`id`, `name`) SELECT `id`, `name` FROM `old`.`orders`;" + ) + } + + /// The statement runs in the target's database, so an unqualified source name would resolve + /// there: the copy would read the table it was about to write. + func testACrossDatabaseCopyNamesTheDatabase() { + let sql = ObjectCopyServerSideInsert.statement( + input(source: endpoint("shop"), target: endpoint("shop_copy")), + driver: ServerSideInsertDriver() + ) + XCTAssertEqual( + sql, + "INSERT INTO `orders` (`id`, `name`) SELECT `id`, `name` FROM `shop`.`orders`;" + ) + } + + /// `Shop` and `shop` are one database on a case-insensitive server and two on a case-sensitive + /// one, so the qualifier is written whenever the engine has one rather than when the strings + /// look different. + func testTheDatabaseIsNamedEvenWhenBothSidesSpellItTheSame() { + let sql = ObjectCopyServerSideInsert.statement( + input(source: endpoint("shop"), target: endpoint("shop")), + driver: ServerSideInsertDriver() + ) + XCTAssertEqual(sql?.contains("FROM `shop`.`orders`"), true) + } + + func testSQLServerNamesTheDatabaseAndTheSchema() { + let sql = ObjectCopyServerSideInsert.statement( + input( + source: endpoint("shop", schema: "sales", type: .mssql), + target: endpoint("shop_copy", schema: "dbo", type: .mssql), + sourceSchema: "sales", + targetSchema: "dbo" + ), + driver: ServerSideInsertDriver() + ) + XCTAssertEqual(sql?.contains("FROM `shop`.`sales`.`orders`"), true) + } + + func testTheRowScopeReachesTheStatement() { + let sql = ObjectCopyServerSideInsert.statement( + input( + source: endpoint("shop"), + target: endpoint("shop_copy"), + scope: PluginExportRowScope(filter: "total > 10", rowLimit: 100) + ), + driver: ServerSideInsertDriver() + ) + XCTAssertEqual(sql?.contains("WHERE total > 10"), true) + XCTAssertEqual(sql?.hasSuffix("LIMIT 100;"), true) + } + + /// The same rule the streamed path follows: a filter is one expression, and text carrying a + /// second statement is refused rather than spliced in. + func testAFilterCarryingASecondStatementIsNotSpliced() { + let sql = ObjectCopyServerSideInsert.statement( + input( + source: endpoint("shop"), + target: endpoint("shop_copy"), + scope: PluginExportRowScope(filter: "1=1; DROP TABLE orders") + ), + driver: ServerSideInsertDriver() + ) + XCTAssertEqual(sql?.contains("DROP TABLE"), false) + XCTAssertEqual(sql?.contains("WHERE"), false) + } + + /// A mismatch means the plan and the statement disagree about which columns are written, which + /// would put values in the wrong columns. + func testMismatchedColumnListsProduceNoStatement() { + let mismatched = ObjectCopyServerSideInsert.Input( + source: endpoint("shop"), + target: endpoint("shop_copy"), + sourceTable: "orders", + sourceSchema: nil, + targetTable: "orders", + targetSchema: nil, + sourceColumns: ["id"], + targetColumns: ["id", "name"], + scope: nil + ) + XCTAssertNil(ObjectCopyServerSideInsert.statement(mismatched, driver: ServerSideInsertDriver())) + } +} diff --git a/TableProTests/Core/ObjectCopy/ObjectCopySessionTests.swift b/TableProTests/Core/ObjectCopy/ObjectCopySessionTests.swift index 3d8b8b2a1..48321f8ed 100644 --- a/TableProTests/Core/ObjectCopy/ObjectCopySessionTests.swift +++ b/TableProTests/Core/ObjectCopy/ObjectCopySessionTests.swift @@ -92,18 +92,71 @@ final class ObjectCopySessionTests: XCTestCase { XCTAssertEqual(subject.request?.target.database, "shop_copy") } - /// Neither half crosses engines. The row writer emits `INSERT … VALUES`, which a target of - /// another engine either cannot parse or reads against a namespace it does not have, so a - /// data-only copy is refused just as a structural one is. - func testNeitherHalfCrossesEngines() { + /// Both halves cross engines. The types are said in the target's own words before any DDL is + /// generated and the values are reshaped on the way in, so what is left to refuse is a target + /// that does not speak SQL at all. + func testBothHalvesCrossEngines() { let subject = session() subject.target = endpoint("analytics", type: .postgresql, connectionId: UUID()) subject.content = .structureAndData - XCTAssertNotNil(subject.reviewDisabledReason) + XCTAssertNil(subject.reviewDisabledReason) subject.content = .data + XCTAssertNil(subject.reviewDisabledReason) + } + + /// A filter is one expression. Text carrying a second statement is refused rather than spliced + /// into the `SELECT` the copy runs. + func testAFilterCarryingASecondStatementHoldsTheCopy() { + let subject = session() + subject.target = endpoint("shop_copy") + guard let table = subject.availableObjects.first(where: { $0.kind.carriesRows }) else { + return XCTFail("The catalog must offer a table to filter") + } + + /// `isUnrestricted` answers yes to a refused filter, so a scope that drops itself on that + /// answer would throw away exactly what the refusal is for. + subject.setRowScope(PluginExportRowScope(filter: "1=1; DROP TABLE orders"), for: table) + XCTAssertTrue(subject.hasRowFilter(for: table)) XCTAssertNotNil(subject.reviewDisabledReason) + + subject.setRowScope(PluginExportRowScope(filter: "total > 10"), for: table) + XCTAssertNil(subject.reviewDisabledReason) + XCTAssertEqual(subject.request?.rowScopes[table.id]?.sanitizedFilter, "total > 10") + } + + /// A filter set and then deselected, or left behind by a switch to structure only, reaches no + /// query: the planner keys them by id and one carried through would narrow a copy the user did + /// not narrow. + func testOnlyTheFiltersThatStillApplyReachTheRequest() { + let subject = session() + subject.target = endpoint("shop_copy") + guard let table = subject.availableObjects.first(where: { $0.kind.carriesRows }) else { + return XCTFail("The catalog must offer a table to filter") + } + + subject.setRowScope(PluginExportRowScope(filter: "total > 10"), for: table) + subject.content = .structure + XCTAssertTrue(subject.request?.rowScopes.isEmpty ?? false) + + subject.content = .structureAndData + subject.setSelected(table, false) + XCTAssertTrue(subject.request?.rowScopes.isEmpty ?? false) + } + + /// Clearing a filter removes it rather than storing an empty one, so nothing downstream has to + /// tell "no filter" from "a filter that narrows nothing". + func testClearingAFilterRemovesIt() { + let subject = session() + guard let table = subject.availableObjects.first(where: { $0.kind.carriesRows }) else { + return XCTFail("The catalog must offer a table to filter") + } + + subject.setRowScope(PluginExportRowScope(filter: "total > 10"), for: table) + subject.setRowScope(.unrestricted, for: table) + XCTAssertTrue(subject.rowScopes.isEmpty) + XCTAssertTrue(subject.rowScope(for: table).isUnrestricted) } // MARK: - Duplicate diff --git a/TableProUITests/CopyObjectsUITests.swift b/TableProUITests/CopyObjectsUITests.swift index 4159895eb..4bb99587f 100644 --- a/TableProUITests/CopyObjectsUITests.swift +++ b/TableProUITests/CopyObjectsUITests.swift @@ -84,8 +84,93 @@ final class CopyObjectsUITests: UITestCase { ) } + /// A filter narrows one table's rows without leaving the object list, and the row has to say so + /// afterwards: a filter that is set and invisible is a copy that quietly carries less than the + /// user thinks it does. + func testAPerTableFilterIsSetFromTheObjectListAndShownOnTheRow() throws { + let app = try launchWithSampleDatabase() + let window = try readyWindow(of: app) + try openCopyToSheet(onRow: "Album", in: window, of: app) + + let funnel = window.descendants(matching: .any) + .matching(identifier: "copy-objects-row-filter-Album").firstMatch + XCTAssertTrue(funnel.waitToExist(timeout: 20), "A table row must offer a row filter") + XCTAssertTrue(waitUntilHittable(funnel, timeout: 20)) + funnel.click() + + let field = app.descendants(matching: .any).matching(identifier: "row-scope-filter").firstMatch + XCTAssertTrue(field.waitToExist(timeout: 20), "The filter popover must offer a WHERE field") + XCTAssertTrue(waitUntilHittable(field, timeout: 20)) + field.click() + app.typeText("AlbumId > 10") + + let done = app.buttons["Done"].firstMatch + XCTAssertTrue(done.waitToExist(timeout: 10)) + done.click() + + let summary = window.descendants(matching: .any) + .matching(identifier: "copy-objects-row-scope-Album").firstMatch + XCTAssertTrue( + waitForPredicate(timeout: 20) { summary.exists }, + "The row must show the filter it now carries" + ) + let spoken = [summary.label, (summary.value as? String) ?? ""].joined(separator: " ") + XCTAssertTrue( + spoken.contains("AlbumId"), + "The summary must name the filter. label=\(summary.label) value=\(String(describing: summary.value))" + ) + + app.typeKey(.escape, modifierFlags: []) + } + + /// A filter is one expression. Text carrying a second statement is refused rather than spliced + /// into the `SELECT` the copy runs, and Continue stays out of reach until it is gone. + func testAFilterCarryingASecondStatementBlocksContinue() throws { + let app = try launchWithSampleDatabase() + let window = try readyWindow(of: app) + try openCopyToSheet(onRow: "Album", in: window, of: app) + + let funnel = window.descendants(matching: .any) + .matching(identifier: "copy-objects-row-filter-Album").firstMatch + XCTAssertTrue(funnel.waitToExist(timeout: 20)) + XCTAssertTrue(waitUntilHittable(funnel, timeout: 20)) + funnel.click() + + let field = app.descendants(matching: .any).matching(identifier: "row-scope-filter").firstMatch + XCTAssertTrue(field.waitToExist(timeout: 20)) + XCTAssertTrue(waitUntilHittable(field, timeout: 20)) + field.click() + app.typeText("1=1; DROP TABLE Album") + + let done = app.buttons["Done"].firstMatch + XCTAssertTrue(done.waitToExist(timeout: 10)) + done.click() + + let cont = window.buttons["Continue"].firstMatch + XCTAssertTrue( + waitForPredicate(timeout: 20) { cont.exists && !cont.isEnabled }, + "Continue must stay unavailable while a filter holds a second statement" + ) + + app.typeKey(.escape, modifierFlags: []) + } + // MARK: - Helpers + private func openCopyToSheet( + onRow name: String, + in window: XCUIElement, + of app: XCUIApplication + ) throws { + openContextMenu(onRow: name, in: window, of: app) + let item = contextMenuItem(copyToTitle, in: app) + XCTAssertTrue(item.waitToExist(timeout: 10)) + item.click() + let list = window.descendants(matching: .any) + .matching(identifier: "copy-objects-list").firstMatch + XCTAssertTrue(list.waitToExist(timeout: 20), "Copy To must open its object list") + } + private func readyWindow(of app: XCUIApplication) throws -> XCUIElement { let window = app.windows.firstMatch XCTAssertTrue(window.waitToExist(timeout: 30)) diff --git a/docs/features/copy-objects.mdx b/docs/features/copy-objects.mdx index c7b4ec33a..0b67bd93e 100644 --- a/docs/features/copy-objects.mdx +++ b/docs/features/copy-objects.mdx @@ -1,6 +1,6 @@ --- title: Copy and duplicate -description: Copy tables and databases to another database or another connection, structure, data, or both +description: Copy tables and databases to another database, another connection, or another engine, structure, data, or both --- Right-click what you want and pick where it goes. Rows stream from one connection to the other in @@ -25,6 +25,9 @@ written until you have read the script. Structure only, data only, or both. Tick the objects taking part. The count under the list counts the whole selection, not the part the search is showing. + + The funnel beside a table narrows the rows it contributes. See + [Copying part of a table](#copying-part-of-a-table). **Continue** reads both databases and shows the DDL that will run, the rows each table expects, @@ -44,8 +47,9 @@ written until you have read the script. A data-only copy leaves views, routines and triggers out and says so in the review step: they hold no rows. -Generated and computed columns are dropped from the write. The server recomputes them, and every -engine that has them rejects an `INSERT` that names one. +Within one engine, generated and computed columns are dropped from the write. The server recomputes +them, and every engine that has them rejects an `INSERT` that names one. Crossing to another engine +they arrive as ordinary columns carrying the values they held on the source. ## When the target already has the object @@ -63,11 +67,66 @@ the target does not have is not written; one the source does not have keeps its Pick **Skip it** unless you are refreshing a copy you made earlier. It is the only one of the three that cannot lose anything already in the target. -## One engine, and the namespace rule +## Copying part of a table + +The funnel beside a table in the object list opens a `WHERE` and a row limit for that table alone. +Both narrow what the copy reads; neither changes the structure it writes, so a copy set to carry +structure and data still creates every column. + +Write the filter in the source engine's own dialect, without the keyword: `status = 'active'`, +`created_at > '2026-01-01'`. It is one expression. A semicolon means a second statement, and **Copy** +stays disabled until it is gone. + +A filtered table shows `Unknown` for its row count in the review step. A row limit on its own shows +a number, since it is a ceiling the copy will not pass. + +**Replace it** still empties the whole target table, filter or no filter. Pair a filter with **Add +rows to it** when you mean to top up a table rather than rebuild it. + +## Crossing engines + +A table crosses. MySQL to PostgreSQL, SQL Server to MySQL, Oracle to SQLite: its columns, keys and +indexes are said again in the target's own types before any DDL is generated, and its rows are +reshaped where the source and the target spell a value differently. `TINYINT(1)` arrives as `BOOLEAN`, +`LONGTEXT` as `text`, `INT UNSIGNED` as `BIGINT`, and a PostgreSQL `t` arrives in a MySQL +`TINYINT(1)` as `1`. + +Nothing is dropped for want of a type. A type the target does not have becomes its widest text +column, so the values arrive as text rather than the column going missing. + +The review step lists every column and index the crossing changed, one line each. It is the only +place those conversions appear, and it is shown before **Copy** does anything. + +| Line | Means | What to do | +|---|---|---| +| An arrow, in grey | Widened. `uuid` to `CHAR(36)`, `INT UNSIGNED` to `BIGINT`. Every source value still fits | Nothing | +| A triangle, in orange | Approximated. A time zone dropped, an `ENUM` become text, an index left out | Read it. This is where a copy loses something | + +Views, materialized views, procedures, functions and triggers do not cross. Their definitions are +the source engine's own SQL, unparsed and uncorrected, so each is listed under **Left out** and the +tables beside it copy anyway. + +Character sets, collations, storage engines and `ON UPDATE CURRENT_TIMESTAMP` are dropped; each +names something only the source engine has. A MySQL `0000-00-00` is written as `NULL`, the only +value the rest accept for it. + +MySQL and MariaDB count as one engine, and so do PostgreSQL, Redshift, CockroachDB and PGlite. A copy +between two of those runs unchanged and reports no conversions. + +## When both sides are one connection -A copy stays inside one engine. Column data types are the driver's own strings and the row writer -emits that driver's own SQL, so neither structure nor data crosses from MySQL to PostgreSQL. The -sheet refuses before it reads anything. MySQL and MariaDB count as one engine. +A copy whose source and target are the same connection is done by the server: one +`INSERT INTO … SELECT` per table, shown in the review step in place of the query it would otherwise +have walked. The rows never reach the app, which on a large table is the difference between minutes +and seconds. + +Two things go with it. There is no row-by-row progress, and **Stop** cannot interrupt a statement the +server has already started. The plan says so above the script. + +PostgreSQL takes this path only between two schemas of one database. MySQL, MariaDB, SQL Server and +ClickHouse take it across databases too. Everything else streams through the app as before. + +## The namespace rule A database-level copy covers every schema. Each schema's objects are read and written in their own scope, and a duplicate recreates each of them under the same name in the new database. A new @@ -124,7 +183,17 @@ have none, so the command does not appear on them. ## Limitations Copying is SQL only. MongoDB, Redis, DynamoDB, Elasticsearch, Kafka, etcd and SurrealDB have no -`CREATE TABLE` and no row writer to copy through, so neither command appears on them. +`CREATE TABLE` and no row writer to copy through, so neither command appears on them, and neither +end of a copy may be one of them. + +A crossing translates types, defaults and indexes, and nothing else. Check constraints, partitioning, +table options and anything the source expressed as an expression stay behind. A copied table is a +table with the same columns and the same rows, not a replica of the original. + +Redshift takes PostgreSQL's type names and not all of PostgreSQL's types. A copy from PostgreSQL +into it is treated as one engine and passes its types through, so a `jsonb`, `uuid` or array column +is refused by the server with its own error. Change those columns to `varchar` on the source, or +create the target table first and use **Add rows to it**. Duplicate Database needs a driver that creates one. Where it does not, the command still appears and the sheet names the engine that cannot rather than failing part way in. @@ -134,8 +203,10 @@ declared `GENERATED ALWAYS AS IDENTITY`, the server refuses an explicit value an appears in the result. Copy the structure, then the data with the identity column removed from the target, or use **Add rows to it** against a table whose key is plain. -PostgreSQL columns declared `SERIAL` carry a default that names a sequence. The sequence is not -copied, so those tables need theirs created first. +A PostgreSQL `SERIAL` column's default names a sequence. Within PostgreSQL the sequence is copied +ahead of the table. Crossing to another engine there is no sequence to copy, so the column is created +with that engine's own auto-increment and the counter starts where the target starts it, not where +the source left off. Two databases on one connection can be copied between only where the driver opens a second connection of its own. DuckDB and PGlite hold their database inside the driver instance, so the