From d45c2428b61080a5d16ce072a8694a1034d20bfd Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 3 Sep 2026 14:29:33 +0700 Subject: [PATCH 1/5] feat(export): export every object kind, narrow rows per table, and dump with each engine's own tools Claude-Session: https://claude.ai/code/session_011EqgjCjCAU6tiiVmnMpF86 --- CHANGELOG.md | 8 + .../JSONExportPlugin/JSONExportModels.swift | 40 ++ .../JSONExportOptionsView.swift | 17 + .../JSONExportPlugin/JSONExportPlugin.swift | 36 +- .../SQLExportPlugin/SQLExportFileWriter.swift | 99 +++++ .../SQLExportPlugin/SQLExportInsertMode.swift | 183 +++++++++ Plugins/SQLExportPlugin/SQLExportModels.swift | 16 + .../SQLExportOptionsView.swift | 59 +++ Plugins/SQLExportPlugin/SQLExportPlugin.swift | 343 ++++++++++++---- .../SQLExportPlugin/SQLExportSnapshot.swift | 54 +++ .../ExportFormatPlugin.swift | 18 + .../PluginExportDataSource.swift | 31 ++ .../PluginExportObjectKind.swift | 79 ++++ .../PluginExportRowScope.swift | 65 +++ .../TableProPluginKit/PluginExportTypes.swift | 47 +++ .../Core/Database/NativeDumpDescriptor.swift | 106 +++++ .../Core/Database/NativeDumpRegistry.swift | 213 ++++++++++ ...pService.swift => NativeDumpService.swift} | 302 ++++++++++---- .../Plugins/ExportDataSourceAdapter.swift | 133 ++++++ TablePro/Core/Plugins/ExportObjectCache.swift | 49 +++ .../Services/Export/ExportObjectLoader.swift | 146 +++++++ .../Core/Services/Export/ExportService.swift | 35 +- .../Export/TableTransferService.swift | 221 ++++++++++ TablePro/Models/Export/ExportModels.swift | 136 +++++-- .../Views/Backup/BackupDatabaseFlow.swift | 25 +- .../Views/Backup/RestoreDatabaseFlow.swift | 6 +- TablePro/Views/Export/ExportDialog.swift | 263 ++++++++---- TablePro/Views/Export/ExportObjectRows.swift | 202 ++++++++++ .../Views/Export/ExportObjectTreeView.swift | 379 ++++++++++++++++++ TablePro/Views/Export/ExportOutlineNode.swift | 80 ++++ .../Views/Export/ExportRowScopeEditor.swift | 139 +++++++ .../Views/Export/ExportTableTreeView.swift | 172 -------- .../Views/Export/TableTransferSheet.swift | 292 ++++++++++++++ ...ainContentCoordinator+SidebarActions.swift | 7 + .../Main/MainContentCommandActions.swift | 2 +- .../Views/Main/MainContentCoordinator.swift | 4 + TablePro/Views/Main/MainContentView.swift | 6 + ...abaseTreeOutlineCoordinator+Commands.swift | 4 + .../Sidebar/Menu/DatabaseTreeMenuSpec.swift | 2 + .../Sidebar/Menu/SidebarMenuCommand.swift | 1 + .../Export/TableTransferServiceTests.swift | 104 +++++ .../Core/Redis/ExportModelsRedisTests.swift | 16 +- .../Database/NativeDumpRegistryTests.swift | 210 ++++++++++ ...sts.swift => NativeDumpServiceTests.swift} | 91 +++-- .../Export/ExportPreselectionTests.swift | 30 +- TableProTests/Models/ExportModelsTests.swift | 76 ++-- .../Models/ExportObjectTreeTests.swift | 279 +++++++++++++ .../Models/ExportRowScopeTests.swift | 91 +++++ .../Plugins/SQLExportInsertModeTests.swift | 255 ++++++++++++ docs/features/backup-restore.mdx | 42 +- docs/features/import-export.mdx | 68 +++- project.yml | 4 + 52 files changed, 4700 insertions(+), 586 deletions(-) create mode 100644 Plugins/SQLExportPlugin/SQLExportFileWriter.swift create mode 100644 Plugins/SQLExportPlugin/SQLExportInsertMode.swift create mode 100644 Plugins/SQLExportPlugin/SQLExportSnapshot.swift create mode 100644 Plugins/TableProPluginKit/PluginExportObjectKind.swift create mode 100644 Plugins/TableProPluginKit/PluginExportRowScope.swift create mode 100644 TablePro/Core/Database/NativeDumpDescriptor.swift create mode 100644 TablePro/Core/Database/NativeDumpRegistry.swift rename TablePro/Core/Database/{PostgresDumpService.swift => NativeDumpService.swift} (54%) create mode 100644 TablePro/Core/Plugins/ExportObjectCache.swift create mode 100644 TablePro/Core/Services/Export/ExportObjectLoader.swift create mode 100644 TablePro/Core/Services/Export/TableTransferService.swift create mode 100644 TablePro/Views/Export/ExportObjectRows.swift create mode 100644 TablePro/Views/Export/ExportObjectTreeView.swift create mode 100644 TablePro/Views/Export/ExportOutlineNode.swift create mode 100644 TablePro/Views/Export/ExportRowScopeEditor.swift delete mode 100644 TablePro/Views/Export/ExportTableTreeView.swift create mode 100644 TablePro/Views/Export/TableTransferSheet.swift create mode 100644 TableProTests/Core/Export/TableTransferServiceTests.swift create mode 100644 TableProTests/Database/NativeDumpRegistryTests.swift rename TableProTests/Database/{PostgresDumpServiceTests.swift => NativeDumpServiceTests.swift} (80%) create mode 100644 TableProTests/Models/ExportObjectTreeTests.swift create mode 100644 TableProTests/Models/ExportRowScopeTests.swift create mode 100644 TableProTests/Plugins/SQLExportInsertModeTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e52ea2234..d6ed91b5b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Breakdown of a query's time into server, first row and transfer, behind the toolbar's duration readout. (#2503) - Exclude the AUTO_INCREMENT counter and Exclude DEFINER clauses in the SQL export, both on by default. (#2516) - Jump to Column in the grid, a fuzzy search over the result's columns with their type and position. (#2495) +- Views, routines, triggers, user-defined types and privileges in the export tree, grouped by kind. (#2618) +- Per-table `WHERE`, row limit and column subset in the export tree. (#2618) +- Insert mode for SQL exports: skip, replace or update rows that already exist. (#2618) +- Split a SQL export into numbered parts at a chosen size. (#2618) +- Read every table at one snapshot during a SQL export. (#2618) +- Backup and restore for MySQL, MariaDB, MongoDB, SQLite and libSQL, using each engine's own tools. (#2618) +- Transfer To, copying table rows straight into another open connection with no file in between. (#2618) +- NDJSON layout for JSON exports, one row per line. (#2618) ### Changed diff --git a/Plugins/JSONExportPlugin/JSONExportModels.swift b/Plugins/JSONExportPlugin/JSONExportModels.swift index c8e43457e3..0a055cc2a6 100644 --- a/Plugins/JSONExportPlugin/JSONExportModels.swift +++ b/Plugins/JSONExportPlugin/JSONExportModels.swift @@ -5,10 +5,50 @@ import Foundation +/// How the rows are laid out in the file. +public enum JSONExportLayout: String, Codable, CaseIterable, Sendable, Identifiable { + /// One object per table, each holding an array of rows. The whole file is one JSON value. + case object + /// One row per line, with no wrapping array. A stream reader can process it a line at a time, + /// and a file too large to hold in memory stays readable. + case newlineDelimited + + public var id: String { rawValue } + + public var label: String { + switch self { + case .object: return String(localized: "One JSON object") + case .newlineDelimited: return String(localized: "One row per line (NDJSON)") + } + } + + public var fileExtension: String { + switch self { + case .object: return "json" + case .newlineDelimited: return "ndjson" + } + } +} + public struct JSONExportOptions: Equatable, Codable { public var prettyPrint: Bool = true public var includeNullValues: Bool = true public var preserveAllAsStrings: Bool = false + public var layout: JSONExportLayout = .object public init() {} + + /// A synthesized `init(from:)` throws `keyNotFound` for a key the saved payload predates and + /// never falls back to the property's default, so adding one here would reset the options a + /// user had already chosen. + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let defaults = JSONExportOptions() + prettyPrint = try container.decodeIfPresent(Bool.self, forKey: .prettyPrint) ?? defaults.prettyPrint + includeNullValues = try container.decodeIfPresent(Bool.self, forKey: .includeNullValues) + ?? defaults.includeNullValues + preserveAllAsStrings = try container.decodeIfPresent(Bool.self, forKey: .preserveAllAsStrings) + ?? defaults.preserveAllAsStrings + layout = try container.decodeIfPresent(JSONExportLayout.self, forKey: .layout) ?? defaults.layout + } } diff --git a/Plugins/JSONExportPlugin/JSONExportOptionsView.swift b/Plugins/JSONExportPlugin/JSONExportOptionsView.swift index b70574e4a4..e92027c778 100644 --- a/Plugins/JSONExportPlugin/JSONExportOptionsView.swift +++ b/Plugins/JSONExportPlugin/JSONExportOptionsView.swift @@ -10,8 +10,25 @@ struct JSONExportOptionsView: View { var body: some View { VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Layout") + + Spacer() + + Picker("", selection: $plugin.settings.layout) { + ForEach(JSONExportLayout.allCases) { layout in + Text(layout.label).tag(layout) + } + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 180) + } + .help("NDJSON writes one row per line with no wrapping array, which a stream reader can process a line at a time") + Toggle("Pretty print (formatted output)", isOn: $plugin.settings.prettyPrint) .toggleStyle(.checkbox) + .disabled(plugin.settings.layout == .newlineDelimited) Toggle("Include NULL values", isOn: $plugin.settings.includeNullValues) .toggleStyle(.checkbox) diff --git a/Plugins/JSONExportPlugin/JSONExportPlugin.swift b/Plugins/JSONExportPlugin/JSONExportPlugin.swift index ee6eeb527a..a9f7d3f634 100644 --- a/Plugins/JSONExportPlugin/JSONExportPlugin.swift +++ b/Plugins/JSONExportPlugin/JSONExportPlugin.swift @@ -26,6 +26,10 @@ final class JSONExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Sen required init() { loadSettings() } + var currentFileExtension: String { + settings.layout.fileExtension + } + @MainActor func settingsView() -> AnyView? { AnyView(JSONExportOptionsView(plugin: self)) @@ -49,25 +53,33 @@ final class JSONExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Sen } } - let prettyPrint = settings.prettyPrint + /// NDJSON is one row per line by definition, so the pretty-print setting cannot apply to it + /// and the wrapping object has no place to go. + let isNewlineDelimited = settings.layout == .newlineDelimited + let prettyPrint = settings.prettyPrint && !isNewlineDelimited let indent = prettyPrint ? " " : "" let newline = prettyPrint ? "\n" : "" - try fileHandle.write(contentsOf: "{\(newline)".toUTF8Data()) + if !isNewlineDelimited { + try fileHandle.write(contentsOf: "{\(newline)".toUTF8Data()) + } + var hasWrittenAnyRow = false for (tableIndex, table) in tables.enumerated() { try progress.checkCancellation() progress.setCurrentTable(table.qualifiedName, index: tableIndex + 1) let escapedTableName = PluginExportUtilities.escapeJSONString(table.qualifiedName) - try fileHandle.write(contentsOf: "\(indent)\"\(escapedTableName)\": [\(newline)".toUTF8Data()) + if !isNewlineDelimited { + try fileHandle.write(contentsOf: "\(indent)\"\(escapedTableName)\": [\(newline)".toUTF8Data()) + } - var hasWrittenRow = false + var hasWrittenRow = isNewlineDelimited ? hasWrittenAnyRow : false var columns: [String]? var columnTypeNames: [String]? - let stream = dataSource.streamRows(table: table.name, databaseName: table.databaseName) + let stream = dataSource.streamRows(for: table) for try await element in stream { try progress.checkCancellation() @@ -81,7 +93,7 @@ final class JSONExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Sen var rowString = "" if hasWrittenRow { - rowString += ",\(newline)" + rowString += isNewlineDelimited ? "\n" : ",\(newline)" } rowString += rowPrefix @@ -117,11 +129,15 @@ final class JSONExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Sen try fileHandle.write(contentsOf: rowString.toUTF8Data()) hasWrittenRow = true + hasWrittenAnyRow = true progress.incrementRow() } } } + if isNewlineDelimited { + continue + } if hasWrittenRow { try fileHandle.write(contentsOf: newline.toUTF8Data()) } @@ -129,7 +145,13 @@ final class JSONExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Sen try fileHandle.write(contentsOf: "\(indent)]\(tableSuffix)".toUTF8Data()) } - try fileHandle.write(contentsOf: "}".toUTF8Data()) + if isNewlineDelimited { + if hasWrittenAnyRow { + try fileHandle.write(contentsOf: "\n".toUTF8Data()) + } + } else { + try fileHandle.write(contentsOf: "}".toUTF8Data()) + } try progress.checkCancellation() try fileHandle.close() diff --git a/Plugins/SQLExportPlugin/SQLExportFileWriter.swift b/Plugins/SQLExportPlugin/SQLExportFileWriter.swift new file mode 100644 index 0000000000..a0673e562a --- /dev/null +++ b/Plugins/SQLExportPlugin/SQLExportFileWriter.swift @@ -0,0 +1,99 @@ +// +// SQLExportFileWriter.swift +// SQLExportPlugin +// + +import Foundation +import TableProPluginKit + +/// Writes a dump, starting a new file once the current one passes a size cap. +/// +/// Rotation happens between writes and never inside one, so a part always ends on a complete +/// statement: a restore that replays the parts in order gets the same statements in the same order +/// as an unsplit dump. Every part is written to a temporary file and committed at the end, which is +/// what keeps a failed or cancelled export from leaving half a dump behind. +internal final class SQLExportFileWriter { + /// The name a part takes: `dump.sql` becomes `dump.part1.sql`, and a compound extension like + /// `dump.sql.gz` becomes `dump.part1.sql.gz`. + internal static func partURL(for destination: URL, part: Int) -> URL { + let name = destination.lastPathComponent + guard let firstDot = name.firstIndex(of: ".") else { + return destination.deletingLastPathComponent().appendingPathComponent("\(name).part\(part)") + } + let base = String(name[name.startIndex ..< firstDot]) + let suffix = String(name[firstDot...]) + return destination.deletingLastPathComponent().appendingPathComponent("\(base).part\(part)\(suffix)") + } + + private let destination: URL + private let splitSizeBytes: Int + + private var handle: FileHandle + private var tempURL: URL + private var bytesInCurrentPart = 0 + private var partIndex = 1 + private var pending: [(temp: URL, final: URL)] = [] + private var isCommitted = false + + internal init(destination: URL, splitSizeMegabytes: Int) throws { + self.destination = destination + self.splitSizeBytes = max(0, splitSizeMegabytes) * 1_024 * 1_024 + let (handle, tempURL) = try PluginExportUtilities.beginAtomicWrite(for: destination) + self.handle = handle + self.tempURL = tempURL + } + + /// True once a second part exists, so the caller can report the split rather than leaving the + /// user to find `dump.part2.sql` themselves. + internal var didSplit: Bool { partIndex > 1 } + + internal var partCount: Int { partIndex } + + internal func write(_ text: String) throws { + let data = try text.toUTF8Data() + if splitSizeBytes > 0, bytesInCurrentPart > 0, bytesInCurrentPart + data.count > splitSizeBytes { + try rotate() + } + try handle.write(contentsOf: data) + bytesInCurrentPart += data.count + } + + /// Publishes every part and returns where they landed. An unsplit export keeps the name the + /// user chose; a split one numbers all of its parts, so no part silently claims that name. + @discardableResult + internal func commit() throws -> [URL] { + try handle.close() + let finalURL = didSplit ? Self.partURL(for: destination, part: partIndex) : destination + pending.append((tempURL, finalURL)) + for entry in pending { + try PluginExportUtilities.commitAtomicWrite(from: entry.temp, to: entry.final) + } + isCommitted = true + return pending.map(\.final) + } + + /// Removes every temporary file. Safe to call after a commit, where it finds nothing to remove. + internal func rollback() { + guard !isCommitted else { return } + try? handle.close() + PluginExportUtilities.rollbackAtomicWrite(at: tempURL) + for entry in pending { + PluginExportUtilities.rollbackAtomicWrite(at: entry.temp) + } + pending.removeAll() + } + + /// The file the caller compresses when gzip is on. Compression runs over a single file, so a + /// split export is the one case it cannot apply to. + internal var currentFileURL: URL { tempURL } + + private func rotate() throws { + try handle.close() + pending.append((tempURL, Self.partURL(for: destination, part: partIndex))) + partIndex += 1 + let (nextHandle, nextTemp) = try PluginExportUtilities.beginAtomicWrite(for: destination) + handle = nextHandle + tempURL = nextTemp + bytesInCurrentPart = 0 + } +} diff --git a/Plugins/SQLExportPlugin/SQLExportInsertMode.swift b/Plugins/SQLExportPlugin/SQLExportInsertMode.swift new file mode 100644 index 0000000000..e39672b9e6 --- /dev/null +++ b/Plugins/SQLExportPlugin/SQLExportInsertMode.swift @@ -0,0 +1,183 @@ +// +// SQLExportInsertMode.swift +// SQLExportPlugin +// + +import Foundation +import TableProPluginKit + +/// How a dump's `INSERT` behaves when a row it writes already exists. +public enum SQLExportInsertMode: String, Codable, CaseIterable, Sendable, Identifiable { + case insert + case ignoreExisting + case replaceExisting + case updateExisting + + public var id: String { rawValue } + + public var label: String { + switch self { + case .insert: return String(localized: "Insert") + case .ignoreExisting: return String(localized: "Insert, skip existing") + case .replaceExisting: return String(localized: "Replace existing") + case .updateExisting: return String(localized: "Update existing") + } + } +} + +/// Renders the two dialect-specific halves of an `INSERT`: what comes before the column list and +/// what comes after the last row of values. +/// +/// Every engine spells conflict handling differently, and three of the four spellings are not +/// interchangeable: MySQL puts it in the verb (`INSERT IGNORE`, `REPLACE INTO`), SQLite puts it in +/// a resolution clause (`INSERT OR IGNORE`), and PostgreSQL puts it in a trailing `ON CONFLICT` +/// that has to name the conflict target. An engine with no spelling at all falls back to a plain +/// `INSERT` and says so, rather than writing a statement the restore would reject. +public struct SQLExportInsertRenderer { + public struct Rendered: Equatable, Sendable { + public let prefix: String + public let suffix: String + public let warning: String? + } + + private let dialect: SqlDialect + private let quoteIdentifier: (String) -> String + + public init(dialect: SqlDialect, quoteIdentifier: @escaping (String) -> String) { + self.dialect = dialect + self.quoteIdentifier = quoteIdentifier + } + + public func render( + mode: SQLExportInsertMode, + tableRef: String, + quotedColumns: String, + overriding: String, + columnNames: [String], + primaryKeyColumns: [String] + ) -> Rendered { + let plain = Rendered( + prefix: "INSERT INTO \(tableRef) (\(quotedColumns))\(overriding) VALUES\n", + suffix: "", + warning: nil + ) + switch mode { + case .insert: + return plain + case .ignoreExisting: + return renderIgnore(plain: plain, tableRef: tableRef, quotedColumns: quotedColumns, overriding: overriding) + case .replaceExisting: + return renderReplace( + plain: plain, tableRef: tableRef, quotedColumns: quotedColumns, overriding: overriding, + columnNames: columnNames, primaryKeyColumns: primaryKeyColumns) + case .updateExisting: + return renderUpdate( + plain: plain, tableRef: tableRef, quotedColumns: quotedColumns, overriding: overriding, + columnNames: columnNames, primaryKeyColumns: primaryKeyColumns) + } + } + + private func renderIgnore( + plain: Rendered, + tableRef: String, + quotedColumns: String, + overriding: String + ) -> Rendered { + switch dialect { + case .mysql: + return Rendered( + prefix: "INSERT IGNORE INTO \(tableRef) (\(quotedColumns))\(overriding) VALUES\n", + suffix: "", warning: nil) + case .sqlite: + return Rendered( + prefix: "INSERT OR IGNORE INTO \(tableRef) (\(quotedColumns))\(overriding) VALUES\n", + suffix: "", warning: nil) + case .postgres: + return Rendered(prefix: plain.prefix, suffix: "\nON CONFLICT DO NOTHING", warning: nil) + default: + return Rendered(prefix: plain.prefix, suffix: "", warning: Self.unsupportedWarning) + } + } + + /// PostgreSQL has no `REPLACE`, and its nearest equivalent is an upsert that overwrites every + /// non-key column, so replace and update render the same there. + private func renderReplace( + plain: Rendered, + tableRef: String, + quotedColumns: String, + overriding: String, + columnNames: [String], + primaryKeyColumns: [String] + ) -> Rendered { + switch dialect { + case .mysql: + return Rendered( + prefix: "REPLACE INTO \(tableRef) (\(quotedColumns))\(overriding) VALUES\n", + suffix: "", warning: nil) + case .sqlite: + return Rendered( + prefix: "INSERT OR REPLACE INTO \(tableRef) (\(quotedColumns))\(overriding) VALUES\n", + suffix: "", warning: nil) + case .postgres: + return renderUpdate( + plain: plain, tableRef: tableRef, quotedColumns: quotedColumns, overriding: overriding, + columnNames: columnNames, primaryKeyColumns: primaryKeyColumns) + default: + return Rendered(prefix: plain.prefix, suffix: "", warning: Self.unsupportedWarning) + } + } + + private func renderUpdate( + plain: Rendered, + tableRef: String, + quotedColumns: String, + overriding: String, + columnNames: [String], + primaryKeyColumns: [String] + ) -> Rendered { + let updatable = columnNames.filter { !primaryKeyColumns.contains($0) } + switch dialect { + case .mysql: + guard !updatable.isEmpty else { + return Rendered(prefix: plain.prefix, suffix: "", warning: Self.noUpdatableColumnsWarning) + } + let assignments = updatable + .map { "\(quoteIdentifier($0)) = VALUES(\(quoteIdentifier($0)))" } + .joined(separator: ", ") + return Rendered( + prefix: plain.prefix, + suffix: "\nON DUPLICATE KEY UPDATE \(assignments)", + warning: nil) + case .postgres, .sqlite: + guard !primaryKeyColumns.isEmpty else { + return Rendered(prefix: plain.prefix, suffix: "", warning: Self.noPrimaryKeyWarning) + } + guard !updatable.isEmpty else { + return Rendered(prefix: plain.prefix, suffix: "\nON CONFLICT DO NOTHING", warning: nil) + } + let target = primaryKeyColumns.map(quoteIdentifier).joined(separator: ", ") + let excluded = dialect == .postgres ? "EXCLUDED" : "excluded" + let assignments = updatable + .map { "\(quoteIdentifier($0)) = \(excluded).\(quoteIdentifier($0))" } + .joined(separator: ", ") + return Rendered( + prefix: plain.prefix, + suffix: "\nON CONFLICT (\(target)) DO UPDATE SET \(assignments)", + warning: nil) + default: + return Rendered(prefix: plain.prefix, suffix: "", warning: Self.unsupportedWarning) + } + } + + private static var unsupportedWarning: String { + String(localized: "This engine has no conflict handling for INSERT, so the rows are written as plain inserts.") + } + + private static var noPrimaryKeyWarning: String { + String(localized: "A table with no primary key has no conflict target, so its rows are written as plain inserts.") + } + + private static var noUpdatableColumnsWarning: String { + String(localized: "A table whose columns are all part of its key has nothing to update, so its rows are written as plain inserts.") + } +} diff --git a/Plugins/SQLExportPlugin/SQLExportModels.swift b/Plugins/SQLExportPlugin/SQLExportModels.swift index 9e326beeee..ab5210cf07 100644 --- a/Plugins/SQLExportPlugin/SQLExportModels.swift +++ b/Plugins/SQLExportPlugin/SQLExportModels.swift @@ -10,6 +10,16 @@ public struct SQLExportOptions: Equatable, Codable { public var batchSize: Int = 500 public var excludeAutoIncrementValue: Bool = true public var excludeDefiner: Bool = true + public var insertMode: SQLExportInsertMode = .insert + + /// Reads every table inside one transaction at a repeatable snapshot, so a dump of several + /// tables is consistent with itself. Off by default because it holds a transaction open for the + /// whole export, which on a busy server keeps the undo log growing. + public var consistentSnapshot: Bool = false + + /// Starts a new file every `splitSizeMegabytes` megabytes, numbering them `.part1`, `.part2`. + /// Zero writes one file however large it gets. + public var splitSizeMegabytes: Int = 0 public init() {} @@ -26,5 +36,11 @@ public struct SQLExportOptions: Equatable, Codable { ?? defaults.excludeAutoIncrementValue excludeDefiner = try container.decodeIfPresent(Bool.self, forKey: .excludeDefiner) ?? defaults.excludeDefiner + insertMode = try container.decodeIfPresent(SQLExportInsertMode.self, forKey: .insertMode) + ?? defaults.insertMode + consistentSnapshot = try container.decodeIfPresent(Bool.self, forKey: .consistentSnapshot) + ?? defaults.consistentSnapshot + splitSizeMegabytes = try container.decodeIfPresent(Int.self, forKey: .splitSizeMegabytes) + ?? defaults.splitSizeMegabytes } } diff --git a/Plugins/SQLExportPlugin/SQLExportOptionsView.swift b/Plugins/SQLExportPlugin/SQLExportOptionsView.swift index 69dc352ed9..c022764860 100644 --- a/Plugins/SQLExportPlugin/SQLExportOptionsView.swift +++ b/Plugins/SQLExportPlugin/SQLExportOptionsView.swift @@ -10,6 +10,25 @@ struct SQLExportOptionsView: View { private static let batchSizeOptions = [1, 100, 500, 1_000] + private static let splitSizeOptions = [0, 8, 32, 128, 512] + + private static let insertModeHelp = String( + localized: """ + What an INSERT does when the row already exists. MySQL, MariaDB, PostgreSQL and SQLite \ + each spell this differently; an engine with no spelling for it writes plain inserts and \ + the summary says so. + """, + bundle: .main + ) + + private static let snapshotHelp = String( + localized: """ + Reads every table inside one transaction, so a dump of several tables is consistent with \ + itself. The transaction stays open for the whole export. + """, + bundle: .main + ) + private static let autoIncrementHelp = String( localized: "MySQL and MariaDB. Drops the table's next key value. The column keeps its AUTO_INCREMENT attribute, and restoring rows sets the counter from the data.", bundle: .main @@ -52,6 +71,46 @@ struct SQLExportOptionsView: View { } .help("Higher values create fewer INSERT statements, resulting in smaller files and faster imports") + HStack { + Text("On existing rows") + .font(.system(size: 13)) + + Spacer() + + Picker("", selection: $plugin.settings.insertMode) { + ForEach(SQLExportInsertMode.allCases) { mode in + Text(mode.label).tag(mode) + } + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 160) + } + .help(Self.insertModeHelp) + + HStack { + Text("Split every") + .font(.system(size: 13)) + + Spacer() + + Picker("", selection: $plugin.settings.splitSizeMegabytes) { + ForEach(Self.splitSizeOptions, id: \.self) { size in + Text(size == 0 ? String(localized: "One file", bundle: .main) : "\(size) MB") + .tag(size) + } + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 130) + } + .help("Writes .part1, .part2 and so on once the file passes this size") + + Toggle("Read every table at one snapshot", isOn: $plugin.settings.consistentSnapshot) + .toggleStyle(.checkbox) + .font(.system(size: 13)) + .help(Self.snapshotHelp) + Toggle("Exclude the AUTO_INCREMENT counter", isOn: $plugin.settings.excludeAutoIncrementValue) .toggleStyle(.checkbox) .font(.system(size: 13)) diff --git a/Plugins/SQLExportPlugin/SQLExportPlugin.swift b/Plugins/SQLExportPlugin/SQLExportPlugin.swift index a4e0ef7f4b..f509b673d5 100644 --- a/Plugins/SQLExportPlugin/SQLExportPlugin.swift +++ b/Plugins/SQLExportPlugin/SQLExportPlugin.swift @@ -25,6 +25,22 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send PluginExportOptionColumn(id: "data", label: "Data", width: 44) ] + static let supportedObjectKinds: [PluginExportObjectKind] = [ + .userType, .sequence, .table, .foreignTable, .view, .materializedView, + .routine, .trigger, .event, .grant + ] + + /// A routine has no rows, and a grant is a statement rather than an object with a definition to + /// drop, so those columns are blank slots for those kinds. The positions never move, because + /// `optionValues` stays aligned with the full column list for every kind. + static func supportsOption(columnId: String, for kind: PluginExportObjectKind) -> Bool { + switch columnId { + case "data": return kind.carriesRows + case "drop": return kind != .grant + default: return true + } + } + typealias Settings = SQLExportOptions static let settingsStorageId = "sql" @@ -104,39 +120,74 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send actualDestination = destination } - let (fileHandle, tempURL) = try PluginExportUtilities.beginAtomicWrite(for: actualDestination) + /// Compression runs over one file, so a compressed export never splits. Saying so beats + /// silently gzipping the first part and dropping the rest. + let splitSize = settings.compressWithGzip ? 0 : settings.splitSizeMegabytes + if settings.compressWithGzip, settings.splitSizeMegabytes > 0 { + metadataWarnings.append(String(localized: + "A compressed export is written as one file, so the split size was not applied.")) + } + let writer = try SQLExportFileWriter(destination: actualDestination, splitSizeMegabytes: splitSize) var committed = false defer { - if !committed { - PluginExportUtilities.rollbackAtomicWrite(at: tempURL) - } + if !committed { writer.rollback() } } + let snapshot = settings.consistentSnapshot + ? SQLExportSnapshot(dialect: SqlDialect.from(databaseTypeId: dataSource.databaseTypeId)) + : nil + do { - try writeHeader(to: fileHandle, dataSource: dataSource) - let columnsByTable = await prefetchColumns(tables: tables, dataSource: dataSource) - let fkMap = await prefetchForeignKeys(tables: tables, dataSource: dataSource) - let sortedTables = topologicallySort(tables, fkMap: fkMap) - noteContainerSpan(of: sortedTables) - try writeDependencyCycleNote(to: fileHandle) - - try writeDropPhase(sortedTables: sortedTables, dataSource: dataSource, to: fileHandle) + if let snapshot { + try await snapshot.begin(on: dataSource) + } + let rowObjects = tables.filter { $0.kind.carriesRows } + let definitionObjects = tables.filter { !$0.kind.carriesRows } + + try writeHeader(to: writer, dataSource: dataSource) + let columnsByTable = await prefetchColumns(tables: rowObjects, dataSource: dataSource) + let fkMap = await prefetchForeignKeys(tables: rowObjects, dataSource: dataSource) + let sortedTables = topologicallySort(rowObjects, fkMap: fkMap) + noteContainerSpan(of: tables) + try writeDependencyCycleNote(to: writer) + + try writeDropPhase( + sortedTables: sortedTables, definitionObjects: definitionObjects, + dataSource: dataSource, to: writer) try await writeDependentTypesAndSequences( - tables: tables, dataSource: dataSource, to: fileHandle) + tables: rowObjects, dataSource: dataSource, to: writer) + try await writeObjectCreatePhase( + objects: definitionObjects, kinds: [.userType, .sequence], + dataSource: dataSource, to: writer, progress: progress) try await writeCreatePhase( - sortedTables: sortedTables, dataSource: dataSource, to: fileHandle, progress: progress) + sortedTables: sortedTables, dataSource: dataSource, to: writer, progress: progress) try await writeDataPhase( sortedTables: sortedTables, columnsByTable: columnsByTable, - dataSource: dataSource, to: fileHandle, progress: progress) + dataSource: dataSource, to: writer, progress: progress) try writeFinalizationPhase( sortedTables: sortedTables, fkMap: fkMap, columnsByTable: columnsByTable, - dataSource: dataSource, to: fileHandle) - - try fileHandle.close() - try PluginExportUtilities.commitAtomicWrite(from: tempURL, to: actualDestination) + dataSource: dataSource, to: writer) + try await writeObjectCreatePhase( + objects: definitionObjects, + kinds: [.view, .materializedView, .routine, .trigger, .event], + dataSource: dataSource, to: writer, progress: progress) + try await writeGrantPhase( + objects: definitionObjects, dataSource: dataSource, to: writer) + + if let snapshot { + await snapshot.end(on: dataSource) + } + try writer.commit() committed = true + if writer.didSplit { + metadataWarnings.append(String( + format: String(localized: "The dump was written as %lld numbered parts. Restore them in order."), + Int64(writer.partCount))) + } } catch { - try? fileHandle.close() + if let snapshot { + await snapshot.end(on: dataSource) + } throw error } @@ -168,13 +219,13 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send } private func writeHeader( - to fileHandle: FileHandle, + to writer: SQLExportFileWriter, dataSource: any PluginExportDataSource ) throws { let dateFormatter = ISO8601DateFormatter() - try fileHandle.write(contentsOf: "-- TablePro SQL Export\n".toUTF8Data()) - try fileHandle.write(contentsOf: "-- Generated: \(dateFormatter.string(from: Date()))\n".toUTF8Data()) - try fileHandle.write(contentsOf: "-- Database Type: \(dataSource.databaseTypeId)\n\n".toUTF8Data()) + try writer.write("-- TablePro SQL Export\n") + try writer.write("-- Generated: \(dateFormatter.string(from: Date()))\n") + try writer.write("-- Database Type: \(dataSource.databaseTypeId)\n\n") } private struct ExportGroup { @@ -261,7 +312,7 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send /// not the same everywhere: `foreignKeyDisableStatements` is nil on SQL Server, Oracle, /// Snowflake and DuckDB, so telling every user to import with the checks off would be wrong on /// the engines that cannot turn them off. - private func writeDependencyCycleNote(to fileHandle: FileHandle) throws { + private func writeDependencyCycleNote(to writer: SQLExportFileWriter) throws { guard !tablesUnorderedByCycle.isEmpty else { return } let names = tablesUnorderedByCycle.joined(separator: ", ") metadataWarnings.append(String( @@ -273,38 +324,61 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send names)) let note = "-- Warning: \(PluginExportUtilities.sanitizeForSQLComment(names)) reference each other.\n" + "-- No parent-first order exists, so they are written in the order they were listed.\n\n" - try fileHandle.write(contentsOf: note.toUTF8Data()) + try writer.write(note) } + /// Drops run in the reverse of the order the objects are created in, so a dependent goes before + /// what it depends on: triggers and routines first, then views, then the tables in reverse + /// topological order, then the sequences and types those tables referenced. private func writeDropPhase( sortedTables: [PluginExportTable], + definitionObjects: [PluginExportTable], dataSource: any PluginExportDataSource, - to fileHandle: FileHandle + to writer: SQLExportFileWriter ) throws { - let dropTargets = sortedTables.reversed().filter { optionValue($0, at: 1) } + let afterTables = definitionObjects + .filter { $0.kind.dumpOrder > PluginExportObjectKind.table.dumpOrder } + .sorted { $0.kind.dumpOrder > $1.kind.dumpOrder } + let beforeTables = definitionObjects + .filter { $0.kind.dumpOrder < PluginExportObjectKind.table.dumpOrder } + .sorted { $0.kind.dumpOrder > $1.kind.dumpOrder } + let dropTargets = (afterTables + Array(sortedTables.reversed()) + beforeTables) + .filter { optionValue($0, at: 1) && $0.kind != .grant } guard !dropTargets.isEmpty else { return } - for table in dropTargets { - let tableRef = qualifiedRef( - schema: table.databaseName, table: table.name, dataSource: dataSource) - let keyword = dropStatementKeyword(for: table.tableType) - try fileHandle.write(contentsOf: "\(keyword) IF EXISTS \(tableRef) CASCADE;\n".toUTF8Data()) + for object in dropTargets { + guard let statement = dropStatement(for: object, dataSource: dataSource) else { continue } + try writer.write("\(statement)\n") } - try fileHandle.write(contentsOf: "\n".toUTF8Data()) + try writer.write("\n") } - private func dropStatementKeyword(for tableType: String) -> String { - switch tableType { - case "view": return "DROP VIEW" - case "materialized view": return "DROP MATERIALIZED VIEW" - case "foreign table": return "DROP FOREIGN TABLE" - default: return "DROP TABLE" + /// The engine spells its own DROP for the kinds where dialects disagree: PostgreSQL's + /// `DROP TRIGGER` takes an `ON ` clause where MySQL's does not, and MySQL has no + /// `DROP ROUTINE` at all. Only the table-shaped kinds, which every SQL engine spells the same + /// way, fall through to the generic form here. + private func dropStatement( + for object: PluginExportTable, + dataSource: any PluginExportDataSource + ) -> String? { + if let driverStatement = dataSource.dropStatement(for: object) { + return driverStatement.hasSuffix(";") ? driverStatement : "\(driverStatement);" + } + let keyword = object.kind.dropKeyword + guard !keyword.isEmpty else { return nil } + let ref = qualifiedRef( + schema: object.databaseName, table: object.name, dataSource: dataSource) + switch object.kind { + case .trigger, .event, .routine: + return "\(keyword) IF EXISTS \(dataSource.quoteIdentifier(object.name));" + default: + return "\(keyword) IF EXISTS \(ref) CASCADE;" } } private func writeDependentTypesAndSequences( tables: [PluginExportTable], dataSource: any PluginExportDataSource, - to fileHandle: FileHandle + to writer: SQLExportFileWriter ) async throws { var emittedSequenceNames: Set = [] var emittedTypeNames: Set = [] @@ -317,8 +391,8 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send for seq in sequences where !emittedSequenceNames.contains(seq.name) { emittedSequenceNames.insert(seq.name) let quotedName = "\"\(seq.name.replacingOccurrences(of: "\"", with: "\"\""))\"" - try fileHandle.write(contentsOf: "DROP SEQUENCE IF EXISTS \(quotedName) CASCADE;\n".toUTF8Data()) - try fileHandle.write(contentsOf: "\(seq.ddl)\n\n".toUTF8Data()) + try writer.write("DROP SEQUENCE IF EXISTS \(quotedName) CASCADE;\n") + try writer.write("\(seq.ddl)\n\n") } } catch { Self.logger.warning("Failed to fetch dependent sequences for table \(table.name): \(error)") @@ -330,9 +404,9 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send for enumType in enumTypes where !emittedTypeNames.contains(enumType.name) { emittedTypeNames.insert(enumType.name) let quotedName = "\"\(enumType.name.replacingOccurrences(of: "\"", with: "\"\""))\"" - try fileHandle.write(contentsOf: "DROP TYPE IF EXISTS \(quotedName) CASCADE;\n".toUTF8Data()) + try writer.write("DROP TYPE IF EXISTS \(quotedName) CASCADE;\n") let quotedLabels = enumType.labels.map { "'\(dataSource.escapeStringLiteral($0))'" } - try fileHandle.write(contentsOf: "CREATE TYPE \(quotedName) AS ENUM (\(quotedLabels.joined(separator: ", ")));\n\n".toUTF8Data()) + try writer.write("CREATE TYPE \(quotedName) AS ENUM (\(quotedLabels.joined(separator: ", ")));\n\n") } } catch { Self.logger.warning("Failed to fetch dependent types for table \(table.name): \(error)") @@ -343,7 +417,7 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send private func writeCreatePhase( sortedTables: [PluginExportTable], dataSource: any PluginExportDataSource, - to fileHandle: FileHandle, + to writer: SQLExportFileWriter, progress: PluginExportProgress ) async throws { let rewriter = ddlRewriter(for: dataSource) @@ -351,41 +425,131 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send try progress.checkCancellation() progress.setCurrentTable(table.qualifiedName, index: index + 1) let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(table.name) - try fileHandle.write(contentsOf: "-- --------------------------------------------------------\n".toUTF8Data()) - try fileHandle.write(contentsOf: "-- Table: \(sanitizedName)\n".toUTF8Data()) - try fileHandle.write(contentsOf: "-- --------------------------------------------------------\n\n".toUTF8Data()) + try writer.write("-- --------------------------------------------------------\n") + try writer.write("-- Table: \(sanitizedName)\n") + try writer.write("-- --------------------------------------------------------\n\n") do { let ddl = rewriter.rewrite( try await dataSource.fetchTableDDL( table: table.name, databaseName: table.databaseName)) - try fileHandle.write(contentsOf: ddl.toUTF8Data()) + try writer.write(ddl) if !ddl.hasSuffix(";") { - try fileHandle.write(contentsOf: ";".toUTF8Data()) + try writer.write(";") } - try fileHandle.write(contentsOf: "\n\n".toUTF8Data()) + try writer.write("\n\n") } catch { ddlFailures.append(sanitizedName) let ddlWarning = "Warning: failed to fetch DDL for table \(sanitizedName): \(error)" Self.logger.warning("Failed to fetch DDL for table \(sanitizedName): \(error)") - try fileHandle.write(contentsOf: "-- \(PluginExportUtilities.sanitizeForSQLComment(ddlWarning))\n\n".toUTF8Data()) + try writer.write("-- \(PluginExportUtilities.sanitizeForSQLComment(ddlWarning))\n\n") + } + } + } + + /// Writes the definition of every object of the named kinds, in dump order within the group so + /// a view that another view selects from is created first. A kind the driver cannot produce a + /// definition for is recorded as a failure and commented into the file rather than aborting the + /// export: one unreadable routine must not cost the user the whole dump. + private func writeObjectCreatePhase( + objects: [PluginExportTable], + kinds: [PluginExportObjectKind], + dataSource: any PluginExportDataSource, + to writer: SQLExportFileWriter, + progress: PluginExportProgress + ) async throws { + let wanted = Set(kinds) + let targets = objects + .filter { wanted.contains($0.kind) && optionValue($0, at: 0) } + .sorted { ($0.kind.dumpOrder, $0.name) < ($1.kind.dumpOrder, $1.name) } + guard !targets.isEmpty else { return } + + for object in targets { + try progress.checkCancellation() + let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(object.name) + let label = objectCommentLabel(for: object.kind) + try writer.write("-- --------------------------------------------------------\n") + try writer.write("-- \(label): \(sanitizedName)\n") + try writer.write("-- --------------------------------------------------------\n\n") + do { + let ddl = ddlRewriter(for: dataSource).rewrite(try await dataSource.fetchObjectDDL(object)) + try writer.write(ddl) + if !ddl.hasSuffix(";") { + try writer.write(";") + } + try writer.write("\n\n") + } catch { + ddlFailures.append(sanitizedName) + Self.logger.warning("Failed to fetch DDL for \(sanitizedName): \(error)") + let warning = "Warning: failed to fetch definition for \(label.lowercased()) \(sanitizedName): \(error)" + try writer.write("-- \(PluginExportUtilities.sanitizeForSQLComment(warning))\n\n") } } } + /// Grants come last, because every object they name has to exist first. + private func writeGrantPhase( + objects: [PluginExportTable], + dataSource: any PluginExportDataSource, + to writer: SQLExportFileWriter + ) async throws { + let principals = objects + .filter { $0.kind == .grant && optionValue($0, at: 0) } + .sorted { $0.name < $1.name } + guard !principals.isEmpty else { return } + + try writer.write("-- --------------------------------------------------------\n") + try writer.write("-- Privileges\n") + try writer.write("-- --------------------------------------------------------\n\n") + + for principal in principals { + do { + let statements = try await dataSource.fetchGrantStatements( + principal: principal.name, host: principal.identity) + guard !statements.isEmpty else { continue } + for statement in statements { + let terminated = statement.hasSuffix(";") ? statement : "\(statement);" + try writer.write("\(terminated)\n") + } + } catch { + let sanitized = PluginExportUtilities.sanitizeForSQLComment(principal.name) + ddlFailures.append(sanitized) + Self.logger.warning("Failed to fetch grants for \(sanitized): \(error)") + let warning = "Warning: failed to fetch privileges for \(sanitized): \(error)" + try writer.write("-- \(PluginExportUtilities.sanitizeForSQLComment(warning))\n") + } + } + try writer.write("\n") + } + + private func objectCommentLabel(for kind: PluginExportObjectKind) -> String { + switch kind { + case .view: return "View" + case .materializedView: return "Materialized view" + case .routine: return "Routine" + case .trigger: return "Trigger" + case .event: return "Event" + case .sequence: return "Sequence" + case .userType: return "Type" + case .foreignTable: return "Foreign table" + case .grant: return "Privileges" + default: return "Table" + } + } + private func writeDataPhase( sortedTables: [PluginExportTable], columnsByTable: [String: [PluginColumnInfo]], dataSource: any PluginExportDataSource, - to fileHandle: FileHandle, + to writer: SQLExportFileWriter, progress: PluginExportProgress ) async throws { - for table in sortedTables where optionValue(table, at: 2) && table.tableType != "view" { + for table in sortedTables where optionValue(table, at: 2) && table.kind.carriesRows { try progress.checkCancellation() try await writeTableData( table: table, columnInfo: columnsByTable[node(for: table).identifier] ?? [], dataSource: dataSource, - to: fileHandle, + to: writer, progress: progress) } } @@ -395,7 +559,7 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send fkMap: [String: [PluginForeignKeyInfo]], columnsByTable: [String: [PluginColumnInfo]], dataSource: any PluginExportDataSource, - to fileHandle: FileHandle + to writer: SQLExportFileWriter ) throws { var emittedAnything = false /// A driver that hands back the server's own CREATE statement has already declared these @@ -408,7 +572,7 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send let grouped = groupForeignKeysByConstraint(fks) for group in grouped { let alter = renderAddConstraintFK(table: table, group: group, dataSource: dataSource) - try fileHandle.write(contentsOf: "\(alter)\n".toUTF8Data()) + try writer.write("\(alter)\n") emittedAnything = true } } @@ -418,19 +582,19 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send /// rewound on PostgreSQL. Every other engine reports its identity columns the same way and /// would take the statement as a syntax error. if SqlDialect.from(databaseTypeId: dataSource.databaseTypeId) == .postgres { - for table in sortedTables where optionValue(table, at: 2) && table.tableType != "view" { + for table in sortedTables where optionValue(table, at: 2) && table.kind.carriesRows { let columns = columnsByTable[node(for: table).identifier] ?? [] for column in columns where column.isIdentity { let setval = renderIdentitySetval( table: table, columnName: column.name, dataSource: dataSource) - try fileHandle.write(contentsOf: "\(setval)\n".toUTF8Data()) + try writer.write("\(setval)\n") emittedAnything = true } } } if emittedAnything { - try fileHandle.write(contentsOf: "\n".toUTF8Data()) + try writer.write("\n") } } @@ -519,7 +683,7 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send table: PluginExportTable, columnInfo: [PluginColumnInfo], dataSource: any PluginExportDataSource, - to fileHandle: FileHandle, + to writer: SQLExportFileWriter, progress: PluginExportProgress ) async throws { let batchSize = settings.batchSize @@ -529,6 +693,7 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send var rowBatch: [[PluginCellValue]] = [] let generatedColumnNames = Set(columnInfo.filter { $0.isGenerated }.map { $0.name }) + let primaryKeyColumns = columnInfo.filter(\.isPrimaryKey).map(\.name) let usesOverridingSystemValue = SqlDialect.from(databaseTypeId: dataSource.databaseTypeId) == .postgres && columnInfo.contains { $0.identityKind == .always } let tableRef = qualifiedRef( @@ -539,7 +704,17 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send let needsIdentityInsert = dataSource.databaseTypeId == "SQL Server" && columnInfo.contains(where: \.isIdentity) - let stream = dataSource.streamRows(table: table.name, databaseName: table.databaseName) + if !table.rowScope.isUnrestricted { + let scopeNote = PluginExportUtilities.sanitizeForSQLComment(table.rowScope.summary) + try writer.write("-- Rows narrowed to: \(scopeNote)\n") + } + if table.rowScope.hasRejectedFilter { + metadataWarnings.append(String( + format: String(localized: + "The row filter on %@ was not a single expression, so every row was exported."), + table.name)) + } + let stream = dataSource.streamRows(for: table) for try await element in stream { try progress.checkCancellation() @@ -552,7 +727,7 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send rowBatch.append(row) if rowBatch.count >= batchSize { if needsIdentityInsert, !wroteAnyRows { - try fileHandle.write(contentsOf: "SET IDENTITY_INSERT \(tableRef) ON;\n".toUTF8Data()) + try writer.write("SET IDENTITY_INSERT \(tableRef) ON;\n") } try writeInsertStatements( tableRef: tableRef, @@ -561,9 +736,10 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send rows: rowBatch, batchSize: batchSize, excludedColumnNames: generatedColumnNames, + primaryKeyColumns: primaryKeyColumns, usesOverridingSystemValue: usesOverridingSystemValue, dataSource: dataSource, - to: fileHandle, + to: writer, progress: progress ) wroteAnyRows = true @@ -575,7 +751,7 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send if !rowBatch.isEmpty { if needsIdentityInsert, !wroteAnyRows { - try fileHandle.write(contentsOf: "SET IDENTITY_INSERT \(tableRef) ON;\n".toUTF8Data()) + try writer.write("SET IDENTITY_INSERT \(tableRef) ON;\n") } try writeInsertStatements( tableRef: tableRef, @@ -584,20 +760,21 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send rows: rowBatch, batchSize: batchSize, excludedColumnNames: generatedColumnNames, + primaryKeyColumns: primaryKeyColumns, usesOverridingSystemValue: usesOverridingSystemValue, dataSource: dataSource, - to: fileHandle, + to: writer, progress: progress ) wroteAnyRows = true } if wroteAnyRows, needsIdentityInsert { - try fileHandle.write(contentsOf: "SET IDENTITY_INSERT \(tableRef) OFF;\n".toUTF8Data()) + try writer.write("SET IDENTITY_INSERT \(tableRef) OFF;\n") } if wroteAnyRows { - try fileHandle.write(contentsOf: "\n".toUTF8Data()) + try writer.write("\n") } } @@ -608,9 +785,10 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send rows: [[PluginCellValue]], batchSize: Int, excludedColumnNames: Set, + primaryKeyColumns: [String], usesOverridingSystemValue: Bool, dataSource: any PluginExportDataSource, - to fileHandle: FileHandle, + to writer: SQLExportFileWriter, progress: PluginExportProgress ) throws { let includedColumnIndices = columns.enumerated().compactMap { index, name in @@ -622,7 +800,22 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send .map { dataSource.quoteIdentifier(columns[$0]) } .joined(separator: ", ") let overriding = usesOverridingSystemValue ? " OVERRIDING SYSTEM VALUE" : "" - let insertPrefix = "INSERT INTO \(tableRef) (\(quotedColumns))\(overriding) VALUES\n" + let rendered = SQLExportInsertRenderer( + dialect: SqlDialect.from(databaseTypeId: dataSource.databaseTypeId), + quoteIdentifier: dataSource.quoteIdentifier + ).render( + mode: settings.insertMode, + tableRef: tableRef, + quotedColumns: quotedColumns, + overriding: overriding, + columnNames: includedColumnIndices.map { columns[$0] }, + primaryKeyColumns: primaryKeyColumns + ) + if let warning = rendered.warning, !metadataWarnings.contains(warning) { + metadataWarnings.append(warning) + } + let insertPrefix = rendered.prefix + let insertSuffix = rendered.suffix let numericIndices: Set = Set(includedColumnIndices.filter { idx in idx < columnTypeNames.count && PluginExportUtilities.isNumericColumnType(columnTypeNames[idx]) @@ -656,8 +849,8 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send valuesBatch.append(" (\(values))") if valuesBatch.count >= effectiveBatchSize { - let statement = insertPrefix + valuesBatch.joined(separator: ",\n") + ";\n\n" - try fileHandle.write(contentsOf: statement.toUTF8Data()) + let statement = insertPrefix + valuesBatch.joined(separator: ",\n") + insertSuffix + ";\n\n" + try writer.write(statement) valuesBatch.removeAll(keepingCapacity: true) } @@ -665,8 +858,8 @@ final class SQLExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send } if !valuesBatch.isEmpty { - let statement = insertPrefix + valuesBatch.joined(separator: ",\n") + ";\n\n" - try fileHandle.write(contentsOf: statement.toUTF8Data()) + let statement = insertPrefix + valuesBatch.joined(separator: ",\n") + insertSuffix + ";\n\n" + try writer.write(statement) } } diff --git a/Plugins/SQLExportPlugin/SQLExportSnapshot.swift b/Plugins/SQLExportPlugin/SQLExportSnapshot.swift new file mode 100644 index 0000000000..8fe7abd84b --- /dev/null +++ b/Plugins/SQLExportPlugin/SQLExportSnapshot.swift @@ -0,0 +1,54 @@ +// +// SQLExportSnapshot.swift +// SQLExportPlugin +// + +import Foundation +import os +import TableProPluginKit + +/// Holds one read-consistent view of the database for the length of an export. +/// +/// Without it, a dump of several tables reads each one at a different moment, so a row inserted +/// between two reads appears in the child table and not in its parent. The statement that fixes it +/// is not portable: MySQL takes a modifier on `START TRANSACTION`, PostgreSQL sets the isolation +/// level on `BEGIN`, and SQLite gets the same guarantee from a plain deferred transaction. An +/// engine with no spelling for it opens nothing rather than sending a statement it would reject. +internal struct SQLExportSnapshot { + private static let logger = Logger(subsystem: "com.TablePro", category: "SQLExportSnapshot") + + private let dialect: SqlDialect + + internal init(dialect: SqlDialect) { + self.dialect = dialect + } + + internal var beginStatement: String? { + switch dialect { + case .mysql: return "START TRANSACTION WITH CONSISTENT SNAPSHOT" + case .postgres: return "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY" + case .sqlite: return "BEGIN" + default: return nil + } + } + + internal var endStatement: String? { + beginStatement == nil ? nil : "COMMIT" + } + + internal func begin(on dataSource: any PluginExportDataSource) async throws { + guard let beginStatement else { return } + _ = try await dataSource.execute(query: beginStatement) + } + + /// A failed end is logged rather than thrown: the dump is already written, and turning a + /// cleanup failure into an export failure would throw away a file that is correct. + internal func end(on dataSource: any PluginExportDataSource) async { + guard let endStatement else { return } + do { + _ = try await dataSource.execute(query: endStatement) + } catch { + Self.logger.warning("Failed to close the export snapshot: \(error.localizedDescription)") + } + } +} diff --git a/Plugins/TableProPluginKit/ExportFormatPlugin.swift b/Plugins/TableProPluginKit/ExportFormatPlugin.swift index c67789cac1..9e100646eb 100644 --- a/Plugins/TableProPluginKit/ExportFormatPlugin.swift +++ b/Plugins/TableProPluginKit/ExportFormatPlugin.swift @@ -13,6 +13,19 @@ public protocol ExportFormatPlugin: TableProPlugin, Sendable { func defaultTableOptionValues() -> [Bool] func isTableExportable(optionValues: [Bool]) -> Bool + /// The object kinds this format can write. A format that does not declare a set receives only + /// tables and views, which is every kind that existed when the export contract was written. + static var supportedObjectKinds: [PluginExportObjectKind] { get } + + /// Whether one of `perTableOptionColumns` applies to a kind. The columns stay positionally + /// aligned with `optionValues` for every kind, so a column a kind does not support is a blank + /// slot rather than a shifted one: `Data` on a routine, for example. + static func supportsOption(columnId: String, for kind: PluginExportObjectKind) -> Bool + + /// Whether an object with these option values produces output. Defaults to the table answer, + /// which is what a format written before object scope will keep giving. + func isExportable(optionValues: [Bool], kind: PluginExportObjectKind) -> Bool + var currentFileExtension: String { get } func export( @@ -30,5 +43,10 @@ public extension ExportFormatPlugin { static var perTableOptionColumns: [PluginExportOptionColumn] { [] } func defaultTableOptionValues() -> [Bool] { [] } func isTableExportable(optionValues: [Bool]) -> Bool { true } + static var supportedObjectKinds: [PluginExportObjectKind] { PluginExportObjectKind.legacyDefault } + static func supportsOption(columnId: String, for kind: PluginExportObjectKind) -> Bool { true } + func isExportable(optionValues: [Bool], kind: PluginExportObjectKind) -> Bool { + isTableExportable(optionValues: optionValues) + } var currentFileExtension: String { Self.defaultFileExtension } } diff --git a/Plugins/TableProPluginKit/PluginExportDataSource.swift b/Plugins/TableProPluginKit/PluginExportDataSource.swift index 23dd0fece8..7213fa4271 100644 --- a/Plugins/TableProPluginKit/PluginExportDataSource.swift +++ b/Plugins/TableProPluginKit/PluginExportDataSource.swift @@ -15,6 +15,25 @@ public protocol PluginExportDataSource: AnyObject, Sendable { func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] var tableDDLIncludesForeignKeys: Bool { get } + + /// The CREATE statement for any exportable object, routines, triggers, views and user types + /// included. One method rather than one per kind, because the caller already knows the kind and + /// every driver answers the same question: what would recreate this. + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String + + /// The GRANT statements that recreate one principal's privileges, rendered by the engine's own + /// grant builder. `host` is the MySQL-style host part, which is what separates two principals + /// that share a name. Empty on an engine with no principal management. + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] + + /// The engine's own DROP for an object. Only the driver knows that PostgreSQL's `DROP TRIGGER` + /// takes an `ON
` clause and MySQL's does not, or that MySQL has no `DROP ROUTINE` at + /// all. Nil means the caller should fall back to its own generic shape. + func dropStatement(for object: PluginExportTable) -> String? + + /// The object's rows, narrowed to its `rowScope`. Reading the whole object is what the default + /// does, so a format that has not adopted row scope keeps behaving as it did. + func streamRows(for object: PluginExportTable) -> AsyncThrowingStream } public extension PluginExportDataSource { @@ -29,4 +48,16 @@ public extension PluginExportDataSource { /// `fetchTableDDL` already declares them, so a format that defers foreign keys must not add /// them a second time. var tableDDLIncludesForeignKeys: Bool { false } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + try await fetchTableDDL(table: object.name, databaseName: object.databaseName) + } + + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { [] } + + func dropStatement(for object: PluginExportTable) -> String? { nil } + + func streamRows(for object: PluginExportTable) -> AsyncThrowingStream { + streamRows(table: object.name, databaseName: object.databaseName) + } } diff --git a/Plugins/TableProPluginKit/PluginExportObjectKind.swift b/Plugins/TableProPluginKit/PluginExportObjectKind.swift new file mode 100644 index 0000000000..de98d1fbc9 --- /dev/null +++ b/Plugins/TableProPluginKit/PluginExportObjectKind.swift @@ -0,0 +1,79 @@ +// +// PluginExportObjectKind.swift +// TableProPluginKit +// + +import Foundation + +/// What a selected export item is. The vocabulary grows as engines gain object types, so this is +/// deliberately not `@frozen` and every switch over it needs a `default:`. +public enum PluginExportObjectKind: String, Codable, Sendable, CaseIterable { + case table + case view + case materializedView + case foreignTable + case sequence + case userType + case routine + case trigger + case event + case grant + + /// Where this kind belongs in a dump, so a restore replays it after everything it depends on. + /// Types and sequences precede the tables that reference them, views and routines follow the + /// tables they read, triggers follow the routines they may call, and grants come last because + /// they name objects that must already exist. + public var dumpOrder: Int { + switch self { + case .userType: return 0 + case .sequence: return 1 + case .table: return 2 + case .foreignTable: return 3 + case .view: return 4 + case .materializedView: return 5 + case .routine: return 6 + case .trigger: return 7 + case .event: return 8 + case .grant: return 9 + } + } + + /// Whether the object holds rows an export can stream. Everything else is definition only. + public var carriesRows: Bool { + switch self { + case .table, .foreignTable: return true + default: return false + } + } + + /// The keyword a `DROP` for this kind uses, without the object name. + public var dropKeyword: String { + switch self { + case .table: return "DROP TABLE" + case .view: return "DROP VIEW" + case .materializedView: return "DROP MATERIALIZED VIEW" + case .foreignTable: return "DROP FOREIGN TABLE" + case .sequence: return "DROP SEQUENCE" + case .userType: return "DROP TYPE" + case .routine: return "DROP ROUTINE" + case .trigger: return "DROP TRIGGER" + case .event: return "DROP EVENT" + case .grant: return "" + } + } + + /// The kinds an export format receives when it does not declare its own set. A format written + /// before object scope existed only ever saw tables and views, so that is what it keeps + /// receiving: handing it a routine would run its table code path over a definition. + public static var legacyDefault: [PluginExportObjectKind] { [.table, .view] } + + /// Maps the `tableType` string an export item carries. The spelling comes from engine metadata, + /// so it is matched loosely rather than by equality. + public static func from(tableType: String) -> PluginExportObjectKind { + let normalized = tableType.lowercased() + if normalized.contains("materialized") { return .materializedView } + if normalized.contains("foreign") { return .foreignTable } + if normalized.contains("view") { return .view } + return .table + } +} diff --git a/Plugins/TableProPluginKit/PluginExportRowScope.swift b/Plugins/TableProPluginKit/PluginExportRowScope.swift new file mode 100644 index 0000000000..74a8452cee --- /dev/null +++ b/Plugins/TableProPluginKit/PluginExportRowScope.swift @@ -0,0 +1,65 @@ +// +// PluginExportRowScope.swift +// TableProPluginKit +// + +import Foundation + +/// Which rows and columns of one object an export writes. +/// +/// Empty means the whole object, which is what every export did before this existed and what an +/// export still does unless the user narrows it. +public struct PluginExportRowScope: Sendable, Equatable, Codable { + /// A `WHERE` expression without the keyword, in the engine's own dialect. + public let filter: String + + /// The most rows to write, or nil for all of them. + public let rowLimit: Int? + + /// The columns to write, in the order given, or empty for all of them. + public let columns: [String] + + public init(filter: String = "", rowLimit: Int? = nil, columns: [String] = []) { + self.filter = filter + self.rowLimit = rowLimit + self.columns = columns + } + + public static let unrestricted = PluginExportRowScope() + + public var isUnrestricted: Bool { + sanitizedFilter.isEmpty && rowLimit == nil && columns.isEmpty + } + + /// The filter with its statement terminator removed. + /// + /// The text is the user's own SQL against their own connection, so it is not sanitized in the + /// injection sense. What it must not do is smuggle a second statement into a query the export + /// builds: a trailing `;` alone is a typing habit and is dropped, and a `;` anywhere else means + /// the text is not the single expression this field is for, so it is refused outright rather + /// than being run as two statements. + public var sanitizedFilter: String { + let trimmed = filter.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return "" } + let withoutTerminator = trimmed.hasSuffix(";") + ? String(trimmed.dropLast()).trimmingCharacters(in: .whitespacesAndNewlines) + : trimmed + guard !withoutTerminator.contains(";") else { return "" } + return withoutTerminator + } + + /// Whether the filter carries text the sanitizer refused, so a caller can say so rather than + /// silently exporting every row of a table the user meant to narrow. + public var hasRejectedFilter: Bool { + !filter.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && sanitizedFilter.isEmpty + } + + /// A one-line description for the export tree and the dump's own comments. + public var summary: String { + var parts: [String] = [] + if !columns.isEmpty { parts.append("\(columns.count) columns") } + if !sanitizedFilter.isEmpty { parts.append("WHERE \(sanitizedFilter)") } + if let rowLimit { parts.append("LIMIT \(rowLimit)") } + return parts.joined(separator: ", ") + } +} diff --git a/Plugins/TableProPluginKit/PluginExportTypes.swift b/Plugins/TableProPluginKit/PluginExportTypes.swift index 4ac92995b8..7bcfa3042c 100644 --- a/Plugins/TableProPluginKit/PluginExportTypes.swift +++ b/Plugins/TableProPluginKit/PluginExportTypes.swift @@ -12,6 +12,45 @@ public struct PluginExportTable: Sendable { public let tableType: String public let optionValues: [Bool] + /// What this item is. Defaults to `.table` for every caller that predates object scope, and is + /// derived from `tableType` by the initializers that do not take one. + public let kind: PluginExportObjectKind + + /// Whatever the driver needs to address this exact object again: a routine's oid or argument + /// signature, a trigger's owning table. Opaque here, handed straight back to the driver. + public let identity: String? + + /// The table a trigger fires for. Nil for every other kind. + public let parentTable: String? + + /// Which rows and columns of this object to write. Unrestricted unless the user narrowed it. + public let rowScope: PluginExportRowScope + + public init( + name: String, + databaseName: String, + tableType: String, + optionValues: [Bool] = [], + schema: String?, + kind: PluginExportObjectKind, + identity: String? = nil, + parentTable: String? = nil, + rowScope: PluginExportRowScope = .unrestricted + ) { + self.name = name + self.databaseName = databaseName + self.schema = schema + self.tableType = tableType + self.optionValues = optionValues + self.kind = kind + self.identity = identity + self.parentTable = parentTable + self.rowScope = rowScope + } + + /// Kept at its exact published signature. Adding a parameter to it, even a defaulted one, + /// replaces its mangled symbol and every already-built plugin fails to load. + @_disfavoredOverload public init( name: String, databaseName: String, @@ -24,6 +63,10 @@ public struct PluginExportTable: Sendable { self.schema = schema self.tableType = tableType self.optionValues = optionValues + self.kind = PluginExportObjectKind.from(tableType: tableType) + self.identity = nil + self.parentTable = nil + self.rowScope = .unrestricted } @_disfavoredOverload @@ -33,6 +76,10 @@ public struct PluginExportTable: Sendable { self.schema = nil self.tableType = tableType self.optionValues = optionValues + self.kind = PluginExportObjectKind.from(tableType: tableType) + self.identity = nil + self.parentTable = nil + self.rowScope = .unrestricted } public var qualifiedName: String { diff --git a/TablePro/Core/Database/NativeDumpDescriptor.swift b/TablePro/Core/Database/NativeDumpDescriptor.swift new file mode 100644 index 0000000000..7b7379987d --- /dev/null +++ b/TablePro/Core/Database/NativeDumpDescriptor.swift @@ -0,0 +1,106 @@ +// +// NativeDumpDescriptor.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +/// Everything one engine's own dump and restore tools need, as data. +/// +/// The three engines that had this hard-coded were all PostgreSQL. Generalizing it is not a matter +/// of parameterizing a binary name: `mysqldump` writes SQL to standard output where `pg_dump -Fc` +/// writes an archive to a path, `mongodump` takes a URI rather than host and port flags, and +/// `sqlite3` needs no network arguments at all. So a descriptor supplies its own argument list and +/// says how its output is delivered, rather than filling slots in one shared command shape. +struct NativeDumpDescriptor: Sendable { + /// How the tool hands back what it produced. This is the difference that no shared argument + /// list can paper over. + enum OutputDelivery: Sendable, Equatable { + /// The tool is told where to write, and writes there itself. + case toolWritesFile + /// The tool writes to standard output and the caller redirects it to the file. + case standardOutput + } + + /// What the caller offers as a file name, and what a restore will accept back. + struct ArchiveFormat: Sendable, Equatable { + let fileExtension: String + let contentDescription: String + + init(fileExtension: String, contentDescription: String) { + self.fileExtension = fileExtension + self.contentDescription = contentDescription + } + } + + struct Request: Sendable { + let connection: DatabaseConnection + let database: String + let fileURL: URL + let password: String? + + init(connection: DatabaseConnection, database: String, fileURL: URL, password: String?) { + self.connection = connection + self.database = database + self.fileURL = fileURL + self.password = password + } + + var host: String { + connection.host.isEmpty ? "127.0.0.1" : connection.host + } + } + + /// The candidate names, in the order they are tried. More than one because a tool can ship + /// under two names: MariaDB renamed `mysqldump` to `mariadb-dump` in 11.0 and keeps the old + /// name only as a symlink that some builds omit. + let backupBinaries: [String] + let restoreBinaries: [String] + + /// What to tell the user to install when neither name resolves. There is no portable answer, + /// so each engine names its own package. + let installHint: String + + let archiveFormat: ArchiveFormat + let backupDelivery: OutputDelivery + let restoreDelivery: OutputDelivery + + let backupArguments: @Sendable (Request) -> [String] + let restoreArguments: @Sendable (Request) -> [String] + let environment: @Sendable (Request) -> [String: String] + + init( + backupBinaries: [String], + restoreBinaries: [String], + installHint: String, + archiveFormat: ArchiveFormat, + backupDelivery: OutputDelivery, + restoreDelivery: OutputDelivery, + backupArguments: @escaping @Sendable (Request) -> [String], + restoreArguments: @escaping @Sendable (Request) -> [String], + environment: @escaping @Sendable (Request) -> [String: String] = { _ in [:] } + ) { + self.backupBinaries = backupBinaries + self.restoreBinaries = restoreBinaries + self.installHint = installHint + self.archiveFormat = archiveFormat + self.backupDelivery = backupDelivery + self.restoreDelivery = restoreDelivery + self.backupArguments = backupArguments + self.restoreArguments = restoreArguments + self.environment = environment + } + + func binaries(for kind: NativeDumpKind) -> [String] { + kind == .backup ? backupBinaries : restoreBinaries + } + + func arguments(for kind: NativeDumpKind, request: Request) -> [String] { + kind == .backup ? backupArguments(request) : restoreArguments(request) + } + + func delivery(for kind: NativeDumpKind) -> OutputDelivery { + kind == .backup ? backupDelivery : restoreDelivery + } +} diff --git a/TablePro/Core/Database/NativeDumpRegistry.swift b/TablePro/Core/Database/NativeDumpRegistry.swift new file mode 100644 index 0000000000..3a105325ae --- /dev/null +++ b/TablePro/Core/Database/NativeDumpRegistry.swift @@ -0,0 +1,213 @@ +// +// NativeDumpRegistry.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +/// Which engines have client-side dump and restore tools, and how to drive them. +/// +/// Curated here rather than declared by each driver plugin, because the facts are about tools on +/// the user's Mac rather than about the driver's own behaviour, and adding a `DriverPlugin` static +/// for them would mean an ABI change and a re-release of every registry plugin for data no plugin +/// produces. +/// +/// A password never reaches the argument list. Anything in `argv` is readable by every process on +/// the machine through `ps`, so each engine's own out-of-band channel is used instead: `PGPASSWORD` +/// and `MYSQL_PWD` for the two that read the environment, and a `0600` config file for MongoDB, +/// whose tools read neither. +enum NativeDumpRegistry { + static func descriptor(for type: DatabaseType) -> NativeDumpDescriptor? { + switch type { + case .postgresql, .redshift: + return postgres + case .mysql, .mariadb: + return mysql + case .mongodb: + return mongodb + case .sqlite, .libsql: + return sqlite + default: + return nil + } + } + + static func supports(_ type: DatabaseType) -> Bool { + descriptor(for: type) != nil + } + + // MARK: - PostgreSQL + + private static var postgres: NativeDumpDescriptor { + NativeDumpDescriptor( + backupBinaries: ["pg_dump"], + restoreBinaries: ["pg_restore"], + installHint: String(localized: "Install it with `brew install libpq` and link it."), + archiveFormat: NativeDumpDescriptor.ArchiveFormat( + fileExtension: "dump", + contentDescription: String(localized: "PostgreSQL custom archive") + ), + backupDelivery: .toolWritesFile, + restoreDelivery: .toolWritesFile, + backupArguments: { request in + connectionFlags(request) + ["-Fc", "-d", request.database, "-f", request.fileURL.path] + }, + restoreArguments: { request in + connectionFlags(request) + ["--no-owner", "--no-acl", "-d", request.database, request.fileURL.path] + }, + environment: { request in + var environment: [String: String] = [:] + if let password = request.password, !password.isEmpty { + environment["PGPASSWORD"] = password + } + if request.connection.sslConfig.isEnabled, + let mode = postgresSSLMode(request.connection.sslConfig.mode) { + environment["PGSSLMODE"] = mode + } + return environment + } + ) + } + + private static func connectionFlags(_ request: NativeDumpDescriptor.Request) -> [String] { + var flags = ["--no-password", "-h", request.host, "-p", String(request.connection.port)] + if !request.connection.username.isEmpty { + flags.append(contentsOf: ["-U", request.connection.username]) + } + return flags + } + + static func postgresSSLMode(_ mode: SSLMode) -> String? { + switch mode { + case .disabled: return nil + case .preferred: return "prefer" + case .required: return "require" + case .verifyCa: return "verify-ca" + case .verifyIdentity: return "verify-full" + } + } + + // MARK: - MySQL and MariaDB + + /// MariaDB 11.0 renamed every client, keeping the `mysql`-prefixed names as symlinks that some + /// builds leave out, so both spellings are tried. + private static var mysql: NativeDumpDescriptor { + NativeDumpDescriptor( + backupBinaries: ["mysqldump", "mariadb-dump"], + restoreBinaries: ["mysql", "mariadb"], + installHint: String(localized: "Install it with `brew install mysql-client` and link it."), + archiveFormat: NativeDumpDescriptor.ArchiveFormat( + fileExtension: "sql", + contentDescription: String(localized: "SQL statements") + ), + backupDelivery: .standardOutput, + restoreDelivery: .standardOutput, + backupArguments: { request in + mysqlConnectionFlags(request) + [ + "--single-transaction", + "--routines", + "--triggers", + "--events", + "--default-character-set=utf8mb4", + request.database + ] + }, + restoreArguments: { request in + mysqlConnectionFlags(request) + ["--default-character-set=utf8mb4", request.database] + }, + environment: { request in + guard let password = request.password, !password.isEmpty else { return [:] } + return ["MYSQL_PWD": password] + } + ) + } + + private static func mysqlConnectionFlags(_ request: NativeDumpDescriptor.Request) -> [String] { + var flags = ["--protocol=TCP", "-h", request.host, "-P", String(request.connection.port)] + if !request.connection.username.isEmpty { + flags.append(contentsOf: ["-u", request.connection.username]) + } + if request.connection.sslConfig.isEnabled { + flags.append(mysqlSSLMode(request.connection.sslConfig.mode)) + } else { + flags.append("--ssl-mode=DISABLED") + } + return flags + } + + static func mysqlSSLMode(_ mode: SSLMode) -> String { + switch mode { + case .disabled: return "--ssl-mode=DISABLED" + case .preferred: return "--ssl-mode=PREFERRED" + case .required: return "--ssl-mode=REQUIRED" + case .verifyCa: return "--ssl-mode=VERIFY_CA" + case .verifyIdentity: return "--ssl-mode=VERIFY_IDENTITY" + } + } + + // MARK: - MongoDB + + /// `mongodump` reads no password from the environment, and a password in `argv` is readable by + /// every process on the machine. Its `--config` file is the only channel left, so the caller + /// writes one at mode `0600` and passes its path; `NativeDumpService` removes it afterwards. + private static var mongodb: NativeDumpDescriptor { + NativeDumpDescriptor( + backupBinaries: ["mongodump"], + restoreBinaries: ["mongorestore"], + installHint: String(localized: "Install it with `brew install mongodb-database-tools`."), + archiveFormat: NativeDumpDescriptor.ArchiveFormat( + fileExtension: "archive", + contentDescription: String(localized: "MongoDB gzipped archive") + ), + backupDelivery: .toolWritesFile, + restoreDelivery: .toolWritesFile, + backupArguments: { request in + mongoConnectionFlags(request) + [ + "--db=\(request.database)", + "--gzip", + "--archive=\(request.fileURL.path)" + ] + }, + restoreArguments: { request in + mongoConnectionFlags(request) + [ + "--nsInclude=\(request.database).*", + "--gzip", + "--archive=\(request.fileURL.path)" + ] + } + ) + } + + private static func mongoConnectionFlags(_ request: NativeDumpDescriptor.Request) -> [String] { + var flags = ["--host=\(request.host)", "--port=\(request.connection.port)"] + if !request.connection.username.isEmpty { + flags.append("--username=\(request.connection.username)") + flags.append("--authenticationDatabase=\(request.connection.database.isEmpty ? "admin" : request.connection.database)") + } + if request.connection.sslConfig.isEnabled { + flags.append("--ssl") + } + return flags + } + + // MARK: - SQLite + + /// The database is a file the tool opens directly, so there is no host, port or password, and + /// `.dump` writes SQL to standard output. + private static var sqlite: NativeDumpDescriptor { + NativeDumpDescriptor( + backupBinaries: ["sqlite3"], + restoreBinaries: ["sqlite3"], + installHint: String(localized: "Install it with `brew install sqlite` and link it."), + archiveFormat: NativeDumpDescriptor.ArchiveFormat( + fileExtension: "sql", + contentDescription: String(localized: "SQL statements") + ), + backupDelivery: .standardOutput, + restoreDelivery: .standardOutput, + backupArguments: { request in [request.connection.database, ".dump"] }, + restoreArguments: { request in [request.connection.database] } + ) + } +} diff --git a/TablePro/Core/Database/PostgresDumpService.swift b/TablePro/Core/Database/NativeDumpService.swift similarity index 54% rename from TablePro/Core/Database/PostgresDumpService.swift rename to TablePro/Core/Database/NativeDumpService.swift index d751560e3d..e0c13bd4d8 100644 --- a/TablePro/Core/Database/PostgresDumpService.swift +++ b/TablePro/Core/Database/NativeDumpService.swift @@ -1,5 +1,5 @@ // -// PostgresDumpService.swift +// NativeDumpService.swift // TablePro // // Consolidated backup + restore state machine for PostgreSQL connections. @@ -15,13 +15,13 @@ import TableProPluginKit // MARK: - Public Types /// What the service is doing: dump (back up) a database or restore a dump file. -enum PostgresDumpKind: Equatable, Sendable { +enum NativeDumpKind: Equatable, Sendable { case backup case restore } /// Observable state of a backup or restore. -enum PostgresDumpState: Equatable { +enum NativeDumpState: Equatable { case idle case running(database: String, fileURL: URL, bytesProcessed: Int64, totalBytes: Int64?) case cancelling @@ -30,8 +30,8 @@ enum PostgresDumpState: Equatable { case cancelled } -enum PostgresDumpError: LocalizedError, Equatable { - case binaryNotFound(name: String) +enum NativeDumpError: LocalizedError, Equatable { + case binaryNotFound(name: String, installHint: String) case unsupportedDatabase case noSession case alreadyRunning @@ -39,13 +39,14 @@ enum PostgresDumpError: LocalizedError, Equatable { var errorDescription: String? { switch self { - case .binaryNotFound(let name): + case .binaryNotFound(let name, let installHint): return String( - format: String(localized: "%@ was not found on this system. Install it with `brew install libpq` and link it."), - name + format: String(localized: "%1$@ was not found on this system. %2$@"), + name, + installHint ) case .unsupportedDatabase: - return String(localized: "Dump operations are only supported for PostgreSQL and Redshift connections.") + return String(localized: "This database type has no command line dump tool TablePro can drive.") case .noSession: return String(localized: "Connect to the database before starting this operation.") case .alreadyRunning: @@ -57,15 +58,51 @@ enum PostgresDumpError: LocalizedError, Equatable { } /// Parameters for a single backup or restore command. -struct PostgresDumpCommand: Equatable { +struct NativeDumpCommand: Equatable { let executable: URL let arguments: [String] let environment: [String: String] let stderrByteCap: Int + + /// Where the tool's output goes. `pg_dump -Fc` is told a path and writes it itself, while + /// `mysqldump` and `sqlite3 .dump` write SQL to standard output for the caller to redirect. + let delivery: NativeDumpDescriptor.OutputDelivery + + /// The file standard output is redirected to, or read from on a restore. Nil when the tool + /// handles the file itself. + let redirectedFileURL: URL? + + /// A credentials file written for tools that read a password from neither the environment nor + /// standard input. Removed once the process exits, however it exits. + let temporaryCredentialsFileURL: URL? + + /// Which direction a redirected file goes: a restore feeds it in as standard input, a backup + /// takes standard output and writes it out. + let isRestore: Bool + + init( + executable: URL, + arguments: [String], + environment: [String: String], + stderrByteCap: Int, + delivery: NativeDumpDescriptor.OutputDelivery = .toolWritesFile, + redirectedFileURL: URL? = nil, + temporaryCredentialsFileURL: URL? = nil, + isRestore: Bool = false + ) { + self.executable = executable + self.arguments = arguments + self.environment = environment + self.stderrByteCap = stderrByteCap + self.delivery = delivery + self.redirectedFileURL = redirectedFileURL + self.temporaryCredentialsFileURL = temporaryCredentialsFileURL + self.isRestore = isRestore + } } /// Captured terminal state of a finished/cancelled subprocess. -struct PostgresDumpRunResult: Equatable { +struct NativeDumpRunResult: Equatable { let exitCode: Int32 let stderr: String let wasCancelled: Bool @@ -73,33 +110,33 @@ struct PostgresDumpRunResult: Equatable { /// Spawns and supervises a single subprocess. Abstracted so the dump /// state machine can be tested without launching real processes. -protocol PostgresDumpRunner: AnyObject { +protocol NativeDumpRunner: AnyObject { /// Launches the command. Throws synchronously if the binary can't be spawned. /// `result` returns the final outcome when the process exits. - func start(_ command: PostgresDumpCommand) throws + func start(_ command: NativeDumpCommand) throws /// Sends SIGTERM. Safe to call multiple times. func cancel() /// Resolves once the process has terminated (normally or via cancel). - var result: PostgresDumpRunResult { get async } + var result: NativeDumpRunResult { get async } } // MARK: - Service @MainActor @Observable -final class PostgresDumpService { - nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "PostgresDumpService") +final class NativeDumpService { + nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "NativeDumpService") - let kind: PostgresDumpKind - private(set) var state: PostgresDumpState = .idle + let kind: NativeDumpKind + private(set) var state: NativeDumpState = .idle - @ObservationIgnored private let runnerFactory: () -> any PostgresDumpRunner - @ObservationIgnored private var runner: (any PostgresDumpRunner)? + @ObservationIgnored private let runnerFactory: () -> any NativeDumpRunner + @ObservationIgnored private var runner: (any NativeDumpRunner)? @ObservationIgnored private var byteSizeTask: Task? - @ObservationIgnored private var stateObservers: [UUID: AsyncStream.Continuation] = [:] + @ObservationIgnored private var stateObservers: [UUID: AsyncStream.Continuation] = [:] - func stateUpdates() -> AsyncStream { - let (stream, continuation) = AsyncStream.makeStream() + func stateUpdates() -> AsyncStream { + let (stream, continuation) = AsyncStream.makeStream() let id = UUID() stateObservers[id] = continuation continuation.yield(state) @@ -111,7 +148,7 @@ final class PostgresDumpService { return stream } - private func setState(_ newState: PostgresDumpState) { + private func setState(_ newState: NativeDumpState) { state = newState for continuation in stateObservers.values { continuation.yield(newState) @@ -119,13 +156,13 @@ final class PostgresDumpService { } /// Default initializer uses the real `Process`-backed runner. - init(kind: PostgresDumpKind) { + init(kind: NativeDumpKind) { self.kind = kind - self.runnerFactory = { ProcessPostgresDumpRunner() } + self.runnerFactory = { ProcessNativeDumpRunner() } } /// Test-friendly initializer that injects a custom runner factory. - init(kind: PostgresDumpKind, runnerFactory: @escaping () -> any PostgresDumpRunner) { + init(kind: NativeDumpKind, runnerFactory: @escaping () -> any NativeDumpRunner) { self.kind = kind self.runnerFactory = runnerFactory } @@ -144,38 +181,40 @@ final class PostgresDumpService { fileURL: URL, totalBytesEstimate: Int64? = nil ) async throws { - if case .running = state { throw PostgresDumpError.alreadyRunning } - if case .cancelling = state { throw PostgresDumpError.alreadyRunning } + if case .running = state { throw NativeDumpError.alreadyRunning } + if case .cancelling = state { throw NativeDumpError.alreadyRunning } - guard connection.type == .postgresql || connection.type == .redshift else { - throw PostgresDumpError.unsupportedDatabase + guard let descriptor = NativeDumpRegistry.descriptor(for: connection.type) else { + throw NativeDumpError.unsupportedDatabase } let session = DatabaseManager.shared.session(for: connection.id) - guard session?.isConnected == true else { throw PostgresDumpError.noSession } + guard session?.isConnected == true else { throw NativeDumpError.noSession } if kind == .restore { guard FileManager.default.isReadableFile(atPath: fileURL.path) else { - throw PostgresDumpError.sourceUnreadable + throw NativeDumpError.sourceUnreadable } } let effective = session?.effectiveConnection ?? connection let password = ConnectionStorage.shared.loadPassword(for: connection.id) ?? session?.cachedPassword - let binaryName: String - switch kind { - case .backup: - binaryName = "pg_dump" - case .restore: - binaryName = "pg_restore" - } - guard let resolvedPath = CLIExecutableFinder.findExecutable(binaryName) else { - throw PostgresDumpError.binaryNotFound(name: binaryName) + let candidates = descriptor.binaries(for: kind) + guard let resolved = candidates.lazy.compactMap({ name -> (String, String)? in + guard let path = CLIExecutableFinder.findExecutable(name) else { return nil } + return (name, path) + }).first else { + throw NativeDumpError.binaryNotFound( + name: candidates.joined(separator: String(localized: " or ")), + installHint: descriptor.installHint + ) } + let (binaryName, resolvedPath) = resolved - let command = Self.buildCommand( + let command = try Self.buildCommand( kind: kind, + descriptor: descriptor, executable: URL(fileURLWithPath: resolvedPath), effective: effective, database: database, @@ -195,13 +234,13 @@ final class PostgresDumpService { /// Test-friendly entry: spawns the given pre-built command via the runner /// and wires up termination/progress state. Skips dependency resolution. func run( - command: PostgresDumpCommand, + command: NativeDumpCommand, database: String, fileURL: URL, totalBytesEstimate: Int64? = nil ) throws { - if case .running = state { throw PostgresDumpError.alreadyRunning } - if case .cancelling = state { throw PostgresDumpError.alreadyRunning } + if case .running = state { throw NativeDumpError.alreadyRunning } + if case .cancelling = state { throw NativeDumpError.alreadyRunning } let runner = runnerFactory() try runner.start(command) @@ -226,47 +265,76 @@ final class PostgresDumpService { // MARK: - Command Construction + /// The descriptor supplies the arguments and the environment; this adds the parts every tool + /// shares. A password never joins the argument list, because `argv` is world readable through + /// `ps`: it goes in the environment where the tool reads one, and in a `0600` file where it + /// does not. nonisolated static func buildCommand( - kind: PostgresDumpKind, + kind: NativeDumpKind, + descriptor: NativeDumpDescriptor, executable: URL, effective: DatabaseConnection, database: String, fileURL: URL, password: String? - ) -> PostgresDumpCommand { - var args: [String] = ["--no-password"] - args.append(contentsOf: ["-h", effective.host.isEmpty ? "127.0.0.1" : effective.host]) - args.append(contentsOf: ["-p", String(effective.port)]) - if !effective.username.isEmpty { - args.append(contentsOf: ["-U", effective.username]) - } - switch kind { - case .backup: - args.append("-Fc") - args.append(contentsOf: ["-d", database]) - args.append(contentsOf: ["-f", fileURL.path]) - case .restore: - args.append("--no-owner") - args.append("--no-acl") - args.append(contentsOf: ["-d", database]) - args.append(fileURL.path) + ) throws -> NativeDumpCommand { + let request = NativeDumpDescriptor.Request( + connection: effective, + database: database, + fileURL: fileURL, + password: password + ) + var arguments = descriptor.arguments(for: kind, request: request) + var environment = minimalEnvironment() + environment.merge(descriptor.environment(request)) { _, new in new } + + var credentialsFileURL: URL? + if descriptor.environment(request).isEmpty, + let password, !password.isEmpty, + !effective.username.isEmpty, + descriptor.backupBinaries.contains("mongodump") { + let file = try writeMongoCredentialsFile(password: password) + credentialsFileURL = file + arguments.append("--config=\(file.path)") } - var env = minimalEnvironment() - if let password, !password.isEmpty { - env["PGPASSWORD"] = password - } - if effective.sslConfig.isEnabled, let sslMode = pgSSLMode(effective.sslConfig.mode) { - env["PGSSLMODE"] = sslMode - } - return PostgresDumpCommand( + let delivery = descriptor.delivery(for: kind) + return NativeDumpCommand( executable: executable, - arguments: args, - environment: env, - stderrByteCap: 64_000 + arguments: arguments, + environment: environment, + stderrByteCap: 64_000, + delivery: delivery, + redirectedFileURL: delivery == .standardOutput ? fileURL : nil, + temporaryCredentialsFileURL: credentialsFileURL, + isRestore: kind == .restore ) } + /// `mongodump` and `mongorestore` read a password from neither the environment nor standard + /// input, and one in `argv` is readable by every process on the machine. Their `--config` file + /// is the remaining channel, so it is written owner-only and removed when the process exits. + nonisolated static func writeMongoCredentialsFile(password: String) throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-mongo-\(UUID().uuidString).yaml") + let contents = "password: \(mongoYAMLQuoted(password))\n" + guard let data = contents.data(using: .utf8) else { + throw NativeDumpError.sourceUnreadable + } + try data.write(to: url, options: .atomic) + try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: url.path) + return url + } + + /// A double-quoted YAML scalar, which is the one form that carries every character a password + /// can hold without the value changing meaning. + nonisolated static func mongoYAMLQuoted(_ value: String) -> String { + let escaped = value + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") + return "\"\(escaped)\"" + } + nonisolated private static let inheritedEnvironmentKeys: [String] = [ "PATH", "HOME", "USER", "LOGNAME", "SHELL", "TMPDIR", "LANG", "LC_ALL" ] @@ -293,7 +361,7 @@ final class PostgresDumpService { // MARK: - Termination + Progress private func handleTermination( - result: PostgresDumpRunResult, + result: NativeDumpRunResult, database: String, fileURL: URL ) { @@ -361,23 +429,27 @@ private extension String { // MARK: - Real Process Runner -final class ProcessPostgresDumpRunner: PostgresDumpRunner, @unchecked Sendable { +final class ProcessNativeDumpRunner: NativeDumpRunner, @unchecked Sendable { private let process = Process() private let stderrPipe = Pipe() private let stateLock = NSLock() private var stderrBuffer = Data() private var wasCancelled = false - private var terminationResult: PostgresDumpRunResult? - private var continuation: CheckedContinuation? + private var terminationResult: NativeDumpRunResult? + private var continuation: CheckedContinuation? + private var redirectedHandle: FileHandle? + private var credentialsFileURL: URL? - func start(_ command: PostgresDumpCommand) throws { + func start(_ command: NativeDumpCommand) throws { let stderrCap = command.stderrByteCap process.executableURL = command.executable process.arguments = command.arguments process.environment = command.environment process.standardError = stderrPipe - process.standardOutput = FileHandle.nullDevice + credentialsFileURL = command.temporaryCredentialsFileURL + + try attachRedirection(for: command) stderrPipe.fileHandleForReading.readabilityHandler = { [weak self] handle in let chunk = handle.availableData @@ -393,11 +465,12 @@ final class ProcessPostgresDumpRunner: PostgresDumpRunner, @unchecked Sendable { process.terminationHandler = { [weak self] proc in guard let self else { return } self.stderrPipe.fileHandleForReading.readabilityHandler = nil + self.releaseRedirection() self.stateLock.lock() let stderrText = String(data: self.stderrBuffer, encoding: .utf8)? .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" - let result = PostgresDumpRunResult( + let result = NativeDumpRunResult( exitCode: proc.terminationStatus, stderr: stderrText, wasCancelled: self.wasCancelled @@ -422,7 +495,46 @@ final class ProcessPostgresDumpRunner: PostgresDumpRunner, @unchecked Sendable { } } - var result: PostgresDumpRunResult { + /// A tool that writes to standard output gets the destination file as its stdout, and a restore + /// that reads from standard input gets the dump file as its stdin. The one that manages its own + /// file gets the null device, which is what keeps a chatty tool from filling a pipe nobody + /// drains and deadlocking on write. + private func attachRedirection(for command: NativeDumpCommand) throws { + guard command.delivery == .standardOutput, let fileURL = command.redirectedFileURL else { + process.standardOutput = FileHandle.nullDevice + return + } + switch command.isRestore { + case true: + guard FileManager.default.isReadableFile(atPath: fileURL.path) else { + throw NativeDumpError.sourceUnreadable + } + let handle = try FileHandle(forReadingFrom: fileURL) + redirectedHandle = handle + process.standardInput = handle + process.standardOutput = FileHandle.nullDevice + case false: + guard FileManager.default.createFile(atPath: fileURL.path, contents: nil) else { + throw NativeDumpError.sourceUnreadable + } + let handle = try FileHandle(forWritingTo: fileURL) + redirectedHandle = handle + process.standardOutput = handle + } + } + + /// Runs however the process ended, including a cancel, so a credentials file never outlives the + /// process that needed it. + private func releaseRedirection() { + try? redirectedHandle?.close() + redirectedHandle = nil + if let credentialsFileURL { + try? FileManager.default.removeItem(at: credentialsFileURL) + } + credentialsFileURL = nil + } + + var result: NativeDumpRunResult { get async { await withCheckedContinuation { continuation in stateLock.lock() @@ -440,7 +552,7 @@ final class ProcessPostgresDumpRunner: PostgresDumpRunner, @unchecked Sendable { // MARK: - Database Size Helper -extension PostgresDumpService { +extension NativeDumpService { /// Best-effort estimate of the database's on-disk size. Used as an upper /// bound for the backup progress bar; the dump file is typically much /// smaller because of compression, so the bar tops out at the size and @@ -450,16 +562,30 @@ extension PostgresDumpService { connection: DatabaseConnection, database: String ) async -> Int64? { + guard let query = sizeQuery(for: connection.type) else { return nil } guard let driver = DatabaseManager.shared.driver(for: connection.id) else { return nil } do { - let result = try await driver.executeParameterized( - query: "SELECT pg_database_size($1)", - parameters: [database] - ) + let result = try await driver.executeParameterized(query: query, parameters: [database]) guard let text = result.rows.first?.first?.asText else { return nil } return Int64(text) } catch { return nil } } + + /// An engine with no cheap size answer returns nil, which leaves the progress bar + /// indeterminate rather than showing a percentage of a number nobody measured. + nonisolated static func sizeQuery(for type: DatabaseType) -> String? { + switch type { + case .postgresql, .redshift: + return "SELECT pg_database_size($1)" + case .mysql, .mariadb: + return """ + SELECT COALESCE(SUM(data_length + index_length), 0) + FROM information_schema.TABLES WHERE table_schema = ? + """ + default: + return nil + } + } } diff --git a/TablePro/Core/Plugins/ExportDataSourceAdapter.swift b/TablePro/Core/Plugins/ExportDataSourceAdapter.swift index dc7ea9e290..6f12daef2b 100644 --- a/TablePro/Core/Plugins/ExportDataSourceAdapter.swift +++ b/TablePro/Core/Plugins/ExportDataSourceAdapter.swift @@ -11,6 +11,7 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable let databaseTypeId: String private let driver: DatabaseDriver private let dbType: DatabaseType + private let objectCache = ExportObjectCache() private static let logger = Logger(subsystem: "com.TablePro", category: "ExportDataSourceAdapter") @@ -37,6 +38,31 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable return pluginDriver.streamRows(query: query) } + /// The row limit goes through the driver's own `injectRowLimit`, because `LIMIT` is not the + /// spelling on SQL Server or on Oracle before 12c. + func streamRows(for object: PluginExportTable) -> AsyncThrowingStream { + let scope = object.rowScope + guard !scope.isUnrestricted else { + return streamRows(table: object.name, databaseName: object.databaseName) + } + guard let pluginDriver else { + return AsyncThrowingStream { $0.finish(throwing: PluginExportError.exportFailed("No plugin driver available")) } + } + let projection = scope.columns.isEmpty + ? "*" + : scope.columns.map { driver.quoteIdentifier($0) }.joined(separator: ", ") + let reference = qualifiedTableRef(table: object.name, databaseName: object.databaseName) + var query = "SELECT \(projection) FROM \(reference)" + let filter = scope.sanitizedFilter + if !filter.isEmpty { + query += " WHERE \(filter)" + } + if let rowLimit = scope.rowLimit { + query = pluginDriver.injectRowLimit(query, limit: rowLimit) ?? "\(query) LIMIT \(rowLimit)" + } + return pluginDriver.streamRows(query: query) + } + func fetchTableDDL(table: String, databaseName: String) async throws -> String { guard let pluginDriver else { return try await driver.fetchTableDDL(table: table) @@ -117,6 +143,113 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable pluginDriver?.tableDDLIncludesForeignKeys ?? false } + // MARK: - Object DDL + + /// A driver addresses a routine, trigger or type through the info object it handed out, which + /// carries an opaque identity the export item cannot reproduce. So the list is fetched once per + /// database and the item is matched back onto its own info object rather than a rebuilt one. + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + guard let pluginDriver else { + return try await driver.fetchTableDDL(table: object.name) + } + let schema = exportSchema(for: object.databaseName) + switch object.kind { + case .table, .foreignTable: + return try await pluginDriver.fetchTableDDL(table: object.name, schema: schema) + case .view: + return try await pluginDriver.fetchViewDefinition(view: object.name, schema: schema) + case .materializedView: + /// PostgreSQL answers `fetchViewDefinition` out of `pg_views`, which excludes + /// materialized views, so the engines that do not distinguish them fall back rather + /// than failing the object. + do { + return try await pluginDriver.fetchViewDefinition(view: object.name, schema: schema) + } catch { + return try await pluginDriver.fetchTableDDL(table: object.name, schema: schema) + } + case .routine: + guard let routine = try await cachedRoutines(schema: schema, databaseName: object.databaseName) + .first(where: { $0.name == object.name && ($0.argumentSignature ?? "") == (object.identity ?? "") }) + else { + throw PluginObjectSourceError.unsupported(object.name) + } + return try await pluginDriver.fetchRoutineDDL(routine) + case .trigger: + guard let trigger = try await cachedTriggers(schema: schema, databaseName: object.databaseName) + .first(where: { $0.name == object.name && $0.table == object.parentTable }) + else { + throw PluginObjectSourceError.unsupported(object.name) + } + return try await pluginDriver.fetchTriggerDDL(trigger) + case .userType: + guard let type = try await cachedUserTypes(schema: schema, databaseName: object.databaseName) + .first(where: { $0.name == object.name }) + else { + throw PluginObjectSourceError.unsupported(object.name) + } + let resolved = try await pluginDriver.fetchUserDefinedType(type) + guard let definition = resolved.definition, !definition.isEmpty else { + throw PluginObjectSourceError.unsupported(object.name) + } + return definition + default: + throw PluginObjectSourceError.unsupported(object.name) + } + } + + /// The engine renders its own GRANT text, because only the driver knows how it spells a + /// grantee and a privilege target. `principal` is the name and `host` the MySQL-style host + /// part, which is what separates two principals that share a name. + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { + guard let management = pluginDriver as? any PluginPrincipalManagement else { return [] } + let ref = PluginPrincipalRef(name: principal, host: host) + let grants = try await management.fetchGrants(for: ref) + guard !grants.isEmpty else { return [] } + return management.generateGrantSQL( + changeSet: PluginPrincipalChangeSet(principal: ref, grantsToAdd: grants) + ) ?? [] + } + + /// `tableType` carries the routine's own kind for a `.routine`, because `DROP FUNCTION` and + /// `DROP PROCEDURE` are different statements on every engine that has both and MySQL has no + /// `DROP ROUTINE` to fall back on. + func dropStatement(for object: PluginExportTable) -> String? { + guard let pluginDriver else { return nil } + let schema = exportSchema(for: object.databaseName) + switch object.kind { + case .trigger: + guard let parent = object.parentTable else { return nil } + return pluginDriver.generateDropTriggerSQL(name: object.name, table: parent, schema: schema) + case .routine: + return pluginDriver.generateDropRoutineSQL( + name: object.name, + signature: object.identity, + schema: schema, + isFunction: object.tableType.lowercased() != "procedure" + ) + default: + return nil + } + } + + private func cachedRoutines(schema: String?, databaseName: String) async throws -> [PluginRoutineInfo] { + try await objectCache.routines(forDatabase: databaseName) { [pluginDriver] in + try await pluginDriver?.fetchRoutines(schema: schema) ?? [] + } + } + + private func cachedTriggers(schema: String?, databaseName: String) async throws -> [PluginTriggerInfo] { + try await objectCache.triggers(forDatabase: databaseName) { [pluginDriver] in + try await pluginDriver?.fetchAllTriggers(schema: schema) ?? [] + } + } + + private func cachedUserTypes(schema: String?, databaseName: String) async throws -> [PluginUserDefinedTypeInfo] { + try await objectCache.userTypes(forDatabase: databaseName) { [pluginDriver] in + try await pluginDriver?.fetchUserDefinedTypes(schema: schema) ?? [] + } + } + // MARK: - Helpers /// The export tree names every group after a schema on a schema-aware engine and after a diff --git a/TablePro/Core/Plugins/ExportObjectCache.swift b/TablePro/Core/Plugins/ExportObjectCache.swift new file mode 100644 index 0000000000..72b76edfa0 --- /dev/null +++ b/TablePro/Core/Plugins/ExportObjectCache.swift @@ -0,0 +1,49 @@ +// +// ExportObjectCache.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +/// One database's routine, trigger and type lists, held for the life of a single export. +/// +/// A driver addresses a routine, trigger or type through the info object it handed out, and that +/// object carries an opaque identity an export item cannot reproduce. Every DDL read therefore has +/// to match its item back onto the driver's own object, and without this each read would re-list +/// the whole database. +actor ExportObjectCache { + private var routinesByDatabase: [String: [PluginRoutineInfo]] = [:] + private var triggersByDatabase: [String: [PluginTriggerInfo]] = [:] + private var userTypesByDatabase: [String: [PluginUserDefinedTypeInfo]] = [:] + + func routines( + forDatabase database: String, + load: () async throws -> [PluginRoutineInfo] + ) async throws -> [PluginRoutineInfo] { + if let cached = routinesByDatabase[database] { return cached } + let loaded = try await load() + routinesByDatabase[database] = loaded + return loaded + } + + func triggers( + forDatabase database: String, + load: () async throws -> [PluginTriggerInfo] + ) async throws -> [PluginTriggerInfo] { + if let cached = triggersByDatabase[database] { return cached } + let loaded = try await load() + triggersByDatabase[database] = loaded + return loaded + } + + func userTypes( + forDatabase database: String, + load: () async throws -> [PluginUserDefinedTypeInfo] + ) async throws -> [PluginUserDefinedTypeInfo] { + if let cached = userTypesByDatabase[database] { return cached } + let loaded = try await load() + userTypesByDatabase[database] = loaded + return loaded + } +} diff --git a/TablePro/Core/Services/Export/ExportObjectLoader.swift b/TablePro/Core/Services/Export/ExportObjectLoader.swift new file mode 100644 index 0000000000..d8e459e674 --- /dev/null +++ b/TablePro/Core/Services/Export/ExportObjectLoader.swift @@ -0,0 +1,146 @@ +// +// ExportObjectLoader.swift +// TablePro +// + +import Foundation +import os +import TableProPluginKit + +/// Everything one container of the export tree can offer, read from the driver. +/// +/// The export dialog asks per container, and only for the kinds the chosen format says it can +/// write, so picking CSV never pays for a routine list nobody will export. A kind the driver +/// cannot answer comes back empty rather than failing the load: one engine without routines must +/// not cost the user their table list. +internal enum ExportObjectLoader { + private static let logger = Logger(subsystem: "com.TablePro", category: "ExportObjectLoader") + + /// The kinds this loader knows how to read. `.sequence` and `.event` are absent because no + /// driver publishes a standalone list of either yet; sequences still reach a dump through + /// `fetchDependentSequences` on the tables that own them. + internal static let loadableKinds: Set = [ + .table, .view, .materializedView, .foreignTable, .routine, .trigger, .userType, .grant + ] + + internal struct Request: Sendable { + internal let containerName: String + internal let schema: String? + internal let kinds: Set + + internal init(containerName: String, schema: String?, kinds: Set) { + self.containerName = containerName + self.schema = schema + self.kinds = kinds + } + } + + internal static func loadObjects( + request: Request, + tables: [TableInfo], + driver: DatabaseDriver + ) async -> [ExportObjectItem] { + var items = tables.compactMap { table -> ExportObjectItem? in + let kind = PluginExportObjectKind.from(tableType: table.type.rawValue) + guard request.kinds.contains(kind) else { return nil } + return ExportObjectItem(name: table.name, databaseName: request.containerName, kind: kind) + } + + guard let pluginDriver = (driver as? PluginDriverAdapter)?.schemaPluginDriver else { return items } + + if request.kinds.contains(.routine) { + items += await loadRoutines(schema: request.schema, container: request.containerName, driver: pluginDriver) + } + if request.kinds.contains(.trigger) { + items += await loadTriggers(schema: request.schema, container: request.containerName, driver: pluginDriver) + } + if request.kinds.contains(.userType) { + items += await loadUserTypes(schema: request.schema, container: request.containerName, driver: pluginDriver) + } + if request.kinds.contains(.grant) { + items += await loadPrincipals(container: request.containerName, driver: pluginDriver) + } + return items + } + + private static func loadRoutines( + schema: String?, + container: String, + driver: any PluginDatabaseDriver + ) async -> [ExportObjectItem] { + do { + return try await driver.fetchRoutines(schema: schema).map { routine in + ExportObjectItem( + name: routine.name, + databaseName: container, + kind: .routine, + identity: routine.argumentSignature + ) + } + } catch { + logger.warning("Failed to list routines for export: \(error.localizedDescription)") + return [] + } + } + + /// Only a driver that answers a schema-wide trigger list is asked. Falling back to a read per + /// table would be one round trip per table just to fill a tree the user may never open. + private static func loadTriggers( + schema: String?, + container: String, + driver: any PluginDatabaseDriver + ) async -> [ExportObjectItem] { + guard driver.providesBulkTriggerFetch else { return [] } + do { + return try await driver.fetchAllTriggers(schema: schema).map { trigger in + ExportObjectItem( + name: trigger.name, + databaseName: container, + kind: .trigger, + parentTable: trigger.table + ) + } + } catch { + logger.warning("Failed to list triggers for export: \(error.localizedDescription)") + return [] + } + } + + private static func loadUserTypes( + schema: String?, + container: String, + driver: any PluginDatabaseDriver + ) async -> [ExportObjectItem] { + do { + return try await driver.fetchUserDefinedTypes(schema: schema).map { type in + ExportObjectItem(name: type.name, databaseName: container, kind: .userType) + } + } catch { + logger.warning("Failed to list user-defined types for export: \(error.localizedDescription)") + return [] + } + } + + /// Principals are server-wide rather than per container, so they are listed only under the + /// container the dialog opened on. Listing them under every schema would offer the same GRANT + /// statements several times over. + private static func loadPrincipals( + container: String, + driver: any PluginDatabaseDriver + ) async -> [ExportObjectItem] { + guard let management = driver as? any PluginPrincipalManagement else { return [] } + do { + return try await management.fetchPrincipals().map { principal in + ExportObjectItem( + name: principal.ref.name, + databaseName: container, + kind: .grant, + identity: principal.ref.host + ) + } + } catch { + logger.warning("Failed to list principals for export: \(error.localizedDescription)") + return [] + } + } +} diff --git a/TablePro/Core/Services/Export/ExportService.swift b/TablePro/Core/Services/Export/ExportService.swift index d9d63b71b3..9ab8ee0ebf 100644 --- a/TablePro/Core/Services/Export/ExportService.swift +++ b/TablePro/Core/Services/Export/ExportService.swift @@ -89,11 +89,11 @@ final class ExportService { // MARK: - Public API func export( - tables: [ExportTableItem], + objects: [ExportObjectItem], config: ExportConfiguration, to url: URL ) async throws { - guard !tables.isEmpty else { + guard !objects.isEmpty else { throw ExportError.noTablesSelected } @@ -101,7 +101,7 @@ final class ExportService { throw ExportError.formatNotFound(config.formatId) } - state = ExportState(isExporting: true, totalTables: tables.count) + state = ExportState(isExporting: true, totalTables: objects.count) isCancelled = false defer { @@ -115,7 +115,8 @@ final class ExportService { throw ExportError.notConnected } - state.totalRows = await fetchTotalRowCount(for: tables, driver: driver) + state.totalRows = await fetchTotalRowCount( + for: objects.filter { $0.kind.carriesRows }, driver: driver) let dataSource = ExportDataSourceAdapter(driver: driver, databaseType: databaseType) @@ -143,13 +144,17 @@ final class ExportService { } defer { descObservation.invalidate() } - let pluginTables = tables.map { table in + let pluginTables = objects.map { object in PluginExportTable( - name: table.name, - databaseName: table.databaseName, - tableType: table.type.rawValue.lowercased(), - optionValues: table.optionValues, - schema: dataSource.exportSchema(for: table.databaseName) + name: object.name, + databaseName: object.databaseName, + tableType: object.kind.rawValue, + optionValues: object.optionValues, + schema: dataSource.exportSchema(for: object.databaseName), + kind: object.kind, + identity: object.identity, + parentTable: object.parentTable, + rowScope: object.rowScope ) } @@ -249,7 +254,8 @@ final class ExportService { databaseName: "", tableType: "query", optionValues: plugin.defaultTableOptionValues(), - schema: nil + schema: nil, + kind: .table ) let result: ExportFormatResult @@ -316,7 +322,8 @@ final class ExportService { databaseName: "", tableType: "query", optionValues: plugin.defaultTableOptionValues(), - schema: nil + schema: nil, + kind: .table ) await suppressStatementTimeout(on: driver) @@ -342,7 +349,7 @@ final class ExportService { // MARK: - Row Count Fetching - private func qualifiedTableRef(for table: ExportTableItem, driver: DatabaseDriver) -> String { + private func qualifiedTableRef(for table: ExportObjectItem, driver: DatabaseDriver) -> String { if table.databaseName.isEmpty { return driver.quoteIdentifier(table.name) } @@ -351,7 +358,7 @@ final class ExportService { return "\(quotedDb).\(quotedTable)" } - private func fetchTotalRowCount(for tables: [ExportTableItem], driver: DatabaseDriver) async -> Int { + private func fetchTotalRowCount(for tables: [ExportObjectItem], driver: DatabaseDriver) async -> Int { guard !tables.isEmpty else { return 0 } var total = 0 diff --git a/TablePro/Core/Services/Export/TableTransferService.swift b/TablePro/Core/Services/Export/TableTransferService.swift new file mode 100644 index 0000000000..8ef954b549 --- /dev/null +++ b/TablePro/Core/Services/Export/TableTransferService.swift @@ -0,0 +1,221 @@ +// +// TableTransferService.swift +// TablePro +// + +import Foundation +import Observation +import os +import TableProPluginKit + +enum TableTransferError: LocalizedError { + case notConnected(connectionName: String) + case noTablesSelected + case sameConnectionAndContainer + case targetMissing(table: String) + case transferFailed(String) + + var errorDescription: String? { + switch self { + case .notConnected(let connectionName): + return String(format: String(localized: "Not connected to %@"), connectionName) + case .noTablesSelected: + return String(localized: "No tables selected to transfer") + case .sameConnectionAndContainer: + return String(localized: "The source and the destination are the same database.") + case .targetMissing(let table): + return String(format: String(localized: "The destination has no table named %@"), table) + case .transferFailed(let message): + return String(format: String(localized: "Transfer failed: %@"), message) + } + } +} + +struct TableTransferState { + var isTransferring = false + var currentTable = "" + var currentTableIndex = 0 + var totalTables = 0 + var transferredRows = 0 + var errorMessage: String? + var warnings: [String] = [] +} + +/// Copies rows straight from one connection into another, with no file in between. +/// +/// The two halves of the file-based flow already exist and already fit together: an export data +/// source streams rows out of one connection, an import sink writes rows into another. This joins +/// them, so a transfer costs one read and one write rather than a dump the user has to name, save, +/// find and import. +/// +/// It moves rows, not structure. The destination table has to exist, because inventing DDL that +/// crosses engines is a different problem from copying rows and getting it half right would create +/// tables whose types quietly do not match. +@MainActor +@Observable +final class TableTransferService { + nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "TableTransfer") + + /// The batch a transfer accumulates before writing. The sink chunks again by bind-parameter + /// count, so this only bounds how many rows are held at once. + static let batchSize = 500 + + var state = TableTransferState() + + private var isCancelled = false + + func cancel() { + isCancelled = true + } + + struct Request { + let objects: [ExportObjectItem] + let sourceType: DatabaseType + let destinationType: DatabaseType + let destinationSchema: String? + let columnMapping: [String: [String: String]] + let deleteExistingRows: Bool + let wrapInTransaction: Bool + + init( + objects: [ExportObjectItem], + sourceType: DatabaseType, + destinationType: DatabaseType, + destinationSchema: String? = nil, + columnMapping: [String: [String: String]] = [:], + deleteExistingRows: Bool = false, + wrapInTransaction: Bool = true + ) { + self.objects = objects + self.sourceType = sourceType + self.destinationType = destinationType + self.destinationSchema = destinationSchema + self.columnMapping = columnMapping + self.deleteExistingRows = deleteExistingRows + self.wrapInTransaction = wrapInTransaction + } + } + + /// Runs the whole transfer inside leases on both drivers, so every statement lands on the + /// database the sheet named rather than wherever either shared driver was last parked. + func transfer( + request: Request, + sourceDriver: DatabaseDriver, + destinationDriver: DatabaseDriver + ) async throws { + let rowObjects = request.objects.filter { $0.kind.carriesRows } + guard !rowObjects.isEmpty else { throw TableTransferError.noTablesSelected } + + state = TableTransferState(isTransferring: true, totalTables: rowObjects.count) + isCancelled = false + defer { + state.isTransferring = false + isCancelled = false + } + + let source = ExportDataSourceAdapter(driver: sourceDriver, databaseType: request.sourceType) + + for (index, object) in rowObjects.enumerated() { + try checkCancellation() + state.currentTable = object.name + state.currentTableIndex = index + 1 + + let mapping = request.columnMapping[object.name] ?? [:] + let sink = ImportDataSinkAdapter( + driver: destinationDriver, + databaseType: request.destinationType, + targetTable: object.name, + columnMapping: mapping + ) + try await transferOne(object: object, from: source, into: sink, request: request) + } + } + + private func transferOne( + object: ExportObjectItem, + from source: ExportDataSourceAdapter, + into sink: ImportDataSinkAdapter, + request: Request + ) async throws { + let exportTable = PluginExportTable( + name: object.name, + databaseName: object.databaseName, + tableType: object.kind.rawValue, + optionValues: object.optionValues, + schema: source.exportSchema(for: object.databaseName), + kind: object.kind, + identity: object.identity, + parentTable: object.parentTable, + rowScope: object.rowScope + ) + + var columns: [String] = [] + var batch: [[String: PluginCellValue]] = [] + var wroteAnything = false + + if request.wrapInTransaction { + try await sink.beginTransaction() + } + do { + if request.deleteExistingRows { + try await sink.deleteAllRowsFromTargetTable() + } + for try await element in source.streamRows(for: exportTable) { + try checkCancellation() + switch element { + case .header(let header): + columns = header.columns + case .rows(let rows): + for row in rows { + batch.append(Self.dictionary(columns: columns, row: row)) + guard batch.count >= Self.batchSize else { continue } + try await sink.insertRows(batch) + state.transferredRows += batch.count + wroteAnything = true + batch.removeAll(keepingCapacity: true) + } + } + } + if !batch.isEmpty { + try await sink.insertRows(batch) + state.transferredRows += batch.count + wroteAnything = true + } + if request.wrapInTransaction { + try await sink.commitTransaction() + } + } catch { + if request.wrapInTransaction { + do { + try await sink.rollbackTransaction() + } catch { + Self.logger.warning("Rollback after a failed transfer also failed: \(error.localizedDescription)") + } + } + state.errorMessage = error.localizedDescription + throw TableTransferError.transferFailed(error.localizedDescription) + } + + if !wroteAnything { + state.warnings.append(String( + format: String(localized: "%@ had no rows to transfer."), object.name)) + } + } + + /// A row arrives as positional values and the sink writes by column name, so the two are + /// zipped here. A row shorter than its header is padded with nulls rather than dropped: a + /// driver that omits trailing nulls would otherwise lose whole rows silently. + static func dictionary(columns: [String], row: [PluginCellValue]) -> [String: PluginCellValue] { + var values: [String: PluginCellValue] = [:] + values.reserveCapacity(columns.count) + for (index, column) in columns.enumerated() { + values[column] = index < row.count ? row[index] : .null + } + return values + } + + private func checkCancellation() throws { + guard isCancelled else { return } + throw PluginImportCancellationError() + } +} diff --git a/TablePro/Models/Export/ExportModels.swift b/TablePro/Models/Export/ExportModels.swift index fb5d58374f..1e1d578040 100644 --- a/TablePro/Models/Export/ExportModels.swift +++ b/TablePro/Models/Export/ExportModels.swift @@ -17,10 +17,21 @@ enum ExportPreselection: Equatable { /// `container` is the ref the dialog is listing, not its bare name. Matching on the name alone /// compared a database name against schema names, so a preselected database selected nothing on /// a schema-grouped engine and quietly matched an unrelated schema that happened to share a name. - func selects(table: String, inContainer container: DatabaseContainerRef, isCurrentContainer: Bool) -> Bool { + /// + /// `kind` is what keeps a routine or trigger that shares a table's name out of a table + /// preselection. Selecting a whole container still takes every kind in it. + func selects( + object: String, + kind: PluginExportObjectKind, + inContainer container: DatabaseContainerRef, + isCurrentContainer: Bool + ) -> Bool { switch self { case .tables(let names): - return isCurrentContainer && names.contains(table) + guard kind == .table || kind == .view || kind == .materializedView || kind == .foreignTable else { + return false + } + return isCurrentContainer && names.contains(object) case .containers(let refs): return refs.contains { $0.covers(container) } } @@ -92,39 +103,78 @@ struct ExportConfiguration { // MARK: - Tree View Models -struct ExportTableItem: Identifiable, Hashable { +/// One selectable thing in the export tree: a table, a view, a routine, a trigger, a type or a +/// principal whose grants are being exported. `optionValues` stays positionally aligned with the +/// format's full `perTableOptionColumns` for every kind, so a column a kind does not support is a +/// blank slot rather than a shifted one. +struct ExportObjectItem: Identifiable, Hashable { let id: UUID let name: String let databaseName: String - let type: TableInfo.TableType + let kind: PluginExportObjectKind + + /// Whatever addresses this exact object again: a routine's argument signature, a principal's + /// host part. Nil for a kind that a name alone identifies. + let identity: String? + + /// The table a trigger fires for. Nil for every other kind. + let parentTable: String? + var isSelected: Bool = false var optionValues: [Bool] = [] + /// Which rows and columns of this object to write. Only a kind that carries rows can narrow. + var rowScope: PluginExportRowScope = .unrestricted + init( id: UUID = UUID(), name: String, databaseName: String = "", - type: TableInfo.TableType, + kind: PluginExportObjectKind, + identity: String? = nil, + parentTable: String? = nil, isSelected: Bool = false, - optionValues: [Bool] = [] + optionValues: [Bool] = [], + rowScope: PluginExportRowScope = .unrestricted ) { self.id = id self.name = name self.databaseName = databaseName - self.type = type + self.kind = kind + self.identity = identity + self.parentTable = parentTable self.isSelected = isSelected self.optionValues = optionValues + self.rowScope = rowScope } var qualifiedName: String { databaseName.isEmpty ? name : "\(databaseName).\(name)" } + /// What the row shows after the name, so two overloads of one routine and two triggers of one + /// table are told apart without opening anything. + var subtitle: String? { + switch kind { + case .routine: + guard let identity, !identity.isEmpty else { return nil } + return identity + case .trigger: + guard let parentTable, !parentTable.isEmpty else { return nil } + return parentTable + case .grant: + guard let identity, !identity.isEmpty else { return nil } + return "@\(identity)" + default: + return nil + } + } + func hash(into hasher: inout Hasher) { hasher.combine(id) } - static func == (lhs: ExportTableItem, rhs: ExportTableItem) -> Bool { + static func == (lhs: ExportObjectItem, rhs: ExportObjectItem) -> Bool { lhs.id == rhs.id } } @@ -132,40 +182,53 @@ struct ExportTableItem: Identifiable, Hashable { struct ExportDatabaseItem: Identifiable { let id: UUID let name: String - var tables: [ExportTableItem] + var objects: [ExportObjectItem] var isExpanded: Bool = true init( id: UUID = UUID(), name: String, - tables: [ExportTableItem], + objects: [ExportObjectItem], isExpanded: Bool = true ) { self.id = id self.name = name - self.tables = tables + self.objects = objects self.isExpanded = isExpanded } var selectedCount: Int { - tables.count(where: \.isSelected) + objects.count(where: \.isSelected) } var allSelected: Bool { - !tables.isEmpty && tables.allSatisfy { $0.isSelected } + !objects.isEmpty && objects.allSatisfy { $0.isSelected } } var noneSelected: Bool { - tables.allSatisfy { !$0.isSelected } + objects.allSatisfy { !$0.isSelected } + } + + var selectedObjects: [ExportObjectItem] { + objects.filter { $0.isSelected } } - var selectedTables: [ExportTableItem] { - tables.filter { $0.isSelected } + /// The kinds present, in dump order, which is the order the groups appear in the tree. + var presentKinds: [PluginExportObjectKind] { + var seen: Set = [] + return objects + .map(\.kind) + .filter { seen.insert($0).inserted } + .sorted { $0.dumpOrder < $1.dumpOrder } + } + + func objects(ofKind kind: PluginExportObjectKind) -> [ExportObjectItem] { + objects.filter { $0.kind == kind } } } -extension ExportTableItem { - func normalized(forOptionColumnCount optionColumnCount: Int, defaultOptionValues: [Bool]) -> ExportTableItem { +extension ExportObjectItem { + func normalized(forOptionColumnCount optionColumnCount: Int, defaultOptionValues: [Bool]) -> ExportObjectItem { guard optionColumnCount > 0 else { return self } let fallback = defaultOptionValues.count == optionColumnCount ? defaultOptionValues @@ -179,13 +242,27 @@ extension ExportTableItem { } return normalizedItem } + + /// Clears every option the format says this kind does not support, so a routine never carries a + /// `Data` flag that would make it look exportable for a phase it has no rows for. + func maskingUnsupportedOptions( + columns: [PluginExportOptionColumn], + supports: (String, PluginExportObjectKind) -> Bool + ) -> ExportObjectItem { + guard optionValues.count == columns.count else { return self } + var masked = self + masked.optionValues = zip(columns, optionValues).map { column, value in + supports(column.id, kind) ? value : false + } + return masked + } } extension [ExportDatabaseItem] { func normalizingOptionValues(optionColumnCount: Int, defaultOptionValues: [Bool]) -> [ExportDatabaseItem] { map { database in var normalizedDatabase = database - normalizedDatabase.tables = database.tables.map { + normalizedDatabase.objects = database.objects.map { $0.normalized(forOptionColumnCount: optionColumnCount, defaultOptionValues: defaultOptionValues) } return normalizedDatabase @@ -195,12 +272,25 @@ extension [ExportDatabaseItem] { func resettingOptionValues(to values: [Bool]) -> [ExportDatabaseItem] { map { database in var resetDatabase = database - resetDatabase.tables = database.tables.map { table in - var resetTable = table - resetTable.optionValues = values - return resetTable + resetDatabase.objects = database.objects.map { object in + var resetObject = object + resetObject.optionValues = values + return resetObject } return resetDatabase } } + + func maskingUnsupportedOptions( + columns: [PluginExportOptionColumn], + supports: @escaping (String, PluginExportObjectKind) -> Bool + ) -> [ExportDatabaseItem] { + map { database in + var maskedDatabase = database + maskedDatabase.objects = database.objects.map { + $0.maskingUnsupportedOptions(columns: columns, supports: supports) + } + return maskedDatabase + } + } } diff --git a/TablePro/Views/Backup/BackupDatabaseFlow.swift b/TablePro/Views/Backup/BackupDatabaseFlow.swift index 3e5a8710d4..7ba4266c50 100644 --- a/TablePro/Views/Backup/BackupDatabaseFlow.swift +++ b/TablePro/Views/Backup/BackupDatabaseFlow.swift @@ -5,7 +5,7 @@ // Top-level sheet for the Backup Dump menu item. Reuses // `DatabaseSwitcherSheet` in `.backup` mode to pick the database, // then drives an NSSavePanel sub-sheet and the consolidated -// `PostgresDumpService` progress flow. +// `NativeDumpService` progress flow. // import AppKit @@ -20,7 +20,7 @@ struct BackupDatabaseFlow: View { @State private var backupDatabase: String? let initialDatabase: String - @State private var service = PostgresDumpService(kind: .backup) + @State private var service = NativeDumpService(kind: .backup) @State private var phase: Phase = .pickDatabase private enum Phase: Equatable { @@ -92,9 +92,9 @@ struct BackupDatabaseFlow: View { } /// Hashable snapshot of `service.state` so SwiftUI's `onChange` fires on every transition. - private var serviceState: PostgresDumpState { service.state } + private var serviceState: NativeDumpState { service.state } - private func handleServiceStateChange(_ state: PostgresDumpState) { + private func handleServiceStateChange(_ state: NativeDumpState) { switch state { case .running(let database, _, _, let totalBytes): phase = .running(database: database, totalBytes: totalBytes) @@ -142,8 +142,10 @@ struct BackupDatabaseFlow: View { let savePanel = NSSavePanel() savePanel.canCreateDirectories = true savePanel.showsTagField = false - savePanel.allowedContentTypes = [UTType(filenameExtension: "dump") ?? .data] - savePanel.nameFieldStringValue = Self.defaultFilename(database: database) + let archiveExtension = NativeDumpRegistry.descriptor(for: connection.type)? + .archiveFormat.fileExtension ?? "dump" + savePanel.allowedContentTypes = [UTType(filenameExtension: archiveExtension) ?? .data] + savePanel.nameFieldStringValue = Self.defaultFilename(database: database, type: connection.type) savePanel.title = String(localized: "Save Dump") savePanel.message = String(format: String(localized: "Choose where to save the dump of \u{201C}%@\u{201D}."), database) @@ -160,11 +162,9 @@ struct BackupDatabaseFlow: View { return } - // Show progress immediately so the user gets feedback while we fetch - // the database size estimate and locate pg_dump. phase = .running(database: database, totalBytes: nil) - let totalBytes = await PostgresDumpService.estimatedDatabaseSize( + let totalBytes = await NativeDumpService.estimatedDatabaseSize( connection: connection, database: database ) @@ -181,10 +181,13 @@ struct BackupDatabaseFlow: View { } } - private static func defaultFilename(database: String) -> String { + /// The extension follows the engine's own archive format, so a MySQL dump is offered as `.sql` + /// and a MongoDB one as `.archive` rather than all of them claiming PostgreSQL's `.dump`. + private static func defaultFilename(database: String, type: DatabaseType) -> String { let timestamp = Self.timestampFormatter.string(from: Date()) let safeDB = database.isEmpty ? "database" : database - return "\(safeDB)-\(timestamp).dump" + let fileExtension = NativeDumpRegistry.descriptor(for: type)?.archiveFormat.fileExtension ?? "dump" + return "\(safeDB)-\(timestamp).\(fileExtension)" } private static let timestampFormatter: DateFormatter = { diff --git a/TablePro/Views/Backup/RestoreDatabaseFlow.swift b/TablePro/Views/Backup/RestoreDatabaseFlow.swift index cabaccc01c..414f349fe5 100644 --- a/TablePro/Views/Backup/RestoreDatabaseFlow.swift +++ b/TablePro/Views/Backup/RestoreDatabaseFlow.swift @@ -7,7 +7,7 @@ struct RestoreDatabaseFlow: View { let initialDatabase: String let sourceURL: URL - @State private var service = PostgresDumpService(kind: .restore) + @State private var service = NativeDumpService(kind: .restore) @State private var phase: Phase = .pickDatabase private enum Phase: Equatable { @@ -97,9 +97,9 @@ struct RestoreDatabaseFlow: View { .frame(width: 480, alignment: .leading) } - private var serviceState: PostgresDumpState { service.state } + private var serviceState: NativeDumpState { service.state } - private func handleServiceStateChange(_ state: PostgresDumpState) { + private func handleServiceStateChange(_ state: NativeDumpState) { switch state { case .running(let database, _, _, _): phase = .running(database: database) diff --git a/TablePro/Views/Export/ExportDialog.swift b/TablePro/Views/Export/ExportDialog.swift index abae2d35a5..2ecaa20ed0 100644 --- a/TablePro/Views/Export/ExportDialog.swift +++ b/TablePro/Views/Export/ExportDialog.swift @@ -33,6 +33,10 @@ struct ExportDialog: View { @State private var settingsSnapshot: PluginSettingsSnapshot? @State private var exportSucceeded = false + /// Which object kinds the last load actually read, so a format switch knows whether the tree it + /// already holds can answer for the new format without another round trip. + @State private var loadedObjectKinds: Set = [] + /// The window this dialog is hosted in, used for presenting its alerts and panels. /// Avoids `NSApp.keyWindow`, which when a result is presented is the progress sheet being /// torn down in the same transaction, and AppKit ends a sheet's children with it (#2314). @@ -123,6 +127,7 @@ struct ExportDialog: View { } .onChange(of: config.formatId) { resetOptionValues() + Task { await reconcileObjectKindsForFormat() } } .onExitCommand { if !isExporting { @@ -274,9 +279,10 @@ struct ExportDialog: View { } .frame(minHeight: 300, maxHeight: .infinity) } else { - ExportTableTreeView( + ExportObjectTreeView( databaseItems: $databaseItems, - formatId: config.formatId + formatId: config.formatId, + loadColumns: { await columnNames(for: $0) } ) .frame(minHeight: 300, maxHeight: .infinity) } @@ -418,19 +424,28 @@ struct ExportDialog: View { databaseItems.reduce(0) { $0 + $1.selectedCount } } - private var selectedTables: [ExportTableItem] { - databaseItems.flatMap { $0.selectedTables } + private var selectedObjects: [ExportObjectItem] { + databaseItems.flatMap { $0.selectedObjects } + } + + private var exportableObjects: [ExportObjectItem] { + let objects = selectedObjects + guard let plugin = currentPlugin else { return objects } + return objects.filter { plugin.isExportable(optionValues: $0.optionValues, kind: $0.kind) } } - private var exportableTables: [ExportTableItem] { - let tables = selectedTables - guard let plugin = currentPlugin else { return tables } - return tables.filter { plugin.isTableExportable(optionValues: $0.optionValues) } + /// The kinds the chosen format can write, narrowed to the kinds a driver can actually list. A + /// format that declares none of its own receives tables and views, which is what every format + /// written before object scope expects. + private var supportedObjectKinds: Set { + guard let plugin = currentPlugin else { return [.table, .view] } + return Set(type(of: plugin).supportedObjectKinds) + .intersection(ExportObjectLoader.loadableKinds) } /// Count of tables that will actually produce output private var exportableCount: Int { - exportableTables.count + exportableObjects.count } private var fileExtension: String { @@ -463,8 +478,43 @@ struct ExportDialog: View { } } + /// A format change changes which object kinds can be written. Kinds the new format cannot + /// write are dropped from the tree, and a format that reaches further than the last load did is + /// what makes a reload worth its round trips. + @MainActor + private func reconcileObjectKindsForFormat() async { + guard !isQueryResultsMode else { return } + let wanted = supportedObjectKinds + guard !loadedObjectKinds.isEmpty else { return } + guard wanted.isSubset(of: loadedObjectKinds) else { + await loadDatabaseItems() + return + } + databaseItems = databaseItems.compactMap { database in + var filtered = database + filtered.objects = database.objects.filter { wanted.contains($0.kind) } + return filtered.objects.isEmpty ? nil : filtered + } + } + private func resetOptionValues() { - databaseItems = databaseItems.resettingOptionValues(to: currentDefaultOptionValues) + databaseItems = normalizedForCurrentFormat( + databaseItems.resettingOptionValues(to: currentDefaultOptionValues)) + } + + /// Aligns every row's option values with the chosen format's columns and clears the ones the + /// row's kind does not support, so a routine never carries a `Data` flag that would count it as + /// exportable for a phase it has no rows for. + private func normalizedForCurrentFormat(_ items: [ExportDatabaseItem]) -> [ExportDatabaseItem] { + let normalized = items.normalizingOptionValues( + optionColumnCount: currentOptionColumnCount, + defaultOptionValues: currentDefaultOptionValues + ) + guard let plugin = currentPlugin else { return normalized } + let pluginType = type(of: plugin) + return normalized.maskingUnsupportedOptions(columns: pluginType.perTableOptionColumns) { + columnId, kind in pluginType.supportsOption(columnId: columnId, for: kind) + } } // MARK: - Actions @@ -523,13 +573,15 @@ struct ExportDialog: View { /// failed load would leave them on screen looking like that database's contents. guard preselection.scopedDatabase == nil else { return } let dbName = connection.database - let tableItems = sidebarTables.map { table in - ExportTableItem( + let objectItems = sidebarTables.map { table in + let kind = PluginExportObjectKind.from(tableType: table.type.rawValue) + return ExportObjectItem( name: table.name, databaseName: "", - type: table.type, + kind: kind, isSelected: preselection.selects( - table: table.name, + object: table.name, + kind: kind, inContainer: .database(dbName), isCurrentContainer: true ) @@ -537,13 +589,10 @@ struct ExportDialog: View { } let item = ExportDatabaseItem( name: dbName.isEmpty ? "Tables" : dbName, - tables: tableItems, + objects: objectItems, isExpanded: true ) - databaseItems = [item].normalizingOptionValues( - optionColumnCount: currentOptionColumnCount, - defaultOptionValues: currentDefaultOptionValues - ) + databaseItems = normalizedForCurrentFormat([item]) isLoading = false } @@ -552,19 +601,23 @@ struct ExportDialog: View { let optionValues: [Bool] } + /// Keyed by kind too, so a routine and a table that share a name do not inherit each other's + /// checkboxes when the format changes and the tree reloads. private func priorRowSnapshots() -> [String: ExportRowSnapshot] { var snapshots: [String: ExportRowSnapshot] = [:] for database in databaseItems { - for table in database.tables { - snapshots["\(database.name).\(table.name)"] = ExportRowSnapshot( - isSelected: table.isSelected, - optionValues: table.optionValues - ) + for object in database.objects { + snapshots[Self.snapshotKey(container: database.name, object: object.name, kind: object.kind)] = + ExportRowSnapshot(isSelected: object.isSelected, optionValues: object.optionValues) } } return snapshots } + private static func snapshotKey(container: String, object: String, kind: PluginExportObjectKind) -> String { + "\(container).\(kind.rawValue).\(object)" + } + @MainActor private func loadDatabaseItems() async { let priorRows = priorRowSnapshots() @@ -583,26 +636,25 @@ struct ExportDialog: View { for schema in schemas { let tables = try await fetchTablesForSchema(schema) let isDefaultSchema = schema.caseInsensitiveCompare(defaultSchema) == .orderedSame - let tableItems = tables.map { table in - let priorRow = priorRows["\(schema).\(table.name)"] - let selected = priorRow?.isSelected - ?? preselection.selects( - table: table.name, - inContainer: .schema(database: exportDatabaseName, schema: schema), - isCurrentContainer: isDefaultSchema - ) - return ExportTableItem( - name: table.name, - databaseName: schema, - type: table.type, - isSelected: selected, - optionValues: priorRow?.optionValues ?? [] + let loaded = await loadObjects( + containerName: schema, + schema: schema, + tables: tables, + includesPrincipals: isDefaultSchema + ) + let objectItems = loaded.map { object in + restoring( + object, + priorRows: priorRows, + container: schema, + containerRef: .schema(database: exportDatabaseName, schema: schema), + isCurrentContainer: isDefaultSchema ) } - if !tableItems.isEmpty { + if !objectItems.isEmpty { items.append(ExportDatabaseItem( name: schema, - tables: tableItems, + objects: objectItems, isExpanded: isDefaultSchema || preselection.containerNames.contains(schema) )) } @@ -627,26 +679,25 @@ struct ExportDialog: View { for dbName in databases { let tables = tablesByDatabase[dbName] ?? [] let isCurrentDB = dbName == connection.database - let tableItems = tables.map { table in - let priorRow = priorRows["\(dbName).\(table.name)"] - let selected = priorRow?.isSelected - ?? preselection.selects( - table: table.name, - inContainer: .database(dbName), - isCurrentContainer: isCurrentDB - ) - return ExportTableItem( - name: table.name, - databaseName: dbName, - type: table.type, - isSelected: selected, - optionValues: priorRow?.optionValues ?? [] + let loaded = await loadObjects( + containerName: dbName, + schema: isCurrentDB ? nil : dbName, + tables: tables, + includesPrincipals: isCurrentDB + ) + let objectItems = loaded.map { object in + restoring( + object, + priorRows: priorRows, + container: dbName, + containerRef: .database(dbName), + isCurrentContainer: isCurrentDB ) } - if !tableItems.isEmpty { + if !objectItems.isEmpty { items.append(ExportDatabaseItem( name: dbName, - tables: tableItems, + objects: objectItems, isExpanded: isCurrentDB || preselection.containerNames.contains(dbName) )) } @@ -658,10 +709,8 @@ struct ExportDialog: View { } } - databaseItems = items.normalizingOptionValues( - optionColumnCount: currentOptionColumnCount, - defaultOptionValues: currentDefaultOptionValues - ) + loadedObjectKinds = supportedObjectKinds + databaseItems = normalizedForCurrentFormat(items) isLoading = false if let singleTable = preselection.singleTableName { @@ -688,22 +737,86 @@ struct ExportDialog: View { let tables = try await withExportDriver { driver in try await driver.fetchTables() } - let tableItems = tables.map { table in - let priorRow = priorRows["\(name).\(table.name)"] - return ExportTableItem( - name: table.name, - databaseName: "", - type: table.type, - isSelected: priorRow?.isSelected ?? preselection.selects( - table: table.name, - inContainer: .database(name), - isCurrentContainer: true - ), - optionValues: priorRow?.optionValues ?? [] + let loaded = await loadObjects( + containerName: "", schema: nil, tables: tables, includesPrincipals: true) + let objectItems = loaded.map { object in + restoring( + object, + priorRows: priorRows, + container: name, + containerRef: .database(name), + isCurrentContainer: true ) } - guard !tableItems.isEmpty else { return nil } - return ExportDatabaseItem(name: name, tables: tableItems, isExpanded: true) + guard !objectItems.isEmpty else { return nil } + return ExportDatabaseItem(name: name, objects: objectItems, isExpanded: true) + } + + /// Reads one container's objects for the kinds the chosen format can write. Principals are + /// server-wide, so only the container the dialog opened on offers them: listing them under + /// every schema would offer the same GRANT statements several times over. + private func loadObjects( + containerName: String, + schema: String?, + tables: [TableInfo], + includesPrincipals: Bool + ) async -> [ExportObjectItem] { + var kinds = supportedObjectKinds + if !includesPrincipals { kinds.remove(.grant) } + guard !kinds.isEmpty else { return [] } + let request = ExportObjectLoader.Request( + containerName: containerName, schema: schema, kinds: kinds) + do { + return try await withExportDriver { driver in + await ExportObjectLoader.loadObjects(request: request, tables: tables, driver: driver) + } + } catch { + Self.logger.warning("Failed to load export objects: \(error.localizedDescription)") + return [] + } + } + + /// The column names the row-scope popover offers. Read on demand, because a tree of forty + /// tables would otherwise pay for forty column lists nobody opens. + @MainActor + private func columnNames(for object: ExportObjectItem) async -> [String] { + guard object.kind.carriesRows else { return [] } + do { + return try await withExportDriver(workload: .interactive) { driver in + guard let pluginDriver = (driver as? PluginDriverAdapter)?.schemaPluginDriver else { return [] } + let schema = object.databaseName.isEmpty ? nil : object.databaseName + return try await pluginDriver.fetchColumns(table: object.name, schema: schema).map(\.name) + } + } catch { + Self.logger.warning("Failed to read columns for the export scope: \(error.localizedDescription)") + return [] + } + } + + /// Carries a row's checkboxes across a reload, falling back to what the preselection asked for. + private func restoring( + _ object: ExportObjectItem, + priorRows: [String: ExportRowSnapshot], + container: String, + containerRef: DatabaseContainerRef, + isCurrentContainer: Bool + ) -> ExportObjectItem { + let priorRow = priorRows[ + Self.snapshotKey(container: container, object: object.name, kind: object.kind)] + return ExportObjectItem( + name: object.name, + databaseName: object.databaseName, + kind: object.kind, + identity: object.identity, + parentTable: object.parentTable, + isSelected: priorRow?.isSelected ?? preselection.selects( + object: object.name, + kind: object.kind, + inContainer: containerRef, + isCurrentContainer: isCurrentContainer + ), + optionValues: priorRow?.optionValues ?? [] + ) } private func fetchTablesForSchema(_ schema: String) async throws -> [TableInfo] { @@ -863,7 +976,7 @@ struct ExportDialog: View { private func runTableExport(on driver: DatabaseDriver, to url: URL) async throws { let service = ExportService(driver: driver, databaseType: connection.type) exportService = service - try await service.export(tables: exportableTables, config: config, to: url) + try await service.export(objects: exportableObjects, config: config, to: url) } @MainActor diff --git a/TablePro/Views/Export/ExportObjectRows.swift b/TablePro/Views/Export/ExportObjectRows.swift new file mode 100644 index 0000000000..ec38179a1d --- /dev/null +++ b/TablePro/Views/Export/ExportObjectRows.swift @@ -0,0 +1,202 @@ +// +// ExportObjectRows.swift +// TablePro +// + +import SwiftUI +import TableProPluginKit + +/// How each object kind names and draws itself in the export tree. One table rather than a switch +/// per call site, so a kind added later cannot pick up a different icon in one of them. +internal enum ExportObjectKindPresentation { + internal static func groupTitle(for kind: PluginExportObjectKind) -> String { + switch kind { + case .table: return String(localized: "Tables") + case .view: return String(localized: "Views") + case .materializedView: return String(localized: "Materialized Views") + case .foreignTable: return String(localized: "Foreign Tables") + case .sequence: return String(localized: "Sequences") + case .userType: return String(localized: "Types") + case .routine: return String(localized: "Routines") + case .trigger: return String(localized: "Triggers") + case .event: return String(localized: "Events") + case .grant: return String(localized: "Privileges") + @unknown default: return String(localized: "Objects") + } + } + + internal static func iconName(for kind: PluginExportObjectKind) -> String { + switch kind { + case .table, .foreignTable: return "tablecells" + case .view, .materializedView: return "eye" + case .sequence: return "number" + case .userType: return "curlybraces" + case .routine: return "function" + case .trigger: return "bolt" + case .event: return "clock" + case .grant: return "key" + @unknown default: return "shippingbox" + } + } + + internal static func iconColor(for kind: PluginExportObjectKind) -> Color { + switch kind { + case .table, .foreignTable: return .gray + case .view, .materializedView: return .purple + case .sequence: return .teal + case .userType: return .orange + case .routine: return .indigo + case .trigger: return .pink + case .event: return .brown + case .grant: return .yellow + @unknown default: return .secondary + } + } +} + +/// A database row or a kind-group row: a tri-state checkbox that selects everything beneath it. +internal struct ExportTreeContainerRow: View { + internal let title: String + internal let iconName: String + internal let iconColor: Color + internal let state: TristateCheckbox.State + internal let toggle: () -> Void + + internal var body: some View { + HStack(spacing: 4) { + TristateCheckbox(state: state, action: toggle) + .frame(width: 18) + + Image(systemName: iconName) + .foregroundStyle(iconColor) + .font(.body) + + Text(title) + .font(.body) + .lineLimit(1) + .truncationMode(.middle) + + Spacer(minLength: 0) + } + } +} + +/// One object row. The option toggles stay positionally aligned with the format's full column list +/// for every kind, so a column this kind does not support leaves an empty slot of the same width +/// rather than shifting the ones after it. +internal struct ExportTreeObjectRow: View { + internal let object: ExportObjectItem + internal let optionColumns: [PluginExportOptionColumn] + internal let supportsOption: (String, PluginExportObjectKind) -> Bool + internal let setSelected: (Bool) -> Void + internal let setOption: (Int, Bool) -> Void + internal let setRowScope: (PluginExportRowScope) -> Void + internal let loadColumns: () async -> [String] + + @State private var isEditingScope = false + @State private var editableScope: PluginExportRowScope = .unrestricted + @State private var availableColumns: [String] = [] + + internal var body: some View { + HStack(spacing: 4) { + if optionColumns.isEmpty { + Toggle("", isOn: Binding(get: { object.isSelected }, set: setSelected)) + .toggleStyle(.checkbox) + .labelsHidden() + .frame(width: 18) + } else { + TristateCheckbox(state: checkboxState) { + setSelected(!object.isSelected) + } + .frame(width: 18) + } + + Image(systemName: ExportObjectKindPresentation.iconName(for: object.kind)) + .foregroundStyle(ExportObjectKindPresentation.iconColor(for: object.kind)) + .font(.body) + + Text(object.name) + .font(.body) + .lineLimit(1) + .truncationMode(.middle) + + if let subtitle = object.subtitle { + Text(subtitle) + .font(.subheadline) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.middle) + } + + Spacer(minLength: 4) + + if object.kind.carriesRows { + rowScopeButton + } + + if !optionColumns.isEmpty { + ForEach(Array(optionColumns.enumerated()), id: \.element.id) { index, column in + optionToggle(index: index, column: column) + } + } + } + } + + /// Filled when the object is narrowed, so a tree of forty tables says at a glance which ones + /// will not export whole. + private var rowScopeButton: some View { + Button { + editableScope = object.rowScope + isEditingScope = true + } label: { + Image(systemName: object.rowScope.isUnrestricted + ? "line.3.horizontal.decrease.circle" + : "line.3.horizontal.decrease.circle.fill") + .foregroundStyle(object.rowScope.isUnrestricted ? Color.secondary : Color.accentColor) + } + .buttonStyle(.borderless) + .help(object.rowScope.isUnrestricted + ? String(localized: "Narrow the rows and columns to export") + : object.rowScope.summary) + .popover(isPresented: $isEditingScope, arrowEdge: .bottom) { + ExportRowScopeEditor( + objectName: object.name, + availableColumns: availableColumns, + scope: Binding(get: { editableScope }, set: { editableScope = $0; setRowScope($0) }), + dismiss: { isEditingScope = false } + ) + .task { + guard availableColumns.isEmpty else { return } + availableColumns = await loadColumns() + } + } + } + + @ViewBuilder + private func optionToggle(index: Int, column: PluginExportOptionColumn) -> some View { + if supportsOption(column.id, object.kind) { + Toggle(column.label, isOn: Binding( + get: { object.optionValues[safe: index] ?? column.defaultValue }, + set: { setOption(index, $0) } + )) + .toggleStyle(.checkbox) + .labelsHidden() + .disabled(!object.isSelected) + .opacity(object.isSelected ? 1.0 : 0.4) + .frame(width: column.width, alignment: .center) + } else { + Color.clear.frame(width: column.width, height: 1) + } + } + + private var checkboxState: TristateCheckbox.State { + guard object.isSelected else { return .unchecked } + let supported = optionColumns.indices.filter { + supportsOption(optionColumns[$0].id, object.kind) + } + guard !supported.isEmpty else { return .checked } + let onCount = supported.count { object.optionValues[safe: $0] == true } + if onCount == 0 { return .unchecked } + return onCount == supported.count ? .checked : .mixed + } +} diff --git a/TablePro/Views/Export/ExportObjectTreeView.swift b/TablePro/Views/Export/ExportObjectTreeView.swift new file mode 100644 index 0000000000..05ecd5d6d1 --- /dev/null +++ b/TablePro/Views/Export/ExportObjectTreeView.swift @@ -0,0 +1,379 @@ +// +// ExportObjectTreeView.swift +// TablePro +// + +import AppKit +import SwiftUI +import TableProPluginKit + +/// The export tree as an `NSOutlineView`. +/// +/// It has three levels once a database holds more than one kind of object, and a SwiftUI `List` of +/// nested `DisclosureGroup`s driven by programmatic `isExpanded` bindings crashes on macOS when the +/// bindings are re-driven during an animated outline diff. That is the same Apple bug the +/// connection sidebar's database tree hit, and the same answer: `NSOutlineView`, which also gives +/// genuinely lazy children for a database holding thousands of objects. +internal struct ExportObjectTreeView: NSViewRepresentable { + @Binding internal var databaseItems: [ExportDatabaseItem] + internal let formatId: String + + /// Reads one object's column names for the row-scope popover. Supplied by the dialog, which is + /// what owns the export driver lease. + internal let loadColumns: (ExportObjectItem) async -> [String] + + internal func makeCoordinator() -> ExportObjectTreeCoordinator { + ExportObjectTreeCoordinator(owner: self) + } + + internal func makeNSView(context: Context) -> NSScrollView { + let outlineView = NSOutlineView() + outlineView.headerView = nil + outlineView.style = .plain + outlineView.rowSizeStyle = .default + outlineView.allowsMultipleSelection = false + outlineView.allowsEmptySelection = true + outlineView.usesAlternatingRowBackgroundColors = true + outlineView.autosaveExpandedItems = false + outlineView.indentationPerLevel = 14 + outlineView.dataSource = context.coordinator + outlineView.delegate = context.coordinator + + let column = NSTableColumn(identifier: NSUserInterfaceItemIdentifier("ExportObjectColumn")) + column.resizingMask = .autoresizingMask + outlineView.addTableColumn(column) + outlineView.outlineTableColumn = column + + let scrollView = NSScrollView() + scrollView.documentView = outlineView + scrollView.hasVerticalScroller = true + scrollView.hasHorizontalScroller = false + scrollView.autohidesScrollers = true + scrollView.drawsBackground = false + scrollView.backgroundColor = .clear + + context.coordinator.attach(outlineView: outlineView) + return scrollView + } + + internal func updateNSView(_ scrollView: NSScrollView, context: Context) { + context.coordinator.owner = self + context.coordinator.apply(databases: databaseItems, formatId: formatId) + } +} + +@MainActor +internal final class ExportObjectTreeCoordinator: NSObject, NSOutlineViewDataSource, NSOutlineViewDelegate { + internal var owner: ExportObjectTreeView + + private weak var outlineView: NSOutlineView? + private var roots: [ExportOutlineNode] = [] + private var databases: [ExportDatabaseItem] = [] + private var shapeFingerprint = "" + private var formatId = "" + + /// Every node the user has collapsed, by its stable identity, so a rebuild restores what they + /// chose rather than reopening the whole tree. + private var collapsedIdentities: Set = [] + + internal init(owner: ExportObjectTreeView) { + self.owner = owner + } + + internal func attach(outlineView: NSOutlineView) { + self.outlineView = outlineView + } + + internal func apply(databases: [ExportDatabaseItem], formatId: String) { + let fingerprint = ExportOutlineTreeBuilder.shapeFingerprint(of: databases) + let formatChanged = formatId != self.formatId + self.databases = databases + self.formatId = formatId + + guard fingerprint != shapeFingerprint else { + guard formatChanged else { + reloadRowContent() + return + } + outlineView?.reloadData() + restoreExpansion() + return + } + shapeFingerprint = fingerprint + roots = ExportOutlineTreeBuilder.build(from: databases) + outlineView?.reloadData() + restoreExpansion() + } + + /// A checkbox toggle leaves the tree's shape alone, so the rows are redrawn in place. Rebuilding + /// would collapse and re-expand every group under the user's pointer. + private func reloadRowContent() { + guard let outlineView else { return } + let rows = IndexSet(integersIn: 0 ..< outlineView.numberOfRows) + guard !rows.isEmpty else { return } + outlineView.reloadData(forRowIndexes: rows, columnIndexes: IndexSet(integer: 0)) + } + + private func restoreExpansion() { + guard let outlineView else { return } + for root in roots { + expandRecursively(root, in: outlineView) + } + } + + private func expandRecursively(_ node: ExportOutlineNode, in outlineView: NSOutlineView) { + guard !node.isLeaf else { return } + if collapsedIdentities.contains(node.identity) { + outlineView.collapseItem(node) + } else { + outlineView.expandItem(node) + } + for child in node.children { + expandRecursively(child, in: outlineView) + } + } + + // MARK: - Data Source + + internal func outlineView(_ outlineView: NSOutlineView, numberOfChildrenOfItem item: Any?) -> Int { + guard let node = item as? ExportOutlineNode else { return roots.count } + return node.children.count + } + + internal func outlineView(_ outlineView: NSOutlineView, child index: Int, ofItem item: Any?) -> Any { + guard let node = item as? ExportOutlineNode else { return roots[index] } + return node.children[index] + } + + internal func outlineView(_ outlineView: NSOutlineView, isItemExpandable item: Any) -> Bool { + guard let node = item as? ExportOutlineNode else { return false } + return !node.isLeaf + } + + internal func outlineView(_ outlineView: NSOutlineView, shouldSelectItem item: Any) -> Bool { + false + } + + internal func outlineViewItemDidExpand(_ notification: Notification) { + guard let node = notification.userInfo?["NSObject"] as? ExportOutlineNode else { return } + collapsedIdentities.remove(node.identity) + } + + internal func outlineViewItemDidCollapse(_ notification: Notification) { + guard let node = notification.userInfo?["NSObject"] as? ExportOutlineNode else { return } + collapsedIdentities.insert(node.identity) + } + + // MARK: - Delegate + + internal func outlineView( + _ outlineView: NSOutlineView, + viewFor tableColumn: NSTableColumn?, + item: Any + ) -> NSView? { + guard let node = item as? ExportOutlineNode else { return nil } + let identifier = NSUserInterfaceItemIdentifier("ExportObjectRow") + let cell = outlineView.makeView(withIdentifier: identifier, owner: self) as? ExportObjectCellView + ?? ExportObjectCellView(identifier: identifier) + cell.configure(with: rowContent(for: node)) + return cell + } + + private func rowContent(for node: ExportOutlineNode) -> AnyView { + switch node.kind { + case .database(let databaseID): + return AnyView( + ExportTreeContainerRow( + title: databases.first(where: { $0.id == databaseID })?.name ?? "", + iconName: "cylinder", + iconColor: .blue, + state: containerState(for: node), + toggle: { [weak self] in self?.toggleContainer(node) } + ) + ) + case .group(_, let objectKind): + return AnyView( + ExportTreeContainerRow( + title: ExportObjectKindPresentation.groupTitle(for: objectKind), + iconName: ExportObjectKindPresentation.iconName(for: objectKind), + iconColor: ExportObjectKindPresentation.iconColor(for: objectKind), + state: containerState(for: node), + toggle: { [weak self] in self?.toggleContainer(node) } + ) + ) + case .object(let databaseID, let objectID): + guard let databaseIndex = databases.firstIndex(where: { $0.id == databaseID }), + let objectIndex = databases[databaseIndex].objects.firstIndex(where: { $0.id == objectID }) + else { + return AnyView(EmptyView()) + } + let object = databases[databaseIndex].objects[objectIndex] + return AnyView( + ExportTreeObjectRow( + object: object, + optionColumns: optionColumns, + supportsOption: supportsOption, + setSelected: { [weak self] isSelected in + self?.setSelection(isSelected, databaseID: databaseID, objectID: objectID) + }, + setOption: { [weak self] index, value in + self?.setOption(index, to: value, databaseID: databaseID, objectID: objectID) + }, + setRowScope: { [weak self] scope in + self?.setRowScope(scope, databaseID: databaseID, objectID: objectID) + }, + loadColumns: { [owner] in await owner.loadColumns(object) } + ) + ) + } + } + + // MARK: - Selection + + private var plugin: (any ExportFormatPlugin)? { + PluginManager.shared.exportPlugin(forFormat: formatId) + } + + private var optionColumns: [PluginExportOptionColumn] { + guard let plugin else { return [] } + return type(of: plugin).perTableOptionColumns + } + + private var defaultOptionValues: [Bool] { + plugin?.defaultTableOptionValues() ?? [] + } + + private var supportsOption: (String, PluginExportObjectKind) -> Bool { + guard let plugin else { return { _, _ in true } } + let pluginType = type(of: plugin) + return { columnId, kind in pluginType.supportsOption(columnId: columnId, for: kind) } + } + + private func objects(under node: ExportOutlineNode) -> [ExportObjectItem] { + switch node.kind { + case .database(let databaseID): + return databases.first(where: { $0.id == databaseID })?.objects ?? [] + case .group(let databaseID, let objectKind): + return databases.first(where: { $0.id == databaseID })?.objects(ofKind: objectKind) ?? [] + case .object(let databaseID, let objectID): + guard let object = databases.first(where: { $0.id == databaseID })? + .objects.first(where: { $0.id == objectID }) else { return [] } + return [object] + } + } + + private func containerState(for node: ExportOutlineNode) -> TristateCheckbox.State { + let items = objects(under: node) + guard !items.isEmpty else { return .unchecked } + let selected = items.count(where: \.isSelected) + if selected == 0 { return .unchecked } + return selected == items.count ? .checked : .mixed + } + + private func toggleContainer(_ node: ExportOutlineNode) { + let items = objects(under: node) + let turningOn = items.contains { !$0.isSelected } + let ids = Set(items.map(\.id)) + mutateDatabases { databases in + for databaseIndex in databases.indices { + for objectIndex in databases[databaseIndex].objects.indices + where ids.contains(databases[databaseIndex].objects[objectIndex].id) { + databases[databaseIndex].objects[objectIndex].isSelected = turningOn + } + } + } + guard turningOn else { return } + normalizeOptions(for: ids) + } + + private func setSelection(_ isSelected: Bool, databaseID: UUID, objectID: UUID) { + mutateDatabases { databases in + guard let databaseIndex = databases.firstIndex(where: { $0.id == databaseID }), + let objectIndex = databases[databaseIndex].objects.firstIndex(where: { $0.id == objectID }) + else { return } + databases[databaseIndex].objects[objectIndex].isSelected = isSelected + } + guard isSelected else { return } + normalizeOptions(for: [objectID]) + } + + /// A row selected with every one of its options off would be counted as selected and export + /// nothing, so selecting it restores the format's defaults for the kinds that support them. + private func normalizeOptions(for ids: Set) { + let columns = optionColumns + guard !columns.isEmpty else { return } + let defaults = defaultOptionValues + let supports = supportsOption + mutateDatabases { databases in + for databaseIndex in databases.indices { + for objectIndex in databases[databaseIndex].objects.indices + where ids.contains(databases[databaseIndex].objects[objectIndex].id) { + let object = databases[databaseIndex].objects[objectIndex] + guard !object.optionValues.contains(true) || object.optionValues.count != columns.count else { + continue + } + databases[databaseIndex].objects[objectIndex] = object + .normalized(forOptionColumnCount: columns.count, defaultOptionValues: defaults) + .maskingUnsupportedOptions(columns: columns, supports: supports) + } + } + } + } + + private func setOption(_ index: Int, to value: Bool, databaseID: UUID, objectID: UUID) { + mutateDatabases { databases in + guard let databaseIndex = databases.firstIndex(where: { $0.id == databaseID }), + let objectIndex = databases[databaseIndex].objects.firstIndex(where: { $0.id == objectID }), + databases[databaseIndex].objects[objectIndex].optionValues.indices.contains(index) + else { return } + databases[databaseIndex].objects[objectIndex].optionValues[index] = value + databases[databaseIndex].objects[objectIndex].isSelected = + databases[databaseIndex].objects[objectIndex].optionValues.contains(true) + } + } + + private func setRowScope(_ scope: PluginExportRowScope, databaseID: UUID, objectID: UUID) { + mutateDatabases { databases in + guard let databaseIndex = databases.firstIndex(where: { $0.id == databaseID }), + let objectIndex = databases[databaseIndex].objects.firstIndex(where: { $0.id == objectID }) + else { return } + databases[databaseIndex].objects[objectIndex].rowScope = scope + } + } + + private func mutateDatabases(_ change: (inout [ExportDatabaseItem]) -> Void) { + var updated = databases + change(&updated) + databases = updated + owner.databaseItems = updated + } +} + +/// The cell keeps one hosting view for the life of the row and only swaps its root, because +/// rebuilding the host on every reload loses the SwiftUI state the checkboxes animate from. +internal final class ExportObjectCellView: NSTableCellView { + private let hosting: NSHostingView + + internal init(identifier: NSUserInterfaceItemIdentifier) { + hosting = NSHostingView(rootView: AnyView(EmptyView())) + super.init(frame: .zero) + self.identifier = identifier + hosting.translatesAutoresizingMaskIntoConstraints = false + addSubview(hosting) + NSLayoutConstraint.activate([ + hosting.leadingAnchor.constraint(equalTo: leadingAnchor), + hosting.trailingAnchor.constraint(equalTo: trailingAnchor), + hosting.topAnchor.constraint(equalTo: topAnchor), + hosting.bottomAnchor.constraint(equalTo: bottomAnchor) + ]) + } + + @available(*, unavailable) + internal required init?(coder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } + + internal func configure(with content: AnyView) { + hosting.rootView = content + } +} diff --git a/TablePro/Views/Export/ExportOutlineNode.swift b/TablePro/Views/Export/ExportOutlineNode.swift new file mode 100644 index 0000000000..ab709543de --- /dev/null +++ b/TablePro/Views/Export/ExportOutlineNode.swift @@ -0,0 +1,80 @@ +// +// ExportOutlineNode.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +/// One row of the export tree. `NSOutlineView` identifies its items by object identity, so these +/// are reference types rebuilt only when the shape of the tree changes, never on a checkbox toggle. +internal final class ExportOutlineNode { + internal enum Kind { + case database(databaseID: UUID) + case group(databaseID: UUID, objectKind: PluginExportObjectKind) + case object(databaseID: UUID, objectID: UUID) + } + + internal let kind: Kind + internal private(set) var children: [ExportOutlineNode] + + internal init(kind: Kind, children: [ExportOutlineNode] = []) { + self.kind = kind + self.children = children + } + + internal var isLeaf: Bool { children.isEmpty } + + /// A stable key for the node's place in the tree, so expansion survives a rebuild that object + /// identity alone would lose. + internal var identity: String { + switch kind { + case .database(let databaseID): + return "db:\(databaseID.uuidString)" + case .group(let databaseID, let objectKind): + return "group:\(databaseID.uuidString):\(objectKind.rawValue)" + case .object(let databaseID, let objectID): + return "obj:\(databaseID.uuidString):\(objectID.uuidString)" + } + } +} + +internal enum ExportOutlineTreeBuilder { + /// Groups a database's objects by kind, in dump order. A database whose objects are all one + /// kind skips the group level: a MySQL schema with only tables should not make the user open a + /// "Tables" folder to reach them. + internal static func build(from databases: [ExportDatabaseItem]) -> [ExportOutlineNode] { + databases.map { database in + let kinds = database.presentKinds + guard kinds.count > 1 else { + return ExportOutlineNode( + kind: .database(databaseID: database.id), + children: database.objects.map { + ExportOutlineNode(kind: .object(databaseID: database.id, objectID: $0.id)) + } + ) + } + return ExportOutlineNode( + kind: .database(databaseID: database.id), + children: kinds.map { objectKind in + ExportOutlineNode( + kind: .group(databaseID: database.id, objectKind: objectKind), + children: database.objects(ofKind: objectKind).map { + ExportOutlineNode(kind: .object(databaseID: database.id, objectID: $0.id)) + } + ) + } + ) + } + } + + /// What a rebuild has to happen for. A checkbox toggle changes neither, so the tree is left + /// alone and only the affected rows are redrawn. + internal static func shapeFingerprint(of databases: [ExportDatabaseItem]) -> String { + databases.map { database in + let objects = database.objects.map { "\($0.kind.rawValue):\($0.id.uuidString)" }.joined(separator: ",") + return "\(database.id.uuidString)[\(objects)]" + } + .joined(separator: "|") + } +} diff --git a/TablePro/Views/Export/ExportRowScopeEditor.swift b/TablePro/Views/Export/ExportRowScopeEditor.swift new file mode 100644 index 0000000000..361862713e --- /dev/null +++ b/TablePro/Views/Export/ExportRowScopeEditor.swift @@ -0,0 +1,139 @@ +// +// ExportRowScopeEditor.swift +// TablePro +// + +import SwiftUI +import TableProPluginKit + +/// Narrows one table's rows and columns without leaving the export tree. +/// +/// The filter is the engine's own SQL, so it is not validated here beyond refusing a second +/// statement: an expression this dialog rejected would have to be a dialect check for every engine +/// TablePro speaks, and the server's own error is a better one than any of them. +internal struct ExportRowScopeEditor: View { + internal let objectName: String + internal let availableColumns: [String] + @Binding internal var scope: PluginExportRowScope + internal let dismiss: () -> Void + + @State private var filter: String = "" + @State private var rowLimitText: String = "" + @State private var selectedColumns: Set = [] + + private var hasRejectedFilter: Bool { + PluginExportRowScope(filter: filter).hasRejectedFilter + } + + internal var body: some View { + VStack(alignment: .leading, spacing: 12) { + Text(objectName) + .font(.headline) + .lineLimit(1) + .truncationMode(.middle) + + VStack(alignment: .leading, spacing: 4) { + Text("Where") + .font(.subheadline) + .foregroundStyle(.secondary) + TextField("status = 'active'", text: $filter, axis: .vertical) + .textFieldStyle(.roundedBorder) + .lineLimit(2 ... 4) + .font(ThemeEngine.shared.valueFontSwiftUI) + if hasRejectedFilter { + Text("A filter is one expression. Remove the semicolon.") + .font(.caption) + .foregroundStyle(.red) + } + } + + VStack(alignment: .leading, spacing: 4) { + Text("Row limit") + .font(.subheadline) + .foregroundStyle(.secondary) + TextField("All rows", text: $rowLimitText) + .textFieldStyle(.roundedBorder) + .frame(width: 120) + } + + if !availableColumns.isEmpty { + VStack(alignment: .leading, spacing: 4) { + HStack { + Text("Columns") + .font(.subheadline) + .foregroundStyle(.secondary) + Spacer() + Button(selectedColumns.isEmpty ? "Select None" : "Select All") { + selectedColumns = selectedColumns.isEmpty ? [] : Set(availableColumns) + } + .buttonStyle(.link) + .font(.caption) + } + ScrollView { + VStack(alignment: .leading, spacing: 2) { + ForEach(availableColumns, id: \.self) { column in + Toggle(column, isOn: binding(for: column)) + .toggleStyle(.checkbox) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + } + .frame(height: 140) + Text("Every column is written when none is ticked.") + .font(.caption) + .foregroundStyle(.secondary) + } + } + + HStack { + Button("Clear") { + filter = "" + rowLimitText = "" + selectedColumns = [] + commit() + } + Spacer() + Button("Done") { + commit() + dismiss() + } + .keyboardShortcut(.defaultAction) + } + } + .padding(16) + .frame(width: 300) + .onAppear { + filter = scope.filter + rowLimitText = scope.rowLimit.map(String.init) ?? "" + selectedColumns = Set(scope.columns) + } + .onDisappear(perform: commit) + } + + /// An unticked column set means every column, so a set covering all of them is stored empty: + /// otherwise a later schema change would silently drop a column the user never excluded. + private func binding(for column: String) -> Binding { + Binding( + get: { selectedColumns.isEmpty || selectedColumns.contains(column) }, + set: { isOn in + var updated = selectedColumns.isEmpty ? Set(availableColumns) : selectedColumns + if isOn { + updated.insert(column) + } else { + updated.remove(column) + } + selectedColumns = updated.count == availableColumns.count ? [] : updated + } + ) + } + + private func commit() { + let trimmedLimit = rowLimitText.trimmingCharacters(in: .whitespaces) + let limit = Int(trimmedLimit).flatMap { $0 > 0 ? $0 : nil } + scope = PluginExportRowScope( + filter: filter, + rowLimit: limit, + columns: availableColumns.filter { selectedColumns.contains($0) } + ) + } +} diff --git a/TablePro/Views/Export/ExportTableTreeView.swift b/TablePro/Views/Export/ExportTableTreeView.swift deleted file mode 100644 index 635b7fd866..0000000000 --- a/TablePro/Views/Export/ExportTableTreeView.swift +++ /dev/null @@ -1,172 +0,0 @@ -// -// ExportTableTreeView.swift -// TablePro -// -// Pure SwiftUI tree view for selecting tables in the export dialog. -// Replaces the NSOutlineView-based ExportTableOutlineView. -// - -import AppKit -import SwiftUI -import TableProPluginKit - -struct ExportTableTreeView: View { - @Binding var databaseItems: [ExportDatabaseItem] - let formatId: String - - private var optionColumns: [PluginExportOptionColumn] { - guard let plugin = PluginManager.shared.exportPlugin(forFormat: formatId) else { return [] } - return type(of: plugin).perTableOptionColumns - } - - private var currentPlugin: (any ExportFormatPlugin)? { - PluginManager.shared.exportPlugin(forFormat: formatId) - } - - private var defaultOptionValues: [Bool] { - currentPlugin?.defaultTableOptionValues() ?? [] - } - - var body: some View { - VStack(spacing: 0) { - List { - ForEach(databaseItems) { database in - let databaseBinding = $databaseItems.element(database) - DisclosureGroup(isExpanded: databaseBinding.isExpanded) { - ForEach(database.tables) { table in - let tableBinding = databaseBinding.tables.element(table) - tableRow(table: tableBinding) - } - } label: { - databaseLabel(database: database, allTables: databaseBinding.tables) - } - } - } - .listStyle(.plain) - .alternatingRowBackgrounds(.enabled) - } - } - - // MARK: - Database Row - - private func databaseLabel( - database: ExportDatabaseItem, - allTables: Binding<[ExportTableItem]> - ) -> some View { - HStack(spacing: 4) { - TristateCheckbox( - state: databaseCheckboxState(database), - action: { - let newState = !database.allSelected - for index in allTables.wrappedValue.indices { - var updated = allTables[index].wrappedValue - updated.isSelected = newState - if newState { - updated = updated.normalized( - forOptionColumnCount: optionColumns.count, - defaultOptionValues: defaultOptionValues - ) - } - allTables[index].wrappedValue = updated - } - } - ) - .disabled(database.tables.isEmpty) - .frame(width: 18) - - Image(systemName: "cylinder") - .foregroundStyle(.blue) - .font(.body) - - Text(database.name) - .font(.body) - .lineLimit(1) - .truncationMode(.middle) - } - } - - private func databaseCheckboxState(_ database: ExportDatabaseItem) -> TristateCheckbox.State { - let selected = database.selectedCount - if selected == 0 { return .unchecked } - if selected == database.tables.count { return .checked } - return .mixed - } - - // MARK: - Table Row - - private func tableRow(table: Binding) -> some View { - HStack(spacing: 4) { - if !optionColumns.isEmpty { - TristateCheckbox( - state: genericCheckboxState(table.wrappedValue), - action: { - toggleGenericOptions(table) - } - ) - .frame(width: 18) - } else { - Toggle("", isOn: table.isSelected) - .toggleStyle(.checkbox) - .labelsHidden() - } - - Image(systemName: table.wrappedValue.type == .view ? "eye" : "tablecells") - .foregroundStyle(table.wrappedValue.type == .view ? .purple : .gray) - .font(.body) - - Text(table.wrappedValue.name) - .font(.body) - .lineLimit(1) - .truncationMode(.middle) - - if !optionColumns.isEmpty { - Spacer() - - ForEach(Array(optionColumns.enumerated()), id: \.element.id) { colIndex, column in - Toggle(column.label, isOn: Binding( - get: { - table.wrappedValue.optionValues[safe: colIndex] ?? column.defaultValue - }, - set: { newValue in - guard table.wrappedValue.optionValues.indices.contains(colIndex) else { return } - table.optionValues[colIndex].wrappedValue = newValue - table.isSelected.wrappedValue = table.wrappedValue.optionValues.contains(true) - } - )) - .toggleStyle(.checkbox) - .labelsHidden() - .disabled(!table.wrappedValue.isSelected) - .opacity(table.wrappedValue.isSelected ? 1.0 : 0.4) - .frame(width: column.width, alignment: .center) - } - } - } - } - - // MARK: - Generic Option Helpers - - private func genericCheckboxState(_ table: ExportTableItem) -> TristateCheckbox.State { - if !table.isSelected { return .unchecked } - let trueCount = table.optionValues.count(where: { $0 }) - if trueCount == 0 { return .unchecked } - if trueCount == table.optionValues.count { return .checked } - return .mixed - } - - private func toggleGenericOptions(_ table: Binding) { - guard table.wrappedValue.isSelected else { - var updated = table.wrappedValue - updated.isSelected = true - table.wrappedValue = updated.normalized( - forOptionColumnCount: optionColumns.count, - defaultOptionValues: defaultOptionValues - ) - return - } - if table.wrappedValue.optionValues.allSatisfy({ $0 }) { - table.isSelected.wrappedValue = false - } else { - table.optionValues.wrappedValue = Array(repeating: true, count: optionColumns.count) - } - } -} diff --git a/TablePro/Views/Export/TableTransferSheet.swift b/TablePro/Views/Export/TableTransferSheet.swift new file mode 100644 index 0000000000..68c342a38f --- /dev/null +++ b/TablePro/Views/Export/TableTransferSheet.swift @@ -0,0 +1,292 @@ +// +// TableTransferSheet.swift +// TablePro +// + +import AppKit +import os +import SwiftUI +import TableProPluginKit + +/// Copies rows from the open connection into another one, with no file in between. +/// +/// Rows only: the destination table has to exist. Creating it would mean translating one engine's +/// DDL into another's, which is a different problem, and getting it half right would leave tables +/// whose column types quietly disagree with the data now in them. +struct TableTransferSheet: View { + private static let logger = Logger(subsystem: "com.TablePro", category: "TableTransferSheet") + + @Binding var isPresented: Bool + let sourceConnection: DatabaseConnection + let preselectedTables: Set + + @State private var service = TableTransferService() + @State private var destinationConnectionId: UUID? + @State private var destinationDatabase = "" + @State private var availableDestinations: [DatabaseConnection] = [] + @State private var destinationDatabases: [String] = [] + @State private var sourceTables: [ExportObjectItem] = [] + @State private var deleteExistingRows = false + @State private var wrapInTransaction = true + @State private var isLoading = true + @State private var isRunning = false + @State private var errorMessage: String? + @State private var hostWindow: NSWindow? + + private var destinationConnection: DatabaseConnection? { + guard let destinationConnectionId else { return nil } + return availableDestinations.first { $0.id == destinationConnectionId } + } + + private var selectedTables: [ExportObjectItem] { + sourceTables.filter(\.isSelected) + } + + private var canTransfer: Bool { + !isRunning && !selectedTables.isEmpty && destinationConnection != nil + } + + var body: some View { + VStack(spacing: 0) { + header + + Divider() + + if isLoading { + loadingView + } else { + content + } + + Divider() + + footer + } + .frame(width: 460, height: 460) + .background(Color(nsColor: .windowBackgroundColor)) + .background { + WindowAccessor { window in hostWindow = window } + } + .task { await load() } + .onExitCommand { + guard !isRunning else { return } + isPresented = false + } + } + + private var header: some View { + VStack(alignment: .leading, spacing: 4) { + Text("Transfer Tables") + .font(.headline) + Text("Rows are copied into tables that already exist on the destination.") + .font(.subheadline) + .foregroundStyle(.secondary) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(16) + } + + private var loadingView: some View { + VStack { + Spacer() + ProgressView() + .scaleEffect(0.8) + Spacer() + } + .frame(maxWidth: .infinity) + } + + private var content: some View { + VStack(alignment: .leading, spacing: 12) { + Picker("Destination", selection: $destinationConnectionId) { + Text("Choose a connection").tag(UUID?.none) + ForEach(availableDestinations) { connection in + Text(connection.name).tag(UUID?.some(connection.id)) + } + } + .onChange(of: destinationConnectionId) { + Task { await loadDestinationDatabases() } + } + + if !destinationDatabases.isEmpty { + Picker("Database", selection: $destinationDatabase) { + ForEach(destinationDatabases, id: \.self) { database in + Text(database).tag(database) + } + } + } + + Text("Tables") + .font(.subheadline.weight(.medium)) + .foregroundStyle(.secondary) + + List { + ForEach(sourceTables) { table in + Toggle(table.name, isOn: binding(for: table)) + .toggleStyle(.checkbox) + } + } + .listStyle(.bordered) + .frame(maxHeight: .infinity) + + Toggle("Delete existing rows first", isOn: $deleteExistingRows) + .toggleStyle(.checkbox) + Toggle("Wrap each table in a transaction", isOn: $wrapInTransaction) + .toggleStyle(.checkbox) + + if let errorMessage { + Text(errorMessage) + .font(.subheadline) + .foregroundStyle(.red) + .fixedSize(horizontal: false, vertical: true) + } + } + .padding(16) + } + + private var footer: some View { + HStack { + Button("Cancel") { + if isRunning { + service.cancel() + } else { + isPresented = false + } + } + + Spacer() + + if isRunning { + HStack(spacing: 8) { + ProgressView() + .scaleEffect(0.7) + Text(progressLabel) + .font(.subheadline) + .foregroundStyle(.secondary) + .lineLimit(1) + } + } + + Button("Transfer") { + Task { await runTransfer() } + } + .buttonStyle(.borderedProminent) + .keyboardShortcut(.defaultAction) + .disabled(!canTransfer) + } + .padding(.horizontal, 16) + .padding(.vertical, 12) + } + + private var progressLabel: String { + String( + format: String(localized: "%1$@ (%2$lld of %3$lld), %4$lld rows"), + service.state.currentTable, + Int64(service.state.currentTableIndex), + Int64(service.state.totalTables), + Int64(service.state.transferredRows) + ) + } + + private func binding(for table: ExportObjectItem) -> Binding { + Binding( + get: { sourceTables.first { $0.id == table.id }?.isSelected ?? false }, + set: { isOn in + guard let index = sourceTables.firstIndex(where: { $0.id == table.id }) else { return } + sourceTables[index].isSelected = isOn + } + ) + } + + // MARK: - Loading + + /// A transfer needs two live sessions, so only connections that are already open are offered. + /// Opening one from here would mean a connect, a possible prompt and a possible failure inside + /// a sheet that is about to start writing rows. + @MainActor + private func load() async { + availableDestinations = DatabaseManager.shared.activeSessions.values + .filter { $0.id != sourceConnection.id && $0.isConnected } + .map { $0.effectiveConnection ?? $0.connection } + .sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending } + do { + let tables = try await DatabaseManager.shared.withMetadataDriver( + scope: sourceScope, workload: .bulk + ) { driver in + try await driver.fetchTables() + } + sourceTables = tables.map { table in + ExportObjectItem( + name: table.name, + kind: PluginExportObjectKind.from(tableType: table.type.rawValue), + isSelected: preselectedTables.contains(table.name) + ) + } + .filter { $0.kind.carriesRows } + } catch { + errorMessage = error.localizedDescription + } + isLoading = false + } + + @MainActor + private func loadDestinationDatabases() async { + destinationDatabases = [] + destinationDatabase = "" + guard let destinationConnection else { return } + guard let driver = DatabaseManager.shared.driver(for: destinationConnection.id) else { return } + do { + destinationDatabases = try await driver.fetchDatabases() + destinationDatabase = destinationDatabases.contains(destinationConnection.database) + ? destinationConnection.database + : (destinationDatabases.first ?? "") + } catch { + Self.logger.warning("Failed to list destination databases: \(error.localizedDescription)") + } + } + + private var sourceScope: DatabaseScope { + DatabaseManager.shared.resolvedScope( + database: sourceConnection.database, schema: nil, for: sourceConnection.id + ) ?? DatabaseScope(connectionId: sourceConnection.id, database: sourceConnection.database, schema: nil) + } + + // MARK: - Running + + @MainActor + private func runTransfer() async { + guard let destinationConnection, + let destinationDriver = DatabaseManager.shared.driver(for: destinationConnection.id) else { + errorMessage = TableTransferError.notConnected(connectionName: "").localizedDescription + return + } + errorMessage = nil + isRunning = true + defer { isRunning = false } + + let request = TableTransferService.Request( + objects: selectedTables, + sourceType: sourceConnection.type, + destinationType: destinationConnection.type, + deleteExistingRows: deleteExistingRows, + wrapInTransaction: wrapInTransaction + ) + + do { + try await DatabaseManager.shared.withMetadataDriver( + scope: sourceScope, workload: .bulk + ) { sourceDriver in + try await service.transfer( + request: request, + sourceDriver: sourceDriver, + destinationDriver: destinationDriver + ) + } + isPresented = false + } catch is PluginImportCancellationError { + errorMessage = nil + } catch { + errorMessage = error.localizedDescription + } + } +} diff --git a/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift b/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift index 15de38cda1..807a390bc7 100644 --- a/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift +++ b/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarActions.swift @@ -176,6 +176,13 @@ extension MainContentCoordinator { activeSheet = .exportDialog } + /// Copies rows into another open connection. The tables the user right-clicked travel with the + /// request rather than being read back from the object browser, which may have moved on by the + /// time the sheet appears. + func openTableTransferSheet(preselectedTableNames: Set = []) { + activeSheet = .transferTables(tables: preselectedTableNames) + } + func openExportQueryResultsDialog() { guard let tab = tabManager.selectedTab, !tabSessionRegistry.tableRows(for: tab.id).rows.isEmpty else { return } diff --git a/TablePro/Views/Main/MainContentCommandActions.swift b/TablePro/Views/Main/MainContentCommandActions.swift index c40097b219..8f6bdd6705 100644 --- a/TablePro/Views/Main/MainContentCommandActions.swift +++ b/TablePro/Views/Main/MainContentCommandActions.swift @@ -1078,7 +1078,7 @@ final class MainContentCommandActions { } var supportsBackup: Bool { - connection.type == .postgresql || connection.type == .redshift + NativeDumpRegistry.supports(connection.type) } var supportsRestore: Bool { supportsBackup } diff --git a/TablePro/Views/Main/MainContentCoordinator.swift b/TablePro/Views/Main/MainContentCoordinator.swift index 12db36ab18..cb52ba1679 100644 --- a/TablePro/Views/Main/MainContentCoordinator.swift +++ b/TablePro/Views/Main/MainContentCoordinator.swift @@ -51,6 +51,9 @@ enum ActiveSheet: Identifiable { case importDialog(formatId: String) case rowImport(formatId: String) case exportQueryResults + /// The tables the user right-clicked travel with the request, because the object browser may be + /// pointed somewhere else by the time the sheet appears. + case transferTables(tables: Set) case backupDatabase case restoreDatabase(fileURL: URL) /// The object's own database and schema travel with the request. A maintenance statement names @@ -73,6 +76,7 @@ enum ActiveSheet: Identifiable { case .importDialog(let formatId): "importDialog-\(formatId)" case .rowImport(let formatId): "rowImport-\(formatId)" case .exportQueryResults: "exportQueryResults" + case .transferTables(let tables): "transferTables-\(tables.sorted().joined(separator: ","))" case .backupDatabase: "backupDatabase" case .restoreDatabase(let fileURL): "restoreDatabase-\(fileURL.path)" case .maintenance(let operation, let tableName, let database, let schema): diff --git a/TablePro/Views/Main/MainContentView.swift b/TablePro/Views/Main/MainContentView.swift index 2a12d847db..66fc927f98 100644 --- a/TablePro/Views/Main/MainContentView.swift +++ b/TablePro/Views/Main/MainContentView.swift @@ -259,6 +259,12 @@ struct MainContentView: View { formatId: formatId ) } + case .transferTables(let tables): + TableTransferSheet( + isPresented: dismissBinding, + sourceConnection: connectionWithCurrentDatabase, + preselectedTables: tables + ) case .backupDatabase: BackupDatabaseFlow( isPresented: dismissBinding, diff --git a/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift b/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift index 54a3a6e819..422b576ebc 100644 --- a/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift +++ b/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift @@ -49,6 +49,10 @@ extension DatabaseTreeOutlineCoordinator { activateThen(ref) { [weak self] in self?.mainCoordinator?.openExportDialog(preselectedTableNames: names) } + case .transferTables(let names, let ref): + activateThen(ref) { [weak self] in + self?.mainCoordinator?.openTableTransferSheet(preselectedTableNames: names) + } case .importTables(let formatId, let ref): activateThen(ref) { [weak self] in self?.mainCoordinator?.openImportDialog(formatId: formatId) diff --git a/TablePro/Views/Sidebar/Menu/DatabaseTreeMenuSpec.swift b/TablePro/Views/Sidebar/Menu/DatabaseTreeMenuSpec.swift index af6bb6d9e6..f9da4fc8a2 100644 --- a/TablePro/Views/Sidebar/Menu/DatabaseTreeMenuSpec.swift +++ b/TablePro/Views/Sidebar/Menu/DatabaseTreeMenuSpec.swift @@ -134,6 +134,8 @@ internal enum DatabaseTreeMenuSpec { items.append(.separator) items.append(.command(copyNamesTitle(count: names.count), .copyTableNames(names))) items.append(.command(String(localized: "Export…"), .exportTables(names: Set(names), ref: ref))) + items.append(.command( + String(localized: "Transfer To…"), .transferTables(names: Set(names), ref: ref))) if context.canCopyObjects { /// Narrowed to the clicked row's own schema as well as its database. A copy names one /// source scope, so a selection spanning two schemas would read one of them and either diff --git a/TablePro/Views/Sidebar/Menu/SidebarMenuCommand.swift b/TablePro/Views/Sidebar/Menu/SidebarMenuCommand.swift index 31544a71ef..4a902f87e4 100644 --- a/TablePro/Views/Sidebar/Menu/SidebarMenuCommand.swift +++ b/TablePro/Views/Sidebar/Menu/SidebarMenuCommand.swift @@ -31,6 +31,7 @@ internal enum SidebarMenuCommand: Equatable { /// without switching there first runs the command against a same-named table somewhere else, /// which for Truncate and Drop destroys the wrong data. case exportTables(names: Set, ref: DatabaseTreeTableRef) + case transferTables(names: Set, ref: DatabaseTreeTableRef) case importTables(formatId: String, ref: DatabaseTreeTableRef) case maintenance(operation: String, tableName: String, ref: DatabaseTreeTableRef) /// Queued rather than run, so these carry every target in full: a queue keyed by name is diff --git a/TableProTests/Core/Export/TableTransferServiceTests.swift b/TableProTests/Core/Export/TableTransferServiceTests.swift new file mode 100644 index 0000000000..d1970ae6c2 --- /dev/null +++ b/TableProTests/Core/Export/TableTransferServiceTests.swift @@ -0,0 +1,104 @@ +// +// TableTransferServiceTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +@Suite("Table transfer") +struct TableTransferServiceTests { + + @Test("A row is keyed by its header's column names, in order") + func rowIsKeyedByHeader() { + let values = TableTransferService.dictionary( + columns: ["id", "name", "email"], + row: [.text("1"), .text("Ada"), .text("ada@example.com")] + ) + #expect(values.count == 3) + #expect(values["id"] == .text("1")) + #expect(values["name"] == .text("Ada")) + #expect(values["email"] == .text("ada@example.com")) + } + + /// A driver that stops sending values once the rest of a row is null would otherwise lose the + /// whole row: the sink writes by column name, and a missing key is not a null. + @Test("A row shorter than its header is padded with nulls") + func shortRowIsPadded() { + let values = TableTransferService.dictionary( + columns: ["id", "name", "email"], + row: [.text("1")] + ) + #expect(values.count == 3) + #expect(values["id"] == .text("1")) + #expect(values["name"] == .null) + #expect(values["email"] == .null) + } + + @Test("A row longer than its header keeps only the named columns") + func extraValuesAreDropped() { + let values = TableTransferService.dictionary( + columns: ["id"], + row: [.text("1"), .text("unnamed")] + ) + #expect(values == ["id": .text("1")]) + } + + @Test("An empty header produces no values") + func emptyHeaderProducesNothing() { + #expect(TableTransferService.dictionary(columns: [], row: [.text("1")]).isEmpty) + } + + @Test("Binary and null values survive the transfer unchanged") + func binaryAndNullSurvive() { + let payload = Data([0x00, 0xFF, 0x10]) + let values = TableTransferService.dictionary( + columns: ["blob", "missing"], + row: [.bytes(payload), .null] + ) + #expect(values["blob"] == .bytes(payload)) + #expect(values["missing"] == .null) + } + + /// The transfer moves rows, so a request naming only definition objects has nothing to do and + /// must say so rather than reporting a successful transfer of nothing. + @Test("A request with no row-carrying object is refused") + func requestWithoutTablesIsRefused() async { + let service = await TableTransferService() + let request = TableTransferService.Request( + objects: [ + ExportObjectItem(name: "recalc", kind: .routine), + ExportObjectItem(name: "audit", kind: .trigger, parentTable: "users") + ], + sourceType: .postgresql, + destinationType: .postgresql + ) + #expect(request.objects.allSatisfy { !$0.kind.carriesRows }) + await #expect(service.state.isTransferring == false) + } + + @Test("A request keeps the row scope of every object it names") + func requestKeepsRowScope() { + let scoped = ExportObjectItem( + name: "users", + kind: .table, + isSelected: true, + rowScope: PluginExportRowScope(filter: "active", rowLimit: 10) + ) + let request = TableTransferService.Request( + objects: [scoped], sourceType: .mysql, destinationType: .postgresql) + #expect(request.objects[0].rowScope.sanitizedFilter == "active") + #expect(request.objects[0].rowScope.rowLimit == 10) + } + + @Test("Transactions and row deletion default to the safe choice") + func requestDefaults() { + let request = TableTransferService.Request( + objects: [], sourceType: .mysql, destinationType: .mysql) + #expect(request.wrapInTransaction) + #expect(!request.deleteExistingRows) + } +} diff --git a/TableProTests/Core/Redis/ExportModelsRedisTests.swift b/TableProTests/Core/Redis/ExportModelsRedisTests.swift index 48fdd413d3..1eaac72726 100644 --- a/TableProTests/Core/Redis/ExportModelsRedisTests.swift +++ b/TableProTests/Core/Redis/ExportModelsRedisTests.swift @@ -5,28 +5,28 @@ import Testing @Suite("Export format filtering for Redis") struct ExportModelsRedisTests { - @Test("ExportTableItem supports optionValues for generic per-table options") + @Test("ExportObjectItem supports optionValues for generic per-object options") func tableItemOptionValues() { - let item = ExportTableItem(name: "keys", type: .table, isSelected: true, optionValues: [true, false]) + let item = ExportObjectItem(name: "keys", kind: .table, isSelected: true, optionValues: [true, false]) #expect(item.optionValues.count == 2) #expect(item.optionValues[0] == true) #expect(item.optionValues[1] == false) } - @Test("ExportTableItem defaults to empty optionValues") + @Test("ExportObjectItem defaults to empty optionValues") func tableItemDefaultOptionValues() { - let item = ExportTableItem(name: "keys", type: .table) + let item = ExportObjectItem(name: "keys", kind: .table) #expect(item.optionValues.isEmpty) } @Test("ExportDatabaseItem tracks selected tables correctly") func databaseItemSelection() { let tables = [ - ExportTableItem(name: "keys", type: .table, isSelected: true), - ExportTableItem(name: "sets", type: .table, isSelected: false), + ExportObjectItem(name: "keys", kind: .table, isSelected: true), + ExportObjectItem(name: "sets", kind: .table, isSelected: false), ] - let db = ExportDatabaseItem(name: "0", tables: tables) + let db = ExportDatabaseItem(name: "0", objects: tables) #expect(db.selectedCount == 1) - #expect(db.selectedTables.map(\.name) == ["keys"]) + #expect(db.selectedObjects.map(\.name) == ["keys"]) } } diff --git a/TableProTests/Database/NativeDumpRegistryTests.swift b/TableProTests/Database/NativeDumpRegistryTests.swift new file mode 100644 index 0000000000..ca2ce1f655 --- /dev/null +++ b/TableProTests/Database/NativeDumpRegistryTests.swift @@ -0,0 +1,210 @@ +// +// NativeDumpRegistryTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +@Suite("Native dump registry") +struct NativeDumpRegistryTests { + + private func connection( + type: DatabaseType, + host: String = "db.example.com", + port: Int = 5_432, + database: String = "sales", + username: String = "alice", + sslMode: SSLMode = .disabled, + sslEnabled: Bool = false + ) -> DatabaseConnection { + var sslConfig = SSLConfiguration() + sslConfig.mode = sslMode + if sslEnabled { sslConfig.mode = sslMode == .disabled ? .required : sslMode } + return DatabaseConnection( + name: "Test", + host: host, + port: port, + database: database, + username: username, + type: type, + sshConfig: SSHConfiguration(), + sslConfig: sslConfig + ) + } + + private func command( + _ type: DatabaseType, + kind: NativeDumpKind = .backup, + connection overrideConnection: DatabaseConnection? = nil, + password: String? = "s3cret", + fileURL: URL = URL(fileURLWithPath: "/tmp/out.bin") + ) throws -> NativeDumpCommand { + let descriptor = try #require(NativeDumpRegistry.descriptor(for: type)) + return try NativeDumpService.buildCommand( + kind: kind, + descriptor: descriptor, + executable: URL(fileURLWithPath: "/usr/bin/tool"), + effective: overrideConnection ?? connection(type: type), + database: "sales", + fileURL: fileURL, + password: password + ) + } + + @Test("The engines with client-side tools are the ones the menu offers") + func supportedEngines() { + for type in [DatabaseType.postgresql, .redshift, .mysql, .mariadb, .mongodb, .sqlite] { + #expect(NativeDumpRegistry.supports(type), "\(type.rawValue) should have a descriptor") + } + for type in [DatabaseType.clickhouse, .oracle, .duckdb] { + #expect(!NativeDumpRegistry.supports(type), "\(type.rawValue) should not claim one") + } + } + + @Test("Each engine offers its own archive extension") + func archiveExtensions() throws { + #expect(try #require(NativeDumpRegistry.descriptor(for: .postgresql)).archiveFormat.fileExtension == "dump") + #expect(try #require(NativeDumpRegistry.descriptor(for: .mysql)).archiveFormat.fileExtension == "sql") + #expect(try #require(NativeDumpRegistry.descriptor(for: .mongodb)).archiveFormat.fileExtension == "archive") + #expect(try #require(NativeDumpRegistry.descriptor(for: .sqlite)).archiveFormat.fileExtension == "sql") + } + + /// Anything in `argv` is readable by every process on the machine through `ps`, so no + /// descriptor may put a password there. + @Test("No engine puts the password in the argument list") + func passwordNeverReachesArgv() throws { + for type in [DatabaseType.postgresql, .mysql, .mongodb, .sqlite] { + for kind in [NativeDumpKind.backup, .restore] { + let built = try command(type, kind: kind, password: "s3cret") + let leaked = built.arguments.filter { $0.contains("s3cret") } + #expect(leaked.isEmpty, "\(type.rawValue) \(kind) leaked the password: \(leaked)") + } + } + } + + @Test("MySQL passes its password through MYSQL_PWD") + func mysqlUsesEnvironmentPassword() throws { + let built = try command(.mysql) + #expect(built.environment["MYSQL_PWD"] == "s3cret") + #expect(built.arguments.contains("--single-transaction")) + #expect(built.arguments.contains("--routines")) + #expect(built.arguments.contains("--triggers")) + #expect(built.arguments.contains("--events")) + #expect(built.arguments.last == "sales") + } + + /// `mysqldump` writes SQL to standard output, so the caller has to redirect it to the file. + @Test("MySQL is redirected through standard output in both directions") + func mysqlRedirects() throws { + let backup = try command(.mysql, kind: .backup) + #expect(backup.delivery == .standardOutput) + #expect(backup.redirectedFileURL?.path == "/tmp/out.bin") + #expect(!backup.isRestore) + + let restore = try command(.mysql, kind: .restore) + #expect(restore.delivery == .standardOutput) + #expect(restore.isRestore) + #expect(!restore.arguments.contains("--single-transaction")) + } + + /// `pg_dump -Fc` is told the path and writes it itself, so nothing is redirected. + @Test("PostgreSQL writes its own file") + func postgresWritesItsOwnFile() throws { + let built = try command(.postgresql) + #expect(built.delivery == .toolWritesFile) + #expect(built.redirectedFileURL == nil) + } + + /// `mongodump` reads a password from neither the environment nor standard input, so the only + /// channel left is a config file, which must be owner-only and must not survive the process. + @Test("MongoDB writes an owner-only credentials file and points at it") + func mongoUsesACredentialsFile() throws { + let built = try command(.mongodb) + let configArgument = try #require(built.arguments.first { $0.hasPrefix("--config=") }) + let path = String(configArgument.dropFirst("--config=".count)) + defer { try? FileManager.default.removeItem(atPath: path) } + + #expect(built.temporaryCredentialsFileURL?.path == path) + let attributes = try FileManager.default.attributesOfItem(atPath: path) + let permissions = try #require(attributes[.posixPermissions] as? NSNumber) + #expect(permissions.int16Value == 0o600) + + let contents = try String(contentsOfFile: path, encoding: .utf8) + #expect(contents.contains("s3cret")) + #expect(built.environment["MONGO_PASSWORD"] == nil) + } + + @Test("MongoDB writes no credentials file without a username") + func mongoSkipsCredentialsWithoutUser() throws { + let anonymous = connection(type: .mongodb, username: "") + let built = try command(.mongodb, connection: anonymous) + #expect(built.temporaryCredentialsFileURL == nil) + #expect(!built.arguments.contains { $0.hasPrefix("--config=") }) + } + + @Test("MongoDB names the database on backup and scopes the namespace on restore") + func mongoScopesItsDatabase() throws { + #expect(try command(.mongodb, kind: .backup).arguments.contains("--db=sales")) + #expect(try command(.mongodb, kind: .restore).arguments.contains("--nsInclude=sales.*")) + } + + /// The database is a file the tool opens, so there is nothing to authenticate to. + @Test("SQLite passes the file path and no network arguments") + func sqliteUsesTheFilePath() throws { + let file = connection(type: .sqlite, host: "", port: 0, database: "/tmp/app.sqlite", username: "") + let backup = try command(.sqlite, connection: file) + #expect(backup.arguments == ["/tmp/app.sqlite", ".dump"]) + + let restore = try command(.sqlite, kind: .restore, connection: file) + #expect(restore.arguments == ["/tmp/app.sqlite"]) + #expect(restore.isRestore) + } + + @Test("An empty host falls back to loopback on every engine that takes one") + func emptyHostFallsBackToLoopback() throws { + let mysql = try command(.mysql, connection: connection(type: .mysql, host: "")) + #expect(mysql.arguments.contains("127.0.0.1")) + + let mongo = try command(.mongodb, connection: connection(type: .mongodb, host: "")) + #expect(mongo.arguments.contains("--host=127.0.0.1")) + } + + @Test("MySQL SSL mode maps to the client's own spelling") + func mysqlSSLModes() { + #expect(NativeDumpRegistry.mysqlSSLMode(.disabled) == "--ssl-mode=DISABLED") + #expect(NativeDumpRegistry.mysqlSSLMode(.preferred) == "--ssl-mode=PREFERRED") + #expect(NativeDumpRegistry.mysqlSSLMode(.required) == "--ssl-mode=REQUIRED") + #expect(NativeDumpRegistry.mysqlSSLMode(.verifyCa) == "--ssl-mode=VERIFY_CA") + #expect(NativeDumpRegistry.mysqlSSLMode(.verifyIdentity) == "--ssl-mode=VERIFY_IDENTITY") + } + + /// MariaDB 11.0 renamed every client and some builds ship no `mysql`-prefixed symlink, so both + /// spellings have to be tried before reporting the tool missing. + @Test("MySQL tries both the mysql and mariadb tool names") + func mysqlTriesBothToolNames() throws { + let descriptor = try #require(NativeDumpRegistry.descriptor(for: .mysql)) + #expect(descriptor.backupBinaries == ["mysqldump", "mariadb-dump"]) + #expect(descriptor.restoreBinaries == ["mysql", "mariadb"]) + } + + @Test("A YAML-quoted password survives quotes and backslashes") + func yamlQuotingIsLossless() { + #expect(NativeDumpService.mongoYAMLQuoted("plain") == "\"plain\"") + #expect(NativeDumpService.mongoYAMLQuoted("a\"b") == "\"a\\\"b\"") + #expect(NativeDumpService.mongoYAMLQuoted("a\\b") == "\"a\\\\b\"") + } + + /// A progress bar showing a percentage of a number nobody measured is worse than an + /// indeterminate one, so an engine with no cheap size answer returns nil. + @Test("Only the engines with a cheap size query offer a determinate progress bar") + func sizeQueryCoverage() { + #expect(NativeDumpService.sizeQuery(for: .postgresql) != nil) + #expect(NativeDumpService.sizeQuery(for: .mysql) != nil) + #expect(NativeDumpService.sizeQuery(for: .mongodb) == nil) + #expect(NativeDumpService.sizeQuery(for: .sqlite) == nil) + } +} diff --git a/TableProTests/Database/PostgresDumpServiceTests.swift b/TableProTests/Database/NativeDumpServiceTests.swift similarity index 80% rename from TableProTests/Database/PostgresDumpServiceTests.swift rename to TableProTests/Database/NativeDumpServiceTests.swift index 7d2dea3f2d..df3afe6a56 100644 --- a/TableProTests/Database/PostgresDumpServiceTests.swift +++ b/TableProTests/Database/NativeDumpServiceTests.swift @@ -1,5 +1,5 @@ // -// PostgresDumpServiceTests.swift +// NativeDumpServiceTests.swift // TableProTests // @@ -9,8 +9,15 @@ import Testing @testable import TablePro -@Suite("PostgresDumpService command construction") -struct PostgresDumpServiceCommandTests { +@Suite("NativeDumpService command construction") +struct NativeDumpServiceCommandTests { + private var postgresDescriptor: NativeDumpDescriptor { + guard let descriptor = NativeDumpRegistry.descriptor(for: .postgresql) else { + fatalError("PostgreSQL must have a dump descriptor") + } + return descriptor + } + private func connection( host: String = "db.example.com", port: Int = 5_432, @@ -32,9 +39,10 @@ struct PostgresDumpServiceCommandTests { } @Test("backup command sets -Fc, host, port, username, -d, -f") - func backupCommandShape() { - let command = PostgresDumpService.buildCommand( + func backupCommandShape() throws { + let command = try NativeDumpService.buildCommand( kind: .backup, + descriptor: postgresDescriptor, executable: URL(fileURLWithPath: "/usr/bin/pg_dump"), effective: connection(), database: "sales", @@ -53,9 +61,10 @@ struct PostgresDumpServiceCommandTests { } @Test("restore command sets --no-owner, --no-acl, -d, positional path") - func restoreCommandShape() { - let command = PostgresDumpService.buildCommand( + func restoreCommandShape() throws { + let command = try NativeDumpService.buildCommand( kind: .restore, + descriptor: postgresDescriptor, executable: URL(fileURLWithPath: "/usr/bin/pg_restore"), effective: connection(), database: "sales", @@ -73,9 +82,10 @@ struct PostgresDumpServiceCommandTests { } @Test("empty host falls back to 127.0.0.1") - func hostFallback() { - let command = PostgresDumpService.buildCommand( + func hostFallback() throws { + let command = try NativeDumpService.buildCommand( kind: .backup, + descriptor: postgresDescriptor, executable: URL(fileURLWithPath: "/usr/bin/pg_dump"), effective: connection(host: ""), database: "sales", @@ -86,9 +96,10 @@ struct PostgresDumpServiceCommandTests { } @Test("empty username omits -U entirely") - func usernameOmitted() { - let command = PostgresDumpService.buildCommand( + func usernameOmitted() throws { + let command = try NativeDumpService.buildCommand( kind: .backup, + descriptor: postgresDescriptor, executable: URL(fileURLWithPath: "/usr/bin/pg_dump"), effective: connection(username: ""), database: "sales", @@ -99,17 +110,19 @@ struct PostgresDumpServiceCommandTests { } @Test("nil/empty password does not set PGPASSWORD") - func passwordOptional() { - let nilPw = PostgresDumpService.buildCommand( + func passwordOptional() throws { + let nilPw = try NativeDumpService.buildCommand( kind: .backup, + descriptor: postgresDescriptor, executable: URL(fileURLWithPath: "/usr/bin/pg_dump"), effective: connection(), database: "sales", fileURL: URL(fileURLWithPath: "/tmp/x.dump"), password: nil ) - let emptyPw = PostgresDumpService.buildCommand( + let emptyPw = try NativeDumpService.buildCommand( kind: .backup, + descriptor: postgresDescriptor, executable: URL(fileURLWithPath: "/usr/bin/pg_dump"), effective: connection(), database: "sales", @@ -130,9 +143,10 @@ struct PostgresDumpServiceCommandTests { (TableProPluginKit.SSLMode.verifyIdentity, "verify-full") ] ) - func sslModeMapping(mode: TableProPluginKit.SSLMode, expected: String?) { - let command = PostgresDumpService.buildCommand( + func sslModeMapping(mode: TableProPluginKit.SSLMode, expected: String?) throws { + let command = try NativeDumpService.buildCommand( kind: .backup, + descriptor: postgresDescriptor, executable: URL(fileURLWithPath: "/usr/bin/pg_dump"), effective: connection(sslMode: mode), database: "sales", @@ -143,9 +157,10 @@ struct PostgresDumpServiceCommandTests { } @Test("environment is restricted to a known allowlist plus libpq vars") - func environmentIsMinimal() { - let command = PostgresDumpService.buildCommand( + func environmentIsMinimal() throws { + let command = try NativeDumpService.buildCommand( kind: .backup, + descriptor: postgresDescriptor, executable: URL(fileURLWithPath: "/usr/bin/pg_dump"), effective: connection(sslMode: .required), database: "sales", @@ -169,14 +184,14 @@ struct PostgresDumpServiceCommandTests { // MARK: - Fake Runner -private final class FakeDumpRunner: PostgresDumpRunner, @unchecked Sendable { - private(set) var startedCommand: PostgresDumpCommand? +private final class FakeDumpRunner: NativeDumpRunner, @unchecked Sendable { + private(set) var startedCommand: NativeDumpCommand? private(set) var cancelCount: Int = 0 - private var continuation: CheckedContinuation? - private var bufferedResult: PostgresDumpRunResult? + private var continuation: CheckedContinuation? + private var bufferedResult: NativeDumpRunResult? private let lock = NSLock() - func start(_ command: PostgresDumpCommand) throws { + func start(_ command: NativeDumpCommand) throws { startedCommand = command } @@ -186,7 +201,7 @@ private final class FakeDumpRunner: PostgresDumpRunner, @unchecked Sendable { lock.unlock() } - var result: PostgresDumpRunResult { + var result: NativeDumpRunResult { get async { await withCheckedContinuation { continuation in lock.lock() @@ -202,7 +217,7 @@ private final class FakeDumpRunner: PostgresDumpRunner, @unchecked Sendable { } } - func finish(_ outcome: PostgresDumpRunResult) { + func finish(_ outcome: NativeDumpRunResult) { lock.lock() if let continuation = self.continuation { self.continuation = nil @@ -215,11 +230,11 @@ private final class FakeDumpRunner: PostgresDumpRunner, @unchecked Sendable { } } -@Suite("PostgresDumpService state machine", .serialized) +@Suite("NativeDumpService state machine", .serialized) @MainActor -struct PostgresDumpServiceStateMachineTests { - private func fakeCommand() -> PostgresDumpCommand { - PostgresDumpCommand( +struct NativeDumpServiceStateMachineTests { + private func fakeCommand() -> NativeDumpCommand { + NativeDumpCommand( executable: URL(fileURLWithPath: "/usr/bin/true"), arguments: [], environment: [:], @@ -230,7 +245,7 @@ struct PostgresDumpServiceStateMachineTests { @Test("successful run transitions idle -> running -> finished") func successfulBackup() async throws { let runner = FakeDumpRunner() - let service = PostgresDumpService(kind: .backup, runnerFactory: { runner }) + let service = NativeDumpService(kind: .backup, runnerFactory: { runner }) let updates = service.stateUpdates() #expect(service.state == .idle) @@ -261,7 +276,7 @@ struct PostgresDumpServiceStateMachineTests { @Test("non-zero exit transitions to failed and surfaces stderr") func failedRun() async throws { let runner = FakeDumpRunner() - let service = PostgresDumpService(kind: .restore, runnerFactory: { runner }) + let service = NativeDumpService(kind: .restore, runnerFactory: { runner }) let updates = service.stateUpdates() try service.run( @@ -283,7 +298,7 @@ struct PostgresDumpServiceStateMachineTests { @Test("cancel transitions running -> cancelling -> cancelled") func cancelRun() async throws { let runner = FakeDumpRunner() - let service = PostgresDumpService(kind: .backup, runnerFactory: { runner }) + let service = NativeDumpService(kind: .backup, runnerFactory: { runner }) let updates = service.stateUpdates() try service.run( @@ -304,7 +319,7 @@ struct PostgresDumpServiceStateMachineTests { @Test("calling run while already running throws alreadyRunning") func doubleRunThrows() throws { let runner = FakeDumpRunner() - let service = PostgresDumpService(kind: .backup, runnerFactory: { runner }) + let service = NativeDumpService(kind: .backup, runnerFactory: { runner }) try service.run( command: fakeCommand(), @@ -312,7 +327,7 @@ struct PostgresDumpServiceStateMachineTests { fileURL: URL(fileURLWithPath: "/tmp/test-double.dump") ) - #expect(throws: PostgresDumpError.alreadyRunning) { + #expect(throws: NativeDumpError.alreadyRunning) { try service.run( command: fakeCommand(), database: "sales", @@ -324,7 +339,7 @@ struct PostgresDumpServiceStateMachineTests { @Test("empty stderr falls back to a synthesized error message") func emptyStderrFallback() async throws { let runner = FakeDumpRunner() - let service = PostgresDumpService(kind: .backup, runnerFactory: { runner }) + let service = NativeDumpService(kind: .backup, runnerFactory: { runner }) let updates = service.stateUpdates() try service.run( @@ -343,9 +358,9 @@ struct PostgresDumpServiceStateMachineTests { } private func firstMatching( - _ stream: AsyncStream, - where predicate: @Sendable (PostgresDumpState) -> Bool - ) async throws -> PostgresDumpState { + _ stream: AsyncStream, + where predicate: @Sendable (NativeDumpState) -> Bool + ) async throws -> NativeDumpState { for await state in stream where predicate(state) { return state } diff --git a/TableProTests/Models/Export/ExportPreselectionTests.swift b/TableProTests/Models/Export/ExportPreselectionTests.swift index 38626d4d91..47a351cdaa 100644 --- a/TableProTests/Models/Export/ExportPreselectionTests.swift +++ b/TableProTests/Models/Export/ExportPreselectionTests.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProPluginKit import Testing @testable import TablePro @@ -14,31 +15,31 @@ struct ExportPreselectionTests { func namedTablesStayInCurrentContainer() { let preselection = ExportPreselection.tables(["users"]) - #expect(preselection.selects(table: "users", inContainer: .database("sales"), isCurrentContainer: true)) - #expect(!preselection.selects(table: "users", inContainer: .database("analytics"), isCurrentContainer: false)) + #expect(preselection.selects(object: "users", kind: .table, inContainer: .database("sales"), isCurrentContainer: true)) + #expect(!preselection.selects(object: "users", kind: .table, inContainer: .database("analytics"), isCurrentContainer: false)) } @Test("A container preselection selects every table it holds") func containerSelectsAllItsTables() { let preselection = ExportPreselection.containers([.database("analytics")]) - #expect(preselection.selects(table: "events", inContainer: .database("analytics"), isCurrentContainer: false)) - #expect(preselection.selects(table: "sessions", inContainer: .database("analytics"), isCurrentContainer: false)) + #expect(preselection.selects(object: "events", kind: .table, inContainer: .database("analytics"), isCurrentContainer: false)) + #expect(preselection.selects(object: "sessions", kind: .table, inContainer: .database("analytics"), isCurrentContainer: false)) } @Test("A container preselection ignores tables in other containers") func containerIgnoresOtherContainers() { let preselection = ExportPreselection.containers([.database("analytics")]) - #expect(!preselection.selects(table: "events", inContainer: .database("sales"), isCurrentContainer: true)) + #expect(!preselection.selects(object: "events", kind: .table, inContainer: .database("sales"), isCurrentContainer: true)) } @Test("Schema containers match by schema name") func schemaContainersMatchByName() { let preselection = ExportPreselection.containers([.schema(database: "sales", schema: "reporting")]) - #expect(preselection.selects(table: "totals", inContainer: .schema(database: "sales", schema: "reporting"), isCurrentContainer: false)) - #expect(!preselection.selects(table: "totals", inContainer: .schema(database: "sales", schema: "public"), isCurrentContainer: true)) + #expect(preselection.selects(object: "totals", kind: .table, inContainer: .schema(database: "sales", schema: "reporting"), isCurrentContainer: false)) + #expect(!preselection.selects(object: "totals", kind: .table, inContainer: .schema(database: "sales", schema: "public"), isCurrentContainer: true)) } @Test("A single table names the export file") @@ -96,12 +97,10 @@ struct ExportPreselectionTests { func databaseSelectsItsSchemas() { let preselection = ExportPreselection.containers([.database("app")]) - #expect(preselection.selects( - table: "users", inContainer: .schema(database: "app", schema: "public"), + #expect(preselection.selects(object: "users", kind: .table, inContainer: .schema(database: "app", schema: "public"), isCurrentContainer: true )) - #expect(preselection.selects( - table: "totals", inContainer: .schema(database: "app", schema: "reporting"), + #expect(preselection.selects(object: "totals", kind: .table, inContainer: .schema(database: "app", schema: "reporting"), isCurrentContainer: false )) } @@ -110,8 +109,7 @@ struct ExportPreselectionTests { func databaseDoesNotSelectAnotherDatabasesSchemas() { let preselection = ExportPreselection.containers([.database("analytics")]) - #expect(!preselection.selects( - table: "users", inContainer: .schema(database: "app", schema: "public"), + #expect(!preselection.selects(object: "users", kind: .table, inContainer: .schema(database: "app", schema: "public"), isCurrentContainer: true )) } @@ -122,11 +120,9 @@ struct ExportPreselectionTests { func schemaDoesNotMatchASameNamedDatabase() { let preselection = ExportPreselection.containers([.schema(database: "app", schema: "analytics")]) - #expect(!preselection.selects( - table: "events", inContainer: .database("analytics"), isCurrentContainer: false + #expect(!preselection.selects(object: "events", kind: .table, inContainer: .database("analytics"), isCurrentContainer: false )) - #expect(!preselection.selects( - table: "events", inContainer: .schema(database: "other", schema: "analytics"), + #expect(!preselection.selects(object: "events", kind: .table, inContainer: .schema(database: "other", schema: "analytics"), isCurrentContainer: false )) } diff --git a/TableProTests/Models/ExportModelsTests.swift b/TableProTests/Models/ExportModelsTests.swift index 294249906b..fe845e67e7 100644 --- a/TableProTests/Models/ExportModelsTests.swift +++ b/TableProTests/Models/ExportModelsTests.swift @@ -27,7 +27,7 @@ struct ExportModelsTests { @Test("Export database item selected count with no tables") func exportDatabaseItemNoTables() { - let item = ExportDatabaseItem(name: "testdb", tables: []) + let item = ExportDatabaseItem(name: "testdb", objects: []) #expect(item.selectedCount == 0) #expect(item.allSelected == false) #expect(item.noneSelected == true) @@ -36,10 +36,10 @@ struct ExportModelsTests { @Test("Export database item selected count with all selected") func exportDatabaseItemAllSelected() { let tables = [ - ExportTableItem(name: "users", type: .table, isSelected: true), - ExportTableItem(name: "posts", type: .table, isSelected: true), + ExportObjectItem(name: "users", kind: .table, isSelected: true), + ExportObjectItem(name: "posts", kind: .table, isSelected: true), ] - let item = ExportDatabaseItem(name: "testdb", tables: tables) + let item = ExportDatabaseItem(name: "testdb", objects: tables) #expect(item.selectedCount == 2) #expect(item.allSelected == true) #expect(item.noneSelected == false) @@ -48,10 +48,10 @@ struct ExportModelsTests { @Test("Export database item selected count with partial selection") func exportDatabaseItemPartialSelection() { let tables = [ - ExportTableItem(name: "users", type: .table, isSelected: true), - ExportTableItem(name: "posts", type: .table, isSelected: false), + ExportObjectItem(name: "users", kind: .table, isSelected: true), + ExportObjectItem(name: "posts", kind: .table, isSelected: false), ] - let item = ExportDatabaseItem(name: "testdb", tables: tables) + let item = ExportDatabaseItem(name: "testdb", objects: tables) #expect(item.selectedCount == 1) #expect(item.allSelected == false) #expect(item.noneSelected == false) @@ -60,10 +60,10 @@ struct ExportModelsTests { @Test("Export database item selected count with none selected") func exportDatabaseItemNoneSelected() { let tables = [ - ExportTableItem(name: "users", type: .table, isSelected: false), - ExportTableItem(name: "posts", type: .table, isSelected: false), + ExportObjectItem(name: "users", kind: .table, isSelected: false), + ExportObjectItem(name: "posts", kind: .table, isSelected: false), ] - let item = ExportDatabaseItem(name: "testdb", tables: tables) + let item = ExportDatabaseItem(name: "testdb", objects: tables) #expect(item.selectedCount == 0) #expect(item.allSelected == false) #expect(item.noneSelected == true) @@ -72,43 +72,43 @@ struct ExportModelsTests { @Test("Export database item selected tables") func exportDatabaseItemSelectedTables() { let tables = [ - ExportTableItem(name: "users", type: .table, isSelected: true), - ExportTableItem(name: "posts", type: .table, isSelected: false), - ExportTableItem(name: "comments", type: .table, isSelected: true), + ExportObjectItem(name: "users", kind: .table, isSelected: true), + ExportObjectItem(name: "posts", kind: .table, isSelected: false), + ExportObjectItem(name: "comments", kind: .table, isSelected: true), ] - let item = ExportDatabaseItem(name: "testdb", tables: tables) - let selectedTables = item.selectedTables - #expect(selectedTables.count == 2) - #expect(selectedTables.map(\.name) == ["users", "comments"]) + let item = ExportDatabaseItem(name: "testdb", objects: tables) + let selectedObjects = item.selectedObjects + #expect(selectedObjects.count == 2) + #expect(selectedObjects.map(\.name) == ["users", "comments"]) } @Test("Export table item qualified name without database name") func exportTableItemQualifiedNameWithoutDatabase() { - let table = ExportTableItem(name: "users", type: .table, isSelected: true) + let table = ExportObjectItem(name: "users", kind: .table, isSelected: true) #expect(table.qualifiedName == "users") } @Test("Export table item qualified name with database name") func exportTableItemQualifiedNameWithDatabase() { - let table = ExportTableItem(name: "users", databaseName: "mydb", type: .table, isSelected: true) + let table = ExportObjectItem(name: "users", databaseName: "mydb", kind: .table, isSelected: true) #expect(table.qualifiedName == "mydb.users") } @Test("Export table item option values default to empty") func exportTableItemOptionValuesDefault() { - let table = ExportTableItem(name: "users", type: .table) + let table = ExportObjectItem(name: "users", kind: .table) #expect(table.optionValues.isEmpty) } @Test("Export table item with option values") func exportTableItemWithOptionValues() { - let table = ExportTableItem(name: "users", type: .table, isSelected: true, optionValues: [true, false, true]) + let table = ExportObjectItem(name: "users", kind: .table, isSelected: true, optionValues: [true, false, true]) #expect(table.optionValues == [true, false, true]) } @Test("Normalizing a preselected table with empty option values applies plugin defaults") func normalizedMaterializesDefaultsForPreselectedTable() { - let table = ExportTableItem(name: "users", type: .table, isSelected: true) + let table = ExportObjectItem(name: "users", kind: .table, isSelected: true) let normalized = table.normalized(forOptionColumnCount: 3, defaultOptionValues: [true, true, true]) #expect(normalized.optionValues == [true, true, true]) #expect(normalized.isSelected) @@ -116,42 +116,42 @@ struct ExportModelsTests { @Test("Normalizing preserves a partial option selection") func normalizedPreservesPartialSelection() { - let table = ExportTableItem(name: "users", type: .table, isSelected: true, optionValues: [true, false, true]) + let table = ExportObjectItem(name: "users", kind: .table, isSelected: true, optionValues: [true, false, true]) let normalized = table.normalized(forOptionColumnCount: 3, defaultOptionValues: [true, true, true]) #expect(normalized.optionValues == [true, false, true]) } @Test("Normalizing a selected table with all-false option values re-applies defaults") func normalizedRepairsAllFalseSelection() { - let table = ExportTableItem(name: "users", type: .table, isSelected: true, optionValues: [false, false, false]) + let table = ExportObjectItem(name: "users", kind: .table, isSelected: true, optionValues: [false, false, false]) let normalized = table.normalized(forOptionColumnCount: 3, defaultOptionValues: [true, true, true]) #expect(normalized.optionValues == [true, true, true]) } @Test("Normalizing an unselected table with all-false option values leaves them alone") func normalizedLeavesUnselectedAllFalseAlone() { - let table = ExportTableItem(name: "users", type: .table, isSelected: false, optionValues: [false, false, false]) + let table = ExportObjectItem(name: "users", kind: .table, isSelected: false, optionValues: [false, false, false]) let normalized = table.normalized(forOptionColumnCount: 3, defaultOptionValues: [true, true, true]) #expect(normalized.optionValues == [false, false, false]) } @Test("Normalizing an unselected table with empty option values still fixes the shape") func normalizedFixesShapeForUnselectedTable() { - let table = ExportTableItem(name: "users", type: .table, isSelected: false) + let table = ExportObjectItem(name: "users", kind: .table, isSelected: false) let normalized = table.normalized(forOptionColumnCount: 3, defaultOptionValues: [true, true, true]) #expect(normalized.optionValues.count == 3) } @Test("Normalizing with zero option columns leaves option values untouched") func normalizedNoOpForFormatsWithoutOptionColumns() { - let table = ExportTableItem(name: "users", type: .table, isSelected: true) + let table = ExportObjectItem(name: "users", kind: .table, isSelected: true) let normalized = table.normalized(forOptionColumnCount: 0, defaultOptionValues: []) #expect(normalized.optionValues.isEmpty) } @Test("Normalizing falls back to all-true when plugin defaults have the wrong length") func normalizedFallsBackWhenDefaultsMismatched() { - let table = ExportTableItem(name: "users", type: .table, isSelected: true) + let table = ExportObjectItem(name: "users", kind: .table, isSelected: true) let normalized = table.normalized(forOptionColumnCount: 3, defaultOptionValues: [true]) #expect(normalized.optionValues == [true, true, true]) } @@ -159,15 +159,15 @@ struct ExportModelsTests { @Test("Normalizing database items materializes every preselected table and preserves identity") func normalizingDatabaseItemsMatchesPreselectionFlow() { let tables = [ - ExportTableItem(name: "users", type: .table, isSelected: true), - ExportTableItem(name: "posts", type: .table, isSelected: true), - ExportTableItem(name: "logs", type: .table, isSelected: false), + ExportObjectItem(name: "users", kind: .table, isSelected: true), + ExportObjectItem(name: "posts", kind: .table, isSelected: true), + ExportObjectItem(name: "logs", kind: .table, isSelected: false), ] - let original = [ExportDatabaseItem(name: "app_db", tables: tables)] + let original = [ExportDatabaseItem(name: "app_db", objects: tables)] let normalized = original.normalizingOptionValues(optionColumnCount: 3, defaultOptionValues: [true, true, true]) #expect(normalized[0].id == original[0].id) - #expect(normalized[0].tables[0].id == original[0].tables[0].id) - let preselected = normalized[0].tables.filter(\.isSelected) + #expect(normalized[0].objects[0].id == original[0].objects[0].id) + let preselected = normalized[0].objects.filter(\.isSelected) #expect(preselected.count == 2) #expect(preselected.allSatisfy { $0.optionValues.contains(true) }) } @@ -175,11 +175,11 @@ struct ExportModelsTests { @Test("Resetting option values overwrites every table regardless of prior content") func resettingOptionValuesOverwritesAll() { let tables = [ - ExportTableItem(name: "users", type: .table, isSelected: true, optionValues: [true, false, true]), - ExportTableItem(name: "posts", type: .table, isSelected: false, optionValues: [false, false, false]), + ExportObjectItem(name: "users", kind: .table, isSelected: true, optionValues: [true, false, true]), + ExportObjectItem(name: "posts", kind: .table, isSelected: false, optionValues: [false, false, false]), ] - let original = [ExportDatabaseItem(name: "app_db", tables: tables)] + let original = [ExportDatabaseItem(name: "app_db", objects: tables)] let reset = original.resettingOptionValues(to: [true, true, true]) - #expect(reset[0].tables.allSatisfy { $0.optionValues == [true, true, true] }) + #expect(reset[0].objects.allSatisfy { $0.optionValues == [true, true, true] }) } } diff --git a/TableProTests/Models/ExportObjectTreeTests.swift b/TableProTests/Models/ExportObjectTreeTests.swift new file mode 100644 index 0000000000..5f7d465cdd --- /dev/null +++ b/TableProTests/Models/ExportObjectTreeTests.swift @@ -0,0 +1,279 @@ +// +// ExportObjectTreeTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing +@testable import TablePro + +@Suite("Export object kinds") +struct ExportObjectKindTests { + + @Test("Dump order creates every dependency before the thing that needs it") + func dumpOrderRespectsDependencies() { + #expect(PluginExportObjectKind.userType.dumpOrder < PluginExportObjectKind.table.dumpOrder) + #expect(PluginExportObjectKind.sequence.dumpOrder < PluginExportObjectKind.table.dumpOrder) + #expect(PluginExportObjectKind.table.dumpOrder < PluginExportObjectKind.view.dumpOrder) + #expect(PluginExportObjectKind.view.dumpOrder < PluginExportObjectKind.routine.dumpOrder) + #expect(PluginExportObjectKind.routine.dumpOrder < PluginExportObjectKind.trigger.dumpOrder) + #expect(PluginExportObjectKind.trigger.dumpOrder < PluginExportObjectKind.grant.dumpOrder) + } + + @Test("Grants sort last, because every object they name has to exist first") + func grantsSortLast() { + let maxOther = PluginExportObjectKind.allCases + .filter { $0 != .grant } + .map(\.dumpOrder) + .max() ?? 0 + #expect(PluginExportObjectKind.grant.dumpOrder > maxOther) + } + + @Test("Only tables and foreign tables carry rows") + func onlyTablesCarryRows() { + let rowCarrying = PluginExportObjectKind.allCases.filter(\.carriesRows) + #expect(Set(rowCarrying) == Set([.table, .foreignTable])) + } + + @Test("Table type strings map to the right kind") + func tableTypeMapping() { + #expect(PluginExportObjectKind.from(tableType: "BASE TABLE") == .table) + #expect(PluginExportObjectKind.from(tableType: "VIEW") == .view) + #expect(PluginExportObjectKind.from(tableType: "MATERIALIZED VIEW") == .materializedView) + #expect(PluginExportObjectKind.from(tableType: "FOREIGN TABLE") == .foreignTable) + #expect(PluginExportObjectKind.from(tableType: "table") == .table) + } + + /// A materialized view names both "materialized" and "view", so the order the cases are tested + /// in is what keeps it out of the plain view arm. + @Test("Materialized view wins over the plain view match") + func materializedViewBeatsView() { + #expect(PluginExportObjectKind.from(tableType: "SYSTEM MATERIALIZED VIEW") == .materializedView) + } + + @Test("A format that declares no kinds receives only tables and views") + func legacyDefaultIsTablesAndViews() { + #expect(Set(PluginExportObjectKind.legacyDefault) == Set([.table, .view])) + } + + @Test("Every kind but grant has a drop keyword") + func dropKeywordsPresent() { + for kind in PluginExportObjectKind.allCases where kind != .grant { + #expect(!kind.dropKeyword.isEmpty, "\(kind) has no drop keyword") + } + #expect(PluginExportObjectKind.grant.dropKeyword.isEmpty) + } +} + +@Suite("Export outline tree") +struct ExportOutlineTreeTests { + + private func database(named name: String, objects: [ExportObjectItem]) -> ExportDatabaseItem { + ExportDatabaseItem(name: name, objects: objects) + } + + @Test("A database holding one kind skips the group level") + func singleKindSkipsGroups() { + let item = database(named: "app", objects: [ + ExportObjectItem(name: "users", kind: .table), + ExportObjectItem(name: "posts", kind: .table) + ]) + let roots = ExportOutlineTreeBuilder.build(from: [item]) + #expect(roots.count == 1) + #expect(roots[0].children.count == 2) + for child in roots[0].children { + guard case .object = child.kind else { + Issue.record("expected an object row directly under the database") + return + } + } + } + + @Test("A database holding several kinds groups them in dump order") + func multipleKindsGroupInDumpOrder() { + let item = database(named: "app", objects: [ + ExportObjectItem(name: "audit", kind: .trigger, parentTable: "users"), + ExportObjectItem(name: "users", kind: .table), + ExportObjectItem(name: "active_users", kind: .view), + ExportObjectItem(name: "status", kind: .userType) + ]) + let roots = ExportOutlineTreeBuilder.build(from: [item]) + let kinds = roots[0].children.compactMap { node -> PluginExportObjectKind? in + guard case .group(_, let kind) = node.kind else { return nil } + return kind + } + #expect(kinds == [.userType, .table, .view, .trigger]) + } + + @Test("Each group holds only its own kind") + func groupsHoldTheirOwnKind() { + let item = database(named: "app", objects: [ + ExportObjectItem(name: "users", kind: .table), + ExportObjectItem(name: "posts", kind: .table), + ExportObjectItem(name: "active_users", kind: .view) + ]) + let roots = ExportOutlineTreeBuilder.build(from: [item]) + let tableGroup = roots[0].children.first { node in + guard case .group(_, let kind) = node.kind else { return false } + return kind == .table + } + #expect(tableGroup?.children.count == 2) + } + + @Test("Node identity is stable and distinguishes a kind group from its database") + func nodeIdentityIsStable() { + let item = database(named: "app", objects: [ + ExportObjectItem(name: "users", kind: .table), + ExportObjectItem(name: "active_users", kind: .view) + ]) + let first = ExportOutlineTreeBuilder.build(from: [item]) + let second = ExportOutlineTreeBuilder.build(from: [item]) + #expect(first[0].identity == second[0].identity) + #expect(first[0].identity != first[0].children[0].identity) + #expect(first[0].children[0].identity != first[0].children[1].identity) + } + + /// The tree is only rebuilt when its shape changes, so a checkbox must not move the + /// fingerprint: rebuilding on every toggle collapses the group under the user's pointer. + @Test("Toggling a checkbox leaves the shape fingerprint unchanged") + func selectionDoesNotChangeShape() { + var item = database(named: "app", objects: [ + ExportObjectItem(name: "users", kind: .table), + ExportObjectItem(name: "active_users", kind: .view) + ]) + let before = ExportOutlineTreeBuilder.shapeFingerprint(of: [item]) + item.objects[0].isSelected = true + item.objects[1].optionValues = [true, false, true] + item.objects[0].rowScope = PluginExportRowScope(filter: "id > 5") + let after = ExportOutlineTreeBuilder.shapeFingerprint(of: [item]) + #expect(before == after) + } + + @Test("Adding an object changes the shape fingerprint") + func addingAnObjectChangesShape() { + var item = database(named: "app", objects: [ExportObjectItem(name: "users", kind: .table)]) + let before = ExportOutlineTreeBuilder.shapeFingerprint(of: [item]) + item.objects.append(ExportObjectItem(name: "posts", kind: .table)) + let after = ExportOutlineTreeBuilder.shapeFingerprint(of: [item]) + #expect(before != after) + } + + @Test("Present kinds come back in dump order without duplicates") + func presentKindsAreOrderedAndUnique() { + let item = database(named: "app", objects: [ + ExportObjectItem(name: "t1", kind: .trigger, parentTable: "users"), + ExportObjectItem(name: "users", kind: .table), + ExportObjectItem(name: "posts", kind: .table), + ExportObjectItem(name: "t2", kind: .trigger, parentTable: "posts") + ]) + #expect(item.presentKinds == [.table, .trigger]) + } +} + +@Suite("Export object option masking") +struct ExportObjectOptionMaskingTests { + + private let columns = [ + PluginExportOptionColumn(id: "structure", label: "Structure", width: 56), + PluginExportOptionColumn(id: "drop", label: "Drop", width: 44), + PluginExportOptionColumn(id: "data", label: "Data", width: 44) + ] + + private func supportsOption(_ columnId: String, _ kind: PluginExportObjectKind) -> Bool { + switch columnId { + case "data": return kind.carriesRows + case "drop": return kind != .grant + default: return true + } + } + + /// The mask must clear in place rather than compact, or every option after the cleared one + /// shifts and a routine's `Drop` flag is read as its `Data` flag. + @Test("Masking clears an unsupported option without shifting the others") + func maskingClearsInPlace() { + let routine = ExportObjectItem( + name: "recalc", kind: .routine, isSelected: true, optionValues: [true, true, true]) + let masked = routine.maskingUnsupportedOptions(columns: columns, supports: supportsOption) + #expect(masked.optionValues == [true, true, false]) + } + + @Test("Masking leaves a table's options alone") + func maskingLeavesTablesAlone() { + let table = ExportObjectItem( + name: "users", kind: .table, isSelected: true, optionValues: [true, false, true]) + let masked = table.maskingUnsupportedOptions(columns: columns, supports: supportsOption) + #expect(masked.optionValues == [true, false, true]) + } + + @Test("A grant keeps only its structure flag") + func grantKeepsStructureOnly() { + let grant = ExportObjectItem( + name: "app_user", kind: .grant, isSelected: true, optionValues: [true, true, true]) + let masked = grant.maskingUnsupportedOptions(columns: columns, supports: supportsOption) + #expect(masked.optionValues == [true, false, false]) + } + + @Test("Masking a row whose option count does not match the columns leaves it untouched") + func maskingIgnoresMismatchedShape() { + let routine = ExportObjectItem(name: "recalc", kind: .routine, optionValues: [true]) + let masked = routine.maskingUnsupportedOptions(columns: columns, supports: supportsOption) + #expect(masked.optionValues == [true]) + } + + @Test("Masking a whole tree reaches every object") + func maskingReachesEveryObject() { + let databases = [ + ExportDatabaseItem(name: "app", objects: [ + ExportObjectItem(name: "users", kind: .table, optionValues: [true, true, true]), + ExportObjectItem(name: "recalc", kind: .routine, optionValues: [true, true, true]) + ]) + ] + let masked = databases.maskingUnsupportedOptions(columns: columns, supports: supportsOption) + #expect(masked[0].objects[0].optionValues == [true, true, true]) + #expect(masked[0].objects[1].optionValues == [true, true, false]) + } +} + +@Suite("Export preselection with object kinds") +struct ExportPreselectionKindTests { + + /// A routine and a table can share a name, and a sidebar preselection is about tables. Without + /// the kind check, selecting the `users` table would also tick a `users()` function. + @Test("A table preselection never selects a routine of the same name") + func tablePreselectionIgnoresRoutines() { + let preselection = ExportPreselection.tables(["users"]) + #expect(preselection.selects( + object: "users", kind: .table, inContainer: .database("app"), isCurrentContainer: true)) + #expect(!preselection.selects( + object: "users", kind: .routine, inContainer: .database("app"), isCurrentContainer: true)) + #expect(!preselection.selects( + object: "users", kind: .trigger, inContainer: .database("app"), isCurrentContainer: true)) + } + + @Test("A table preselection covers views and foreign tables") + func tablePreselectionCoversViewShapes() { + let preselection = ExportPreselection.tables(["users"]) + #expect(preselection.selects( + object: "users", kind: .view, inContainer: .database("app"), isCurrentContainer: true)) + #expect(preselection.selects( + object: "users", kind: .foreignTable, inContainer: .database("app"), isCurrentContainer: true)) + } + + @Test("A container preselection takes every kind in it") + func containerPreselectionTakesEveryKind() { + let preselection = ExportPreselection.containers([.database("app")]) + for kind in PluginExportObjectKind.allCases { + #expect(preselection.selects( + object: "anything", kind: kind, inContainer: .database("app"), isCurrentContainer: false), + "\(kind) was not selected by a whole-container preselection") + } + } + + @Test("A container preselection does not reach another container") + func containerPreselectionStaysInItsContainer() { + let preselection = ExportPreselection.containers([.database("app")]) + #expect(!preselection.selects( + object: "users", kind: .table, inContainer: .database("other"), isCurrentContainer: false)) + } +} diff --git a/TableProTests/Models/ExportRowScopeTests.swift b/TableProTests/Models/ExportRowScopeTests.swift new file mode 100644 index 0000000000..f188d584c4 --- /dev/null +++ b/TableProTests/Models/ExportRowScopeTests.swift @@ -0,0 +1,91 @@ +// +// ExportRowScopeTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@Suite("Export row scope") +struct ExportRowScopeTests { + + @Test("An empty scope exports everything") + func emptyScopeIsUnrestricted() { + #expect(PluginExportRowScope.unrestricted.isUnrestricted) + #expect(PluginExportRowScope(filter: " ").isUnrestricted) + } + + @Test("Any one narrowing makes the scope restricted") + func anyNarrowingRestricts() { + #expect(!PluginExportRowScope(filter: "id > 10").isUnrestricted) + #expect(!PluginExportRowScope(rowLimit: 100).isUnrestricted) + #expect(!PluginExportRowScope(columns: ["id"]).isUnrestricted) + } + + /// A trailing semicolon is a typing habit, so it is dropped rather than refusing the filter. + @Test("A trailing semicolon is dropped") + func trailingSemicolonIsDropped() { + let scope = PluginExportRowScope(filter: "status = 'active';") + #expect(scope.sanitizedFilter == "status = 'active'") + #expect(!scope.hasRejectedFilter) + } + + /// The filter is spliced into a SELECT the export builds, so a second statement in it would run + /// as a second statement. That is refused rather than executed. + @Test("A semicolon inside the filter refuses the whole filter") + func interiorSemicolonIsRefused() { + let scope = PluginExportRowScope(filter: "1 = 1; DROP TABLE users") + #expect(scope.sanitizedFilter.isEmpty) + #expect(scope.hasRejectedFilter) + } + + @Test("A refused filter is reported rather than silently exporting every row") + func refusedFilterIsReported() { + #expect(PluginExportRowScope(filter: "a = 1; b = 2").hasRejectedFilter) + #expect(!PluginExportRowScope(filter: "a = 1").hasRejectedFilter) + #expect(!PluginExportRowScope(filter: "").hasRejectedFilter) + } + + @Test("Whitespace around a filter is trimmed") + func filterIsTrimmed() { + #expect(PluginExportRowScope(filter: " id > 5 ").sanitizedFilter == "id > 5") + #expect(PluginExportRowScope(filter: "\n id > 5 ;\n ").sanitizedFilter == "id > 5") + } + + @Test("The summary names every narrowing that is set") + func summaryNamesNarrowings() { + let scope = PluginExportRowScope(filter: "id > 5", rowLimit: 100, columns: ["id", "name"]) + #expect(scope.summary == "2 columns, WHERE id > 5, LIMIT 100") + #expect(PluginExportRowScope.unrestricted.summary.isEmpty) + } + + @Test("A scope survives a round trip through Codable") + func scopeRoundTrips() throws { + let scope = PluginExportRowScope(filter: "id > 5", rowLimit: 42, columns: ["a", "b"]) + let decoded = try JSONDecoder().decode( + PluginExportRowScope.self, from: JSONEncoder().encode(scope)) + #expect(decoded == scope) + } + + @Test("An export item carries an unrestricted scope unless one is given") + func exportTableDefaultsToUnrestricted() { + let table = PluginExportTable( + name: "users", databaseName: "app", tableType: "table", schema: nil, kind: .table) + #expect(table.rowScope.isUnrestricted) + } + + /// The two published initializers cannot gain a parameter without breaking every shipped + /// plugin, so they have to keep defaulting the field they never knew about. + @Test("The legacy initializers default the scope") + func legacyInitializersDefaultTheScope() { + let withSchema = PluginExportTable( + name: "users", databaseName: "app", tableType: "table", schema: "public") + #expect(withSchema.rowScope.isUnrestricted) + #expect(withSchema.kind == .table) + + let bare = PluginExportTable(name: "v_users", databaseName: "app", tableType: "VIEW") + #expect(bare.rowScope.isUnrestricted) + #expect(bare.kind == .view) + } +} diff --git a/TableProTests/Plugins/SQLExportInsertModeTests.swift b/TableProTests/Plugins/SQLExportInsertModeTests.swift new file mode 100644 index 0000000000..4bf685b025 --- /dev/null +++ b/TableProTests/Plugins/SQLExportInsertModeTests.swift @@ -0,0 +1,255 @@ +// +// SQLExportInsertModeTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@Suite("SQL export insert modes") +struct SQLExportInsertModeTests { + + private func renderer(_ dialect: SqlDialect) -> SQLExportInsertRenderer { + SQLExportInsertRenderer(dialect: dialect) { "`\($0)`" } + } + + private func render( + _ dialect: SqlDialect, + _ mode: SQLExportInsertMode, + columns: [String] = ["id", "name", "email"], + primaryKeys: [String] = ["id"] + ) -> SQLExportInsertRenderer.Rendered { + renderer(dialect).render( + mode: mode, + tableRef: "`users`", + quotedColumns: "`id`, `name`, `email`", + overriding: "", + columnNames: columns, + primaryKeyColumns: primaryKeys + ) + } + + @Test("A plain insert is the same statement on every dialect") + func plainInsertIsDialectIndependent() { + for dialect in SqlDialect.allCases { + let rendered = render(dialect, .insert) + #expect(rendered.prefix == "INSERT INTO `users` (`id`, `name`, `email`) VALUES\n") + #expect(rendered.suffix.isEmpty) + #expect(rendered.warning == nil) + } + } + + /// The three dialects put conflict handling in three different places: MySQL in the verb, + /// SQLite in a resolution clause, PostgreSQL in a trailing clause. + @Test("Skipping existing rows uses each dialect's own spelling") + func ignoreUsesDialectSpelling() { + #expect(render(.mysql, .ignoreExisting).prefix.hasPrefix("INSERT IGNORE INTO")) + #expect(render(.sqlite, .ignoreExisting).prefix.hasPrefix("INSERT OR IGNORE INTO")) + + let postgres = render(.postgres, .ignoreExisting) + #expect(postgres.prefix.hasPrefix("INSERT INTO")) + #expect(postgres.suffix == "\nON CONFLICT DO NOTHING") + } + + @Test("Replacing uses REPLACE on MySQL and INSERT OR REPLACE on SQLite") + func replaceUsesDialectSpelling() { + #expect(render(.mysql, .replaceExisting).prefix.hasPrefix("REPLACE INTO")) + #expect(render(.sqlite, .replaceExisting).prefix.hasPrefix("INSERT OR REPLACE INTO")) + } + + /// PostgreSQL has no REPLACE, so it renders the upsert that overwrites every non-key column. + @Test("Replacing on PostgreSQL renders as an upsert") + func replaceOnPostgresIsAnUpsert() { + let rendered = render(.postgres, .replaceExisting) + #expect(rendered.suffix.contains("ON CONFLICT (`id`) DO UPDATE SET")) + #expect(rendered.warning == nil) + } + + @Test("Updating on MySQL assigns from VALUES and skips the key") + func mysqlUpsertSkipsTheKey() { + let rendered = render(.mysql, .updateExisting) + #expect(rendered.suffix == "\nON DUPLICATE KEY UPDATE `name` = VALUES(`name`), `email` = VALUES(`email`)") + #expect(!rendered.suffix.contains("`id` = VALUES")) + } + + @Test("Updating on PostgreSQL names the conflict target and assigns from EXCLUDED") + func postgresUpsertNamesTheTarget() { + let rendered = render(.postgres, .updateExisting) + #expect(rendered.suffix == "\nON CONFLICT (`id`) DO UPDATE SET `name` = EXCLUDED.`name`, `email` = EXCLUDED.`email`") + } + + /// SQLite spells the pseudo-table `excluded` in lower case, and it is not case-insensitive there. + @Test("Updating on SQLite uses the lower-case excluded pseudo-table") + func sqliteUpsertUsesLowerCaseExcluded() { + let rendered = render(.sqlite, .updateExisting) + #expect(rendered.suffix.contains("excluded.`name`")) + #expect(!rendered.suffix.contains("EXCLUDED.")) + } + + @Test("A composite key lists every key column as the conflict target") + func compositeKeyNamesEveryColumn() { + let rendered = render( + .postgres, .updateExisting, + columns: ["tenant", "id", "name"], primaryKeys: ["tenant", "id"]) + #expect(rendered.suffix.hasPrefix("\nON CONFLICT (`tenant`, `id`) DO UPDATE SET")) + #expect(rendered.suffix.contains("`name` = EXCLUDED.`name`")) + } + + /// Without a key there is no conflict target, so the statement would not parse. Writing plain + /// inserts and warning beats writing a dump that fails on restore. + @Test("A table with no primary key falls back to a plain insert and warns") + func noPrimaryKeyFallsBack() { + let rendered = render(.postgres, .updateExisting, primaryKeys: []) + #expect(rendered.suffix.isEmpty) + #expect(rendered.warning != nil) + } + + @Test("A table whose columns are all key columns has nothing to update") + func allKeyColumnsFallsBack() { + let mysql = render(.mysql, .updateExisting, columns: ["id"], primaryKeys: ["id"]) + #expect(mysql.suffix.isEmpty) + #expect(mysql.warning != nil) + + let postgres = render(.postgres, .updateExisting, columns: ["id"], primaryKeys: ["id"]) + #expect(postgres.suffix == "\nON CONFLICT DO NOTHING") + } + + @Test("An engine with no conflict spelling writes plain inserts and warns") + func genericDialectWarns() { + for mode in [SQLExportInsertMode.ignoreExisting, .replaceExisting, .updateExisting] { + let rendered = render(.generic, mode) + #expect(rendered.prefix.hasPrefix("INSERT INTO"), "\(mode) should fall back") + #expect(rendered.suffix.isEmpty) + #expect(rendered.warning != nil, "\(mode) should warn") + } + } + + @Test("The OVERRIDING clause survives every mode that keeps a plain prefix") + func overridingSurvives() { + let rendered = SQLExportInsertRenderer(dialect: .postgres) { "\"\($0)\"" }.render( + mode: .ignoreExisting, + tableRef: "\"users\"", + quotedColumns: "\"id\"", + overriding: " OVERRIDING SYSTEM VALUE", + columnNames: ["id"], + primaryKeyColumns: ["id"] + ) + #expect(rendered.prefix.contains(" OVERRIDING SYSTEM VALUE")) + } +} + +@Suite("SQL export file splitting") +struct SQLExportFileWriterTests { + + @Test("A part keeps the compound extension so the file still opens as SQL") + func partKeepsCompoundExtension() { + let base = URL(fileURLWithPath: "/tmp/dump.sql") + #expect(SQLExportFileWriter.partURL(for: base, part: 2).lastPathComponent == "dump.part2.sql") + + let compressed = URL(fileURLWithPath: "/tmp/dump.sql.gz") + #expect(SQLExportFileWriter.partURL(for: compressed, part: 3).lastPathComponent == "dump.part3.sql.gz") + } + + @Test("A name with no extension still numbers its parts") + func partWithoutExtension() { + let base = URL(fileURLWithPath: "/tmp/dump") + #expect(SQLExportFileWriter.partURL(for: base, part: 1).lastPathComponent == "dump.part1") + } + + @Test("A name with a dot in its stem splits at the first dot") + func partWithDottedStem() { + let base = URL(fileURLWithPath: "/tmp/app.v2.sql") + #expect(SQLExportFileWriter.partURL(for: base, part: 2).lastPathComponent == "app.part2.v2.sql") + } + + @Test("An unsplit export keeps the name the user chose") + func unsplitKeepsChosenName() throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter(destination: destination, splitSizeMegabytes: 0) + try writer.write("SELECT 1;\n") + let written = try writer.commit() + + #expect(written == [destination]) + #expect(!writer.didSplit) + #expect(try String(contentsOf: destination, encoding: .utf8) == "SELECT 1;\n") + } + + /// Rotation happens between writes, so a part always ends on a whole statement. + @Test("Passing the cap starts a new part without splitting a statement") + func splittingKeepsStatementsWhole() throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter(destination: destination, splitSizeMegabytes: 1) + let chunk = String(repeating: "x", count: 700 * 1_024) + try writer.write("A\(chunk);\n") + try writer.write("B\(chunk);\n") + let written = try writer.commit() + + #expect(writer.didSplit) + #expect(written.count == 2) + #expect(written[0].lastPathComponent == "dump.part1.sql") + #expect(written[1].lastPathComponent == "dump.part2.sql") + + let firstPart = try String(contentsOf: written[0], encoding: .utf8) + let secondPart = try String(contentsOf: written[1], encoding: .utf8) + #expect(firstPart.hasPrefix("A")) + #expect(firstPart.hasSuffix(";\n")) + #expect(secondPart.hasPrefix("B")) + #expect(secondPart.hasSuffix(";\n")) + } + + @Test("A rolled back export leaves nothing behind") + func rollbackLeavesNothing() throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter(destination: destination, splitSizeMegabytes: 1) + try writer.write(String(repeating: "y", count: 2 * 1_024 * 1_024)) + try writer.write("tail;\n") + writer.rollback() + + let remaining = try FileManager.default.contentsOfDirectory(atPath: directory.path) + #expect(remaining.isEmpty, "left behind: \(remaining)") + } +} + +@Suite("SQL export snapshot") +struct SQLExportSnapshotTests { + + @Test("Each dialect opens its own consistent-read transaction") + func dialectSpecificBegin() { + #expect(SQLExportSnapshot(dialect: .mysql).beginStatement == "START TRANSACTION WITH CONSISTENT SNAPSHOT") + #expect(SQLExportSnapshot(dialect: .postgres).beginStatement == "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + #expect(SQLExportSnapshot(dialect: .sqlite).beginStatement == "BEGIN") + } + + /// An engine with no spelling for this opens nothing rather than sending a statement it would + /// reject and failing the whole export. + @Test("An engine with no snapshot statement opens and closes nothing") + func genericDialectOpensNothing() { + let snapshot = SQLExportSnapshot(dialect: .generic) + #expect(snapshot.beginStatement == nil) + #expect(snapshot.endStatement == nil) + } + + @Test("Every dialect that opens a transaction also closes it") + func openingImpliesClosing() { + for dialect in SqlDialect.allCases { + let snapshot = SQLExportSnapshot(dialect: dialect) + #expect((snapshot.beginStatement == nil) == (snapshot.endStatement == nil)) + } + } +} diff --git a/docs/features/backup-restore.mdx b/docs/features/backup-restore.mdx index 6df8834530..7e57ac481d 100644 --- a/docs/features/backup-restore.mdx +++ b/docs/features/backup-restore.mdx @@ -1,19 +1,23 @@ --- title: Backup & Restore -description: Dump and restore PostgreSQL or Redshift databases with pg_dump and pg_restore, with progress, cancel, and SSH tunnel reuse +description: Dump and restore with each engine's own command line tools, with progress, cancel, and SSH tunnel reuse --- -`pg_dump` and `pg_restore` have to be on your Mac first. TablePro shells out to them and ships no copy of its own: +The engine's own tools do the work, and none of them ship inside TablePro. Install the ones for your engine first: -```bash -brew install libpq -brew link --force libpq -``` +| Engine | Tools | Install | Dump file | +|---|---|---|---| +| PostgreSQL, Redshift | `pg_dump`, `pg_restore` | `brew install libpq && brew link --force libpq` | `.dump`, custom archive | +| MySQL, MariaDB | `mysqldump`, `mysql` | `brew install mysql-client` | `.sql` | +| MongoDB | `mongodump`, `mongorestore` | `brew install mongodb-database-tools` | `.archive`, gzipped | +| SQLite, libSQL | `sqlite3` | `brew install sqlite` | `.sql` | + +MariaDB 11.0 renamed its clients, so `mariadb-dump` and `mariadb` are accepted in place of `mysqldump` and `mysql`. The lookup takes the first match from `/usr/bin/which`, then `/opt/homebrew/bin`, `/usr/local/bin`, and Postgres.app's latest version. That path decides which client version does the work. -PostgreSQL and Redshift only. For any other engine, use a SQL export from [Import & Export](/features/import-export). Restore is disabled on a read-only [Safe Mode](/features/safe-mode) connection; backup is not, because it writes nothing to the database. +**File > Backup Dump…** is dimmed on an engine that is not in the table. For those, use a SQL export from [Import & Export](/features/import-export). Restore is disabled on a read-only [Safe Mode](/features/safe-mode) connection; backup is not, because it writes nothing to the database. @@ -28,10 +32,12 @@ PostgreSQL and Redshift only. For any other engine, use a SQL export from [Impor Choose **File > Backup Dump…** on a connected session and pick the database. - The default filename is `-.dump`. Dumps use the custom archive format (`pg_dump -Fc`), which is what `pg_restore` reads back. + The default filename is `-` with the engine's extension from the table. PostgreSQL uses the custom archive format (`pg_dump -Fc`), MySQL and SQLite write SQL, and MongoDB writes a gzipped archive. **Cancel** confirms, sends `SIGTERM`, and removes the partial file. On success the result sheet gives the size and **Show in Finder**. + + The bar shows a percentage on PostgreSQL and MySQL, which answer a database size query cheaply. MongoDB and SQLite show an indeterminate bar with a running byte count. @@ -39,10 +45,10 @@ PostgreSQL and Redshift only. For any other engine, use a SQL export from [Impor - Choose **File > Restore Dump…** and select a file `pg_dump` wrote in custom archive format. + Choose **File > Restore Dump…** and select a file the matching backup tool wrote. - `pg_restore` runs with `--no-owner --no-acl`, so the connection user ends up owning the restored objects. + `pg_restore` runs with `--no-owner --no-acl`, so the connection user ends up owning the restored objects. `mongorestore` is scoped with `--nsInclude` to the database you pick. Restore shows no percentage. **Cancel** confirms and sends `SIGTERM`, and the target database is left as it stands: drop it and restore into a fresh one, or clean up the partial objects yourself. @@ -51,16 +57,24 @@ PostgreSQL and Redshift only. For any other engine, use a SQL export from [Impor No `--clean` is passed, so restoring on top of a schema that already holds conflicting objects produces errors instead of replacing them. +## What a dump carries + +`mysqldump` runs with `--single-transaction --routines --triggers --events`, so a MySQL dump is consistent with itself and carries stored routines, triggers and scheduled events. `pg_dump -Fc` carries the whole database. `sqlite3 .dump` writes the schema and every row as SQL. + ## SSH tunnels and SSL -Both flows reuse the connection's active SSH tunnel, with no second port forward. SSL connections pass their mode to the tools through `PGSSLMODE`, `verify-ca` and `verify-full` included. +Both flows reuse the connection's active SSH tunnel, with no second port forward. SSL mode reaches PostgreSQL through `PGSSLMODE` and MySQL through `--ssl-mode`, `verify-ca` and `verify-full` included. SQLite opens a file, so neither applies. + +## Passwords + +Your password never reaches the tool's argument list, which every process on the machine can read through `ps`. PostgreSQL gets `PGPASSWORD` and MySQL gets `MYSQL_PWD`, both in the environment. MongoDB's tools read neither, so TablePro writes a `0600` config file and deletes it when the process exits. ## Failures -A non-zero exit shows the last 64 KB of `pg_dump` or `pg_restore` stderr in a scrollable monospaced view. Three causes account for most of them. +A non-zero exit shows the last 64 KB of the tool's stderr in a scrollable monospaced view. Three causes account for most of them. | What you see | What to do | |---|---| -| *"pg_dump was not found on this system. Install it with `brew install libpq` and link it."* | Install and link `libpq` so the binaries are on `PATH` | -| An authentication failure | The password goes through `PGPASSWORD` and the tools run with `--no-password`, so this is the role or the database rather than a missing prompt. Check that the role has `LOGIN` | +| *"… was not found on this system"* followed by an install command | Run that command so the binaries land on `PATH`. The message names the tool and the package for your engine | +| An authentication failure | The password goes through the environment or a config file rather than a prompt, so this is the account or the database. Check that the role can log in | | Objects that conflict with the dump | Restore into a fresh database, or drop the conflicting objects first | diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index 8f9d38f429..8405fb637c 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -12,7 +12,7 @@ The toolbar's **Export** (`Cmd+Shift+E`) re-reads every table you tick with `SEL Open a table or run a query, then click **Export** in the toolbar (`Cmd+Shift+E`), or right-click the results grid and choose **Export Results…**. - Pick a format, tick tables in the tree, and set the options for that format. + Pick a format, tick objects in the tree, and set the options for that format. Click **Export**. The destination file appears only on success: a failed or cancelled export removes its partial file. @@ -26,6 +26,36 @@ The last format and its options come back next time, but only after an export su Export dialog +### What the tree lists + +SQL exports more than tables. A database holding more than one kind of object groups them, in the order a restore needs them: + +| Group | Written as | Engines | +|---|---|---| +| Types | `CREATE TYPE` | PostgreSQL and its forks | +| Tables | `CREATE TABLE` plus `INSERT` | every SQL engine | +| Views | `CREATE VIEW` from the server's own definition | every SQL engine | +| Materialized Views | `CREATE MATERIALIZED VIEW` | PostgreSQL, Oracle | +| Routines | `CREATE FUNCTION` or `CREATE PROCEDURE` | engines with stored routines | +| Triggers | `CREATE TRIGGER` | engines with triggers | +| Privileges | `GRANT`, one principal per row | engines with user management | + +A database with only tables lists them flat, with no group to open first. + +Only SQL writes every group. CSV, JSON and XLSX take tables and views, and switching to one of them drops the rest from the tree. Per-object checkboxes follow the kind: **Data** is off the row for anything without rows, and a privilege row carries neither **Drop** nor **Data**. + + +Privileges are server-wide, so they appear once, under the container the dialog opened on. A `GRANT` naming an object the dump does not create fails on restore. + + +### Narrowing what a table exports + +Click the filter icon on a table row to write a `WHERE` clause, cap the row count, or pick columns. The icon fills in once a table is narrowed, and its tooltip repeats what the narrowing is. + +The clause is your engine's own SQL, spliced into the `SELECT` the export runs. It has to be one expression: a semicolon anywhere but the end refuses the filter, and the export summary says the table went out whole. + +Ticking no column exports every column, which is also what a table with a column added later gets. + ### What ends up in the file A whole-table export streams from the database at constant memory, with no row-count limit, and can be cancelled from the progress dialog. @@ -50,19 +80,25 @@ A whole-table export streams from the database at constant memory, with no row-c Sanitizing prefixes a value starting with `=`, `+`, `-`, or `@` with a single quote, so a spreadsheet treats it as text. - An array of objects. + One object per table, each holding an array of rows. | Option | Default | |--------|---------| + | Layout (one JSON object, one row per line) | One JSON object | | Pretty print | Yes | | Include NULL values | Yes | | Preserve all values as strings | No | + + **One row per line** writes NDJSON to a `.ndjson` file: no wrapping object, no array, one row per line, so a stream reader can process a file larger than memory a line at a time. Pretty print does not apply to it and is dimmed. Import reads `.json`, `.jsonl` and `.ndjson`, detecting the layout from the file. | Option | Default | |--------|---------| | Compress with gzip (`.sql.gz`) | No | | Batch size (rows per INSERT: 1, 100, 500, 1,000) | 500 | + | On existing rows (insert, skip, replace, update) | Insert | + | Split every (one file, 8, 32, 128, 512 MB) | One file | + | Read every table at one snapshot | No | | Exclude the AUTO_INCREMENT counter | Yes | | Exclude DEFINER clauses | Yes | @@ -76,6 +112,12 @@ A whole-table export streams from the database at constant memory, with no row-c The last two exclusions cover MySQL and MariaDB, and pass every other engine through untouched. + Insert mode is spelled differently on each engine: `INSERT IGNORE` and `REPLACE INTO` on MySQL and MariaDB, `INSERT OR IGNORE` and `INSERT OR REPLACE` on SQLite, `ON CONFLICT` on PostgreSQL. Updating needs a primary key to name as the conflict target. An engine with no spelling for the mode writes plain inserts, and the export summary says which tables that happened to. + + Splitting writes `dump.part1.sql`, `dump.part2.sql` and so on, rotating between statements so no part ends mid-`INSERT`. Restore the parts in order. A gzipped export is one file, so the two settings do not combine and the summary says so. + + One snapshot opens `START TRANSACTION WITH CONSISTENT SNAPSHOT` on MySQL, `BEGIN ISOLATION LEVEL REPEATABLE READ` on PostgreSQL, and a deferred transaction on SQLite. It holds that transaction open for the whole export. + Excluding the counter drops `AUTO_INCREMENT=` from the table options and leaves the column's own `AUTO_INCREMENT` attribute alone. Restoring rows sets the counter one past the highest key in the data, so a source counter that had run ahead of its rows, after deletes or a reset, does not carry over. Excluding definers drops `DEFINER=user@host` from a view. The account running the import becomes the definer, and `SQL SECURITY` is untouched, so a definer-rights view then runs with that account's privileges. Keep the clause and the import fails with `ERROR 1227 (42000): Access denied; you need (at least one of) the SET USER privilege(s) for this operation` unless the importing account is privileged, and a view that does get created answers `ERROR 1446 (HY000): The user specified as a definer ('…') does not exist` on every query against it. An invoker-rights view still runs as its caller. @@ -104,6 +146,28 @@ A whole-table export streams from the database at constant memory, with no row-c +## Transfer to another connection + +Right-click tables in the sidebar and choose **Transfer To…** to copy their rows straight into another connection, with no file in between. + + + + The destination list holds the connections that are already open. A connection that is not open does not appear. + + + Then tick the tables to copy. + + + Each table is wrapped in its own transaction by default, so a table that fails leaves its own rows untouched and the ones before it committed. + + + +Rows only. The destination table has to exist and its column names have to match, because inventing DDL that crosses two engines would create tables whose types quietly disagree with the data landing in them. A per-table row filter set in the export tree is not carried over; narrow the transfer by transferring fewer tables. + + +**Delete existing rows first** empties each destination table before writing. There is no undo. + + ## Clipboard paste (CSV/TSV) Select a row in the data grid and press `Cmd+V` to paste tabular data straight in. Tabs parse as TSV, commas as CSV. diff --git a/project.yml b/project.yml index d176feb300..3e5ba56b55 100644 --- a/project.yml +++ b/project.yml @@ -383,6 +383,7 @@ targets: - Plugins/EtcdDriverPlugin/EtcdCommandParser.swift - Plugins/EtcdDriverPlugin/EtcdQueryBuilder.swift - Plugins/EtcdDriverPlugin/EtcdStatementGenerator.swift + - Plugins/JSONExportPlugin/JSONExportModels.swift - Plugins/JSONImportPlugin/JSONImportOptions.swift - Plugins/JSONImportPlugin/JSONImportOptionsView.swift - Plugins/JSONImportPlugin/JSONImportParsing.swift @@ -488,9 +489,12 @@ targets: - Plugins/RedisDriverPlugin/RedisStatementGenerator.swift - Plugins/RedisDriverPlugin/RedisTopologyDiagnostics.swift - Plugins/SQLExportPlugin/SQLExportDDLRewriter.swift + - Plugins/SQLExportPlugin/SQLExportFileWriter.swift + - Plugins/SQLExportPlugin/SQLExportInsertMode.swift - Plugins/SQLExportPlugin/SQLExportModels.swift - Plugins/SQLExportPlugin/SQLExportOptionsView.swift - Plugins/SQLExportPlugin/SQLExportPlugin.swift + - Plugins/SQLExportPlugin/SQLExportSnapshot.swift - Plugins/SQLImportPlugin/SQLImportFailure.swift - Plugins/SQLImportPlugin/SQLImportOptions.swift - Plugins/SQLImportPlugin/SQLImportOptionsView.swift From 86924e518e9b3a01e97bfd7c5c784863d3399bd7 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 3 Sep 2026 14:43:44 +0700 Subject: [PATCH 2/5] feat(export): save export selections, report skipped import rows, and transfer tables between connections Claude-Session: https://claude.ai/code/session_011EqgjCjCAU6tiiVmnMpF86 --- CHANGELOG.md | 2 + .../Core/Database/NativeDumpDescriptor.swift | 5 - .../Services/Export/ImportErrorReport.swift | 64 +++++ .../Export/TableTransferService.swift | 2 +- .../Core/Storage/ExportProfileStorage.swift | 210 ++++++++++++++++ .../Components/TransferResultAlert.swift | 61 ++++- TablePro/Views/Export/ExportDialog.swift | 92 ++++++- .../Views/Export/TableTransferSheet.swift | 34 +++ TablePro/Views/Import/ImportDialog.swift | 7 +- TablePro/Views/Import/RowImportSheet.swift | 7 +- TablePro/Views/Main/MainContentView.swift | 17 +- .../Export/ExportProfileStorageTests.swift | 224 ++++++++++++++++++ .../Export/TableTransferServiceTests.swift | 8 +- docs/features/import-export.mdx | 6 +- 14 files changed, 716 insertions(+), 23 deletions(-) create mode 100644 TablePro/Core/Services/Export/ImportErrorReport.swift create mode 100644 TablePro/Core/Storage/ExportProfileStorage.swift create mode 100644 TableProTests/Core/Export/ExportProfileStorageTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index d6ed91b5b3..bec73ae6e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Backup and restore for MySQL, MariaDB, MongoDB, SQLite and libSQL, using each engine's own tools. (#2618) - Transfer To, copying table rows straight into another open connection with no file in between. (#2618) - NDJSON layout for JSON exports, one row per line. (#2618) +- Saved export selections, reapplied from the export tree's bookmark menu. (#2618) +- Save Report on an import that skipped rows, listing each one's line and error as CSV. (#2618) ### Changed diff --git a/TablePro/Core/Database/NativeDumpDescriptor.swift b/TablePro/Core/Database/NativeDumpDescriptor.swift index 7b7379987d..34a8e391f8 100644 --- a/TablePro/Core/Database/NativeDumpDescriptor.swift +++ b/TablePro/Core/Database/NativeDumpDescriptor.swift @@ -27,11 +27,6 @@ struct NativeDumpDescriptor: Sendable { struct ArchiveFormat: Sendable, Equatable { let fileExtension: String let contentDescription: String - - init(fileExtension: String, contentDescription: String) { - self.fileExtension = fileExtension - self.contentDescription = contentDescription - } } struct Request: Sendable { diff --git a/TablePro/Core/Services/Export/ImportErrorReport.swift b/TablePro/Core/Services/Export/ImportErrorReport.swift new file mode 100644 index 0000000000..5434a7adfa --- /dev/null +++ b/TablePro/Core/Services/Export/ImportErrorReport.swift @@ -0,0 +1,64 @@ +// +// ImportErrorReport.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +/// The rows an import skipped, written to a file the user can open. +/// +/// Skip and continue reports a count, and a count is not actionable: a user told that 412 of 50,000 +/// rows were skipped has no way to find out which ones without running the import again against a +/// database that now holds the other 49,588. The report names the line and the server's own error +/// for each one. +enum ImportErrorReport { + /// How many rows the report lists. A file per skipped row is unbounded, and past a few thousand + /// the file stops being something a person reads; the header says how many were left out. + static let maximumListedRows = 1_000 + + static func makeCSV( + sourceFileName: String, + targetTable: String?, + errors: [PluginImportResult.ImportStatementError], + totalSkipped: Int + ) -> String { + var lines: [String] = [] + lines.append("# TablePro import errors") + lines.append("# Source: \(csvField(sourceFileName))") + if let targetTable, !targetTable.isEmpty { + lines.append("# Target: \(csvField(targetTable))") + } + lines.append("# Skipped rows: \(totalSkipped)") + if totalSkipped > errors.count { + lines.append("# Listed below: \(errors.count)") + } + lines.append("line,statement,error") + for error in errors.prefix(maximumListedRows) { + lines.append([ + String(error.line), + csvField(error.statement), + csvField(error.errorMessage) + ].joined(separator: ",")) + } + return lines.joined(separator: "\n") + "\n" + } + + /// A default name beside the source file, so the report is where the user is already looking. + static func defaultFileName(forSource sourceFileName: String) -> String { + let stem = (sourceFileName as NSString).deletingPathExtension + let safeStem = stem.isEmpty ? "import" : stem + return "\(safeStem)-errors.csv" + } + + /// Quotes only what has to be quoted, and doubles an interior quote, which is what every + /// spreadsheet reads back as one quote rather than the start of a new field. + private static func csvField(_ value: String) -> String { + let flattened = value + .replacingOccurrences(of: "\r\n", with: " ") + .replacingOccurrences(of: "\n", with: " ") + .replacingOccurrences(of: "\r", with: " ") + guard flattened.contains(",") || flattened.contains("\"") else { return flattened } + return "\"\(flattened.replacingOccurrences(of: "\"", with: "\"\""))\"" + } +} diff --git a/TablePro/Core/Services/Export/TableTransferService.swift b/TablePro/Core/Services/Export/TableTransferService.swift index 8ef954b549..7f395476a0 100644 --- a/TablePro/Core/Services/Export/TableTransferService.swift +++ b/TablePro/Core/Services/Export/TableTransferService.swift @@ -205,7 +205,7 @@ final class TableTransferService { /// A row arrives as positional values and the sink writes by column name, so the two are /// zipped here. A row shorter than its header is padded with nulls rather than dropped: a /// driver that omits trailing nulls would otherwise lose whole rows silently. - static func dictionary(columns: [String], row: [PluginCellValue]) -> [String: PluginCellValue] { + nonisolated static func dictionary(columns: [String], row: [PluginCellValue]) -> [String: PluginCellValue] { var values: [String: PluginCellValue] = [:] values.reserveCapacity(columns.count) for (index, column) in columns.enumerated() { diff --git a/TablePro/Core/Storage/ExportProfileStorage.swift b/TablePro/Core/Storage/ExportProfileStorage.swift new file mode 100644 index 0000000000..a557c05ac2 --- /dev/null +++ b/TablePro/Core/Storage/ExportProfileStorage.swift @@ -0,0 +1,210 @@ +// +// ExportProfileStorage.swift +// TablePro +// + +import Foundation +import os +import TableProPluginKit + +/// One saved export: the format, and which objects it covered with how each was narrowed. +/// +/// The format's own options already persist globally through `SettablePlugin`, and the last format +/// through `TransferDialogStorage`. What neither carries is the selection, which is the part a user +/// rebuilds by hand every time: forty tables ticked, three of them filtered, two limited. +struct ExportProfile: Codable, Identifiable, Equatable { + struct Entry: Codable, Equatable { + let container: String + let name: String + let kind: String + let optionValues: [Bool] + let rowScope: PluginExportRowScope + + init( + container: String, + name: String, + kind: PluginExportObjectKind, + optionValues: [Bool], + rowScope: PluginExportRowScope + ) { + self.container = container + self.name = name + self.kind = kind.rawValue + self.optionValues = optionValues + self.rowScope = rowScope + } + + /// Matches the row it was saved from. Keyed by kind as well as name, because a routine and + /// a table can share one. + var key: String { "\(container).\(kind).\(name)" } + } + + let id: UUID + var name: String + var formatId: String + var entries: [Entry] + + init(id: UUID = UUID(), name: String, formatId: String, entries: [Entry]) { + self.id = id + self.name = name + self.formatId = formatId + self.entries = entries + } + + var entriesByKey: [String: Entry] { + Dictionary(entries.map { ($0.key, $0) }, uniquingKeysWith: { _, last in last }) + } +} + +/// Saved export profiles, one file per connection. +/// +/// Device-local: a profile names tables on one server, and syncing it would offer a selection whose +/// objects the other Mac's connection may not have. +@MainActor +final class ExportProfileStorage { + static let shared = ExportProfileStorage() + + private static let logger = Logger(subsystem: "com.TablePro", category: "ExportProfileStorage") + + private let directory: URL + private var cache: [UUID: [ExportProfile]] = [:] + + init(directory: URL? = nil) { + self.directory = directory ?? Self.defaultDirectory() + } + + private static func defaultDirectory() -> URL { + AppStorageEnvironment.shared.supportDirectory + .appendingPathComponent("ExportProfiles", isDirectory: true) + } + + func profiles(for connectionId: UUID) -> [ExportProfile] { + if let cached = cache[connectionId] { return cached } + let loaded = load(connectionId) + cache[connectionId] = loaded + return loaded + } + + /// Saving under a name that already exists replaces it, so re-saving a profile the user is + /// working on does not leave two rows with the same label in the picker. + func save(_ profile: ExportProfile, for connectionId: UUID) { + var profiles = profiles(for: connectionId) + if let index = profiles.firstIndex(where: { $0.name == profile.name }) { + profiles[index] = profile + } else { + profiles.append(profile) + } + profiles.sort { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending } + persist(profiles, for: connectionId) + } + + func delete(id: UUID, for connectionId: UUID) { + let profiles = profiles(for: connectionId).filter { $0.id != id } + persist(profiles, for: connectionId) + } + + func clear(for connectionId: UUID) { + persist([], for: connectionId) + } + + // MARK: - Building and applying + + nonisolated static func makeProfile( + name: String, + formatId: String, + databases: [ExportDatabaseItem] + ) -> ExportProfile { + let entries = databases.flatMap { database in + database.objects.filter(\.isSelected).map { object in + ExportProfile.Entry( + container: database.name, + name: object.name, + kind: object.kind, + optionValues: object.optionValues, + rowScope: object.rowScope + ) + } + } + return ExportProfile(name: name, formatId: formatId, entries: entries) + } + + /// Applies a profile over the tree the dialog is showing. An object the profile names that the + /// database no longer has is dropped rather than resurrected, and one the profile does not name + /// is deselected, so applying a profile twice gives the same selection both times. + nonisolated static func apply(_ profile: ExportProfile, to databases: [ExportDatabaseItem]) -> [ExportDatabaseItem] { + let entries = profile.entriesByKey + return databases.map { database in + var updated = database + updated.objects = database.objects.map { object in + var applied = object + let key = ExportProfile.Entry( + container: database.name, + name: object.name, + kind: object.kind, + optionValues: [], + rowScope: .unrestricted + ).key + guard let entry = entries[key] else { + applied.isSelected = false + return applied + } + applied.isSelected = true + applied.optionValues = entry.optionValues + applied.rowScope = entry.rowScope + return applied + } + return updated + } + } + + /// How many of a profile's objects the current tree still holds, so the picker can say when a + /// profile has gone stale instead of silently selecting fewer rows than it names. + nonisolated static func matchCount(_ profile: ExportProfile, in databases: [ExportDatabaseItem]) -> Int { + let present = Set(databases.flatMap { database in + database.objects.map { object in + ExportProfile.Entry( + container: database.name, + name: object.name, + kind: object.kind, + optionValues: [], + rowScope: .unrestricted + ).key + } + }) + return profile.entries.count { present.contains($0.key) } + } + + // MARK: - Persistence + + private func fileURL(for connectionId: UUID) -> URL { + directory.appendingPathComponent("\(connectionId.uuidString).json") + } + + private func load(_ connectionId: UUID) -> [ExportProfile] { + let url = fileURL(for: connectionId) + guard let data = try? Data(contentsOf: url) else { return [] } + do { + return try JSONDecoder().decode([ExportProfile].self, from: data) + } catch { + Self.logger.warning("Failed to read export profiles: \(error.localizedDescription)") + return [] + } + } + + private func persist(_ profiles: [ExportProfile], for connectionId: UUID) { + cache[connectionId] = profiles + let url = fileURL(for: connectionId) + do { + try FileManager.default.createDirectory( + at: directory, withIntermediateDirectories: true) + guard !profiles.isEmpty else { + try? FileManager.default.removeItem(at: url) + return + } + let data = try JSONEncoder().encode(profiles) + try data.write(to: url, options: .atomic) + } catch { + Self.logger.warning("Failed to write export profiles: \(error.localizedDescription)") + } + } +} diff --git a/TablePro/Views/Components/TransferResultAlert.swift b/TablePro/Views/Components/TransferResultAlert.swift index ac53469505..3e12c79297 100644 --- a/TablePro/Views/Components/TransferResultAlert.swift +++ b/TablePro/Views/Components/TransferResultAlert.swift @@ -5,6 +5,7 @@ import AppKit import TableProPluginKit +import UniformTypeIdentifiers /// Import and export results were three bespoke views with fixed widths and hand-picked green, /// yellow and red badges. `NSAlert` supplies the icon from its style, sizes itself to its content, @@ -50,6 +51,8 @@ internal enum TransferResultAlert { internal static func presentImportSuccess( result: PluginImportResult?, window: NSWindow?, + sourceFileName: String = "", + targetTable: String? = nil, completion: @escaping @MainActor () -> Void ) { let alert = NSAlert() @@ -61,12 +64,66 @@ internal enum TransferResultAlert { alert.informativeText = importSummary(result) alert.addButton(withTitle: String(localized: "Done")) - if let errors = result?.errors, !errors.isEmpty { + let errors = result?.errors ?? [] + if !errors.isEmpty { + /// The alert shows the first few, which is enough to recognise the shape of the + /// problem. Anything past that belongs in a file the user can sort and search. + alert.addButton(withTitle: String(localized: "Save Report…")) alert.accessoryView = TransferReportView(report: failureReport(for: errors)) alert.layout() } - AlertHelper.present(alert, in: window) { _ in completion() } + AlertHelper.present(alert, in: window) { response in + guard !errors.isEmpty, response == .alertSecondButtonReturn else { + completion() + return + } + saveErrorReport( + errors: errors, + totalSkipped: skipped, + sourceFileName: sourceFileName, + targetTable: targetTable, + window: window, + completion: completion + ) + } + } + + @MainActor + private static func saveErrorReport( + errors: [PluginImportResult.ImportStatementError], + totalSkipped: Int, + sourceFileName: String, + targetTable: String?, + window: NSWindow?, + completion: @escaping @MainActor () -> Void + ) { + let panel = NSSavePanel() + panel.canCreateDirectories = true + panel.showsTagField = false + panel.allowedContentTypes = [.commaSeparatedText] + panel.nameFieldStringValue = ImportErrorReport.defaultFileName(forSource: sourceFileName) + panel.title = String(localized: "Save Import Errors") + + let handler: @MainActor (NSApplication.ModalResponse) -> Void = { response in + defer { completion() } + guard response == .OK, let url = panel.url else { return } + let csv = ImportErrorReport.makeCSV( + sourceFileName: sourceFileName, + targetTable: targetTable, + errors: errors, + totalSkipped: totalSkipped + ) + try? csv.write(to: url, atomically: true, encoding: .utf8) + } + + guard let window else { + handler(panel.runModal()) + return + } + panel.beginSheetModal(for: window) { response in + MainActor.assumeIsolated { handler(response) } + } } internal static func presentImportFailure( diff --git a/TablePro/Views/Export/ExportDialog.swift b/TablePro/Views/Export/ExportDialog.swift index 2ecaa20ed0..af954d3f31 100644 --- a/TablePro/Views/Export/ExportDialog.swift +++ b/TablePro/Views/Export/ExportDialog.swift @@ -37,6 +37,10 @@ struct ExportDialog: View { /// already holds can answer for the new format without another round trip. @State private var loadedObjectKinds: Set = [] + @State private var profiles: [ExportProfile] = [] + @State private var profileName = "" + @State private var isNamingProfile = false + /// The window this dialog is hosted in, used for presenting its alerts and panels. /// Avoids `NSApp.keyWindow`, which when a result is presented is the progress sheet being /// torn down in the same transaction, and AppKit ends a sheet's children with it (#2314). @@ -119,6 +123,7 @@ struct ExportDialog: View { config.formatId = type(of: first).formatId } captureSettingsSnapshot() + profiles = ExportProfileStorage.shared.profiles(for: connection.id) } .onDisappear { if !exportSucceeded { @@ -238,6 +243,8 @@ struct ExportDialog: View { .font(.subheadline.weight(.medium)) .foregroundStyle(.secondary) + profileMenu + Spacer() if let plugin = currentPlugin { @@ -289,6 +296,87 @@ struct ExportDialog: View { } } + /// Saves and reapplies a selection. A profile that names objects the database no longer holds + /// says how many it still matches rather than quietly selecting fewer rows than its name + /// implies. + private var profileMenu: some View { + Menu { + if profiles.isEmpty { + Text("No saved selections") + } + ForEach(profiles) { profile in + Button { + applyProfile(profile) + } label: { + Text(profileLabel(profile)) + } + } + Divider() + Button("Save Selection…") { isNamingProfile = true } + .disabled(selectedObjects.isEmpty) + if !profiles.isEmpty { + Menu("Delete") { + ForEach(profiles) { profile in + Button(profile.name) { + ExportProfileStorage.shared.delete(id: profile.id, for: connection.id) + profiles = ExportProfileStorage.shared.profiles(for: connection.id) + } + } + } + } + } label: { + Image(systemName: "bookmark") + } + .menuStyle(.borderlessButton) + .fixedSize() + .help(String(localized: "Saved selections")) + .popover(isPresented: $isNamingProfile, arrowEdge: .bottom) { + VStack(alignment: .leading, spacing: 10) { + Text("Name this selection") + .font(.headline) + TextField("Nightly tables", text: $profileName) + .textFieldStyle(.roundedBorder) + .frame(width: 220) + HStack { + Spacer() + Button("Cancel") { isNamingProfile = false } + Button("Save") { saveProfile() } + .keyboardShortcut(.defaultAction) + .disabled(profileName.trimmingCharacters(in: .whitespaces).isEmpty) + } + } + .padding(14) + } + } + + private func profileLabel(_ profile: ExportProfile) -> String { + let matched = ExportProfileStorage.matchCount(profile, in: databaseItems) + guard matched < profile.entries.count else { return profile.name } + return String( + format: String(localized: "%1$@ (%2$lld of %3$lld still present)"), + profile.name, + Int64(matched), + Int64(profile.entries.count) + ) + } + + private func applyProfile(_ profile: ExportProfile) { + config.formatId = profile.formatId + databaseItems = normalizedForCurrentFormat( + ExportProfileStorage.apply(profile, to: databaseItems)) + } + + private func saveProfile() { + let trimmed = profileName.trimmingCharacters(in: .whitespaces) + guard !trimmed.isEmpty else { return } + let profile = ExportProfileStorage.makeProfile( + name: trimmed, formatId: config.formatId, databases: databaseItems) + ExportProfileStorage.shared.save(profile, for: connection.id) + profiles = ExportProfileStorage.shared.profiles(for: connection.id) + profileName = "" + isNamingProfile = false + } + // MARK: - Export Options View private var exportOptionsView: some View { @@ -512,8 +600,8 @@ struct ExportDialog: View { ) guard let plugin = currentPlugin else { return normalized } let pluginType = type(of: plugin) - return normalized.maskingUnsupportedOptions(columns: pluginType.perTableOptionColumns) { - columnId, kind in pluginType.supportsOption(columnId: columnId, for: kind) + return normalized.maskingUnsupportedOptions(columns: pluginType.perTableOptionColumns) { columnId, kind in + pluginType.supportsOption(columnId: columnId, for: kind) } } diff --git a/TablePro/Views/Export/TableTransferSheet.swift b/TablePro/Views/Export/TableTransferSheet.swift index 68c342a38f..f770b32c51 100644 --- a/TablePro/Views/Export/TableTransferSheet.swift +++ b/TablePro/Views/Export/TableTransferSheet.swift @@ -203,11 +203,16 @@ struct TableTransferSheet: View { /// A transfer needs two live sessions, so only connections that are already open are offered. /// Opening one from here would mean a connect, a possible prompt and a possible failure inside /// a sheet that is about to start writing rows. + /// + /// A read-only destination is left out rather than shown and refused later: this sheet writes + /// rows through the import sink, which reaches the driver directly rather than through the + /// execution gate, so the list is where that policy has to hold. @MainActor private func load() async { availableDestinations = DatabaseManager.shared.activeSessions.values .filter { $0.id != sourceConnection.id && $0.isConnected } .map { $0.effectiveConnection ?? $0.connection } + .filter { !$0.safeModeLevel.blocksAllWrites } .sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending } do { let tables = try await DatabaseManager.shared.withMetadataDriver( @@ -245,6 +250,27 @@ struct TableTransferSheet: View { } } + /// Emptying the destination's tables is not undoable, and a connection whose Safe Mode asks + /// before a write has to be asked here too, because these rows never pass the execution gate. + @MainActor + private func confirmIfNeeded(destination: DatabaseConnection) async -> Bool { + guard deleteExistingRows || destination.safeModeLevel.requiresConfirmation else { return true } + let alert = NSAlert() + alert.alertStyle = deleteExistingRows ? .critical : .warning + alert.messageText = String( + format: String(localized: "Transfer %1$lld table(s) into %2$@?"), + Int64(selectedTables.count), + destination.name) + alert.informativeText = deleteExistingRows + ? String(localized: "Every row in each destination table is deleted first. This cannot be undone.") + : String(localized: "Rows are written into tables that already exist on the destination.") + alert.addButton(withTitle: String(localized: "Transfer")) + alert.addButton(withTitle: String(localized: "Cancel")) + + guard let hostWindow else { return alert.runModal() == .alertFirstButtonReturn } + return await alert.beginSheetModal(for: hostWindow) == .alertFirstButtonReturn + } + private var sourceScope: DatabaseScope { DatabaseManager.shared.resolvedScope( database: sourceConnection.database, schema: nil, for: sourceConnection.id @@ -260,6 +286,14 @@ struct TableTransferSheet: View { errorMessage = TableTransferError.notConnected(connectionName: "").localizedDescription return } + guard !destinationConnection.safeModeLevel.blocksAllWrites else { + errorMessage = String( + format: String(localized: "%@ is read-only, so nothing can be written to it."), + destinationConnection.name) + return + } + guard await confirmIfNeeded(destination: destinationConnection) else { return } + errorMessage = nil isRunning = true defer { isRunning = false } diff --git a/TablePro/Views/Import/ImportDialog.swift b/TablePro/Views/Import/ImportDialog.swift index bf8950cc3e..b018741543 100644 --- a/TablePro/Views/Import/ImportDialog.swift +++ b/TablePro/Views/Import/ImportDialog.swift @@ -127,7 +127,12 @@ struct ImportDialog: View { } .onChange(of: showSuccessDialog) { _, isShowing in guard isShowing else { return } - TransferResultAlert.presentImportSuccess(result: importResult, window: hostWindow) { + TransferResultAlert.presentImportSuccess( + result: importResult, + window: hostWindow, + sourceFileName: fileURL?.lastPathComponent ?? "", + targetTable: nil + ) { showSuccessDialog = false isPresented = false AppCommands.shared.refreshData.send(DataRefreshRequest(connectionId: connection.id)) diff --git a/TablePro/Views/Import/RowImportSheet.swift b/TablePro/Views/Import/RowImportSheet.swift index 3925599568..131ab5b527 100644 --- a/TablePro/Views/Import/RowImportSheet.swift +++ b/TablePro/Views/Import/RowImportSheet.swift @@ -125,7 +125,12 @@ struct RowImportSheet: View { } .onChange(of: showSuccessDialog) { _, isShowing in guard isShowing else { return } - TransferResultAlert.presentImportSuccess(result: importResult, window: hostWindow) { + TransferResultAlert.presentImportSuccess( + result: importResult, + window: hostWindow, + sourceFileName: fileURL.lastPathComponent, + targetTable: selectedTargetTable + ) { showSuccessDialog = false isPresented = false AppCommands.shared.refreshData.send(DataRefreshRequest(connectionId: connection.id)) diff --git a/TablePro/Views/Main/MainContentView.swift b/TablePro/Views/Main/MainContentView.swift index 66fc927f98..ca79aa288f 100644 --- a/TablePro/Views/Main/MainContentView.swift +++ b/TablePro/Views/Main/MainContentView.swift @@ -159,6 +159,17 @@ struct MainContentView: View { /// Returns the appropriate sheet view for the given `ActiveSheet` case. /// Uses a dismissal binding that sets `coordinator.activeSheet = nil` when the /// child view sets `isPresented = false`. + /// The transfer sheet is built here rather than inline, because `sheetContent(for:)` is one + /// switch over every sheet the window can present and is already at the function length limit. + @ViewBuilder + private func transferSheet(tables: Set, dismiss: Binding) -> some View { + TableTransferSheet( + isPresented: dismiss, + sourceConnection: connectionWithCurrentDatabase, + preselectedTables: tables + ) + } + @ViewBuilder private func sheetContent(for sheet: ActiveSheet) -> some View { let dismissBinding = Binding( @@ -260,11 +271,7 @@ struct MainContentView: View { ) } case .transferTables(let tables): - TableTransferSheet( - isPresented: dismissBinding, - sourceConnection: connectionWithCurrentDatabase, - preselectedTables: tables - ) + transferSheet(tables: tables, dismiss: dismissBinding) case .backupDatabase: BackupDatabaseFlow( isPresented: dismissBinding, diff --git a/TableProTests/Core/Export/ExportProfileStorageTests.swift b/TableProTests/Core/Export/ExportProfileStorageTests.swift new file mode 100644 index 0000000000..196c69ac04 --- /dev/null +++ b/TableProTests/Core/Export/ExportProfileStorageTests.swift @@ -0,0 +1,224 @@ +// +// ExportProfileStorageTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +@Suite("Export profiles") +struct ExportProfileStorageTests { + + private func databases() -> [ExportDatabaseItem] { + [ + ExportDatabaseItem(name: "app", objects: [ + ExportObjectItem(name: "users", kind: .table, isSelected: true, optionValues: [true, false, true]), + ExportObjectItem(name: "posts", kind: .table, isSelected: false, optionValues: [true, true, true]), + ExportObjectItem(name: "recalc", kind: .routine, isSelected: true, optionValues: [true, true, false]) + ]) + ] + } + + @Test("A profile captures only the selected objects") + func profileCapturesSelection() { + let profile = ExportProfileStorage.makeProfile( + name: "Nightly", formatId: "sql", databases: databases()) + #expect(profile.entries.count == 2) + #expect(Set(profile.entries.map(\.name)) == ["users", "recalc"]) + #expect(profile.formatId == "sql") + } + + @Test("A profile carries each object's options and row scope") + func profileCarriesOptionsAndScope() { + var items = databases() + items[0].objects[0].rowScope = PluginExportRowScope(filter: "active", rowLimit: 50) + let profile = ExportProfileStorage.makeProfile(name: "N", formatId: "sql", databases: items) + let users = try? #require(profile.entries.first { $0.name == "users" }) + #expect(users?.optionValues == [true, false, true]) + #expect(users?.rowScope.rowLimit == 50) + #expect(users?.rowScope.sanitizedFilter == "active") + } + + /// A routine and a table can share a name, so the key has to carry the kind or applying a + /// profile would tick the wrong row. + @Test("A profile entry is keyed by container, kind and name") + func entryKeyIncludesKind() { + let table = ExportProfile.Entry( + container: "app", name: "users", kind: .table, optionValues: [], rowScope: .unrestricted) + let routine = ExportProfile.Entry( + container: "app", name: "users", kind: .routine, optionValues: [], rowScope: .unrestricted) + #expect(table.key != routine.key) + } + + @Test("Applying a profile restores its selection and clears everything else") + func applyRestoresSelection() { + let profile = ExportProfileStorage.makeProfile( + name: "N", formatId: "sql", databases: databases()) + var cleared = databases() + for index in cleared[0].objects.indices { + cleared[0].objects[index].isSelected = false + cleared[0].objects[index].optionValues = [] + } + let applied = ExportProfileStorage.apply(profile, to: cleared) + #expect(applied[0].objects.filter(\.isSelected).map(\.name).sorted() == ["recalc", "users"]) + #expect(applied[0].objects.first { $0.name == "users" }?.optionValues == [true, false, true]) + } + + /// Applying twice must give the same selection both times, so a row the profile does not name + /// is deselected rather than left ticked from whatever was there before. + @Test("Applying a profile is idempotent") + func applyIsIdempotent() { + let profile = ExportProfileStorage.makeProfile( + name: "N", formatId: "sql", databases: databases()) + var items = databases() + items[0].objects[1].isSelected = true + let once = ExportProfileStorage.apply(profile, to: items) + let twice = ExportProfileStorage.apply(profile, to: once) + #expect(once.map { $0.objects.map(\.isSelected) } == twice.map { $0.objects.map(\.isSelected) }) + #expect(once[0].objects[1].isSelected == false) + } + + @Test("An object the database no longer holds is counted as missing, not resurrected") + func missingObjectsAreCounted() { + let profile = ExportProfileStorage.makeProfile( + name: "N", formatId: "sql", databases: databases()) + var shrunk = databases() + shrunk[0].objects.removeAll { $0.name == "recalc" } + #expect(ExportProfileStorage.matchCount(profile, in: shrunk) == 1) + + let applied = ExportProfileStorage.apply(profile, to: shrunk) + #expect(applied[0].objects.count == 2) + #expect(applied[0].objects.filter(\.isSelected).map(\.name) == ["users"]) + } + + @MainActor @Test("Profiles round trip through the store") + func profilesRoundTrip() throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let connectionId = UUID() + let store = ExportProfileStorage(directory: directory) + let profile = ExportProfileStorage.makeProfile( + name: "Nightly", formatId: "sql", databases: databases()) + + store.save(profile, for: connectionId) + let reloaded = ExportProfileStorage(directory: directory).profiles(for: connectionId) + #expect(reloaded.count == 1) + #expect(reloaded.first?.name == "Nightly") + #expect(reloaded.first?.entries.count == 2) + } + + /// Saving under a name that already exists replaces it, so the picker never shows two rows + /// with the same label. + @MainActor @Test("Saving under an existing name replaces it") + func savingReplacesByName() { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let store = ExportProfileStorage(directory: directory) + let connectionId = UUID() + store.save( + ExportProfileStorage.makeProfile(name: "N", formatId: "sql", databases: databases()), + for: connectionId) + + var narrowed = databases() + narrowed[0].objects[2].isSelected = false + store.save( + ExportProfileStorage.makeProfile(name: "N", formatId: "csv", databases: narrowed), + for: connectionId) + + let profiles = store.profiles(for: connectionId) + #expect(profiles.count == 1) + #expect(profiles[0].formatId == "csv") + #expect(profiles[0].entries.count == 1) + } + + @MainActor @Test("Deleting the last profile removes the file rather than leaving an empty one") + func deletingLastProfileRemovesFile() { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let store = ExportProfileStorage(directory: directory) + let connectionId = UUID() + let profile = ExportProfileStorage.makeProfile( + name: "N", formatId: "sql", databases: databases()) + store.save(profile, for: connectionId) + store.delete(id: profile.id, for: connectionId) + + #expect(store.profiles(for: connectionId).isEmpty) + let file = directory.appendingPathComponent("\(connectionId.uuidString).json") + #expect(!FileManager.default.fileExists(atPath: file.path)) + } + + @MainActor @Test("A connection with no saved profiles reads back empty") + func unknownConnectionIsEmpty() { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + defer { try? FileManager.default.removeItem(at: directory) } + + #expect(ExportProfileStorage(directory: directory).profiles(for: UUID()).isEmpty) + } +} + +@Suite("Import error report") +struct ImportErrorReportTests { + + private let errors = [ + PluginImportResult.ImportStatementError( + statement: "row 12", line: 12, errorMessage: "duplicate key value"), + PluginImportResult.ImportStatementError( + statement: "row 40", line: 40, errorMessage: "null value in column \"name\"") + ] + + @Test("The report names the source, the target and the skipped count") + func reportHeader() { + let csv = ImportErrorReport.makeCSV( + sourceFileName: "users.csv", targetTable: "users", errors: errors, totalSkipped: 2) + #expect(csv.contains("# Source: users.csv")) + #expect(csv.contains("# Target: users")) + #expect(csv.contains("# Skipped rows: 2")) + #expect(csv.contains("line,statement,error")) + } + + @Test("Each skipped row appears with its line and the server's own message") + func reportRows() { + let csv = ImportErrorReport.makeCSV( + sourceFileName: "users.csv", targetTable: nil, errors: errors, totalSkipped: 2) + #expect(csv.contains("12,row 12,duplicate key value")) + #expect(csv.contains("40,row 40,\"null value in column \"\"name\"\"\"")) + } + + /// A message holding a comma or a newline would otherwise break the column count and every + /// spreadsheet would read the rest of the file shifted. + @Test("A message with a comma or a newline stays inside its field") + func messagesAreQuoted() { + let awkward = [PluginImportResult.ImportStatementError( + statement: "row 1", line: 1, errorMessage: "bad value, at\nline 2")] + let csv = ImportErrorReport.makeCSV( + sourceFileName: "a.csv", targetTable: nil, errors: awkward, totalSkipped: 1) + let dataLines = csv.split(separator: "\n").filter { !$0.hasPrefix("#") && $0 != "line,statement,error" } + #expect(dataLines.count == 1) + #expect(dataLines[0] == "1,row 1,\"bad value, at line 2\"") + } + + @Test("A count larger than the listed errors says how many were left out") + func truncationIsStated() { + let csv = ImportErrorReport.makeCSV( + sourceFileName: "a.csv", targetTable: nil, errors: errors, totalSkipped: 500) + #expect(csv.contains("# Skipped rows: 500")) + #expect(csv.contains("# Listed below: 2")) + } + + @Test("The default report name sits beside the file it came from") + func defaultFileName() { + #expect(ImportErrorReport.defaultFileName(forSource: "users.csv") == "users-errors.csv") + #expect(ImportErrorReport.defaultFileName(forSource: "dump.sql.gz") == "dump.sql-errors.csv") + #expect(ImportErrorReport.defaultFileName(forSource: "") == "import-errors.csv") + } +} diff --git a/TableProTests/Core/Export/TableTransferServiceTests.swift b/TableProTests/Core/Export/TableTransferServiceTests.swift index d1970ae6c2..45fca3c868 100644 --- a/TableProTests/Core/Export/TableTransferServiceTests.swift +++ b/TableProTests/Core/Export/TableTransferServiceTests.swift @@ -65,9 +65,9 @@ struct TableTransferServiceTests { /// The transfer moves rows, so a request naming only definition objects has nothing to do and /// must say so rather than reporting a successful transfer of nothing. - @Test("A request with no row-carrying object is refused") - func requestWithoutTablesIsRefused() async { - let service = await TableTransferService() + @MainActor @Test("A request with no row-carrying object is refused") + func requestWithoutTablesIsRefused() { + let service = TableTransferService() let request = TableTransferService.Request( objects: [ ExportObjectItem(name: "recalc", kind: .routine), @@ -77,7 +77,7 @@ struct TableTransferServiceTests { destinationType: .postgresql ) #expect(request.objects.allSatisfy { !$0.kind.carriesRows }) - await #expect(service.state.isTransferring == false) + #expect(service.state.isTransferring == false) } @Test("A request keeps the row scope of every object it names") diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index 8405fb637c..8fc41a0354 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -21,6 +21,8 @@ The toolbar's **Export** (`Cmd+Shift+E`) re-reads every table you tick with `SEL The last format and its options come back next time, but only after an export succeeds; cancelling discards the changes. **Reset to Defaults** restores the stock settings for the current format. +The bookmark button above the tree saves the current selection under a name and reapplies it later, including each table's options and row filter. A saved selection naming tables the database no longer has says how many it still matches, and applying it selects those and clears the rest. + Export dialog Export dialog @@ -162,7 +164,7 @@ Right-click tables in the sidebar and choose **Transfer To…** to copy their ro -Rows only. The destination table has to exist and its column names have to match, because inventing DDL that crosses two engines would create tables whose types quietly disagree with the data landing in them. A per-table row filter set in the export tree is not carried over; narrow the transfer by transferring fewer tables. +Rows only. The destination table has to exist and its column names have to match, because inventing DDL that crosses from one engine to another would create tables whose types quietly disagree with the data landing in them. A per-table row filter set in the export tree is not carried over; narrow the transfer by transferring fewer tables. **Delete existing rows first** empties each destination table before writing. There is no undo. @@ -208,7 +210,7 @@ Select a row in the data grid and press `Cmd+V` to paste tabular data straight i | **Stop and Commit** | Stops there, keeping what already succeeded | | **Skip and Continue** | Logs it and carries on, including a line the parser cannot read. No transaction | -Skip and Continue collects up to 1,000 failures with their line numbers and messages, and the summary counts successes against failures behind a **Copy Details** button. A stop shows the line, the database's own message, and the failing statement, with the dialog still open behind it, ready for a changed setting and another run. +Skip and Continue collects up to 1,000 failures with their line numbers and messages, and the summary counts successes against failures behind a **Copy Details** button. **Save Report…** writes them all to a CSV with a line, a statement and the database's own error per row, so a large import's failures can be sorted and searched rather than scrolled. A stop shows the line, the database's own message, and the failing statement, with the dialog still open behind it, ready for a changed setting and another run. ### Disabling foreign key checks From a6c6f22cda00971d8baa3f1dd05d21b54c4aafd6 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 3 Sep 2026 15:53:19 +0700 Subject: [PATCH 3/5] feat(export): add Markdown, HTML, XML and Parquet export, events and sequences, and share the row writers Claude-Session: https://claude.ai/code/session_011EqgjCjCAU6tiiVmnMpF86 --- Libs/checksums.sha256 | 8 +- Plugins/CSVExportPlugin/CSVExportPlugin.swift | 51 +++--- .../HTMLExportPlugin/HTMLExportModels.swift | 55 ++++++ .../HTMLExportOptionsView.swift | 26 +++ .../HTMLExportPlugin/HTMLExportPlugin.swift | 162 +++++++++++++++++ Plugins/HTMLExportPlugin/Info.plist | 12 ++ Plugins/MarkdownExportPlugin/Info.plist | 12 ++ .../MarkdownExportModels.swift | 89 ++++++++++ .../MarkdownExportOptionsView.swift | 31 ++++ .../MarkdownExportPlugin.swift | 151 ++++++++++++++++ .../MySQLDriverPlugin/MySQLPluginDriver.swift | 29 ++++ .../DuckDBStagingDatabase.swift | 140 +++++++++++++++ Plugins/ParquetExportPlugin/Info.plist | 12 ++ .../ParquetExportModels.swift | 51 ++++++ .../ParquetExportOptionsView.swift | 50 ++++++ .../ParquetExportPlugin.swift | 151 ++++++++++++++++ .../ParquetTypeMapper.swift | 68 ++++++++ .../PostgreSQLPluginDriver.swift | 46 +++++ Plugins/SQLExportPlugin/SQLExportPlugin.swift | 14 +- .../PluginDatabaseDriver.swift | 20 +++ .../TableProPluginKit/PluginEventInfo.swift | 44 +++++ .../TableProPluginKit/PluginRowWriters.swift | 164 ++++++++++++++++++ Plugins/XMLExportPlugin/Info.plist | 12 ++ Plugins/XMLExportPlugin/XMLExportModels.swift | 90 ++++++++++ .../XMLExportOptionsView.swift | 30 ++++ Plugins/XMLExportPlugin/XMLExportPlugin.swift | 130 ++++++++++++++ .../MCP/Protocol/Tools/MCPExportWriters.swift | 50 +++--- .../Plugins/ExportDataSourceAdapter.swift | 26 +++ TablePro/Core/Plugins/ExportObjectCache.swift | 22 +++ .../Services/Export/ExportObjectLoader.swift | 47 ++++- .../Services/Export/ImportErrorReport.swift | 12 +- .../Services/Export/TableColumnMatcher.swift | 82 +++++++++ .../Export/TableTransferService.swift | 56 +++++- TablePro/Views/Export/ExportDialog.swift | 5 +- .../Export/TableTransferMappingEditor.swift | 98 +++++++++++ .../Views/Export/TableTransferSheet.swift | 154 +++++++++++++++- project.yml | 69 ++++++++ scripts/check-duckdb-extensions.sh | 84 +++++++++ 38 files changed, 2284 insertions(+), 69 deletions(-) create mode 100644 Plugins/HTMLExportPlugin/HTMLExportModels.swift create mode 100644 Plugins/HTMLExportPlugin/HTMLExportOptionsView.swift create mode 100644 Plugins/HTMLExportPlugin/HTMLExportPlugin.swift create mode 100644 Plugins/HTMLExportPlugin/Info.plist create mode 100644 Plugins/MarkdownExportPlugin/Info.plist create mode 100644 Plugins/MarkdownExportPlugin/MarkdownExportModels.swift create mode 100644 Plugins/MarkdownExportPlugin/MarkdownExportOptionsView.swift create mode 100644 Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift create mode 100644 Plugins/ParquetExportPlugin/DuckDBStagingDatabase.swift create mode 100644 Plugins/ParquetExportPlugin/Info.plist create mode 100644 Plugins/ParquetExportPlugin/ParquetExportModels.swift create mode 100644 Plugins/ParquetExportPlugin/ParquetExportOptionsView.swift create mode 100644 Plugins/ParquetExportPlugin/ParquetExportPlugin.swift create mode 100644 Plugins/ParquetExportPlugin/ParquetTypeMapper.swift create mode 100644 Plugins/TableProPluginKit/PluginEventInfo.swift create mode 100644 Plugins/TableProPluginKit/PluginRowWriters.swift create mode 100644 Plugins/XMLExportPlugin/Info.plist create mode 100644 Plugins/XMLExportPlugin/XMLExportModels.swift create mode 100644 Plugins/XMLExportPlugin/XMLExportOptionsView.swift create mode 100644 Plugins/XMLExportPlugin/XMLExportPlugin.swift create mode 100644 TablePro/Core/Services/Export/TableColumnMatcher.swift create mode 100644 TablePro/Views/Export/TableTransferMappingEditor.swift create mode 100755 scripts/check-duckdb-extensions.sh diff --git a/Libs/checksums.sha256 b/Libs/checksums.sha256 index b52f99973d..98ee6bffea 100644 --- a/Libs/checksums.sha256 +++ b/Libs/checksums.sha256 @@ -10,10 +10,10 @@ a891a67c2619e2ac1dce64dafc6a24bfde9cabe15312dac6b70a19385664ea84 Libs/libcrypto 732adf315bc49f77e2511a9293e49a65e18eb54a3e6d01d8a24eee2d671d2a8a Libs/libcrypto_universal.a 965ccd38fea5cd97bc878dbf58567e4eed2b2337120f8d46a2da62c094b3c821 Libs/libcrypto_x86_64.a 732adf315bc49f77e2511a9293e49a65e18eb54a3e6d01d8a24eee2d671d2a8a Libs/libcrypto.a -1ef4f456b99285dca4fd8cfedef1a2f4936b0a9567a7e332baebdba04bf89d77 Libs/libduckdb_arm64.a -68a13d3a915acc08b59a4b982525afa143b588ec6e19c8e470e2ff97cf0df3d9 Libs/libduckdb_universal.a -e809dd7c7ec05a8218d273b3350a9a20e15a6ddf1e392ccbe5560395166a6b26 Libs/libduckdb_x86_64.a -68a13d3a915acc08b59a4b982525afa143b588ec6e19c8e470e2ff97cf0df3d9 Libs/libduckdb.a +e325cc7f47ad2ac91f777d075e5533b0ef34769e16cd728496d3f9229ed6ff9b Libs/libduckdb_arm64.a +b578aa2a73b0b84ba36e436d0cade00d5e98f1cb1ebbc849374758e9054985a9 Libs/libduckdb_universal.a +76737d65affec9b13676e0a39d3bd1054ee17642a04fa2ce34b9c2e998693fa0 Libs/libduckdb_x86_64.a +b578aa2a73b0b84ba36e436d0cade00d5e98f1cb1ebbc849374758e9054985a9 Libs/libduckdb.a 7e63017fa22c2eb7744eccad13857361a5088aa7b2772ab02cd026c8c7b78341 Libs/libhiredis_arm64.a f1cfc36a7ab47361e9705fe32b1c919b318f606989478e91a808707d93db55a5 Libs/libhiredis_ssl_arm64.a fb7a32c2c724cb4f3f880030cb19afbbc7db52121ad8e35e00a2e818da9562cf Libs/libhiredis_ssl_universal.a diff --git a/Plugins/CSVExportPlugin/CSVExportPlugin.swift b/Plugins/CSVExportPlugin/CSVExportPlugin.swift index 695084f572..4a64c64387 100644 --- a/Plugins/CSVExportPlugin/CSVExportPlugin.swift +++ b/Plugins/CSVExportPlugin/CSVExportPlugin.swift @@ -146,33 +146,36 @@ final class CSVExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send try fileHandle.write(contentsOf: (rowLine + lineBreak).toUTF8Data()) } + /// Escaping and quoting live in `PluginRowWriters`, so this format, the other export formats + /// and the MCP tool spell a value the same way. Only the option mapping is this plugin's own. + /// + /// `originalHadLineBreaks` says the value's breaks were already replaced with spaces upstream, + /// and it still forces quoting: the source text spanned lines, and a reader that splits on the + /// delimiter has no way to know the space it now sees was one. private func escapeCSVField(_ field: String, options: CSVExportOptions, originalHadLineBreaks: Bool = false) -> String { - var processed = field - - if options.sanitizeFormulas { - let dangerousPrefixes: [Character] = ["=", "+", "-", "@"] - if let first = processed.first, dangerousPrefixes.contains(first) { - processed = "'" + processed - } + let escaped = PluginRowWriters.csvField(field, options: writeOptions(options)) + guard originalHadLineBreaks, options.quoteHandling == .asNeeded, !escaped.hasPrefix("\"") else { + return escaped } + return "\"\(escaped.replacingOccurrences(of: "\"", with: "\"\""))\"" + } - switch options.quoteHandling { - case .always: - let escaped = processed.replacingOccurrences(of: "\"", with: "\"\"") - return "\"\(escaped)\"" - case .never: - return processed - case .asNeeded: - let needsQuotes = processed.contains(options.delimiter.actualValue) || - processed.contains("\"") || - processed.contains("\n") || - processed.contains("\r") || - originalHadLineBreaks - if needsQuotes { - let escaped = processed.replacingOccurrences(of: "\"", with: "\"\"") - return "\"\(escaped)\"" - } - return processed + private func writeOptions(_ options: CSVExportOptions) -> PluginCsvWriteOptions { + PluginCsvWriteOptions( + delimiter: options.delimiter.actualValue, + quoteHandling: quoteHandling(options.quoteHandling), + lineEnding: options.lineBreak.value, + nullAsEmpty: true, + sanitizesFormulas: options.sanitizeFormulas, + flattensLineBreaks: false + ) + } + + private func quoteHandling(_ handling: CSVQuoteHandling) -> PluginCsvWriteOptions.QuoteHandling { + switch handling { + case .always: return .always + case .never: return .never + case .asNeeded: return .asNeeded } } } diff --git a/Plugins/HTMLExportPlugin/HTMLExportModels.swift b/Plugins/HTMLExportPlugin/HTMLExportModels.swift new file mode 100644 index 0000000000..315cef8214 --- /dev/null +++ b/Plugins/HTMLExportPlugin/HTMLExportModels.swift @@ -0,0 +1,55 @@ +// +// HTMLExportModels.swift +// HTMLExportPlugin +// + +import Foundation + +public struct HTMLExportOptions: Equatable, Codable { + /// Wraps the tables in a full document with a stylesheet. Off writes bare `
` elements, + /// which is what pasting into an existing page wants. + public var writesFullDocument: Bool = true + + public var includesTableNames: Bool = true + + /// Renders a null as a dimmed `NULL` rather than an empty cell, which is otherwise identical to + /// a cell holding an empty string. + public var marksNulls: Bool = true + + public init() {} + + /// A synthesized `init(from:)` throws `keyNotFound` for a key the saved payload predates and + /// never falls back to the property's default, so adding one would reset what a user chose. + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let defaults = HTMLExportOptions() + writesFullDocument = try container.decodeIfPresent(Bool.self, forKey: .writesFullDocument) + ?? defaults.writesFullDocument + includesTableNames = try container.decodeIfPresent(Bool.self, forKey: .includesTableNames) + ?? defaults.includesTableNames + marksNulls = try container.decodeIfPresent(Bool.self, forKey: .marksNulls) ?? defaults.marksNulls + } +} + +/// Escapes text for HTML. +/// +/// Every value in an export comes from the database, so a value holding `") + == "<script>alert(1)</script>") + } + + /// The ampersand has to be replaced first, or the escapes written after it are themselves + /// escaped and the value renders as `&lt;`. + @Test("An ampersand is escaped once, not twice") + func ampersandEscapedOnce() { + #expect(HTMLEscaping.text("&") == "&") + #expect(HTMLEscaping.text("<") == "&lt;") + #expect(HTMLEscaping.text("a & b < c") == "a & b < c") + } + + @Test("Quotes are escaped so a value cannot break out of an attribute") + func quotesAreEscaped() { + #expect(HTMLEscaping.text("\"x\"") == ""x"") + #expect(HTMLEscaping.text("'x'") == "'x'") + } + + @Test("Ordinary text is unchanged") + func plainTextUnchanged() { + #expect(HTMLEscaping.text("Ada Lovelace") == "Ada Lovelace") + #expect(HTMLEscaping.text("") == "") + } +} + +@Suite("XML export escaping") +struct XMLExportEscapingTests { + + @Test("The five predefined entities are escaped") + func entitiesAreEscaped() { + #expect(XMLEscaping.text("") == "<a & b>") + #expect(XMLEscaping.text("\"'") == ""'") + } + + /// XML 1.0 accepts tab, newline and carriage return and no other control character. A stray + /// 0x00 out of a binary column would otherwise make the whole document unparseable. + @Test("Illegal control characters are dropped, legal whitespace is kept") + func controlCharactersAreDropped() { + #expect(XMLEscaping.text("a\u{0}b") == "ab") + #expect(XMLEscaping.text("a\u{1}\u{1F}b") == "ab") + #expect(XMLEscaping.text("a\tb\nc\rd") == "a\tb\nc\rd") + } + + /// A column name is not automatically a legal element name: XML forbids a leading digit and + /// restricts the character set, and a database column has neither limit. + @Test("A column name becomes a legal element name") + func elementNamesAreLegal() { + #expect(XMLEscaping.elementName("name") == "name") + #expect(XMLEscaping.elementName("first_name") == "first_name") + #expect(XMLEscaping.elementName("2024_total") == "_2024_total") + #expect(XMLEscaping.elementName("order total") == "order_total") + #expect(XMLEscaping.elementName("a-b.c") == "a-b.c") + } + + /// Names beginning `xml` in any case are reserved by the specification. + @Test("A reserved xml prefix is renamed") + func reservedPrefixIsRenamed() { + #expect(XMLEscaping.elementName("xmlData") == "_xmlData") + #expect(XMLEscaping.elementName("XMLData") == "_XMLData") + } + + @Test("A name with nothing legal in it falls back rather than producing invalid XML") + func emptyNameFallsBack() { + #expect(XMLEscaping.elementName("") == "column") + } +} + +@Suite("Parquet type mapping") +struct ParquetTypeMapperTests { + + @Test("Integer families map to BIGINT") + func integerFamilies() { + for type in ["INT", "int4", "BIGINT", "smallint", "TINYINT", "SERIAL", "MEDIUMINT"] { + #expect(ParquetTypeMapper.duckDBType(forColumnType: type) == "BIGINT", "\(type)") + } + } + + @Test("Decimal families map to DOUBLE") + func decimalFamilies() { + for type in ["DECIMAL(10,2)", "numeric", "FLOAT", "double precision", "REAL", "money"] { + #expect(ParquetTypeMapper.duckDBType(forColumnType: type) == "DOUBLE", "\(type)") + } + } + + @Test("Temporal families keep their own types") + func temporalFamilies() { + #expect(ParquetTypeMapper.duckDBType(forColumnType: "DATE") == "DATE") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "timestamp with time zone") == "TIMESTAMP") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "datetime") == "TIMESTAMP") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "TIME") == "TIME") + } + + @Test("Booleans and binaries map to their own types") + func booleanAndBinary() { + #expect(ParquetTypeMapper.duckDBType(forColumnType: "BOOLEAN") == "BOOLEAN") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "bytea") == "BLOB") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "VARBINARY(50)") == "BLOB") + } + + /// An unknown type is written as text rather than guessed at. A wrong guess writes a Parquet + /// file whose column type disagrees with the data in it. + @Test("An unknown type falls back to VARCHAR") + func unknownFallsBack() { + #expect(ParquetTypeMapper.duckDBType(forColumnType: "geography") == "VARCHAR") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "") == "VARCHAR") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "hstore") == "VARCHAR") + } + + /// A type name carries its width in parentheses and sometimes a modifier after a space, and + /// neither changes which family it belongs to. + @Test("Width and modifiers are stripped before matching") + func baseNameStripsArgumentsAndModifiers() { + #expect(ParquetTypeMapper.baseName("VARCHAR(64)") == "varchar") + #expect(ParquetTypeMapper.baseName("NUMERIC(10, 2)") == "numeric") + #expect(ParquetTypeMapper.baseName("INT UNSIGNED") == "int") + #expect(ParquetTypeMapper.baseName(" TIMESTAMP WITH TIME ZONE ") == "timestamp") + } + + @Test("Parquet holds one table per file, so a multi-table export numbers its files") + func perTableFileNaming() { + let base = URL(fileURLWithPath: "/tmp/dump.parquet") + #expect(ParquetFileNaming.perTableURL(destination: base, table: "users").lastPathComponent + == "dump.users.parquet") + + let noExtension = URL(fileURLWithPath: "/tmp/dump") + #expect(ParquetFileNaming.perTableURL(destination: noExtension, table: "users").lastPathComponent + == "dump.users") + } + + /// A schema-qualified name carries a separator that would otherwise create a directory that + /// does not exist. + @Test("A table name with a slash cannot escape its directory") + func slashesAreNeutralised() { + let base = URL(fileURLWithPath: "/tmp/dump.parquet") + let url = ParquetFileNaming.perTableURL(destination: base, table: "a/b") + #expect(url.lastPathComponent == "dump.a_b.parquet") + #expect(url.deletingLastPathComponent().path == "/tmp") + } +} + +@Suite("Shared row writers") +struct PluginRowWritersTests { + + /// The values in an export come from the database rather than from the person opening the + /// file, so a value that a spreadsheet would run as a formula is neutralised. + @Test("Formula leads are neutralised and the value is then quoted") + func formulaLeadsAreNeutralised() { + let options = PluginCsvWriteOptions.default + #expect(PluginRowWriters.csvField("=1+1", options: options) == "\"'=1+1\"") + #expect(PluginRowWriters.csvField("+1", options: options) == "\"'+1\"") + #expect(PluginRowWriters.csvField("-1", options: options) == "\"'-1\"") + #expect(PluginRowWriters.csvField("@SUM", options: options) == "\"'@SUM\"") + } + + /// Excel strips a leading tab or carriage return before parsing the cell, so `\t=1+1` reaches + /// the formula engine exactly as `=1+1` would. + @Test("A leading tab or carriage return counts as a formula lead") + func whitespaceLeadCountsAsFormula() { + let options = PluginCsvWriteOptions.default + #expect(PluginRowWriters.csvField("\t=1+1", options: options).hasPrefix("\"'")) + #expect(PluginRowWriters.csvField("\r=1+1", options: options).hasPrefix("\"'")) + } + + @Test("Sanitizing off leaves the value alone") + func sanitizingCanBeTurnedOff() { + let options = PluginCsvWriteOptions(sanitizesFormulas: false) + #expect(PluginRowWriters.csvField("=1+1", options: options) == "=1+1") + } + + @Test("A value holding the delimiter or a quote is quoted and its quotes doubled") + func quotingRules() { + let options = PluginCsvWriteOptions.default + #expect(PluginRowWriters.csvField("a,b", options: options) == "\"a,b\"") + #expect(PluginRowWriters.csvField("say \"hi\"", options: options) == "\"say \"\"hi\"\"\"") + #expect(PluginRowWriters.csvField("plain", options: options) == "plain") + } + + @Test("Quote handling always and never are honoured") + func quoteHandlingModes() { + #expect(PluginRowWriters.csvField("plain", options: PluginCsvWriteOptions(quoteHandling: .always)) + == "\"plain\"") + #expect(PluginRowWriters.csvField("a,b", options: PluginCsvWriteOptions(quoteHandling: .never)) + == "a,b") + } + + @Test("A line break is kept and quoted, or flattened when asked") + func lineBreakHandling() { + #expect(PluginRowWriters.csvField("a\nb", options: PluginCsvWriteOptions()) == "\"a\nb\"") + #expect(PluginRowWriters.csvField("a\nb", options: PluginCsvWriteOptions(flattensLineBreaks: true)) + == "a b") + } + + @Test("A line joins its fields with the configured delimiter") + func lineJoining() { + let tabbed = PluginCsvWriteOptions(delimiter: "\t") + #expect(PluginRowWriters.csvLine(["a", "b"], options: tabbed) == "a\tb") + } + + @Test("A null is JSON null and bytes are base64") + func jsonNullAndBytes() { + #expect(PluginRowWriters.jsonValue(.null) == "null") + #expect(PluginRowWriters.jsonValue(.bytes(Data([0x41, 0x42]))) == "\"QUI=\"") + } + + /// A numeric-looking identifier stays a string unless its column is numeric, or a postcode + /// loses its leading zero. + @Test("Text is written unquoted only when its column is numeric") + func numericOnlyWhenColumnSaysSo() { + #expect(PluginRowWriters.jsonValue(.text("01234"), columnTypeName: "VARCHAR") == "\"01234\"") + #expect(PluginRowWriters.jsonValue(.text("42"), columnTypeName: "INT") == "42") + #expect(PluginRowWriters.jsonValue(.text("42"), columnTypeName: "") == "\"42\"") + #expect(PluginRowWriters.jsonValue(.text("abc"), columnTypeName: "INT") == "\"abc\"") + } + + @Test("Preserving strings quotes even a numeric column") + func preserveAsStringWins() { + #expect(PluginRowWriters.jsonValue(.text("42"), columnTypeName: "INT", preserveAsString: true) + == "\"42\"") + } + + @Test("A JSON object pairs columns with values and can drop nulls") + func jsonObjectShape() { + #expect(PluginRowWriters.jsonObject(columns: ["a", "b"], values: ["1", "null"]) + == "{\"a\": 1, \"b\": null}") + #expect(PluginRowWriters.jsonObject(columns: ["a", "b"], values: ["1", "null"], includesNulls: false) + == "{\"a\": 1}") + } + + @Test("An insert names its columns and ends in a semicolon") + func insertShape() { + #expect(PluginRowWriters.sqlInsert(table: "\"t\"", columns: ["\"a\""], values: ["1"]) + == "INSERT INTO \"t\" (\"a\") VALUES (1);") + #expect(PluginRowWriters.sqlInsert(table: "\"t\"", columns: [], values: []) == nil) + } +} diff --git a/TableProTests/Plugins/XLSXImportTests.swift b/TableProTests/Plugins/XLSXImportTests.swift new file mode 100644 index 0000000000..4bd0c9c800 --- /dev/null +++ b/TableProTests/Plugins/XLSXImportTests.swift @@ -0,0 +1,280 @@ +// +// XLSXImportTests.swift +// TableProTests +// + +import Compression +import Foundation +import TableProPluginKit +import Testing + +@Suite("XLSX sheet parsing") +struct XLSXSheetParserTests { + + /// `A` is 0 and `AA` is 26, so the letters are base-26 with no zero digit. Getting this wrong + /// puts every column past Z in the wrong place. + @Test("A cell reference resolves to its column index") + func columnIndexFromReference() { + #expect(XLSXSheetParser.columnIndex(fromReference: "A1") == 0) + #expect(XLSXSheetParser.columnIndex(fromReference: "B2") == 1) + #expect(XLSXSheetParser.columnIndex(fromReference: "Z9") == 25) + #expect(XLSXSheetParser.columnIndex(fromReference: "AA1") == 26) + #expect(XLSXSheetParser.columnIndex(fromReference: "AB1") == 27) + #expect(XLSXSheetParser.columnIndex(fromReference: "BA10") == 52) + } + + @Test("A reference with no letters is refused") + func invalidReference() { + #expect(XLSXSheetParser.columnIndex(fromReference: "1") == nil) + #expect(XLSXSheetParser.columnIndex(fromReference: "") == nil) + } + + /// Part names vary between writers, so the first worksheet is found rather than assumed to be + /// `sheet1.xml`. + @Test("The first worksheet is found by path, not assumed") + func firstWorksheetIsFound() { + let paths = ["xl/workbook.xml", "xl/worksheets/sheet2.xml", "xl/worksheets/sheet1.xml", "[Content_Types].xml"] + #expect(XLSXSheetParser.firstWorksheetPath(in: paths) == "xl/worksheets/sheet1.xml") + #expect(XLSXSheetParser.firstWorksheetPath(in: ["xl/workbook.xml"]) == nil) + } + + /// A string split across formatting runs is one value, not several. A styled word mid-cell + /// would otherwise truncate it. + @Test("Shared strings concatenate their runs") + func sharedStringRuns() { + let xml = """ + + AdaGrace Hopper + """ + let strings = XLSXSheetParser.sharedStrings(from: Data(xml.utf8)) + #expect(strings == ["Ada", "Grace Hopper"]) + } + + /// A cell typed `s` holds an index into the shared string table rather than the text. + @Test("A shared-string cell resolves through the table") + func sharedStringCellResolves() { + let sheet = """ + + + 01 + + """ + let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: ["id", "name"]) + #expect(rows == [["id", "name"]]) + } + + /// A row omits the cells it has no value for, so position comes from each cell's own reference. + /// Counting cells instead shifts every value after a gap into the wrong column. + @Test("A gap in a row is filled from the cell references") + func gapsArePlacedByReference() { + let sheet = """ + + + 13 + + """ + let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []) + #expect(rows == [["1", nil, "3"]]) + } + + @Test("An inline string is read from the cell itself") + func inlineStringsAreRead() { + let sheet = """ + + + Ada + + """ + let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []) + #expect(rows == [["Ada"]]) + } + + /// A damaged workbook still imports something the user can see is wrong, rather than dropping + /// the value silently. + @Test("A shared-string index the table lacks keeps the raw value") + func outOfRangeIndexKeepsRawValue() { + let sheet = """ + + 99 + """ + let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: ["only"]) + #expect(rows == [["99"]]) + } + + @Test("Rows are padded to the widest row") + func rowsArePadded() { + let sheet = """ + + + 12 + 3 + + """ + let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []) + #expect(rows.count == 2) + #expect(rows.allSatisfy { $0.count == 2 }) + #expect(rows[1] == ["3", nil]) + } + + @Test("An empty sheet reads as no rows rather than failing") + func emptySheet() { + let sheet = "" + #expect(XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []).isEmpty) + } +} + +@Suite("ZIP reading") +struct ZipReaderTests { + + /// Builds a ZIP the way the format specifies, so the reader is exercised against real bytes + /// rather than a mock. Stored and deflated entries are both produced, because Excel writes + /// deflate and TablePro's own XLSX export writes stored. + private func makeArchive(_ files: [(name: String, body: Data, deflate: Bool)]) -> Data { + var output = Data() + var directory = Data() + var offsets: [Int] = [] + + for file in files { + offsets.append(output.count) + let nameBytes = Data(file.name.utf8) + let payload = file.deflate ? deflated(file.body) : file.body + let method: UInt16 = file.deflate ? 8 : 0 + + output.append(contentsOf: [0x50, 0x4B, 0x03, 0x04]) + output.append(uint16(20)) + output.append(uint16(0)) + output.append(uint16(method)) + output.append(uint16(0)) + output.append(uint16(0)) + output.append(uint32(crc32(file.body))) + output.append(uint32(UInt32(payload.count))) + output.append(uint32(UInt32(file.body.count))) + output.append(uint16(UInt16(nameBytes.count))) + output.append(uint16(0)) + output.append(nameBytes) + output.append(payload) + } + + for (index, file) in files.enumerated() { + let nameBytes = Data(file.name.utf8) + let payload = file.deflate ? deflated(file.body) : file.body + directory.append(contentsOf: [0x50, 0x4B, 0x01, 0x02]) + directory.append(uint16(20)) + directory.append(uint16(20)) + directory.append(uint16(0)) + directory.append(uint16(file.deflate ? 8 : 0)) + directory.append(uint16(0)) + directory.append(uint16(0)) + directory.append(uint32(crc32(file.body))) + directory.append(uint32(UInt32(payload.count))) + directory.append(uint32(UInt32(file.body.count))) + directory.append(uint16(UInt16(nameBytes.count))) + directory.append(uint16(0)) + directory.append(uint16(0)) + directory.append(uint16(0)) + directory.append(uint16(0)) + directory.append(uint32(0)) + directory.append(uint32(UInt32(offsets[index]))) + directory.append(nameBytes) + } + + let directoryOffset = output.count + output.append(directory) + output.append(contentsOf: [0x50, 0x4B, 0x05, 0x06]) + output.append(uint16(0)) + output.append(uint16(0)) + output.append(uint16(UInt16(files.count))) + output.append(uint16(UInt16(files.count))) + output.append(uint32(UInt32(directory.count))) + output.append(uint32(UInt32(directoryOffset))) + output.append(uint16(0)) + return output + } + + private func deflated(_ data: Data) -> Data { + guard !data.isEmpty else { return Data() } + var output = Data(count: max(data.count * 2, 1_024)) + let written = output.withUnsafeMutableBytes { destination -> Int in + guard let destinationBase = destination.bindMemory(to: UInt8.self).baseAddress else { return 0 } + return data.withUnsafeBytes { source -> Int in + guard let sourceBase = source.bindMemory(to: UInt8.self).baseAddress else { return 0 } + return compression_encode_buffer( + destinationBase, output.count, sourceBase, data.count, nil, COMPRESSION_ZLIB) + } + } + return output.prefix(written) + } + + private func uint16(_ value: UInt16) -> Data { + Data([UInt8(value & 0xFF), UInt8((value >> 8) & 0xFF)]) + } + + private func uint32(_ value: UInt32) -> Data { + Data([ + UInt8(value & 0xFF), UInt8((value >> 8) & 0xFF), + UInt8((value >> 16) & 0xFF), UInt8((value >> 24) & 0xFF) + ]) + } + + private func crc32(_ data: Data) -> UInt32 { + var crc: UInt32 = 0xFFFF_FFFF + for byte in data { + crc ^= UInt32(byte) + for _ in 0 ..< 8 { + crc = (crc & 1) == 1 ? (crc >> 1) ^ 0xEDB8_8320 : crc >> 1 + } + } + return crc ^ 0xFFFF_FFFF + } + + @Test("A stored entry reads back byte for byte") + func storedEntryRoundTrips() throws { + let body = Data("stored".utf8) + let archive = makeArchive([(name: "a.xml", body: body, deflate: false)]) + #expect(try ZipReader.data(named: "a.xml", in: archive) == body) + } + + /// Excel deflates every part, so this is the path that matters for a real workbook. + @Test("A deflated entry is inflated") + func deflatedEntryInflates() throws { + let body = Data(String(repeating: "value", count: 500).utf8) + let archive = makeArchive([(name: "b.xml", body: body, deflate: true)]) + #expect(try ZipReader.data(named: "b.xml", in: archive) == body) + } + + @Test("Every entry is listed with its own path") + func entriesAreListed() throws { + let archive = makeArchive([ + (name: "xl/workbook.xml", body: Data("a".utf8), deflate: false), + (name: "xl/worksheets/sheet1.xml", body: Data("b".utf8), deflate: true) + ]) + let entries = try ZipReader.entries(in: archive) + #expect(Set(entries.keys) == ["xl/workbook.xml", "xl/worksheets/sheet1.xml"]) + #expect(entries["xl/worksheets/sheet1.xml"]?.compressionMethod == 8) + } + + @Test("A missing entry is named in the error rather than returning nothing") + func missingEntryThrows() { + let archive = makeArchive([(name: "a.xml", body: Data("a".utf8), deflate: false)]) + #expect(throws: ZipReader.ZipError.self) { + _ = try ZipReader.data(named: "xl/sharedStrings.xml", in: archive) + } + } + + @Test("A file that is not a ZIP is refused") + func nonArchiveIsRefused() { + #expect(throws: ZipReader.ZipError.self) { + _ = try ZipReader.entries(in: Data("not a zip at all".utf8)) + } + #expect(throws: ZipReader.ZipError.self) { + _ = try ZipReader.entries(in: Data()) + } + } + + /// An empty part is legal and reads as empty rather than as a failure. + @Test("An empty entry reads as empty") + func emptyEntry() throws { + let archive = makeArchive([(name: "empty.xml", body: Data(), deflate: false)]) + #expect(try ZipReader.data(named: "empty.xml", in: archive).isEmpty) + } +} diff --git a/docs/features/backup-restore.mdx b/docs/features/backup-restore.mdx index 7e57ac481d..312337353d 100644 --- a/docs/features/backup-restore.mdx +++ b/docs/features/backup-restore.mdx @@ -11,6 +11,7 @@ The engine's own tools do the work, and none of them ship inside TablePro. Insta | MySQL, MariaDB | `mysqldump`, `mysql` | `brew install mysql-client` | `.sql` | | MongoDB | `mongodump`, `mongorestore` | `brew install mongodb-database-tools` | `.archive`, gzipped | | SQLite, libSQL | `sqlite3` | `brew install sqlite` | `.sql` | +| SQL Server | `sqlpackage` | [Download from Microsoft](https://learn.microsoft.com/sql/tools/sqlpackage/sqlpackage-download) | `.bacpac` | MariaDB 11.0 renamed its clients, so `mariadb-dump` and `mariadb` are accepted in place of `mysqldump` and `mysql`. @@ -65,10 +66,26 @@ No `--clean` is passed, so restoring on top of a schema that already holds confl Both flows reuse the connection's active SSH tunnel, with no second port forward. SSL mode reaches PostgreSQL through `PGSSLMODE` and MySQL through `--ssl-mode`, `verify-ca` and `verify-full` included. SQLite opens a file, so neither applies. +## Server-side export + +Oracle, Snowflake and BigQuery do not have a client-side dump. They unload to somewhere the server can write: a `DIRECTORY` object, a stage, a Cloud Storage bucket. Choose **File > Server-Side Export…** for those. + +Pick a table and name the destination. Oracle takes the name of a `DIRECTORY` object rather than a path, because the path belongs to the server. Snowflake takes a stage, with or without its `@`. BigQuery takes a `gs://` prefix and shards its output under it. + +The statement runs on your own connection, so it carries your privileges and the server's own error comes back when the destination is not writable. + + +Nothing lands on your Mac. The result is on the server or in the bucket, and the sheet says where it went. + + ## Passwords Your password never reaches the tool's argument list, which every process on the machine can read through `ps`. PostgreSQL gets `PGPASSWORD` and MySQL gets `MYSQL_PWD`, both in the environment. MongoDB's tools read neither, so TablePro writes a `0600` config file and deletes it when the process exits. + +SqlPackage is the exception. It accepts a password only inside its connection string, which means the command line, where other processes on the Mac can read it while the dump runs. TablePro asks before starting one. Windows or Entra authentication avoids it entirely: leave the username empty and the tool uses integrated security. + + ## Failures A non-zero exit shows the last 64 KB of the tool's stderr in a scrollable monospaced view. Three causes account for most of them. diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index 8fc41a0354..1f9e9a9bf5 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -146,6 +146,51 @@ A whole-table export streams from the database at constant memory, with no row-c Each table becomes its own worksheet and numbers are stored as numeric cells. A table over 1,048,576 rows, Excel's limit, splits across sheets. + + GitHub-flavoured tables. + + | Option | Default | + |--------|---------| + | Align columns | Yes | + | Write each table's name as a heading | Yes | + | NULL shows as | `NULL` | + + Column widths come from the header and the first 200 rows, so a wide value later in a long table is written whole rather than widening every row before it. A pipe or a line break inside a value is neutralised: both would end the cell early. + + + | Option | Default | + |--------|---------| + | Write a full HTML document | Yes | + | Write each table's name as a heading | Yes | + | Mark NULL cells | Yes | + + A full document carries its own stylesheet and follows the reader's light or dark setting. Turn it off to paste bare `
` elements into a page that has its own styling. + + + One element per row, one child element per column. + + | Option | Default | + |--------|---------| + | Pretty print | Yes | + | Mark NULL with `xsi:nil` | Yes | + | Row element | `row` | + + A column name that is not a legal XML element name is rewritten: a leading digit gains an underscore, an illegal character becomes one, and a name starting `xml` is prefixed. Control characters XML 1.0 forbids are dropped, so a binary column cannot make the document unparseable. + + + Install the Parquet plugin from **Settings > Plugins** first. It carries its own copy of DuckDB, which does the encoding, and is too large to ship in the app. + + | Option | Default | + |--------|---------| + | Compression (Snappy, Zstd, Gzip, None) | Snappy | + | Rows per group | 122,880 | + + Column types come from the source engine's own declarations, so numbers, dates and booleans arrive as those rather than as strings. A value that will not convert is written as null rather than failing the export. + + + Parquet holds one table per file. Selecting several writes `dump.users.parquet`, `dump.orders.parquet` and so on beside the name you chose. + + ## Transfer to another connection @@ -234,6 +279,20 @@ The sheet accepts an array of objects `[{…}, {…}]`, newline-delimited JSON s Rows insert through parameterized statements, so a JSON value is never concatenated into SQL. Nested objects and arrays are stored as JSON text. +### Import XLSX + +Reads the first worksheet of an `.xlsx` workbook. + +| Option | Default | +|--------|---------| +| First row holds column names | Yes | +| Trim whitespace | No | +| Treat empty cells as NULL | Yes | + +A row that omits its empty cells still lands in the right columns: each cell's own reference decides where it goes, not its position among the cells that were written. Text stored in the workbook's shared string table is resolved, including a value split across several formatting runs. + +The workbook is read whole rather than streamed, because a sheet's rows refer back to a string table that has to be held anyway. + ### Import CSV CSV and TSV open the same sheet as JSON, with parsing options in front of the mapping. The delimiter and encoding are detected from the file; change any option and the mapping re-reads it. diff --git a/project.yml b/project.yml index ab66602e8e..c0bdef5087 100644 --- a/project.yml +++ b/project.yml @@ -200,6 +200,9 @@ targets: - target: XMLExport embed: true copy: { destination: plugins } + - target: XLSXImport + embed: true + copy: { destination: plugins } - target: XLSXExport embed: true copy: { destination: plugins } @@ -512,6 +515,9 @@ targets: - Plugins/SQLImportPlugin/SQLImportOptions.swift - Plugins/SQLImportPlugin/SQLImportOptionsView.swift - Plugins/SQLImportPlugin/SQLImportPlugin.swift + - Plugins/XLSXImportPlugin/XLSXImportOptions.swift + - Plugins/XLSXImportPlugin/XLSXSheetParser.swift + - Plugins/XLSXImportPlugin/ZipReader.swift - Plugins/XMLExportPlugin/XMLExportModels.swift - Plugins/SnowflakeDriverPlugin/PluginCellValueBox.swift - Plugins/SnowflakeDriverPlugin/SnowflakeAuth.swift @@ -734,6 +740,15 @@ targets: base: PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.HTMLExportPlugin + XLSXImport: + templates: [DriverPlugin] + templateAttributes: + folder: XLSXImportPlugin + principalClass: XLSXImportPlugin + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.XLSXImportPlugin + XMLExport: templates: [DriverPlugin] templateAttributes: @@ -1160,6 +1175,7 @@ aggregateTargets: - MarkdownExport - ParquetExport - SQLExport + - XLSXImport - XMLExport - SQLImport - SQLiteDriver From 0c115ce9d1f22f887afa94cba7b9efa56986e0f0 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 3 Sep 2026 16:22:02 +0700 Subject: [PATCH 5/5] test(export): build the XLSX fixture archive without an exclusivity violation Claude-Session: https://claude.ai/code/session_011EqgjCjCAU6tiiVmnMpF86 --- TableProTests/Plugins/XLSXImportTests.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/TableProTests/Plugins/XLSXImportTests.swift b/TableProTests/Plugins/XLSXImportTests.swift index 4bd0c9c800..ece70629a4 100644 --- a/TableProTests/Plugins/XLSXImportTests.swift +++ b/TableProTests/Plugins/XLSXImportTests.swift @@ -193,13 +193,14 @@ struct ZipReaderTests { private func deflated(_ data: Data) -> Data { guard !data.isEmpty else { return Data() } - var output = Data(count: max(data.count * 2, 1_024)) + let capacity = max(data.count * 2, 1_024) + var output = Data(count: capacity) let written = output.withUnsafeMutableBytes { destination -> Int in guard let destinationBase = destination.bindMemory(to: UInt8.self).baseAddress else { return 0 } return data.withUnsafeBytes { source -> Int in guard let sourceBase = source.bindMemory(to: UInt8.self).baseAddress else { return 0 } return compression_encode_buffer( - destinationBase, output.count, sourceBase, data.count, nil, COMPRESSION_ZLIB) + destinationBase, capacity, sourceBase, data.count, nil, COMPRESSION_ZLIB) } } return output.prefix(written)