diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index 3cc5e17a61..73b5bdc9ee 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -158,6 +158,18 @@ "category": "database-driver", "homepage": "https://docs.tablepro.app/databases/etcd" }, + "html": { + "target": "HTMLExport", + "bundleName": "HTMLExport", + "bundleId": "com.TablePro.HTMLExportPlugin", + "bundled": true, + "displayName": "HTML Export", + "summary": "Export data as an HTML table", + "databaseTypeIds": null, + "icon": "chevron.left.forwardslash.chevron.right", + "category": "export-format", + "homepage": "https://docs.tablepro.app/features/import-export" + }, "kafka": { "target": "KafkaDriverPlugin", "bundleName": "KafkaDriverPlugin", @@ -187,6 +199,18 @@ "category": "database-driver", "homepage": "https://docs.tablepro.app/databases/libsql" }, + "markdown": { + "target": "MarkdownExport", + "bundleName": "MarkdownExport", + "bundleId": "com.TablePro.MarkdownExportPlugin", + "bundled": true, + "displayName": "Markdown Export", + "summary": "Export data as Markdown tables", + "databaseTypeIds": null, + "icon": "text.alignleft", + "category": "export-format", + "homepage": "https://docs.tablepro.app/features/import-export" + }, "mongodb": { "target": "MongoDBDriver", "bundleName": "MongoDBDriver", @@ -241,6 +265,18 @@ "category": "database-driver", "homepage": "https://docs.tablepro.app/databases/oracle" }, + "parquet": { + "target": "ParquetExport", + "bundleName": "ParquetExport", + "bundleId": "com.TablePro.ParquetExportPlugin", + "bundled": false, + "displayName": "Parquet Export", + "summary": "Export data to Apache Parquet", + "databaseTypeIds": null, + "icon": "square.grid.3x3", + "category": "export-format", + "homepage": "https://docs.tablepro.app/features/import-export" + }, "redis": { "target": "RedisDriver", "bundleName": "RedisDriver", @@ -348,6 +384,30 @@ "icon": "tablecells", "category": "export-format", "homepage": "https://docs.tablepro.app/features/import-export" + }, + "xlsximport": { + "target": "XLSXImport", + "bundleName": "XLSXImport", + "bundleId": "com.TablePro.XLSXImportPlugin", + "bundled": true, + "displayName": "XLSX Import", + "summary": "Import data from Excel workbooks", + "databaseTypeIds": null, + "icon": "tablecells", + "category": "import-format", + "homepage": "https://docs.tablepro.app/features/import-export" + }, + "xml": { + "target": "XMLExport", + "bundleName": "XMLExport", + "bundleId": "com.TablePro.XMLExportPlugin", + "bundled": true, + "displayName": "XML Export", + "summary": "Export data as XML", + "databaseTypeIds": null, + "icon": "doc.badge.gearshape", + "category": "export-format", + "homepage": "https://docs.tablepro.app/features/import-export" } } } diff --git a/CHANGELOG.md b/CHANGELOG.md index b2ba0eae47..8d15483865 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Breakdown of a query's time into server, first row and transfer, behind the toolbar's duration readout. (#2503) - Exclude the AUTO_INCREMENT counter and Exclude DEFINER clauses in the SQL export, both on by default. (#2516) - Jump to Column in the grid, a fuzzy search over the result's columns with their type and position. (#2495) +- Views, routines, triggers, user-defined types and privileges in the export tree, grouped by kind. (#2618) +- Per-table `WHERE`, row limit and column subset in the export tree. (#2618) +- Insert mode for SQL exports: skip, replace or update rows that already exist. (#2618) +- Split a SQL export into numbered parts at a chosen size. (#2618) +- Read every table at one snapshot during a SQL export. (#2618) +- Backup and restore for MySQL, MariaDB, MongoDB, SQLite and libSQL, using each engine's own tools. (#2618) +- Transfer To, copying table rows straight into another open connection with no file in between. (#2618) +- NDJSON layout for JSON exports, one row per line. (#2618) +- Saved export selections, reapplied from the export tree's bookmark menu. (#2618) +- Save Report on an import that skipped rows, listing each one's line and error as CSV. (#2618) +- Markdown, HTML and XML export. (#2618) +- Parquet export, through a plugin installed from Settings > Plugins. (#2618) +- XLSX import, reading the first worksheet of a workbook. (#2618) +- Server-Side Export for Oracle, Snowflake and BigQuery, which unload to a server directory, a stage or a bucket. (#2618) +- Backup and restore for SQL Server through SqlPackage. (#2618) +- MySQL events and PostgreSQL sequences in the export tree. (#2618) - Connection groups in Switch Connection, with `Cmd`-click to open a saved connection in a new window. (#1311) ### Changed @@ -26,6 +42,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- DuckDB aggregate, JSON and Parquet functions failing on a Mac that cannot reach `extensions.duckdb.org`. (#2626) - Last line of a helper process's output lost when it exits right after writing it. - Structure and trigger edits committing or rolling back a transaction left open in a query tab on the same connection. - Composite, range and extension-typed PostgreSQL columns labelled `ENUM(…)` in the structure editor. diff --git a/CLAUDE.md b/CLAUDE.md index 1fd704dc48..af9fdea486 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,8 +22,8 @@ TablePro is a native macOS database client (SwiftUI + AppKit), a fast, lightweig - **Source**: `TablePro/` holds `Core/` (business logic, services), `Views/` (UI), `Models/` (data structures), `ViewModels/`, `Extensions/` and `Theme/` - **Plugins**: `Plugins/` holds the `.tableplugin` bundles plus the `TableProPluginKit` shared framework. - - **Bundled in app** (the 14 targets in the app's `copy: { destination: plugins }` phase in `project.yml`): MySQL, PostgreSQL, SQLite, ClickHouse, Redis, CSV export, JSON export, SQL export, XLSX export, MQL export, SQL import, JSON import, CSV import, CSV inspector. These ship inside the app bundle and their updates normally ride with the next app release. Six of them (`sqlite`, `clickhouse`, `redis`, `xlsx`, `mql`, `sqlimport`) also have registry arms in `build-plugin.yml`, so a bundled plugin can be published when users on an already-shipped app need the fix sooner. `scripts/build-plugin.sh:10` explains the flag that makes that work. - - **Registry-only** (the other 18): MongoDB, Oracle, DuckDB, MSSQL, Cassandra, Etcd, CloudflareD1, DynamoDB, BigQuery, LibSQL, Snowflake, Elasticsearch, Beancount, SurrealDB, Teradata, Trino, Dameng, Kafka. Distributed via [TableProApp/plugins](https://github.com/TableProApp/plugins) `plugins.json`, installed into the user plugins directory. + - **Bundled in app** (the 18 targets in the app's `copy: { destination: plugins }` phase in `project.yml`): MySQL, PostgreSQL, SQLite, ClickHouse, Redis, CSV export, JSON export, SQL export, XLSX export, Markdown export, HTML export, XML export, MQL export, SQL import, JSON import, CSV import, XLSX import, CSV inspector. These ship inside the app bundle and their updates normally ride with the next app release. Six of them (`sqlite`, `clickhouse`, `redis`, `xlsx`, `mql`, `sqlimport`) also have registry arms in `build-plugin.yml`, so a bundled plugin can be published when users on an already-shipped app need the fix sooner. `scripts/build-plugin.sh:10` explains the flag that makes that work. + - **Registry-only** (the other 19): MongoDB, Oracle, DuckDB, MSSQL, Cassandra, Etcd, CloudflareD1, DynamoDB, BigQuery, LibSQL, Snowflake, Elasticsearch, Beancount, SurrealDB, Teradata, Trino, Dameng, Kafka, Parquet export. Parquet is registry-only because it links its own copy of DuckDB, which does the encoding, and that is too large to ship in the app for one format. Distributed via [TableProApp/plugins](https://github.com/TableProApp/plugins) `plugins.json`, installed into the user plugins directory. - **C bridges**: Each plugin contains its own C bridge module (e.g., `Plugins/MySQLDriverPlugin/CMariaDB/`, `Plugins/PostgreSQLDriverPlugin/CLibPQ/`) - **Static libs**: `Libs/` holds pre-built `.a` files and `Libs/ios/` holds the iOS xcframeworks. Both are downloaded by `scripts/download-libs.sh` and are not in git. - **SPM deps**: declared in `project.yml`. Vendored local packages under `LocalPackages/` (CodeEditSourceEditor, CodeEditTextView, CodeEditLanguages) and `Packages/` (TableProCore, TableProOracle); remote packages are Sparkle, swift-certificates and Yams. Revisions are pinned by the tracked `Package.resolved` inside each generated `.xcodeproj`. @@ -103,7 +103,7 @@ git add Libs/ios/checksums.sha256 && git commit -m "build: update iOS xcframewor Run `scripts/generate-project.sh` after editing any of those, and after adding, moving, or deleting a source file: XcodeGen globs sources at generation time, so a new file is not in the project until you regenerate. Changing signing in the Xcode UI is pointless, because the next generate discards it; set `TABLEPRO_DEVELOPMENT_TEAM` and `TABLEPRO_APP_BUNDLE_IDENTIFIER` in `Configs/Secrets.xcconfig` instead. -The 32 plugin bundles share one `DriverPlugin` target template; a plugin declares only its folder, principal class, and any C-library link flags. Every target gets a shared scheme named after it, which is what `scripts/build-plugin.sh [arm64|x86_64|both] [version]` builds. The `AllPlugins` aggregate target compile-checks all 32, including the registry-only ones the app does not embed, and PR CI runs it: the `Compile every plugin` step in the `app-tests` job of `.github/workflows/macos-tests.yml` builds that scheme whenever the change touches `Plugins/` or any other watched path. What PR CI still does not cover is plugin packaging, signing and notarization, which only `build-plugin.yml` does and only on a release tag. +The 37 plugin bundles share one `DriverPlugin` target template; a plugin declares only its folder, principal class, and any C-library link flags. Every target gets a shared scheme named after it, which is what `scripts/build-plugin.sh [arm64|x86_64|both] [version]` builds. The `AllPlugins` aggregate target compile-checks all 37, including the registry-only ones the app does not embed, and PR CI runs it: the `Compile every plugin` step in the `app-tests` job of `.github/workflows/macos-tests.yml` builds that scheme whenever the change touches `Plugins/` or any other watched path. What PR CI still does not cover is plugin packaging, signing and notarization, which only `build-plugin.yml` does and only on a release tag. ### Plugin System diff --git a/Libs/checksums.sha256 b/Libs/checksums.sha256 index b52f99973d..98ee6bffea 100644 --- a/Libs/checksums.sha256 +++ b/Libs/checksums.sha256 @@ -10,10 +10,10 @@ a891a67c2619e2ac1dce64dafc6a24bfde9cabe15312dac6b70a19385664ea84 Libs/libcrypto 732adf315bc49f77e2511a9293e49a65e18eb54a3e6d01d8a24eee2d671d2a8a Libs/libcrypto_universal.a 965ccd38fea5cd97bc878dbf58567e4eed2b2337120f8d46a2da62c094b3c821 Libs/libcrypto_x86_64.a 732adf315bc49f77e2511a9293e49a65e18eb54a3e6d01d8a24eee2d671d2a8a Libs/libcrypto.a -1ef4f456b99285dca4fd8cfedef1a2f4936b0a9567a7e332baebdba04bf89d77 Libs/libduckdb_arm64.a -68a13d3a915acc08b59a4b982525afa143b588ec6e19c8e470e2ff97cf0df3d9 Libs/libduckdb_universal.a -e809dd7c7ec05a8218d273b3350a9a20e15a6ddf1e392ccbe5560395166a6b26 Libs/libduckdb_x86_64.a -68a13d3a915acc08b59a4b982525afa143b588ec6e19c8e470e2ff97cf0df3d9 Libs/libduckdb.a +e325cc7f47ad2ac91f777d075e5533b0ef34769e16cd728496d3f9229ed6ff9b Libs/libduckdb_arm64.a +b578aa2a73b0b84ba36e436d0cade00d5e98f1cb1ebbc849374758e9054985a9 Libs/libduckdb_universal.a +76737d65affec9b13676e0a39d3bd1054ee17642a04fa2ce34b9c2e998693fa0 Libs/libduckdb_x86_64.a +b578aa2a73b0b84ba36e436d0cade00d5e98f1cb1ebbc849374758e9054985a9 Libs/libduckdb.a 7e63017fa22c2eb7744eccad13857361a5088aa7b2772ab02cd026c8c7b78341 Libs/libhiredis_arm64.a f1cfc36a7ab47361e9705fe32b1c919b318f606989478e91a808707d93db55a5 Libs/libhiredis_ssl_arm64.a fb7a32c2c724cb4f3f880030cb19afbbc7db52121ad8e35e00a2e818da9562cf Libs/libhiredis_ssl_universal.a diff --git a/Plugins/CSVExportPlugin/CSVExportPlugin.swift b/Plugins/CSVExportPlugin/CSVExportPlugin.swift index 695084f572..4a64c64387 100644 --- a/Plugins/CSVExportPlugin/CSVExportPlugin.swift +++ b/Plugins/CSVExportPlugin/CSVExportPlugin.swift @@ -146,33 +146,36 @@ final class CSVExportPlugin: ExportFormatPlugin, SettablePlugin, @unchecked Send try fileHandle.write(contentsOf: (rowLine + lineBreak).toUTF8Data()) } + /// Escaping and quoting live in `PluginRowWriters`, so this format, the other export formats + /// and the MCP tool spell a value the same way. Only the option mapping is this plugin's own. + /// + /// `originalHadLineBreaks` says the value's breaks were already replaced with spaces upstream, + /// and it still forces quoting: the source text spanned lines, and a reader that splits on the + /// delimiter has no way to know the space it now sees was one. private func escapeCSVField(_ field: String, options: CSVExportOptions, originalHadLineBreaks: Bool = false) -> String { - var processed = field - - if options.sanitizeFormulas { - let dangerousPrefixes: [Character] = ["=", "+", "-", "@"] - if let first = processed.first, dangerousPrefixes.contains(first) { - processed = "'" + processed - } + let escaped = PluginRowWriters.csvField(field, options: writeOptions(options)) + guard originalHadLineBreaks, options.quoteHandling == .asNeeded, !escaped.hasPrefix("\"") else { + return escaped } + return "\"\(escaped.replacingOccurrences(of: "\"", with: "\"\""))\"" + } - switch options.quoteHandling { - case .always: - let escaped = processed.replacingOccurrences(of: "\"", with: "\"\"") - return "\"\(escaped)\"" - case .never: - return processed - case .asNeeded: - let needsQuotes = processed.contains(options.delimiter.actualValue) || - processed.contains("\"") || - processed.contains("\n") || - processed.contains("\r") || - originalHadLineBreaks - if needsQuotes { - let escaped = processed.replacingOccurrences(of: "\"", with: "\"\"") - return "\"\(escaped)\"" - } - return processed + private func writeOptions(_ options: CSVExportOptions) -> PluginCsvWriteOptions { + PluginCsvWriteOptions( + delimiter: options.delimiter.actualValue, + quoteHandling: quoteHandling(options.quoteHandling), + lineEnding: options.lineBreak.value, + nullAsEmpty: true, + sanitizesFormulas: options.sanitizeFormulas, + flattensLineBreaks: false + ) + } + + private func quoteHandling(_ handling: CSVQuoteHandling) -> PluginCsvWriteOptions.QuoteHandling { + switch handling { + case .always: return .always + case .never: return .never + case .asNeeded: return .asNeeded } } } diff --git a/Plugins/HTMLExportPlugin/HTMLExportModels.swift b/Plugins/HTMLExportPlugin/HTMLExportModels.swift new file mode 100644 index 0000000000..315cef8214 --- /dev/null +++ b/Plugins/HTMLExportPlugin/HTMLExportModels.swift @@ -0,0 +1,55 @@ +// +// HTMLExportModels.swift +// HTMLExportPlugin +// + +import Foundation + +public struct HTMLExportOptions: Equatable, Codable { + /// Wraps the tables in a full document with a stylesheet. Off writes bare `` elements, + /// which is what pasting into an existing page wants. + public var writesFullDocument: Bool = true + + public var includesTableNames: Bool = true + + /// Renders a null as a dimmed `NULL` rather than an empty cell, which is otherwise identical to + /// a cell holding an empty string. + public var marksNulls: Bool = true + + public init() {} + + /// A synthesized `init(from:)` throws `keyNotFound` for a key the saved payload predates and + /// never falls back to the property's default, so adding one would reset what a user chose. + public init(from decoder: any Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let defaults = HTMLExportOptions() + writesFullDocument = try container.decodeIfPresent(Bool.self, forKey: .writesFullDocument) + ?? defaults.writesFullDocument + includesTableNames = try container.decodeIfPresent(Bool.self, forKey: .includesTableNames) + ?? defaults.includesTableNames + marksNulls = try container.decodeIfPresent(Bool.self, forKey: .marksNulls) ?? defaults.marksNulls + } +} + +/// Escapes text for HTML. +/// +/// Every value in an export comes from the database, so a value holding `") + == "<script>alert(1)</script>") + } + + /// The ampersand has to be replaced first, or the escapes written after it are themselves + /// escaped and the value renders as `&lt;`. + @Test("An ampersand is escaped once, not twice") + func ampersandEscapedOnce() { + #expect(HTMLEscaping.text("&") == "&") + #expect(HTMLEscaping.text("<") == "&lt;") + #expect(HTMLEscaping.text("a & b < c") == "a & b < c") + } + + @Test("Quotes are escaped so a value cannot break out of an attribute") + func quotesAreEscaped() { + #expect(HTMLEscaping.text("\"x\"") == ""x"") + #expect(HTMLEscaping.text("'x'") == "'x'") + } + + @Test("Ordinary text is unchanged") + func plainTextUnchanged() { + #expect(HTMLEscaping.text("Ada Lovelace") == "Ada Lovelace") + #expect(HTMLEscaping.text("") == "") + } +} + +@Suite("XML export escaping") +struct XMLExportEscapingTests { + + @Test("The five predefined entities are escaped") + func entitiesAreEscaped() { + #expect(XMLEscaping.text("") == "<a & b>") + #expect(XMLEscaping.text("\"'") == ""'") + } + + /// XML 1.0 accepts tab, newline and carriage return and no other control character. A stray + /// 0x00 out of a binary column would otherwise make the whole document unparseable. + @Test("Illegal control characters are dropped, legal whitespace is kept") + func controlCharactersAreDropped() { + #expect(XMLEscaping.text("a\u{0}b") == "ab") + #expect(XMLEscaping.text("a\u{1}\u{1F}b") == "ab") + #expect(XMLEscaping.text("a\tb\nc\rd") == "a\tb\nc\rd") + } + + /// A column name is not automatically a legal element name: XML forbids a leading digit and + /// restricts the character set, and a database column has neither limit. + @Test("A column name becomes a legal element name") + func elementNamesAreLegal() { + #expect(XMLEscaping.elementName("name") == "name") + #expect(XMLEscaping.elementName("first_name") == "first_name") + #expect(XMLEscaping.elementName("2024_total") == "_2024_total") + #expect(XMLEscaping.elementName("order total") == "order_total") + #expect(XMLEscaping.elementName("a-b.c") == "a-b.c") + } + + /// Names beginning `xml` in any case are reserved by the specification. + @Test("A reserved xml prefix is renamed") + func reservedPrefixIsRenamed() { + #expect(XMLEscaping.elementName("xmlData") == "_xmlData") + #expect(XMLEscaping.elementName("XMLData") == "_XMLData") + } + + @Test("A name with nothing legal in it falls back rather than producing invalid XML") + func emptyNameFallsBack() { + #expect(XMLEscaping.elementName("") == "column") + } +} + +@Suite("Parquet type mapping") +struct ParquetTypeMapperTests { + + @Test("Integer families map to BIGINT") + func integerFamilies() { + for type in ["INT", "int4", "BIGINT", "smallint", "TINYINT", "SERIAL", "MEDIUMINT"] { + #expect(ParquetTypeMapper.duckDBType(forColumnType: type) == "BIGINT", "\(type)") + } + } + + @Test("Decimal families map to DOUBLE") + func decimalFamilies() { + for type in ["DECIMAL(10,2)", "numeric", "FLOAT", "double precision", "REAL", "money"] { + #expect(ParquetTypeMapper.duckDBType(forColumnType: type) == "DOUBLE", "\(type)") + } + } + + @Test("Temporal families keep their own types") + func temporalFamilies() { + #expect(ParquetTypeMapper.duckDBType(forColumnType: "DATE") == "DATE") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "timestamp with time zone") == "TIMESTAMP") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "datetime") == "TIMESTAMP") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "TIME") == "TIME") + } + + @Test("Booleans and binaries map to their own types") + func booleanAndBinary() { + #expect(ParquetTypeMapper.duckDBType(forColumnType: "BOOLEAN") == "BOOLEAN") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "bytea") == "BLOB") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "VARBINARY(50)") == "BLOB") + } + + /// An unknown type is written as text rather than guessed at. A wrong guess writes a Parquet + /// file whose column type disagrees with the data in it. + @Test("An unknown type falls back to VARCHAR") + func unknownFallsBack() { + #expect(ParquetTypeMapper.duckDBType(forColumnType: "geography") == "VARCHAR") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "") == "VARCHAR") + #expect(ParquetTypeMapper.duckDBType(forColumnType: "hstore") == "VARCHAR") + } + + /// A type name carries its width in parentheses and sometimes a modifier after a space, and + /// neither changes which family it belongs to. + @Test("Width and modifiers are stripped before matching") + func baseNameStripsArgumentsAndModifiers() { + #expect(ParquetTypeMapper.baseName("VARCHAR(64)") == "varchar") + #expect(ParquetTypeMapper.baseName("NUMERIC(10, 2)") == "numeric") + #expect(ParquetTypeMapper.baseName("INT UNSIGNED") == "int") + #expect(ParquetTypeMapper.baseName(" TIMESTAMP WITH TIME ZONE ") == "timestamp") + } + + @Test("Parquet holds one table per file, so a multi-table export numbers its files") + func perTableFileNaming() { + let base = URL(fileURLWithPath: "/tmp/dump.parquet") + #expect(ParquetFileNaming.perTableURL(destination: base, table: "users").lastPathComponent + == "dump.users.parquet") + + let noExtension = URL(fileURLWithPath: "/tmp/dump") + #expect(ParquetFileNaming.perTableURL(destination: noExtension, table: "users").lastPathComponent + == "dump.users") + } + + /// A schema-qualified name carries a separator that would otherwise create a directory that + /// does not exist. + @Test("A table name with a slash cannot escape its directory") + func slashesAreNeutralised() { + let base = URL(fileURLWithPath: "/tmp/dump.parquet") + let url = ParquetFileNaming.perTableURL(destination: base, table: "a/b") + #expect(url.lastPathComponent == "dump.a_b.parquet") + #expect(url.deletingLastPathComponent().path == "/tmp") + } +} + +@Suite("Shared row writers") +struct PluginRowWritersTests { + + /// The values in an export come from the database rather than from the person opening the + /// file, so a value that a spreadsheet would run as a formula is neutralised. + @Test("Formula leads are neutralised and the value is then quoted") + func formulaLeadsAreNeutralised() { + let options = PluginCsvWriteOptions.default + #expect(PluginRowWriters.csvField("=1+1", options: options) == "\"'=1+1\"") + #expect(PluginRowWriters.csvField("+1", options: options) == "\"'+1\"") + #expect(PluginRowWriters.csvField("-1", options: options) == "\"'-1\"") + #expect(PluginRowWriters.csvField("@SUM", options: options) == "\"'@SUM\"") + } + + /// Excel strips a leading tab or carriage return before parsing the cell, so `\t=1+1` reaches + /// the formula engine exactly as `=1+1` would. + @Test("A leading tab or carriage return counts as a formula lead") + func whitespaceLeadCountsAsFormula() { + let options = PluginCsvWriteOptions.default + #expect(PluginRowWriters.csvField("\t=1+1", options: options).hasPrefix("\"'")) + #expect(PluginRowWriters.csvField("\r=1+1", options: options).hasPrefix("\"'")) + } + + @Test("Sanitizing off leaves the value alone") + func sanitizingCanBeTurnedOff() { + let options = PluginCsvWriteOptions(sanitizesFormulas: false) + #expect(PluginRowWriters.csvField("=1+1", options: options) == "=1+1") + } + + @Test("A value holding the delimiter or a quote is quoted and its quotes doubled") + func quotingRules() { + let options = PluginCsvWriteOptions.default + #expect(PluginRowWriters.csvField("a,b", options: options) == "\"a,b\"") + #expect(PluginRowWriters.csvField("say \"hi\"", options: options) == "\"say \"\"hi\"\"\"") + #expect(PluginRowWriters.csvField("plain", options: options) == "plain") + } + + @Test("Quote handling always and never are honoured") + func quoteHandlingModes() { + #expect(PluginRowWriters.csvField("plain", options: PluginCsvWriteOptions(quoteHandling: .always)) + == "\"plain\"") + #expect(PluginRowWriters.csvField("a,b", options: PluginCsvWriteOptions(quoteHandling: .never)) + == "a,b") + } + + @Test("A line break is kept and quoted, or flattened when asked") + func lineBreakHandling() { + #expect(PluginRowWriters.csvField("a\nb", options: PluginCsvWriteOptions()) == "\"a\nb\"") + #expect(PluginRowWriters.csvField("a\nb", options: PluginCsvWriteOptions(flattensLineBreaks: true)) + == "a b") + } + + @Test("A line joins its fields with the configured delimiter") + func lineJoining() { + let tabbed = PluginCsvWriteOptions(delimiter: "\t") + #expect(PluginRowWriters.csvLine(["a", "b"], options: tabbed) == "a\tb") + } + + @Test("A null is JSON null and bytes are base64") + func jsonNullAndBytes() { + #expect(PluginRowWriters.jsonValue(.null) == "null") + #expect(PluginRowWriters.jsonValue(.bytes(Data([0x41, 0x42]))) == "\"QUI=\"") + } + + /// A numeric-looking identifier stays a string unless its column is numeric, or a postcode + /// loses its leading zero. + @Test("Text is written unquoted only when its column is numeric") + func numericOnlyWhenColumnSaysSo() { + #expect(PluginRowWriters.jsonValue(.text("01234"), columnTypeName: "VARCHAR") == "\"01234\"") + #expect(PluginRowWriters.jsonValue(.text("42"), columnTypeName: "INT") == "42") + #expect(PluginRowWriters.jsonValue(.text("42"), columnTypeName: "") == "\"42\"") + #expect(PluginRowWriters.jsonValue(.text("abc"), columnTypeName: "INT") == "\"abc\"") + } + + @Test("Preserving strings quotes even a numeric column") + func preserveAsStringWins() { + #expect(PluginRowWriters.jsonValue(.text("42"), columnTypeName: "INT", preserveAsString: true) + == "\"42\"") + } + + @Test("A JSON object pairs columns with values and can drop nulls") + func jsonObjectShape() { + #expect(PluginRowWriters.jsonObject(columns: ["a", "b"], values: ["1", "null"]) + == "{\"a\": 1, \"b\": null}") + #expect(PluginRowWriters.jsonObject(columns: ["a", "b"], values: ["1", "null"], includesNulls: false) + == "{\"a\": 1}") + } + + @Test("An insert names its columns and ends in a semicolon") + func insertShape() { + #expect(PluginRowWriters.sqlInsert(table: "\"t\"", columns: ["\"a\""], values: ["1"]) + == "INSERT INTO \"t\" (\"a\") VALUES (1);") + #expect(PluginRowWriters.sqlInsert(table: "\"t\"", columns: [], values: []) == nil) + } +} diff --git a/TableProTests/Plugins/SQLExportInsertModeTests.swift b/TableProTests/Plugins/SQLExportInsertModeTests.swift new file mode 100644 index 0000000000..4bf685b025 --- /dev/null +++ b/TableProTests/Plugins/SQLExportInsertModeTests.swift @@ -0,0 +1,255 @@ +// +// SQLExportInsertModeTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@Suite("SQL export insert modes") +struct SQLExportInsertModeTests { + + private func renderer(_ dialect: SqlDialect) -> SQLExportInsertRenderer { + SQLExportInsertRenderer(dialect: dialect) { "`\($0)`" } + } + + private func render( + _ dialect: SqlDialect, + _ mode: SQLExportInsertMode, + columns: [String] = ["id", "name", "email"], + primaryKeys: [String] = ["id"] + ) -> SQLExportInsertRenderer.Rendered { + renderer(dialect).render( + mode: mode, + tableRef: "`users`", + quotedColumns: "`id`, `name`, `email`", + overriding: "", + columnNames: columns, + primaryKeyColumns: primaryKeys + ) + } + + @Test("A plain insert is the same statement on every dialect") + func plainInsertIsDialectIndependent() { + for dialect in SqlDialect.allCases { + let rendered = render(dialect, .insert) + #expect(rendered.prefix == "INSERT INTO `users` (`id`, `name`, `email`) VALUES\n") + #expect(rendered.suffix.isEmpty) + #expect(rendered.warning == nil) + } + } + + /// The three dialects put conflict handling in three different places: MySQL in the verb, + /// SQLite in a resolution clause, PostgreSQL in a trailing clause. + @Test("Skipping existing rows uses each dialect's own spelling") + func ignoreUsesDialectSpelling() { + #expect(render(.mysql, .ignoreExisting).prefix.hasPrefix("INSERT IGNORE INTO")) + #expect(render(.sqlite, .ignoreExisting).prefix.hasPrefix("INSERT OR IGNORE INTO")) + + let postgres = render(.postgres, .ignoreExisting) + #expect(postgres.prefix.hasPrefix("INSERT INTO")) + #expect(postgres.suffix == "\nON CONFLICT DO NOTHING") + } + + @Test("Replacing uses REPLACE on MySQL and INSERT OR REPLACE on SQLite") + func replaceUsesDialectSpelling() { + #expect(render(.mysql, .replaceExisting).prefix.hasPrefix("REPLACE INTO")) + #expect(render(.sqlite, .replaceExisting).prefix.hasPrefix("INSERT OR REPLACE INTO")) + } + + /// PostgreSQL has no REPLACE, so it renders the upsert that overwrites every non-key column. + @Test("Replacing on PostgreSQL renders as an upsert") + func replaceOnPostgresIsAnUpsert() { + let rendered = render(.postgres, .replaceExisting) + #expect(rendered.suffix.contains("ON CONFLICT (`id`) DO UPDATE SET")) + #expect(rendered.warning == nil) + } + + @Test("Updating on MySQL assigns from VALUES and skips the key") + func mysqlUpsertSkipsTheKey() { + let rendered = render(.mysql, .updateExisting) + #expect(rendered.suffix == "\nON DUPLICATE KEY UPDATE `name` = VALUES(`name`), `email` = VALUES(`email`)") + #expect(!rendered.suffix.contains("`id` = VALUES")) + } + + @Test("Updating on PostgreSQL names the conflict target and assigns from EXCLUDED") + func postgresUpsertNamesTheTarget() { + let rendered = render(.postgres, .updateExisting) + #expect(rendered.suffix == "\nON CONFLICT (`id`) DO UPDATE SET `name` = EXCLUDED.`name`, `email` = EXCLUDED.`email`") + } + + /// SQLite spells the pseudo-table `excluded` in lower case, and it is not case-insensitive there. + @Test("Updating on SQLite uses the lower-case excluded pseudo-table") + func sqliteUpsertUsesLowerCaseExcluded() { + let rendered = render(.sqlite, .updateExisting) + #expect(rendered.suffix.contains("excluded.`name`")) + #expect(!rendered.suffix.contains("EXCLUDED.")) + } + + @Test("A composite key lists every key column as the conflict target") + func compositeKeyNamesEveryColumn() { + let rendered = render( + .postgres, .updateExisting, + columns: ["tenant", "id", "name"], primaryKeys: ["tenant", "id"]) + #expect(rendered.suffix.hasPrefix("\nON CONFLICT (`tenant`, `id`) DO UPDATE SET")) + #expect(rendered.suffix.contains("`name` = EXCLUDED.`name`")) + } + + /// Without a key there is no conflict target, so the statement would not parse. Writing plain + /// inserts and warning beats writing a dump that fails on restore. + @Test("A table with no primary key falls back to a plain insert and warns") + func noPrimaryKeyFallsBack() { + let rendered = render(.postgres, .updateExisting, primaryKeys: []) + #expect(rendered.suffix.isEmpty) + #expect(rendered.warning != nil) + } + + @Test("A table whose columns are all key columns has nothing to update") + func allKeyColumnsFallsBack() { + let mysql = render(.mysql, .updateExisting, columns: ["id"], primaryKeys: ["id"]) + #expect(mysql.suffix.isEmpty) + #expect(mysql.warning != nil) + + let postgres = render(.postgres, .updateExisting, columns: ["id"], primaryKeys: ["id"]) + #expect(postgres.suffix == "\nON CONFLICT DO NOTHING") + } + + @Test("An engine with no conflict spelling writes plain inserts and warns") + func genericDialectWarns() { + for mode in [SQLExportInsertMode.ignoreExisting, .replaceExisting, .updateExisting] { + let rendered = render(.generic, mode) + #expect(rendered.prefix.hasPrefix("INSERT INTO"), "\(mode) should fall back") + #expect(rendered.suffix.isEmpty) + #expect(rendered.warning != nil, "\(mode) should warn") + } + } + + @Test("The OVERRIDING clause survives every mode that keeps a plain prefix") + func overridingSurvives() { + let rendered = SQLExportInsertRenderer(dialect: .postgres) { "\"\($0)\"" }.render( + mode: .ignoreExisting, + tableRef: "\"users\"", + quotedColumns: "\"id\"", + overriding: " OVERRIDING SYSTEM VALUE", + columnNames: ["id"], + primaryKeyColumns: ["id"] + ) + #expect(rendered.prefix.contains(" OVERRIDING SYSTEM VALUE")) + } +} + +@Suite("SQL export file splitting") +struct SQLExportFileWriterTests { + + @Test("A part keeps the compound extension so the file still opens as SQL") + func partKeepsCompoundExtension() { + let base = URL(fileURLWithPath: "/tmp/dump.sql") + #expect(SQLExportFileWriter.partURL(for: base, part: 2).lastPathComponent == "dump.part2.sql") + + let compressed = URL(fileURLWithPath: "/tmp/dump.sql.gz") + #expect(SQLExportFileWriter.partURL(for: compressed, part: 3).lastPathComponent == "dump.part3.sql.gz") + } + + @Test("A name with no extension still numbers its parts") + func partWithoutExtension() { + let base = URL(fileURLWithPath: "/tmp/dump") + #expect(SQLExportFileWriter.partURL(for: base, part: 1).lastPathComponent == "dump.part1") + } + + @Test("A name with a dot in its stem splits at the first dot") + func partWithDottedStem() { + let base = URL(fileURLWithPath: "/tmp/app.v2.sql") + #expect(SQLExportFileWriter.partURL(for: base, part: 2).lastPathComponent == "app.part2.v2.sql") + } + + @Test("An unsplit export keeps the name the user chose") + func unsplitKeepsChosenName() throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter(destination: destination, splitSizeMegabytes: 0) + try writer.write("SELECT 1;\n") + let written = try writer.commit() + + #expect(written == [destination]) + #expect(!writer.didSplit) + #expect(try String(contentsOf: destination, encoding: .utf8) == "SELECT 1;\n") + } + + /// Rotation happens between writes, so a part always ends on a whole statement. + @Test("Passing the cap starts a new part without splitting a statement") + func splittingKeepsStatementsWhole() throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter(destination: destination, splitSizeMegabytes: 1) + let chunk = String(repeating: "x", count: 700 * 1_024) + try writer.write("A\(chunk);\n") + try writer.write("B\(chunk);\n") + let written = try writer.commit() + + #expect(writer.didSplit) + #expect(written.count == 2) + #expect(written[0].lastPathComponent == "dump.part1.sql") + #expect(written[1].lastPathComponent == "dump.part2.sql") + + let firstPart = try String(contentsOf: written[0], encoding: .utf8) + let secondPart = try String(contentsOf: written[1], encoding: .utf8) + #expect(firstPart.hasPrefix("A")) + #expect(firstPart.hasSuffix(";\n")) + #expect(secondPart.hasPrefix("B")) + #expect(secondPart.hasSuffix(";\n")) + } + + @Test("A rolled back export leaves nothing behind") + func rollbackLeavesNothing() throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + + let destination = directory.appendingPathComponent("dump.sql") + let writer = try SQLExportFileWriter(destination: destination, splitSizeMegabytes: 1) + try writer.write(String(repeating: "y", count: 2 * 1_024 * 1_024)) + try writer.write("tail;\n") + writer.rollback() + + let remaining = try FileManager.default.contentsOfDirectory(atPath: directory.path) + #expect(remaining.isEmpty, "left behind: \(remaining)") + } +} + +@Suite("SQL export snapshot") +struct SQLExportSnapshotTests { + + @Test("Each dialect opens its own consistent-read transaction") + func dialectSpecificBegin() { + #expect(SQLExportSnapshot(dialect: .mysql).beginStatement == "START TRANSACTION WITH CONSISTENT SNAPSHOT") + #expect(SQLExportSnapshot(dialect: .postgres).beginStatement == "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + #expect(SQLExportSnapshot(dialect: .sqlite).beginStatement == "BEGIN") + } + + /// An engine with no spelling for this opens nothing rather than sending a statement it would + /// reject and failing the whole export. + @Test("An engine with no snapshot statement opens and closes nothing") + func genericDialectOpensNothing() { + let snapshot = SQLExportSnapshot(dialect: .generic) + #expect(snapshot.beginStatement == nil) + #expect(snapshot.endStatement == nil) + } + + @Test("Every dialect that opens a transaction also closes it") + func openingImpliesClosing() { + for dialect in SqlDialect.allCases { + let snapshot = SQLExportSnapshot(dialect: dialect) + #expect((snapshot.beginStatement == nil) == (snapshot.endStatement == nil)) + } + } +} diff --git a/TableProTests/Plugins/XLSXImportTests.swift b/TableProTests/Plugins/XLSXImportTests.swift new file mode 100644 index 0000000000..ece70629a4 --- /dev/null +++ b/TableProTests/Plugins/XLSXImportTests.swift @@ -0,0 +1,281 @@ +// +// XLSXImportTests.swift +// TableProTests +// + +import Compression +import Foundation +import TableProPluginKit +import Testing + +@Suite("XLSX sheet parsing") +struct XLSXSheetParserTests { + + /// `A` is 0 and `AA` is 26, so the letters are base-26 with no zero digit. Getting this wrong + /// puts every column past Z in the wrong place. + @Test("A cell reference resolves to its column index") + func columnIndexFromReference() { + #expect(XLSXSheetParser.columnIndex(fromReference: "A1") == 0) + #expect(XLSXSheetParser.columnIndex(fromReference: "B2") == 1) + #expect(XLSXSheetParser.columnIndex(fromReference: "Z9") == 25) + #expect(XLSXSheetParser.columnIndex(fromReference: "AA1") == 26) + #expect(XLSXSheetParser.columnIndex(fromReference: "AB1") == 27) + #expect(XLSXSheetParser.columnIndex(fromReference: "BA10") == 52) + } + + @Test("A reference with no letters is refused") + func invalidReference() { + #expect(XLSXSheetParser.columnIndex(fromReference: "1") == nil) + #expect(XLSXSheetParser.columnIndex(fromReference: "") == nil) + } + + /// Part names vary between writers, so the first worksheet is found rather than assumed to be + /// `sheet1.xml`. + @Test("The first worksheet is found by path, not assumed") + func firstWorksheetIsFound() { + let paths = ["xl/workbook.xml", "xl/worksheets/sheet2.xml", "xl/worksheets/sheet1.xml", "[Content_Types].xml"] + #expect(XLSXSheetParser.firstWorksheetPath(in: paths) == "xl/worksheets/sheet1.xml") + #expect(XLSXSheetParser.firstWorksheetPath(in: ["xl/workbook.xml"]) == nil) + } + + /// A string split across formatting runs is one value, not several. A styled word mid-cell + /// would otherwise truncate it. + @Test("Shared strings concatenate their runs") + func sharedStringRuns() { + let xml = """ + + AdaGrace Hopper + """ + let strings = XLSXSheetParser.sharedStrings(from: Data(xml.utf8)) + #expect(strings == ["Ada", "Grace Hopper"]) + } + + /// A cell typed `s` holds an index into the shared string table rather than the text. + @Test("A shared-string cell resolves through the table") + func sharedStringCellResolves() { + let sheet = """ + + + 01 + + """ + let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: ["id", "name"]) + #expect(rows == [["id", "name"]]) + } + + /// A row omits the cells it has no value for, so position comes from each cell's own reference. + /// Counting cells instead shifts every value after a gap into the wrong column. + @Test("A gap in a row is filled from the cell references") + func gapsArePlacedByReference() { + let sheet = """ + + + 13 + + """ + let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []) + #expect(rows == [["1", nil, "3"]]) + } + + @Test("An inline string is read from the cell itself") + func inlineStringsAreRead() { + let sheet = """ + + + Ada + + """ + let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []) + #expect(rows == [["Ada"]]) + } + + /// A damaged workbook still imports something the user can see is wrong, rather than dropping + /// the value silently. + @Test("A shared-string index the table lacks keeps the raw value") + func outOfRangeIndexKeepsRawValue() { + let sheet = """ + + 99 + """ + let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: ["only"]) + #expect(rows == [["99"]]) + } + + @Test("Rows are padded to the widest row") + func rowsArePadded() { + let sheet = """ + + + 12 + 3 + + """ + let rows = XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []) + #expect(rows.count == 2) + #expect(rows.allSatisfy { $0.count == 2 }) + #expect(rows[1] == ["3", nil]) + } + + @Test("An empty sheet reads as no rows rather than failing") + func emptySheet() { + let sheet = "" + #expect(XLSXSheetParser.rows(from: Data(sheet.utf8), sharedStrings: []).isEmpty) + } +} + +@Suite("ZIP reading") +struct ZipReaderTests { + + /// Builds a ZIP the way the format specifies, so the reader is exercised against real bytes + /// rather than a mock. Stored and deflated entries are both produced, because Excel writes + /// deflate and TablePro's own XLSX export writes stored. + private func makeArchive(_ files: [(name: String, body: Data, deflate: Bool)]) -> Data { + var output = Data() + var directory = Data() + var offsets: [Int] = [] + + for file in files { + offsets.append(output.count) + let nameBytes = Data(file.name.utf8) + let payload = file.deflate ? deflated(file.body) : file.body + let method: UInt16 = file.deflate ? 8 : 0 + + output.append(contentsOf: [0x50, 0x4B, 0x03, 0x04]) + output.append(uint16(20)) + output.append(uint16(0)) + output.append(uint16(method)) + output.append(uint16(0)) + output.append(uint16(0)) + output.append(uint32(crc32(file.body))) + output.append(uint32(UInt32(payload.count))) + output.append(uint32(UInt32(file.body.count))) + output.append(uint16(UInt16(nameBytes.count))) + output.append(uint16(0)) + output.append(nameBytes) + output.append(payload) + } + + for (index, file) in files.enumerated() { + let nameBytes = Data(file.name.utf8) + let payload = file.deflate ? deflated(file.body) : file.body + directory.append(contentsOf: [0x50, 0x4B, 0x01, 0x02]) + directory.append(uint16(20)) + directory.append(uint16(20)) + directory.append(uint16(0)) + directory.append(uint16(file.deflate ? 8 : 0)) + directory.append(uint16(0)) + directory.append(uint16(0)) + directory.append(uint32(crc32(file.body))) + directory.append(uint32(UInt32(payload.count))) + directory.append(uint32(UInt32(file.body.count))) + directory.append(uint16(UInt16(nameBytes.count))) + directory.append(uint16(0)) + directory.append(uint16(0)) + directory.append(uint16(0)) + directory.append(uint16(0)) + directory.append(uint32(0)) + directory.append(uint32(UInt32(offsets[index]))) + directory.append(nameBytes) + } + + let directoryOffset = output.count + output.append(directory) + output.append(contentsOf: [0x50, 0x4B, 0x05, 0x06]) + output.append(uint16(0)) + output.append(uint16(0)) + output.append(uint16(UInt16(files.count))) + output.append(uint16(UInt16(files.count))) + output.append(uint32(UInt32(directory.count))) + output.append(uint32(UInt32(directoryOffset))) + output.append(uint16(0)) + return output + } + + private func deflated(_ data: Data) -> Data { + guard !data.isEmpty else { return Data() } + let capacity = max(data.count * 2, 1_024) + var output = Data(count: capacity) + let written = output.withUnsafeMutableBytes { destination -> Int in + guard let destinationBase = destination.bindMemory(to: UInt8.self).baseAddress else { return 0 } + return data.withUnsafeBytes { source -> Int in + guard let sourceBase = source.bindMemory(to: UInt8.self).baseAddress else { return 0 } + return compression_encode_buffer( + destinationBase, capacity, sourceBase, data.count, nil, COMPRESSION_ZLIB) + } + } + return output.prefix(written) + } + + private func uint16(_ value: UInt16) -> Data { + Data([UInt8(value & 0xFF), UInt8((value >> 8) & 0xFF)]) + } + + private func uint32(_ value: UInt32) -> Data { + Data([ + UInt8(value & 0xFF), UInt8((value >> 8) & 0xFF), + UInt8((value >> 16) & 0xFF), UInt8((value >> 24) & 0xFF) + ]) + } + + private func crc32(_ data: Data) -> UInt32 { + var crc: UInt32 = 0xFFFF_FFFF + for byte in data { + crc ^= UInt32(byte) + for _ in 0 ..< 8 { + crc = (crc & 1) == 1 ? (crc >> 1) ^ 0xEDB8_8320 : crc >> 1 + } + } + return crc ^ 0xFFFF_FFFF + } + + @Test("A stored entry reads back byte for byte") + func storedEntryRoundTrips() throws { + let body = Data("stored".utf8) + let archive = makeArchive([(name: "a.xml", body: body, deflate: false)]) + #expect(try ZipReader.data(named: "a.xml", in: archive) == body) + } + + /// Excel deflates every part, so this is the path that matters for a real workbook. + @Test("A deflated entry is inflated") + func deflatedEntryInflates() throws { + let body = Data(String(repeating: "value", count: 500).utf8) + let archive = makeArchive([(name: "b.xml", body: body, deflate: true)]) + #expect(try ZipReader.data(named: "b.xml", in: archive) == body) + } + + @Test("Every entry is listed with its own path") + func entriesAreListed() throws { + let archive = makeArchive([ + (name: "xl/workbook.xml", body: Data("a".utf8), deflate: false), + (name: "xl/worksheets/sheet1.xml", body: Data("b".utf8), deflate: true) + ]) + let entries = try ZipReader.entries(in: archive) + #expect(Set(entries.keys) == ["xl/workbook.xml", "xl/worksheets/sheet1.xml"]) + #expect(entries["xl/worksheets/sheet1.xml"]?.compressionMethod == 8) + } + + @Test("A missing entry is named in the error rather than returning nothing") + func missingEntryThrows() { + let archive = makeArchive([(name: "a.xml", body: Data("a".utf8), deflate: false)]) + #expect(throws: ZipReader.ZipError.self) { + _ = try ZipReader.data(named: "xl/sharedStrings.xml", in: archive) + } + } + + @Test("A file that is not a ZIP is refused") + func nonArchiveIsRefused() { + #expect(throws: ZipReader.ZipError.self) { + _ = try ZipReader.entries(in: Data("not a zip at all".utf8)) + } + #expect(throws: ZipReader.ZipError.self) { + _ = try ZipReader.entries(in: Data()) + } + } + + /// An empty part is legal and reads as empty rather than as a failure. + @Test("An empty entry reads as empty") + func emptyEntry() throws { + let archive = makeArchive([(name: "empty.xml", body: Data(), deflate: false)]) + #expect(try ZipReader.data(named: "empty.xml", in: archive).isEmpty) + } +} diff --git a/docs/features/backup-restore.mdx b/docs/features/backup-restore.mdx index 6df8834530..312337353d 100644 --- a/docs/features/backup-restore.mdx +++ b/docs/features/backup-restore.mdx @@ -1,19 +1,24 @@ --- title: Backup & Restore -description: Dump and restore PostgreSQL or Redshift databases with pg_dump and pg_restore, with progress, cancel, and SSH tunnel reuse +description: Dump and restore with each engine's own command line tools, with progress, cancel, and SSH tunnel reuse --- -`pg_dump` and `pg_restore` have to be on your Mac first. TablePro shells out to them and ships no copy of its own: +The engine's own tools do the work, and none of them ship inside TablePro. Install the ones for your engine first: -```bash -brew install libpq -brew link --force libpq -``` +| Engine | Tools | Install | Dump file | +|---|---|---|---| +| PostgreSQL, Redshift | `pg_dump`, `pg_restore` | `brew install libpq && brew link --force libpq` | `.dump`, custom archive | +| MySQL, MariaDB | `mysqldump`, `mysql` | `brew install mysql-client` | `.sql` | +| MongoDB | `mongodump`, `mongorestore` | `brew install mongodb-database-tools` | `.archive`, gzipped | +| SQLite, libSQL | `sqlite3` | `brew install sqlite` | `.sql` | +| SQL Server | `sqlpackage` | [Download from Microsoft](https://learn.microsoft.com/sql/tools/sqlpackage/sqlpackage-download) | `.bacpac` | + +MariaDB 11.0 renamed its clients, so `mariadb-dump` and `mariadb` are accepted in place of `mysqldump` and `mysql`. The lookup takes the first match from `/usr/bin/which`, then `/opt/homebrew/bin`, `/usr/local/bin`, and Postgres.app's latest version. That path decides which client version does the work. -PostgreSQL and Redshift only. For any other engine, use a SQL export from [Import & Export](/features/import-export). Restore is disabled on a read-only [Safe Mode](/features/safe-mode) connection; backup is not, because it writes nothing to the database. +**File > Backup Dump…** is dimmed on an engine that is not in the table. For those, use a SQL export from [Import & Export](/features/import-export). Restore is disabled on a read-only [Safe Mode](/features/safe-mode) connection; backup is not, because it writes nothing to the database. @@ -28,10 +33,12 @@ PostgreSQL and Redshift only. For any other engine, use a SQL export from [Impor Choose **File > Backup Dump…** on a connected session and pick the database. - The default filename is `-.dump`. Dumps use the custom archive format (`pg_dump -Fc`), which is what `pg_restore` reads back. + The default filename is `-` with the engine's extension from the table. PostgreSQL uses the custom archive format (`pg_dump -Fc`), MySQL and SQLite write SQL, and MongoDB writes a gzipped archive. **Cancel** confirms, sends `SIGTERM`, and removes the partial file. On success the result sheet gives the size and **Show in Finder**. + + The bar shows a percentage on PostgreSQL and MySQL, which answer a database size query cheaply. MongoDB and SQLite show an indeterminate bar with a running byte count. @@ -39,10 +46,10 @@ PostgreSQL and Redshift only. For any other engine, use a SQL export from [Impor - Choose **File > Restore Dump…** and select a file `pg_dump` wrote in custom archive format. + Choose **File > Restore Dump…** and select a file the matching backup tool wrote. - `pg_restore` runs with `--no-owner --no-acl`, so the connection user ends up owning the restored objects. + `pg_restore` runs with `--no-owner --no-acl`, so the connection user ends up owning the restored objects. `mongorestore` is scoped with `--nsInclude` to the database you pick. Restore shows no percentage. **Cancel** confirms and sends `SIGTERM`, and the target database is left as it stands: drop it and restore into a fresh one, or clean up the partial objects yourself. @@ -51,16 +58,40 @@ PostgreSQL and Redshift only. For any other engine, use a SQL export from [Impor No `--clean` is passed, so restoring on top of a schema that already holds conflicting objects produces errors instead of replacing them. +## What a dump carries + +`mysqldump` runs with `--single-transaction --routines --triggers --events`, so a MySQL dump is consistent with itself and carries stored routines, triggers and scheduled events. `pg_dump -Fc` carries the whole database. `sqlite3 .dump` writes the schema and every row as SQL. + ## SSH tunnels and SSL -Both flows reuse the connection's active SSH tunnel, with no second port forward. SSL connections pass their mode to the tools through `PGSSLMODE`, `verify-ca` and `verify-full` included. +Both flows reuse the connection's active SSH tunnel, with no second port forward. SSL mode reaches PostgreSQL through `PGSSLMODE` and MySQL through `--ssl-mode`, `verify-ca` and `verify-full` included. SQLite opens a file, so neither applies. + +## Server-side export + +Oracle, Snowflake and BigQuery do not have a client-side dump. They unload to somewhere the server can write: a `DIRECTORY` object, a stage, a Cloud Storage bucket. Choose **File > Server-Side Export…** for those. + +Pick a table and name the destination. Oracle takes the name of a `DIRECTORY` object rather than a path, because the path belongs to the server. Snowflake takes a stage, with or without its `@`. BigQuery takes a `gs://` prefix and shards its output under it. + +The statement runs on your own connection, so it carries your privileges and the server's own error comes back when the destination is not writable. + + +Nothing lands on your Mac. The result is on the server or in the bucket, and the sheet says where it went. + + +## Passwords + +Your password never reaches the tool's argument list, which every process on the machine can read through `ps`. PostgreSQL gets `PGPASSWORD` and MySQL gets `MYSQL_PWD`, both in the environment. MongoDB's tools read neither, so TablePro writes a `0600` config file and deletes it when the process exits. + + +SqlPackage is the exception. It accepts a password only inside its connection string, which means the command line, where other processes on the Mac can read it while the dump runs. TablePro asks before starting one. Windows or Entra authentication avoids it entirely: leave the username empty and the tool uses integrated security. + ## Failures -A non-zero exit shows the last 64 KB of `pg_dump` or `pg_restore` stderr in a scrollable monospaced view. Three causes account for most of them. +A non-zero exit shows the last 64 KB of the tool's stderr in a scrollable monospaced view. Three causes account for most of them. | What you see | What to do | |---|---| -| *"pg_dump was not found on this system. Install it with `brew install libpq` and link it."* | Install and link `libpq` so the binaries are on `PATH` | -| An authentication failure | The password goes through `PGPASSWORD` and the tools run with `--no-password`, so this is the role or the database rather than a missing prompt. Check that the role has `LOGIN` | +| *"… was not found on this system"* followed by an install command | Run that command so the binaries land on `PATH`. The message names the tool and the package for your engine | +| An authentication failure | The password goes through the environment or a config file rather than a prompt, so this is the account or the database. Check that the role can log in | | Objects that conflict with the dump | Restore into a fresh database, or drop the conflicting objects first | diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index 8f9d38f429..1f9e9a9bf5 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -12,7 +12,7 @@ The toolbar's **Export** (`Cmd+Shift+E`) re-reads every table you tick with `SEL Open a table or run a query, then click **Export** in the toolbar (`Cmd+Shift+E`), or right-click the results grid and choose **Export Results…**. - Pick a format, tick tables in the tree, and set the options for that format. + Pick a format, tick objects in the tree, and set the options for that format. Click **Export**. The destination file appears only on success: a failed or cancelled export removes its partial file. @@ -21,11 +21,43 @@ The toolbar's **Export** (`Cmd+Shift+E`) re-reads every table you tick with `SEL The last format and its options come back next time, but only after an export succeeds; cancelling discards the changes. **Reset to Defaults** restores the stock settings for the current format. +The bookmark button above the tree saves the current selection under a name and reapplies it later, including each table's options and row filter. A saved selection naming tables the database no longer has says how many it still matches, and applying it selects those and clears the rest. + Export dialog Export dialog +### What the tree lists + +SQL exports more than tables. A database holding more than one kind of object groups them, in the order a restore needs them: + +| Group | Written as | Engines | +|---|---|---| +| Types | `CREATE TYPE` | PostgreSQL and its forks | +| Tables | `CREATE TABLE` plus `INSERT` | every SQL engine | +| Views | `CREATE VIEW` from the server's own definition | every SQL engine | +| Materialized Views | `CREATE MATERIALIZED VIEW` | PostgreSQL, Oracle | +| Routines | `CREATE FUNCTION` or `CREATE PROCEDURE` | engines with stored routines | +| Triggers | `CREATE TRIGGER` | engines with triggers | +| Privileges | `GRANT`, one principal per row | engines with user management | + +A database with only tables lists them flat, with no group to open first. + +Only SQL writes every group. CSV, JSON and XLSX take tables and views, and switching to one of them drops the rest from the tree. Per-object checkboxes follow the kind: **Data** is off the row for anything without rows, and a privilege row carries neither **Drop** nor **Data**. + + +Privileges are server-wide, so they appear once, under the container the dialog opened on. A `GRANT` naming an object the dump does not create fails on restore. + + +### Narrowing what a table exports + +Click the filter icon on a table row to write a `WHERE` clause, cap the row count, or pick columns. The icon fills in once a table is narrowed, and its tooltip repeats what the narrowing is. + +The clause is your engine's own SQL, spliced into the `SELECT` the export runs. It has to be one expression: a semicolon anywhere but the end refuses the filter, and the export summary says the table went out whole. + +Ticking no column exports every column, which is also what a table with a column added later gets. + ### What ends up in the file A whole-table export streams from the database at constant memory, with no row-count limit, and can be cancelled from the progress dialog. @@ -50,19 +82,25 @@ A whole-table export streams from the database at constant memory, with no row-c Sanitizing prefixes a value starting with `=`, `+`, `-`, or `@` with a single quote, so a spreadsheet treats it as text. - An array of objects. + One object per table, each holding an array of rows. | Option | Default | |--------|---------| + | Layout (one JSON object, one row per line) | One JSON object | | Pretty print | Yes | | Include NULL values | Yes | | Preserve all values as strings | No | + + **One row per line** writes NDJSON to a `.ndjson` file: no wrapping object, no array, one row per line, so a stream reader can process a file larger than memory a line at a time. Pretty print does not apply to it and is dimmed. Import reads `.json`, `.jsonl` and `.ndjson`, detecting the layout from the file. | Option | Default | |--------|---------| | Compress with gzip (`.sql.gz`) | No | | Batch size (rows per INSERT: 1, 100, 500, 1,000) | 500 | + | On existing rows (insert, skip, replace, update) | Insert | + | Split every (one file, 8, 32, 128, 512 MB) | One file | + | Read every table at one snapshot | No | | Exclude the AUTO_INCREMENT counter | Yes | | Exclude DEFINER clauses | Yes | @@ -76,6 +114,12 @@ A whole-table export streams from the database at constant memory, with no row-c The last two exclusions cover MySQL and MariaDB, and pass every other engine through untouched. + Insert mode is spelled differently on each engine: `INSERT IGNORE` and `REPLACE INTO` on MySQL and MariaDB, `INSERT OR IGNORE` and `INSERT OR REPLACE` on SQLite, `ON CONFLICT` on PostgreSQL. Updating needs a primary key to name as the conflict target. An engine with no spelling for the mode writes plain inserts, and the export summary says which tables that happened to. + + Splitting writes `dump.part1.sql`, `dump.part2.sql` and so on, rotating between statements so no part ends mid-`INSERT`. Restore the parts in order. A gzipped export is one file, so the two settings do not combine and the summary says so. + + One snapshot opens `START TRANSACTION WITH CONSISTENT SNAPSHOT` on MySQL, `BEGIN ISOLATION LEVEL REPEATABLE READ` on PostgreSQL, and a deferred transaction on SQLite. It holds that transaction open for the whole export. + Excluding the counter drops `AUTO_INCREMENT=` from the table options and leaves the column's own `AUTO_INCREMENT` attribute alone. Restoring rows sets the counter one past the highest key in the data, so a source counter that had run ahead of its rows, after deletes or a reset, does not carry over. Excluding definers drops `DEFINER=user@host` from a view. The account running the import becomes the definer, and `SQL SECURITY` is untouched, so a definer-rights view then runs with that account's privileges. Keep the clause and the import fails with `ERROR 1227 (42000): Access denied; you need (at least one of) the SET USER privilege(s) for this operation` unless the importing account is privileged, and a view that does get created answers `ERROR 1446 (HY000): The user specified as a definer ('…') does not exist` on every query against it. An invoker-rights view still runs as its caller. @@ -102,8 +146,75 @@ A whole-table export streams from the database at constant memory, with no row-c Each table becomes its own worksheet and numbers are stored as numeric cells. A table over 1,048,576 rows, Excel's limit, splits across sheets. + + GitHub-flavoured tables. + + | Option | Default | + |--------|---------| + | Align columns | Yes | + | Write each table's name as a heading | Yes | + | NULL shows as | `NULL` | + + Column widths come from the header and the first 200 rows, so a wide value later in a long table is written whole rather than widening every row before it. A pipe or a line break inside a value is neutralised: both would end the cell early. + + + | Option | Default | + |--------|---------| + | Write a full HTML document | Yes | + | Write each table's name as a heading | Yes | + | Mark NULL cells | Yes | + + A full document carries its own stylesheet and follows the reader's light or dark setting. Turn it off to paste bare `
` elements into a page that has its own styling. + + + One element per row, one child element per column. + + | Option | Default | + |--------|---------| + | Pretty print | Yes | + | Mark NULL with `xsi:nil` | Yes | + | Row element | `row` | + + A column name that is not a legal XML element name is rewritten: a leading digit gains an underscore, an illegal character becomes one, and a name starting `xml` is prefixed. Control characters XML 1.0 forbids are dropped, so a binary column cannot make the document unparseable. + + + Install the Parquet plugin from **Settings > Plugins** first. It carries its own copy of DuckDB, which does the encoding, and is too large to ship in the app. + + | Option | Default | + |--------|---------| + | Compression (Snappy, Zstd, Gzip, None) | Snappy | + | Rows per group | 122,880 | + + Column types come from the source engine's own declarations, so numbers, dates and booleans arrive as those rather than as strings. A value that will not convert is written as null rather than failing the export. + + + Parquet holds one table per file. Selecting several writes `dump.users.parquet`, `dump.orders.parquet` and so on beside the name you chose. + + +## Transfer to another connection + +Right-click tables in the sidebar and choose **Transfer To…** to copy their rows straight into another connection, with no file in between. + + + + The destination list holds the connections that are already open. A connection that is not open does not appear. + + + Then tick the tables to copy. + + + Each table is wrapped in its own transaction by default, so a table that fails leaves its own rows untouched and the ones before it committed. + + + +Rows only. The destination table has to exist and its column names have to match, because inventing DDL that crosses from one engine to another would create tables whose types quietly disagree with the data landing in them. A per-table row filter set in the export tree is not carried over; narrow the transfer by transferring fewer tables. + + +**Delete existing rows first** empties each destination table before writing. There is no undo. + + ## Clipboard paste (CSV/TSV) Select a row in the data grid and press `Cmd+V` to paste tabular data straight in. Tabs parse as TSV, commas as CSV. @@ -144,7 +255,7 @@ Select a row in the data grid and press `Cmd+V` to paste tabular data straight i | **Stop and Commit** | Stops there, keeping what already succeeded | | **Skip and Continue** | Logs it and carries on, including a line the parser cannot read. No transaction | -Skip and Continue collects up to 1,000 failures with their line numbers and messages, and the summary counts successes against failures behind a **Copy Details** button. A stop shows the line, the database's own message, and the failing statement, with the dialog still open behind it, ready for a changed setting and another run. +Skip and Continue collects up to 1,000 failures with their line numbers and messages, and the summary counts successes against failures behind a **Copy Details** button. **Save Report…** writes them all to a CSV with a line, a statement and the database's own error per row, so a large import's failures can be sorted and searched rather than scrolled. A stop shows the line, the database's own message, and the failing statement, with the dialog still open behind it, ready for a changed setting and another run. ### Disabling foreign key checks @@ -168,6 +279,20 @@ The sheet accepts an array of objects `[{…}, {…}]`, newline-delimited JSON s Rows insert through parameterized statements, so a JSON value is never concatenated into SQL. Nested objects and arrays are stored as JSON text. +### Import XLSX + +Reads the first worksheet of an `.xlsx` workbook. + +| Option | Default | +|--------|---------| +| First row holds column names | Yes | +| Trim whitespace | No | +| Treat empty cells as NULL | Yes | + +A row that omits its empty cells still lands in the right columns: each cell's own reference decides where it goes, not its position among the cells that were written. Text stored in the workbook's shared string table is resolved, including a value split across several formatting runs. + +The workbook is read whole rather than streamed, because a sheet's rows refer back to a string table that has to be held anyway. + ### Import CSV CSV and TSV open the same sheet as JSON, with parsing options in front of the mapping. The delimiter and encoding are detected from the file; change any option and the mapping re-reads it. diff --git a/project.yml b/project.yml index d176feb300..c0bdef5087 100644 --- a/project.yml +++ b/project.yml @@ -191,6 +191,18 @@ targets: - target: SQLExport embed: true copy: { destination: plugins } + - target: MarkdownExport + embed: true + copy: { destination: plugins } + - target: HTMLExport + embed: true + copy: { destination: plugins } + - target: XMLExport + embed: true + copy: { destination: plugins } + - target: XLSXImport + embed: true + copy: { destination: plugins } - target: XLSXExport embed: true copy: { destination: plugins } @@ -383,6 +395,11 @@ targets: - Plugins/EtcdDriverPlugin/EtcdCommandParser.swift - Plugins/EtcdDriverPlugin/EtcdQueryBuilder.swift - Plugins/EtcdDriverPlugin/EtcdStatementGenerator.swift + - Plugins/HTMLExportPlugin/HTMLExportModels.swift + - Plugins/JSONExportPlugin/JSONExportModels.swift + - Plugins/MarkdownExportPlugin/MarkdownExportModels.swift + - Plugins/ParquetExportPlugin/ParquetExportModels.swift + - Plugins/ParquetExportPlugin/ParquetTypeMapper.swift - Plugins/JSONImportPlugin/JSONImportOptions.swift - Plugins/JSONImportPlugin/JSONImportOptionsView.swift - Plugins/JSONImportPlugin/JSONImportParsing.swift @@ -488,13 +505,20 @@ targets: - Plugins/RedisDriverPlugin/RedisStatementGenerator.swift - Plugins/RedisDriverPlugin/RedisTopologyDiagnostics.swift - Plugins/SQLExportPlugin/SQLExportDDLRewriter.swift + - Plugins/SQLExportPlugin/SQLExportFileWriter.swift + - Plugins/SQLExportPlugin/SQLExportInsertMode.swift - Plugins/SQLExportPlugin/SQLExportModels.swift - Plugins/SQLExportPlugin/SQLExportOptionsView.swift - Plugins/SQLExportPlugin/SQLExportPlugin.swift + - Plugins/SQLExportPlugin/SQLExportSnapshot.swift - Plugins/SQLImportPlugin/SQLImportFailure.swift - Plugins/SQLImportPlugin/SQLImportOptions.swift - Plugins/SQLImportPlugin/SQLImportOptionsView.swift - Plugins/SQLImportPlugin/SQLImportPlugin.swift + - Plugins/XLSXImportPlugin/XLSXImportOptions.swift + - Plugins/XLSXImportPlugin/XLSXSheetParser.swift + - Plugins/XLSXImportPlugin/ZipReader.swift + - Plugins/XMLExportPlugin/XMLExportModels.swift - Plugins/SnowflakeDriverPlugin/PluginCellValueBox.swift - Plugins/SnowflakeDriverPlugin/SnowflakeAuth.swift - Plugins/SnowflakeDriverPlugin/SnowflakeBindingEncoder.swift @@ -698,6 +722,66 @@ targets: base: PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.JSONExportPlugin + MarkdownExport: + templates: [DriverPlugin] + templateAttributes: + folder: MarkdownExportPlugin + principalClass: MarkdownExportPlugin + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.MarkdownExportPlugin + + HTMLExport: + templates: [DriverPlugin] + templateAttributes: + folder: HTMLExportPlugin + principalClass: HTMLExportPlugin + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.HTMLExportPlugin + + XLSXImport: + templates: [DriverPlugin] + templateAttributes: + folder: XLSXImportPlugin + principalClass: XLSXImportPlugin + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.XLSXImportPlugin + + XMLExport: + templates: [DriverPlugin] + templateAttributes: + folder: XMLExportPlugin + principalClass: XMLExportPlugin + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.XMLExportPlugin + + ParquetExport: + templates: [DriverPlugin] + templateAttributes: + folder: ParquetExportPlugin + principalClass: ParquetExportPlugin + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.ParquetExportPlugin + # DuckDB does the Parquet encoding, so this links the same static library the + # DuckDB driver does. That is why the plugin is registry-only rather than bundled: + # 149MB universal is not worth adding to every app download for one export format. + HEADER_SEARCH_PATHS: + - $(inherited) + - $(SRCROOT)/Plugins/DuckDBDriverPlugin/CDuckDB/include + LIBRARY_SEARCH_PATHS: + - $(inherited) + - $(SRCROOT)/Libs + OTHER_LDFLAGS: + - $(inherited) + - -force_load + - $(SRCROOT)/Libs/libduckdb.a + - -lc++ + SWIFT_INCLUDE_PATHS: $(SRCROOT)/Plugins/DuckDBDriverPlugin/CDuckDB + SQLExport: templates: [DriverPlugin] templateAttributes: @@ -1087,7 +1171,12 @@ aggregateTargets: - OracleDriver - PostgreSQLDriver - RedisDriver + - HTMLExport + - MarkdownExport + - ParquetExport - SQLExport + - XLSXImport + - XMLExport - SQLImport - SQLiteDriver - SnowflakeDriverPlugin diff --git a/scripts/check-duckdb-extensions.sh b/scripts/check-duckdb-extensions.sh new file mode 100755 index 0000000000..c498681053 --- /dev/null +++ b/scripts/check-duckdb-extensions.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# Asserts that the shipped Libs/libduckdb.a actually links the extensions +# scripts/duckdb-macos-extensions.cmake declares. +# +# The two drifted apart once and nothing caught it. The cmake file listed +# core_functions, json, parquet, icu and autocomplete, and the published library +# linked none of them, so `sum`, `avg`, `round`, `json_extract` and +# `COPY ... (FORMAT PARQUET)` all failed with "not in the catalog, but it exists +# in the extension" on any Mac that could not reach extensions.duckdb.org. +# The config was right and the binary was stale, which is invisible unless +# something asks the binary. +# +# Run after bumping DUCKDB_VERSION in build-duckdb.sh, after editing +# duckdb-macos-extensions.cmake, and before publishing a rebuilt libduckdb. +# +# Usage: scripts/check-duckdb-extensions.sh +# +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +HEADERS="$ROOT/Plugins/DuckDBDriverPlugin/CDuckDB/include" +LIB="$ROOT/Libs/libduckdb.a" +EXTENSION_CONFIG="$ROOT/scripts/duckdb-macos-extensions.cmake" + +if [ ! -f "$LIB" ]; then + echo "error: $LIB is missing. Run scripts/download-libs.sh first." >&2 + exit 1 +fi + +# One probe query per extension, chosen so it fails with a catalog error rather +# than a syntax error when the extension is absent. +probe_for() { + case "$1" in + core_functions) echo "SELECT sum(1)" ;; + json) echo "SELECT json_extract('{\\\"a\\\":1}', '\$.a')" ;; + parquet) echo "COPY (SELECT 1 AS a) TO '\$TMP/probe.parquet' (FORMAT PARQUET)" ;; + icu) echo "SELECT strftime(DATE '2020-01-01', '%Y')" ;; + autocomplete) echo "SELECT * FROM sql_auto_complete('SEL')" ;; + *) echo "" ;; + esac +} + +EXTENSIONS="$(sed -nE 's/^duckdb_extension_load\(([a-z_]+)\)?.*/\1/p' "$EXTENSION_CONFIG" | sort -u)" +if [ -z "$EXTENSIONS" ]; then + echo "error: no duckdb_extension_load entries found in $EXTENSION_CONFIG" >&2 + exit 1 +fi + +WORK_DIR="$(mktemp -d /tmp/duckdb-ext-check.XXXXXX)" +trap 'rm -rf "$WORK_DIR"' EXIT + +{ + echo '#include ' + echo '#include ' + echo '#include "duckdb.h"' + echo 'int main(void) {' + echo ' duckdb_database db; duckdb_connection con; int failures = 0;' + echo ' if (duckdb_open(NULL, &db) == DuckDBError) { printf("open failed\n"); return 2; }' + echo ' if (duckdb_connect(db, &con) == DuckDBError) { printf("connect failed\n"); return 2; }' + for extension in $EXTENSIONS; do + query="$(probe_for "$extension")" + [ -z "$query" ] && continue + query="${query//\$TMP/$WORK_DIR}" + echo " { duckdb_result r;" + echo " if (duckdb_query(con, \"$query\", &r) == DuckDBError) {" + echo " printf(\"%-16s MISSING %s\n\", \"$extension\", duckdb_result_error(&r)); failures++;" + echo " } else { printf(\"%-16s linked\n\", \"$extension\"); }" + echo " duckdb_destroy_result(&r); }" + done + echo ' duckdb_disconnect(&con); duckdb_close(&db);' + echo ' if (failures > 0) {' + echo ' printf("\n%d extension(s) the build config declares are not in the library.\n", failures);' + echo ' printf("Rebuild with scripts/build-duckdb.sh both, then publish with scripts/publish-libs.sh.\n");' + echo ' return 1;' + echo ' }' + echo ' printf("\nEvery declared extension is linked in.\n");' + echo ' return 0;' + echo '}' +} > "$WORK_DIR/probe.c" + +ARCH="$(uname -m)" +clang -arch "$ARCH" -I "$HEADERS" "$WORK_DIR/probe.c" "$LIB" -lc++ -o "$WORK_DIR/probe" +"$WORK_DIR/probe"