diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index c1ee902353..00fd3af828 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -69,6 +69,7 @@ let package = Package( .library(name: "TableProGzip", targets: ["TableProGzip"]), .library(name: "TableProKafkaCore", targets: ["TableProKafkaCore"]), .library(name: "TableProCassandraCore", targets: ["TableProCassandraCore"]), + .library(name: "TableProEtcdCore", targets: ["TableProEtcdCore"]), .library(name: "TableProDuckDBCore", targets: ["TableProDuckDBCore"]) ], // Kafka brokers hand back whatever codec the producer stored, and macOS has no zstd. @@ -368,6 +369,13 @@ let package = Package( path: "Sources/TableProCassandraCore", swiftSettings: pluginCoreSettings ), + // Linked by the macOS app and a plugin bundle, so never TableProPluginKit. + .target( + name: "TableProEtcdCore", + dependencies: [], + path: "Sources/TableProEtcdCore", + swiftSettings: pluginCoreSettings + ), // Linked by a plugin bundle, so never TableProPluginKit. .target( name: "TableProDuckDBCore", @@ -620,6 +628,12 @@ let package = Package( path: "Tests/TableProCassandraCoreTests", swiftSettings: pluginCoreSettings ), + .testTarget( + name: "TableProEtcdCoreTests", + dependencies: ["TableProEtcdCore"], + path: "Tests/TableProEtcdCoreTests", + swiftSettings: pluginCoreSettings + ), .testTarget( name: "TableProDuckDBCoreTests", dependencies: ["TableProDuckDBCore"], diff --git a/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdCommandText.swift b/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdCommandText.swift new file mode 100644 index 0000000000..57a609669f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdCommandText.swift @@ -0,0 +1,208 @@ +import Foundation + +// One word of an etcd command line, read two ways. +public struct EtcdCommandToken: Equatable, Sendable { + // As every TablePro release reads the word, with \xHH left as four characters. Commands, flags and names use + // this, so an older host grades the same command the driver runs. + public let text: String + // The bytes of a key or a value, with \xHH as one byte. + public let bytes: Data + + public init(text: String, bytes: Data) { + self.text = text + self.bytes = bytes + } +} + +// One etcd command line as text: the trim, the tokenizer and the quoter share one set of blanks, so an +// argument the quoter leaves bare reads back as the same bytes. +public enum EtcdCommandText { + public static let endOfFlags = "--" + + public static func isBlank(_ scalar: Unicode.Scalar) -> Bool { + scalar.properties.isWhitespace + } + + public static func trimmed(_ text: String) -> String { + let scalars = text.unicodeScalars + guard let first = scalars.firstIndex(where: { !isBlank($0) }), + let last = scalars.lastIndex(where: { !isBlank($0) }) else { + return "" + } + var kept = String.UnicodeScalarView() + kept.append(contentsOf: scalars[first ... last]) + return String(kept) + } + + // Inside quotes, \n, \r, \t, \\, \" and \' are escapes, and \xHH is one byte of a key or a value. + public static func tokenize(_ input: String) -> [EtcdCommandToken] { + let scalars = Array(input.unicodeScalars) + var tokens: [EtcdCommandToken] = [] + var token = TokenBuilder() + var tokenStarted = false + var openQuote: Unicode.Scalar? + var index = 0 + + while index < scalars.count { + let scalar = scalars[index] + index += 1 + + if let quote = openQuote { + if scalar == quote { + openQuote = nil + } else if scalar == "\\" { + index = appendEscape(in: scalars, at: index, to: &token) + } else { + token.append(scalar) + } + } else if scalar == "\"" || scalar == "'" { + openQuote = scalar + tokenStarted = true + } else if isBlank(scalar) { + if tokenStarted { + tokens.append(token.finish()) + tokenStarted = false + } + } else { + token.append(scalar) + tokenStarted = true + } + } + + if tokenStarted { + tokens.append(token.finish()) + } + return tokens + } + + public static func quoted(_ argument: String) -> String { + isBare(argument) ? argument : quotedText(argument) + } + + public static func quoted(_ argument: Data) -> String { + guard let text = text(argument) else { return quotedBytes(argument) } + return quoted(text) + } + + // An argument that starts with "--" would read as a flag, so the flags go first and "--" ends them. + public static func command(_ name: String, arguments: [Data], flags: [String] = []) -> String { + let words = arguments.map { quoted($0) } + guard arguments.contains(where: { $0.starts(with: flagMarker) }) else { + return ([name] + words + flags).joined(separator: " ") + } + return ([name] + flags + [endOfFlags] + words).joined(separator: " ") + } + + // String(data:encoding:) drops a leading byte order mark, which names a different key. + public static func text(_ bytes: Data) -> String? { + let text = String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + return text.utf8.elementsEqual(bytes) ? text : nil + } + + // MARK: - Private + + private static let flagMarker = Data(endOfFlags.utf8) + + private struct TokenBuilder { + private var text = String.UnicodeScalarView() + private var bytes: [UInt8] = [] + + mutating func append(_ scalar: Unicode.Scalar) { + text.append(scalar) + UTF8.encode(scalar) { bytes.append($0) } + } + + mutating func appendHexEscape(_ high: Unicode.Scalar, _ low: Unicode.Scalar, byte: UInt8) { + text.append(contentsOf: ["\\", "x", high, low]) + bytes.append(byte) + } + + mutating func finish() -> EtcdCommandToken { + defer { + text = String.UnicodeScalarView() + bytes.removeAll() + } + return EtcdCommandToken(text: String(text), bytes: Data(bytes)) + } + } + + private static func appendEscape(in scalars: [Unicode.Scalar], at index: Int, to token: inout TokenBuilder) -> Int { + guard index < scalars.count else { + token.append("\\") + return index + } + let escaped = scalars[index] + switch escaped { + case "n": token.append("\n") + case "r": token.append("\r") + case "t": token.append("\t") + case "\\", "\"", "'": token.append(escaped) + case "x" where index + 2 < scalars.count: + let high = scalars[index + 1] + let low = scalars[index + 2] + if let byte = hexByte(high, low) { + token.appendHexEscape(high, low, byte: byte) + return index + 3 + } + token.append("\\") + token.append(escaped) + default: + token.append("\\") + token.append(escaped) + } + return index + 1 + } + + private static func hexByte(_ high: Unicode.Scalar, _ low: Unicode.Scalar) -> UInt8? { + guard high.properties.isASCIIHexDigit, low.properties.isASCIIHexDigit else { return nil } + return UInt8(String(high) + String(low), radix: 16) + } + + // A quote opens a quoted argument and a blank ends one. A control or invisible character is quoted too: + // text fields and editors drop one at an edge, and CharacterSet.whitespacesAndNewlines holds U+200B. + private static func isBare(_ text: String) -> Bool { + guard !text.isEmpty else { return false } + return !text.unicodeScalars.contains { scalar in + isBlank(scalar) || scalar == "\"" || scalar == "'" + || scalar.properties.generalCategory == .control + || scalar.properties.isDefaultIgnorableCodePoint + } + } + + private static func quotedText(_ text: String) -> String { + var result = "\"" + for scalar in text.unicodeScalars { + switch scalar { + case "\\": result += "\\\\" + case "\"": result += "\\\"" + case "\n": result += "\\n" + case "\r": result += "\\r" + case "\t": result += "\\t" + default: + if scalar.value < 0x20 || scalar.value == 0x7F { + result += hexEscape(UInt8(scalar.value)) + } else { + result.unicodeScalars.append(scalar) + } + } + } + return result + "\"" + } + + private static func quotedBytes(_ bytes: Data) -> String { + var result = "\"" + for byte in bytes { + switch byte { + case UInt8(ascii: "\\"): result += "\\\\" + case UInt8(ascii: "\""): result += "\\\"" + case 0x20 ..< 0x7F: result.unicodeScalars.append(Unicode.Scalar(byte)) + default: result += hexEscape(byte) + } + } + return result + "\"" + } + + private static func hexEscape(_ byte: UInt8) -> String { + String(format: "\\x%02x", byte) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdKeyRange.swift b/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdKeyRange.swift new file mode 100644 index 0000000000..96e9975ba6 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdKeyRange.swift @@ -0,0 +1,38 @@ +import Foundation + +public struct EtcdKeyRange: Equatable, Sendable { + public let key: Data + public let rangeEnd: Data? + + // clientv3 WithPrefix: an empty prefix reads every key, which etcd takes as "\0" to "\0". + public init(key: Data, prefix: Bool = false) { + guard prefix else { + self.key = key + rangeEnd = nil + return + } + guard !key.isEmpty else { + self.key = Self.zeroByte + rangeEnd = Self.zeroByte + return + } + self.key = key + rangeEnd = Self.prefixEnd(of: key) + } + + // clientv3 GetPrefixRangeEnd: the first key past every key that starts with the prefix. A prefix of + // only 0xFF bytes has none, so the range runs to the end of the keyspace. + public static func prefixEnd(of prefix: Data) -> Data { + var bytes = Array(prefix) + while let last = bytes.last { + if last < 0xFF { + bytes[bytes.count - 1] = last + 1 + return Data(bytes) + } + bytes.removeLast() + } + return zeroByte + } + + private static let zeroByte = Data([0]) +} diff --git a/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdCommandTextTests.swift b/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdCommandTextTests.swift new file mode 100644 index 0000000000..af710d4cf1 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdCommandTextTests.swift @@ -0,0 +1,152 @@ +import Foundation +import Testing + +@testable import TableProEtcdCore + +private func tokenBytes(_ input: String) -> [Data] { + EtcdCommandText.tokenize(input).map { $0.bytes } +} + +struct EtcdCommandTextRoundTripTests { + @Test("every byte value survives quote then tokenize") + func everyByteSurvives() { + for value in UInt8.min ... UInt8.max { + let argument = Data([value]) + #expect(tokenBytes(EtcdCommandText.quoted(argument)) == [argument]) + } + } + + @Test("binary keys survive quote then tokenize") + func blobsSurvive() { + var seed: UInt64 = 0x9E37_79B9_7F4A_7C15 + func nextByte() -> UInt8 { + seed ^= seed << 13 + seed ^= seed >> 7 + seed ^= seed << 17 + return UInt8(truncatingIfNeeded: seed) + } + for length in 1 ... 200 { + let blob = Data((0 ..< length).map { _ in nextByte() }) + #expect(tokenBytes(EtcdCommandText.quoted(blob)) == [blob]) + } + } + + @Test("a key that is not UTF-8 keeps its bytes") + func nonUTF8Key() { + let key = Data([0x2F, 0x61, 0xFF, 0xFE, 0x2F, 0x62]) + let quoted = EtcdCommandText.quoted(key) + #expect(quoted == #""/a\xff\xfe/b""#) + #expect(tokenBytes("del \(quoted)") == [Data("del".utf8), key]) + } + + @Test("a leading byte order mark is part of the key") + func byteOrderMark() { + let key = Data([0xEF, 0xBB, 0xBF, 0x61]) + #expect(EtcdCommandText.text(key)?.unicodeScalars.first == "\u{FEFF}") + #expect(tokenBytes(EtcdCommandText.quoted(key)) == [key]) + } + + @Test("an empty argument survives") + func emptySurvives() { + #expect(EtcdCommandText.quoted(Data()) == "\"\"") + #expect(tokenBytes("get \"\" --prefix") == [Data("get".utf8), Data(), Data("--prefix".utf8)]) + } + + @Test("every character a trim, the tokenizer or a text field drops survives at either edge of a key") + func edgeCharactersSurvive() { + var checked = 0 + for value in UInt32(0) ... 0x10FFFF { + guard let scalar = Unicode.Scalar(value) else { continue } + let dropped = EtcdCommandText.isBlank(scalar) + || CharacterSet.whitespacesAndNewlines.contains(scalar) + || scalar.properties.isDefaultIgnorableCodePoint + || scalar.properties.generalCategory == .format + || scalar.properties.generalCategory == .control + guard dropped else { continue } + checked += 1 + var key = String.UnicodeScalarView() + key.append(scalar) + key.append(contentsOf: "key".unicodeScalars) + key.append(scalar) + let argument = Data(String(key).utf8) + let line = "del " + EtcdCommandText.quoted(argument) + " \n" + #expect(tokenBytes(EtcdCommandText.trimmed(line)) == [Data("del".utf8), argument]) + let hostTrimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) + #expect(tokenBytes(hostTrimmed) == [Data("del".utf8), argument]) + } + #expect(checked > 4_000) + } +} + +struct EtcdCommandTextQuotingTests { + @Test("a plain key stays bare") + func plainKeyIsBare() { + #expect(EtcdCommandText.quoted("/app/config") == "/app/config") + } + + @Test("non-ASCII text stays readable") + func unicodeStaysReadable() { + #expect(EtcdCommandText.quoted("/app/café") == "/app/café") + } + + @Test("a zero width space is quoted, since a whitespace trim drops it") + func zeroWidthSpaceIsQuoted() { + #expect(EtcdCommandText.quoted("foo\u{200B}") == "\"foo\u{200B}\"") + } + + @Test("a key with a space or a quote is quoted and escaped") + func spacesAndQuotes() { + #expect(EtcdCommandText.quoted("a b") == "\"a b\"") + #expect(EtcdCommandText.quoted("say \"hi\"") == #""say \"hi\"""#) + #expect(EtcdCommandText.quoted("it's") == "\"it's\"") + } + + @Test("a control character is written as a byte escape") + func controlsAreEscaped() { + #expect(EtcdCommandText.quoted("a\u{01}b\nc") == #""a\x01b\nc""#) + } +} + +struct EtcdCommandTextCommandTests { + @Test("a plain command keeps etcdctl's argument order") + func plainOrder() { + let line = EtcdCommandText.command("get", arguments: [Data("/app/".utf8)], flags: ["--prefix"]) + #expect(line == "get /app/ --prefix") + } + + @Test("a key that starts with -- follows the end of the flags") + func dashedKeyFollowsEndOfFlags() { + let line = EtcdCommandText.command( + "put", arguments: [Data("--lease".utf8), Data("v".utf8)], flags: ["--lease=0x7b"] + ) + #expect(line == "put --lease=0x7b -- --lease v") + #expect(tokenBytes(line).count == 5) + } +} + +struct EtcdCommandTextTokenizerTests { + @Test("a hex escape inside quotes is one byte") + func hexEscape() { + #expect(tokenBytes(#""\xff\x00A""#) == [Data([0xFF, 0x00, 0x41])]) + #expect(tokenBytes(#"'\x2F'"#) == [Data("/".utf8)]) + } + + @Test("a hex escape without two hex digits stays as written") + func incompleteHexEscape() { + #expect(tokenBytes(#""\xZZ""#) == [Data(#"\xZZ"#.utf8)]) + #expect(tokenBytes(#""a\x4""#) == [Data(#"a\x4"#.utf8)]) + #expect(tokenBytes(#""\x""#) == [Data(#"\x"#.utf8)]) + } + + @Test("a backslash outside quotes is a literal character") + func backslashOutsideQuotes() { + #expect(tokenBytes(#"C:\xff"#) == [Data(#"C:\xff"#.utf8)]) + } + + @Test("the trim drops only Unicode whitespace") + func trimDropsOnlyWhitespace() { + #expect(EtcdCommandText.trimmed(" \u{3000}get k\n") == "get k") + #expect(EtcdCommandText.trimmed("del foo\u{200B}") == "del foo\u{200B}") + #expect(EtcdCommandText.trimmed(" \t ").isEmpty) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdKeyRangeTests.swift b/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdKeyRangeTests.swift new file mode 100644 index 0000000000..fbeec05c93 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdKeyRangeTests.swift @@ -0,0 +1,71 @@ +import Foundation +import Testing + +@testable import TableProEtcdCore + +struct EtcdKeyRangeTests { + @Test("an ASCII prefix ends one past its last byte") + func asciiPrefix() { + #expect(EtcdKeyRange.prefixEnd(of: Data("/app/".utf8)) == Data("/app0".utf8)) + } + + @Test("a prefix ending in a multibyte character ends one past its last byte, not at every key") + func multibytePrefix() { + let prefix = Data("keyÿ".utf8) + #expect(Array(prefix.suffix(2)) == [0xC3, 0xBF]) + #expect(EtcdKeyRange.prefixEnd(of: prefix) == Data([0x6B, 0x65, 0x79, 0xC3, 0xC0])) + #expect(EtcdKeyRange.prefixEnd(of: Data("café".utf8)) == Data([0x63, 0x61, 0x66, 0xC3, 0xAA])) + } + + @Test("a prefix ending in DEL ends at 0x80, which is not UTF-8 on its own") + func deletePrefix() { + #expect(EtcdKeyRange.prefixEnd(of: Data([0x61, 0x7F])) == Data([0x61, 0x80])) + } + + @Test("trailing 0xFF bytes are dropped before the bump") + func trailingFF() { + #expect(EtcdKeyRange.prefixEnd(of: Data([0x61, 0xFF, 0xFF])) == Data([0x62])) + #expect(EtcdKeyRange.prefixEnd(of: Data([0x61, 0xFE, 0xFF])) == Data([0x61, 0xFF])) + } + + @Test("a prefix of only 0xFF bytes runs to the end of the keyspace") + func allFF() { + #expect(EtcdKeyRange.prefixEnd(of: Data([0xFF, 0xFF])) == Data([0x00])) + } + + @Test("a prefix that is not UTF-8 keeps its bytes") + func nonUTF8Prefix() { + #expect(EtcdKeyRange.prefixEnd(of: Data([0xC3, 0x28, 0x80])) == Data([0xC3, 0x28, 0x81])) + #expect(EtcdKeyRange.prefixEnd(of: Data([0x2F, 0xC3])) == Data([0x2F, 0xC4])) + } + + @Test("a prefix range covers exactly the keys that start with the prefix") + func rangeMatchesPrefix() throws { + let prefix = Data([0x61, 0xC3]) + let range = EtcdKeyRange(key: prefix, prefix: true) + #expect(range.key == prefix) + let end = try #require(range.rangeEnd) + let inside = [Data([0x61, 0xC3]), Data([0x61, 0xC3, 0xFF]), Data([0x61, 0xC3, 0xA9, 0x00])] + let outside = [Data([0x61, 0xC4]), Data([0x61, 0xC2, 0xFF]), Data([0x62])] + for key in inside { + #expect(!key.lexicographicallyPrecedes(range.key) && key.lexicographicallyPrecedes(end)) + } + for key in outside { + #expect(key.lexicographicallyPrecedes(range.key) || !key.lexicographicallyPrecedes(end)) + } + } + + @Test("an empty prefix reads every key") + func emptyPrefix() { + let range = EtcdKeyRange(key: Data(), prefix: true) + #expect(range.key == Data([0x00])) + #expect(range.rangeEnd == Data([0x00])) + } + + @Test("a single key has no range end") + func singleKey() { + let range = EtcdKeyRange(key: Data("/app".utf8)) + #expect(range.key == Data("/app".utf8)) + #expect(range.rangeEnd == nil) + } +} diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdCommandArgument.swift b/apple/Plugins/EtcdDriverPlugin/EtcdCommandArgument.swift deleted file mode 100644 index cefd0ae970..0000000000 --- a/apple/Plugins/EtcdDriverPlugin/EtcdCommandArgument.swift +++ /dev/null @@ -1,33 +0,0 @@ -// -// EtcdCommandArgument.swift -// EtcdDriverPlugin -// - -import Foundation - -internal enum EtcdCommandArgument { - private static let escapeSequences: [Unicode.Scalar: String] = [ - "\\": "\\\\", - "\"": "\\\"", - "\n": "\\n", - "\r": "\\r" - ] - - static func quoted(_ value: String) -> String { - let needsQuoting = value.isEmpty || value.unicodeScalars.contains(where: requiresQuoting) - guard needsQuoting else { return value } - var body = String.UnicodeScalarView() - for scalar in value.unicodeScalars { - if let escapeSequence = escapeSequences[scalar] { - body.append(contentsOf: escapeSequence.unicodeScalars) - } else { - body.append(scalar) - } - } - return "\"" + String(body) + "\"" - } - - private static func requiresQuoting(_ scalar: Unicode.Scalar) -> Bool { - scalar.properties.isWhitespace || scalar == "\"" || scalar == "'" - } -} diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift b/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift index ac352443aa..d079020050 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift @@ -2,20 +2,18 @@ // EtcdCommandParser.swift // TablePro // -// Parses etcdctl-compatible command strings into structured operations. -// Supports: get, put, del, watch, lease, member, endpoint, compaction, auth, user, role commands. -// import Foundation import os +import TableProEtcdCore import TableProPluginKit enum EtcdOperation { // KV - case get(key: String, prefix: Bool, limit: Int64?, keysOnly: Bool, sortOrder: EtcdSortOrder?, sortTarget: EtcdSortTarget?) - case put(key: String, value: String, leaseId: Int64?) - case del(key: String, prefix: Bool) - case watch(key: String, prefix: Bool, timeout: TimeInterval) + case get(key: Data, prefix: Bool, limit: Int64?, keysOnly: Bool, sortOrder: EtcdSortOrder?, sortTarget: EtcdSortTarget?) + case put(key: Data, value: Data, leaseId: Int64?) + case del(key: Data, prefix: Bool) + case watch(key: Data, prefix: Bool, timeout: TimeInterval) // Lease case leaseGrant(ttl: Int64) @@ -48,6 +46,84 @@ enum EtcdOperation { case unknown(command: String, args: [String]) } +enum EtcdOperationTier { + case read, write, destructive +} + +extension EtcdOperation { + // One table for the host's classifier and the driver's own check. Nil for a command the driver does not run. + var tier: EtcdOperationTier? { + switch self { + case .get, .watch, .leaseTimetolive, .leaseList, .memberList, .endpointStatus, .endpointHealth, + .userList, .roleList: + return .read + case .put, .leaseGrant, .leaseKeepAlive, .authEnable, .userAdd, .roleAdd, .userGrantRole, .userRevokeRole: + return .write + case .del, .compaction, .leaseRevoke, .authDisable, .userDelete, .roleDelete: + return .destructive + case .unknown: + return nil + } + } + + func refusal(readOnly: Bool, allowsDestructive: Bool) -> EtcdStatementRefusal? { + switch tier { + case .read, nil: + return nil + case .write: + return readOnly ? .writeWhileReadOnly(commandName) : nil + case .destructive: + if readOnly { return .writeWhileReadOnly(commandName) } + return allowsDestructive ? nil : .destructiveNotApproved(commandName) + } + } + + private var commandName: String { + switch self { + case .get: return "get" + case .put: return "put" + case .del: return "del" + case .watch: return "watch" + case .leaseGrant: return "lease grant" + case .leaseRevoke: return "lease revoke" + case .leaseTimetolive: return "lease timetolive" + case .leaseList: return "lease list" + case .leaseKeepAlive: return "lease keep-alive" + case .memberList: return "member list" + case .endpointStatus: return "endpoint status" + case .endpointHealth: return "endpoint health" + case .compaction: return "compaction" + case .authEnable: return "auth enable" + case .authDisable: return "auth disable" + case .userAdd: return "user add" + case .userDelete: return "user delete" + case .userList: return "user list" + case .roleAdd: return "role add" + case .roleDelete: return "role delete" + case .roleList: return "role list" + case .userGrantRole: return "user grant-role" + case .userRevokeRole: return "user revoke-role" + case .unknown(let command, _): return command + } + } +} + +enum EtcdStatementRefusal: Error, Equatable { + case writeWhileReadOnly(String) + case destructiveNotApproved(String) +} + +extension EtcdStatementRefusal: PluginDriverError { + var pluginErrorMessage: String { + switch self { + case .writeWhileReadOnly(let command): + return String(format: String(localized: "Refused %@: this statement ran as a read."), command) + case .destructiveNotApproved(let command): + return String(format: String(localized: "Refused %@: deleting data was not approved for this statement."), command) + } + } +} + enum EtcdSortOrder: String { case ascend = "ASCEND" case descend = "DESCEND" @@ -88,14 +164,11 @@ struct EtcdCommandParser { // MARK: - Public API static func parse(_ input: String) throws -> EtcdOperation { - let trimmed = input.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { throw EtcdParseError.emptySyntax } - - let tokens = tokenize(trimmed) - guard let first = tokens.first else { throw EtcdParseError.emptySyntax } - - let command = first.lowercased() - let remaining = Array(tokens.dropFirst()) + let trimmed = EtcdCommandText.trimmed(input) + guard let command = commandName(trimmed) else { throw EtcdParseError.emptySyntax } + let arguments = trimmed.unicodeScalars.drop(while: { !EtcdCommandText.isBlank($0) }) + let remaining = EtcdCommandText.tokenize(String(arguments)) + let words = remaining.map(\.text) switch command { case "get": return try parseGet(remaining) @@ -103,23 +176,29 @@ struct EtcdCommandParser { case "del", "delete": return try parseDel(remaining) case "watch": return try parseWatch(remaining) case "lease": return try parseLease(remaining) - case "member": return try parseMember(remaining) - case "endpoint": return try parseEndpoint(remaining) + case "member": return try parseMember(words) + case "endpoint": return try parseEndpoint(words) case "compaction": return try parseCompaction(remaining) - case "auth": return try parseAuth(remaining) - case "user": return try parseUser(remaining) - case "role": return try parseRole(remaining) - default: return .unknown(command: command, args: remaining) + case "auth": return try parseAuth(words) + case "user": return try parseUser(words) + case "role": return try parseRole(words) + default: return .unknown(command: command, args: words) } } + static func commandName(_ input: String) -> String? { + // A verb stays literal so a registry driver cannot reinterpret an older host's command. + let word = EtcdCommandText.trimmed(input).unicodeScalars.prefix { !EtcdCommandText.isBlank($0) } + return word.isEmpty ? nil : String(word).lowercased() + } + // MARK: - KV Commands - private static func parseGet(_ tokens: [String]) throws -> EtcdOperation { + private static func parseGet(_ tokens: [EtcdCommandToken]) throws -> EtcdOperation { var flags = ParsedFlags() let positional = flags.parse(from: tokens) - guard let key = positional.first else { + guard let key = positional.first?.bytes else { throw EtcdParseError.missingArgument("get requires a key") } @@ -160,7 +239,7 @@ struct EtcdCommandParser { ) } - private static func parsePut(_ tokens: [String]) throws -> EtcdOperation { + private static func parsePut(_ tokens: [EtcdCommandToken]) throws -> EtcdOperation { var flags = ParsedFlags() let positional = flags.parse(from: tokens) @@ -168,8 +247,8 @@ struct EtcdCommandParser { throw EtcdParseError.missingArgument("put requires key and value") } - let key = positional[0] - let value = positional[1] + let key = positional[0].bytes + let value = positional[1].bytes var leaseId: Int64? if let leaseStr = flags.value(for: "lease") { @@ -179,11 +258,11 @@ struct EtcdCommandParser { return .put(key: key, value: value, leaseId: leaseId) } - private static func parseDel(_ tokens: [String]) throws -> EtcdOperation { + private static func parseDel(_ tokens: [EtcdCommandToken]) throws -> EtcdOperation { var flags = ParsedFlags() let positional = flags.parse(from: tokens) - guard let key = positional.first else { + guard let key = positional.first?.bytes else { throw EtcdParseError.missingArgument("del requires a key") } @@ -191,11 +270,11 @@ struct EtcdCommandParser { return .del(key: key, prefix: prefix) } - private static func parseWatch(_ tokens: [String]) throws -> EtcdOperation { + private static func parseWatch(_ tokens: [EtcdCommandToken]) throws -> EtcdOperation { var flags = ParsedFlags() let positional = flags.parse(from: tokens) - guard let key = positional.first else { + guard let key = positional.first?.bytes else { throw EtcdParseError.missingArgument("watch requires a key") } @@ -219,12 +298,12 @@ struct EtcdCommandParser { // MARK: - Lease Commands - private static func parseLease(_ tokens: [String]) throws -> EtcdOperation { - guard let subcommand = tokens.first else { + private static func parseLease(_ tokens: [EtcdCommandToken]) throws -> EtcdOperation { + guard let subcommand = tokens.first?.text else { throw EtcdParseError.missingArgument("lease requires a subcommand (grant, revoke, timetolive, list, keep-alive)") } - let args = Array(tokens.dropFirst()) + let args = tokens.dropFirst().map(\.text) switch subcommand.lowercased() { case "grant": @@ -246,7 +325,7 @@ struct EtcdCommandParser { } let leaseId = try parseLeaseId(idStr) var flags = ParsedFlags() - _ = flags.parse(from: Array(args.dropFirst())) + _ = flags.parse(from: Array(tokens.dropFirst(2))) let keys = flags.has("keys") return .leaseTimetolive(leaseId: leaseId, keys: keys) @@ -297,11 +376,11 @@ struct EtcdCommandParser { // MARK: - Maintenance Commands - private static func parseCompaction(_ tokens: [String]) throws -> EtcdOperation { + private static func parseCompaction(_ tokens: [EtcdCommandToken]) throws -> EtcdOperation { var flags = ParsedFlags() let positional = flags.parse(from: tokens) - guard let revisionStr = positional.first, let revision = Int64(revisionStr) else { + guard let revisionStr = positional.first?.text, let revision = Int64(revisionStr) else { throw EtcdParseError.missingArgument("compaction requires a revision (integer)") } @@ -424,91 +503,6 @@ struct EtcdCommandParser { return value } - // MARK: - Tokenizer - - static func tokenize(_ input: String) -> [String] { - var tokens: [String] = [] - var current = String.UnicodeScalarView() - var inQuote = false - var quoteScalar: Unicode.Scalar = "\"" - var escapeNext = false - var tokenStarted = false - - for scalar in input.unicodeScalars { - if escapeNext { - tokenStarted = true - if inQuote { - switch scalar { - case "n": current.append("\n") - case "r": current.append("\r") - case "t": current.append("\t") - case "\\": current.append("\\") - case "\"": current.append("\"") - case "'": current.append("'") - default: - current.append("\\") - current.append(scalar) - } - } else { - current.append("\\") - current.append(scalar) - } - escapeNext = false - continue - } - - if scalar == "\\" { - if inQuote { - escapeNext = true - } else { - current.append(scalar) - tokenStarted = true - } - continue - } - - if inQuote { - if scalar == quoteScalar { - inQuote = false - tokenStarted = true - } else { - current.append(scalar) - tokenStarted = true - } - continue - } - - if scalar == "\"" || scalar == "'" { - inQuote = true - quoteScalar = scalar - tokenStarted = true - continue - } - - if scalar.properties.isWhitespace { - if tokenStarted { - tokens.append(String(current)) - current = String.UnicodeScalarView() - tokenStarted = false - } - continue - } - - current.append(scalar) - tokenStarted = true - } - - if escapeNext { - current.append("\\") - tokenStarted = true - } - - if tokenStarted { - tokens.append(String(current)) - } - - return tokens - } } // MARK: - Flag Parsing @@ -524,27 +518,32 @@ private struct ParsedFlags { private var booleanFlags: Set = [] private var valueFlags: [String: String] = [:] - mutating func parse(from tokens: [String]) -> [String] { - var positional: [String] = [] + // etcdctl reads every token after a bare "--" as an argument, so a key may start with "--". + mutating func parse(from tokens: [EtcdCommandToken]) -> [EtcdCommandToken] { + var positional: [EtcdCommandToken] = [] var index = 0 while index < tokens.count { - let token = tokens[index] - if token.hasPrefix("--") { - let flagContent = String(token.dropFirst(2)) + let token = tokens[index].text + if token == EtcdCommandText.endOfFlags { + positional += tokens[(index + 1)...] + break + } + if token.hasPrefix(EtcdCommandText.endOfFlags) { + let flagContent = String(token.dropFirst(EtcdCommandText.endOfFlags.count)) if let equalsIndex = flagContent.firstIndex(of: "=") { let key = String(flagContent[flagContent.startIndex.. [EtcdWatchEvent] { - try await watch(key: key, prefix: prefix, timeout: timeout, isRetry: false) + func watch(range: EtcdKeyRange, timeout: TimeInterval) async throws -> [EtcdWatchEvent] { + try await watch(range: range, timeout: timeout, isRetry: false) } private func watch( - key: String, - prefix: Bool, + range: EtcdKeyRange, timeout: TimeInterval, isRetry: Bool ) async throws -> [EtcdWatchEvent] { @@ -629,10 +630,7 @@ internal final class EtcdHttpClient: @unchecked Sendable { return authToken } - var createRequest = EtcdWatchCreateRequest(key: Self.base64Encode(key)) - if prefix { - createRequest.rangeEnd = Self.base64Encode(Self.prefixRangeEnd(for: key)) - } + let createRequest = EtcdWatchCreateRequest(key: range.key, rangeEnd: range.rangeEnd) let window = Self.watchWindow(timeout) let watchRequest = try buildRequest( path: apiPath("watch"), @@ -652,7 +650,7 @@ internal final class EtcdHttpClient: @unchecked Sendable { ) guard recovery == .reauthenticateAndRetry else { throw EtcdError.fault(fault) } try await refreshToken(replacing: token) - return try await watch(key: key, prefix: prefix, timeout: timeout, isRetry: true) + return try await watch(range: range, timeout: timeout, isRetry: true) } return Self.parseWatchEvents(from: outcome.data) @@ -1011,39 +1009,13 @@ internal final class EtcdHttpClient: @unchecked Sendable { return events } - // MARK: - Base64 Helpers - - static func base64Encode(_ string: String) -> String { - Data(string.utf8).base64EncodedString() - } - - static func base64Decode(_ string: String) -> String { - guard let data = Data(base64Encoded: string) else { return string } - return String(data: data, encoding: .utf8) ?? "" - } - - static func prefixRangeEnd(for prefix: String) -> String { - // Increment last byte for prefix range queries - var bytes = Array(prefix.utf8) - guard !bytes.isEmpty else { return "\0" } - var i = bytes.count - 1 - while i >= 0 { - if bytes[i] < 0xFF { - bytes[i] += 1 - return String(bytes: Array(bytes[0 ... i]), encoding: .utf8) ?? "\0" - } - i -= 1 - } - return "\0" - } - // MARK: - Empty Body Helper private struct EmptyBody: Encodable {} // MARK: - Request Handle - private static let healthProbeKey = "health" + private static let healthProbeKey = Data("health".utf8) private static let watchTransportGrace = TimeInterval(HttpQueryTimeout.defaultGraceSeconds) private static let maximumWatchWindow = TimeInterval(HttpQueryTimeout.resourceCeilingSeconds) - watchTransportGrace diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdKeyspace.swift b/apple/Plugins/EtcdDriverPlugin/EtcdKeyspace.swift index cd2c88208a..fdb2d44a09 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdKeyspace.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdKeyspace.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProEtcdCore internal struct EtcdKeyspace: Equatable, Sendable { struct Table: Equatable, Sendable { @@ -14,6 +15,7 @@ internal struct EtcdKeyspace: Equatable, Sendable { static let rootTableName = "(root)" private static let separator: Unicode.Scalar = "/" + private static let separatorByte = UInt8(ascii: "/") let root: String @@ -23,21 +25,22 @@ internal struct EtcdKeyspace: Equatable, Sendable { } var keysOnlyListing: String { - "get \(EtcdCommandArgument.quoted(root)) --prefix --keys-only" + EtcdCommandText.command("get", arguments: [Data(root.utf8)], flags: ["--prefix", "--keys-only"]) } - func tableName(forKey key: String) -> String { + // A segment that is not UTF-8 has no table name that reads back as its bytes, so its key stays + // under the root table, which lists every key below the root. + func tableName(forKey key: Data) -> String { let relative = relativeKey(key) - let segmentStart = relative.first == Self.separator - ? relative.index(after: relative.startIndex) - : relative.startIndex - guard let segmentEnd = relative[segmentStart...].firstIndex(of: Self.separator) else { + let segmentStart = relative.first == Self.separatorByte ? relative.index(after: relative.startIndex) : relative.startIndex + guard let segmentEnd = relative[segmentStart...].firstIndex(of: Self.separatorByte), + let name = EtcdCommandText.text(Data(relative[...segmentEnd])) else { return Self.rootTableName } - return String(relative[...segmentEnd]) + return name } - func tables(forKeys keys: [String]) -> [Table] { + func tables(forKeys keys: [Data]) -> [Table] { var keyCounts: [String: Int] = [:] for key in keys { keyCounts[tableName(forKey: key), default: 0] += 1 @@ -55,23 +58,23 @@ internal struct EtcdKeyspace: Equatable, Sendable { } func exportQuery(forTable table: String) -> String { - "get \(EtcdCommandArgument.quoted(prefix(forTable: table))) --prefix" + EtcdCommandText.command("get", arguments: [Data(prefix(forTable: table).utf8)], flags: ["--prefix"]) } func dropStatement(forTable table: String) -> String? { let prefix = prefix(forTable: table) let coversWholeRoot = prefix == root guard !coversWholeRoot else { return nil } - return "del \(EtcdCommandArgument.quoted(prefix)) --prefix" + return EtcdCommandText.command("del", arguments: [Data(prefix.utf8)], flags: ["--prefix"]) } func truncateStatements(forTable table: String) -> [String]? { dropStatement(forTable: table).map { [$0] } } - private func relativeKey(_ key: String) -> Substring.UnicodeScalarView { - let scalars = key[...].unicodeScalars - guard scalars.starts(with: root.unicodeScalars) else { return scalars } - return scalars.dropFirst(root.unicodeScalars.count) + private func relativeKey(_ key: Data) -> Data { + let rootBytes = Data(root.utf8) + guard key.starts(with: rootBytes) else { return key } + return key.dropFirst(rootBytes.count) } } diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift index 0d7bb25791..dc1d869384 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift @@ -2,13 +2,11 @@ // EtcdPluginDriver.swift // EtcdDriverPlugin // -// PluginDatabaseDriver implementation for etcd v3. -// Routes both NoSQL browsing hooks and editor commands through EtcdHttpClient. -// import Foundation import os import OSLog +import TableProEtcdCore import TableProPluginKit private extension Array where Element == String? { @@ -44,10 +42,10 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var supportsTransactions: Bool { false } var capabilities: PluginCapabilities { - [.cancelQuery] + [.cancelQuery, .statementPolicy] } - // etcd has no transaction support — these are no-ops + // etcd has no transactions. func beginTransaction() async throws {} func commitTransaction() async throws {} func rollbackTransaction() async throws {} @@ -102,13 +100,19 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Query Execution func execute(query: String) async throws -> PluginQueryResult { + try await execute(query: query, context: PluginStatementContext()) + } + + // The driver checks the parsed command against what the host approved, since a host can grade the text + // differently from how this driver reads it. + func execute(query: String, context: PluginStatementContext) async throws -> PluginQueryResult { let startTime = Date() guard let client = httpClient else { throw EtcdError.notConnected } - let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + let trimmed = EtcdCommandText.trimmed(query) // Health monitor sends "SELECT 1" as a ping if trimmed.lowercased() == "select 1" { @@ -127,6 +131,9 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } let operation = try EtcdCommandParser.parse(trimmed) + if let refusal = operation.refusal(readOnly: context.readOnly, allowsDestructive: context.allowsDestructive) { + throw refusal + } return try await dispatch(operation, client: client, startTime: startTime) } @@ -134,13 +141,45 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { try await execute(query: query) } + func executeParameterized( + query: String, + parameters: [PluginCellValue], + context: PluginStatementContext + ) async throws -> PluginQueryResult { + try await execute(query: query, context: context) + } + + // The default runs the context-free `execute`, which would drop what the host approved. + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + let result = try await execute(query: query, context: context) + guard let rowCap, rowCap > 0, result.rows.count > rowCap else { return result } + return PluginQueryResult( + columns: result.columns, + columnTypeNames: result.columnTypeNames, + rows: Array(result.rows.prefix(rowCap)), + rowsAffected: result.rowsAffected, + executionTime: result.executionTime, + isTruncated: true, + statusMessage: result.statusMessage + ) + } + // MARK: - Streaming func streamRows(query: String) -> AsyncThrowingStream { + streamRows(query: query, context: PluginStatementContext()) + } + + func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in let streamTask = Task { do { - try await self.performStreamRows(query: query, continuation: continuation) + try await self.performStreamRows(query: query, context: context, continuation: continuation) } catch { continuation.finish(throwing: error) } @@ -154,13 +193,14 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private func performStreamRows( query: String, + context: PluginStatementContext, continuation: AsyncThrowingStream.Continuation ) async throws { guard let client = httpClient else { throw EtcdError.notConnected } - let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + let trimmed = EtcdCommandText.trimmed(query) if let parsed = EtcdQueryBuilder.parseRangeQuery(trimmed) { try await streamRangeRows( @@ -173,7 +213,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return } - let result = try await execute(query: query) + let result = try await execute(query: query, context: context) continuation.yield(.header(PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames, @@ -199,8 +239,8 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { estimatedRowCount: nil ))) - let (b64Key, b64RangeEnd) = Self.allKeysRange(for: prefix) - var req = EtcdRangeRequest(key: b64Key, rangeEnd: b64RangeEnd, limit: Int64(Self.maxKeys)) + let range = Self.prefixRange(prefix) + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: Int64(Self.maxKeys)) req.sortOrder = sortAscending ? "ASCEND" : "DESCEND" req.sortTarget = "KEY" @@ -238,16 +278,16 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { throw EtcdError.notConnected } - let (b64Key, b64RangeEnd) = Self.allKeysRange(for: keyspace.root) + let range = Self.prefixRange(keyspace.root) let response = try await client.rangeRequest(EtcdRangeRequest( - key: b64Key, - rangeEnd: b64RangeEnd, + key: range.key, + rangeEnd: range.rangeEnd, limit: Int64(Self.maxKeys), keysOnly: true )) - let keys = (response.kvs ?? []).map { EtcdHttpClient.base64Decode($0.key) } + let keys = (response.kvs ?? []).map(\.key) return keyspace.tables(forKeys: keys).map { PluginTableInfo(name: $0.name, type: "PREFIX", rowCount: $0.keyCount) } @@ -297,11 +337,12 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let prefix = keyspace.prefix(forTable: table) let count = try await countKeys(prefix: prefix, client: client) + let browse = EtcdCommandText.command("get", arguments: [Data(prefix.utf8)], flags: ["--prefix"]) return """ // etcd key prefix: \(prefix.isEmpty ? "(all keys)" : prefix) // Keys: \(count) - // Use 'get \(prefix.isEmpty ? "\"\"" : prefix) --prefix' to browse keys + // Use '\(browse)' to browse keys """ } @@ -514,16 +555,13 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - KV Dispatch private func dispatchGet( - key: String, prefix: Bool, limit: Int64?, keysOnly: Bool, + key: Data, prefix: Bool, limit: Int64?, keysOnly: Bool, sortOrder: EtcdSortOrder?, sortTarget: EtcdSortTarget?, client: EtcdHttpClient, startTime: Date ) async throws -> PluginQueryResult { - let b64Key = EtcdHttpClient.base64Encode(key) - var req = EtcdRangeRequest(key: b64Key) + let range = EtcdKeyRange(key: key, prefix: prefix) + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd) - if prefix { - req.rangeEnd = EtcdHttpClient.base64Encode(EtcdHttpClient.prefixRangeEnd(for: key)) - } if let limit = limit { req.limit = limit } @@ -540,13 +578,10 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let response = try await client.rangeRequest(req) if keysOnly { - let rowsRaw: [[String?]] = (response.kvs ?? []).map { kv in - [EtcdHttpClient.base64Decode(kv.key)] - } return PluginQueryResult( columns: ["Key"], columnTypeNames: ["String"], - rows: rowsRaw.map { $0.asCells }, + rows: (response.kvs ?? []).map { [PluginCellValue.etcd($0.key)] }, rowsAffected: 0, executionTime: Date().timeIntervalSince(startTime) ) @@ -556,14 +591,10 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } private func dispatchPut( - key: String, value: String, leaseId: Int64?, + key: Data, value: Data, leaseId: Int64?, client: EtcdHttpClient, startTime: Date ) async throws -> PluginQueryResult { - var req = EtcdPutRequest( - key: EtcdHttpClient.base64Encode(key), - value: EtcdHttpClient.base64Encode(value), - prevKv: true - ) + var req = EtcdPutRequest(key: key, value: value, prevKv: true) if let leaseId = leaseId { req.lease = String(leaseId) } @@ -574,23 +605,18 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return PluginQueryResult( columns: ["Key", "Value", "Revision"], columnTypeNames: ["String", "String", "Int64"], - rows: [[key, value, revision].asCells], + rows: [[.etcd(key), .etcd(value), .text(revision)]], rowsAffected: 1, executionTime: Date().timeIntervalSince(startTime) ) } private func dispatchDel( - key: String, prefix: Bool, + key: Data, prefix: Bool, client: EtcdHttpClient, startTime: Date ) async throws -> PluginQueryResult { - var req = EtcdDeleteRequest( - key: EtcdHttpClient.base64Encode(key), - prevKv: true - ) - if prefix { - req.rangeEnd = EtcdHttpClient.base64Encode(EtcdHttpClient.prefixRangeEnd(for: key)) - } + let range = EtcdKeyRange(key: key, prefix: prefix) + let req = EtcdDeleteRequest(key: range.key, rangeEnd: range.rangeEnd, prevKv: true) let response = try await client.deleteRequest(req) let deleted = response.deleted ?? "0" @@ -607,24 +633,25 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Watch Dispatch private func dispatchWatch( - key: String, prefix: Bool, timeout: TimeInterval, + key: Data, prefix: Bool, timeout: TimeInterval, client: EtcdHttpClient, startTime: Date ) async throws -> PluginQueryResult { - let events = try await client.watch(key: key, prefix: prefix, timeout: timeout) + let events = try await client.watch(range: EtcdKeyRange(key: key, prefix: prefix), timeout: timeout) - let rowsRaw: [[String?]] = events.map { event in - let eventType = event.type ?? "UNKNOWN" - let eventKey = event.kv.map { EtcdHttpClient.base64Decode($0.key) } ?? "" - let eventValue = event.kv?.value.map { EtcdHttpClient.base64Decode($0) } ?? "" - let modRevision = event.kv?.modRevision ?? "" - let prevValue = event.prevKv?.value.map { EtcdHttpClient.base64Decode($0) } ?? "" - return [eventType, eventKey, eventValue, modRevision, prevValue] + let rows: [PluginRow] = events.map { event in + [ + .text(event.type ?? "UNKNOWN"), + .etcd(event.kv?.key ?? Data()), + .etcd(event.kv?.value ?? Data()), + .text(event.kv?.modRevision ?? ""), + .etcd(event.prevKv?.value ?? Data()), + ] } return PluginQueryResult( columns: ["Type", "Key", "Value", "ModRevision", "PrevValue"], columnTypeNames: ["String", "String", "String", "Int64", "String"], - rows: rowsRaw.map { $0.asCells }, + rows: rows, rowsAffected: 0, executionTime: Date().timeIntervalSince(startTime) ) @@ -668,7 +695,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let ttl = response.TTL ?? "unknown" let grantedTtl = response.grantedTTL ?? "unknown" let attachedKeys = (response.keys ?? []) - .map { EtcdHttpClient.base64Decode($0) } + .map { EtcdCommandText.text($0) ?? EtcdCommandText.quoted($0) } .joined(separator: ", ") return PluginQueryResult( @@ -813,10 +840,10 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { startTime: Date ) async throws -> PluginQueryResult { let matcher = try filter.matcher() - let (b64Key, b64RangeEnd) = Self.allKeysRange(for: prefix) + let range = Self.prefixRange(prefix) let fetchLimit = filter.isUnfiltered ? Int64(min(offset + limit, Self.maxKeys)) : Int64(Self.maxKeys) - var req = EtcdRangeRequest(key: b64Key, rangeEnd: b64RangeEnd, limit: fetchLimit) + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: fetchLimit) req.sortOrder = sortAscending ? "ASCEND" : "DESCEND" req.sortTarget = "KEY" @@ -831,8 +858,8 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } private func countKeys(prefix: String, client: EtcdHttpClient) async throws -> Int { - let (b64Key, b64RangeEnd) = Self.allKeysRange(for: prefix) - var req = EtcdRangeRequest(key: b64Key, rangeEnd: b64RangeEnd, limit: Int64(Self.maxKeys)) + let range = Self.prefixRange(prefix) + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: Int64(Self.maxKeys)) req.countOnly = true let response = try await client.rangeRequest(req) return Int(response.count ?? "0") ?? 0 @@ -840,24 +867,14 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Helpers - /// Returns (base64Key, base64RangeEnd) for a prefix range query. - /// Empty prefix uses null byte (\0) as key to mean "all keys". - private static func allKeysRange(for prefix: String) -> (key: String, rangeEnd: String) { - if prefix.isEmpty { - // \0 as key = start from beginning, \0 as range_end = all keys - let b64Key = EtcdHttpClient.base64Encode("\0") - let b64RangeEnd = EtcdHttpClient.base64Encode("\0") - return (b64Key, b64RangeEnd) - } - let b64Key = EtcdHttpClient.base64Encode(prefix) - let b64RangeEnd = EtcdHttpClient.base64Encode(EtcdHttpClient.prefixRangeEnd(for: prefix)) - return (b64Key, b64RangeEnd) + private static func prefixRange(_ prefix: String) -> EtcdKeyRange { + EtcdKeyRange(key: Data(prefix.utf8), prefix: true) } private static func filterRow(_ kv: EtcdKeyValue) -> EtcdFilterRow { EtcdFilterRow( - key: EtcdHttpClient.base64Decode(kv.key), - value: kv.value.map { EtcdHttpClient.base64Decode($0) }, + key: kv.key, + value: kv.value, version: kv.version ?? "0", modRevision: kv.modRevision ?? "0", createRevision: kv.createRevision ?? "0", diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdQueryBuilder.swift b/apple/Plugins/EtcdDriverPlugin/EtcdQueryBuilder.swift index 8eb5dfc66d..29eb6bc5df 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdQueryBuilder.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdQueryBuilder.swift @@ -2,10 +2,9 @@ // EtcdQueryBuilder.swift // EtcdDriverPlugin // -// Builds internal query strings for etcd key browsing and filtering. -// import Foundation +import TableProEtcdCore import TableProPluginKit struct EtcdParsedQuery { @@ -82,7 +81,7 @@ struct EtcdQueryBuilder { private static func decodeSegment(_ segment: String) -> String? { guard let data = Data(base64Encoded: segment) else { return nil } - return String(data: data, encoding: .utf8) + return EtcdCommandText.text(data) } private static func decodeFilter(_ segment: String) -> EtcdRowFilter? { @@ -97,6 +96,8 @@ struct EtcdQueryBuilder { let prefix = decodeSegment(parts[0]), let limit = Int(parts[1]), let offset = Int(parts[2]), + limit >= 0, offset >= 0, + !offset.addingReportingOverflow(limit).overflow, let filter = decodeFilter(parts[4]) else { return nil } return EtcdParsedQuery( prefix: prefix, limit: limit, offset: offset, sortAscending: parts[3] == "1", filter: filter diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdRowFilter.swift b/apple/Plugins/EtcdDriverPlugin/EtcdRowFilter.swift index 3af752f744..8cd67ca638 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdRowFilter.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdRowFilter.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProEtcdCore import TableProPluginKit internal enum EtcdColumn: String, Codable, CaseIterable, Sendable { @@ -78,22 +79,44 @@ internal struct EtcdFilterRefusal: Error, Equatable, PluginDriverError { } } +extension PluginCellValue { + // Text only when the bytes are exactly UTF-8, so a grid write sends back the bytes the server holds. + static func etcd(_ bytes: Data) -> PluginCellValue { + EtcdCommandText.text(bytes).map(PluginCellValue.text) ?? .bytes(bytes) + } + + var etcdBytes: Data? { + switch self { + case .null: return nil + case .text(let text): return Data(text.utf8) + case .bytes(let bytes): return bytes + } + } +} + internal struct EtcdFilterRow: Equatable, Sendable { - let key: String - let value: String? + let key: Data + let value: Data? let version: String let modRevision: String let createRevision: String let lease: String var cells: [PluginCellValue] { - EtcdColumn.allCases.map { PluginCellValue.fromOptional(text(of: $0)) } + EtcdColumn.allCases.map { column in + switch column { + case .key: return .etcd(key) + case .value: return value.map(PluginCellValue.etcd) ?? .null + default: return .fromOptional(text(of: column)) + } + } } + // Bytes that are not UTF-8 match as U+FFFD. func text(of column: EtcdColumn) -> String? { switch column { - case .key: return key - case .value: return value + case .key: return Self.readable(key) + case .value: return value.map(Self.readable) case .version: return version case .modRevision: return modRevision case .createRevision: return createRevision @@ -110,6 +133,18 @@ internal struct EtcdFilterRow: Equatable, Sendable { case .lease: return Int64(lease) ?? 0 } } + + func bytes(of column: EtcdColumn) -> Data? { + switch column { + case .key: return key + case .value: return value + default: return text(of: column).map { Data($0.utf8) } + } + } + + private static func readable(_ bytes: Data) -> String { + String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + } } internal struct EtcdRowFilter: Codable, Equatable, Sendable { @@ -304,9 +339,12 @@ private struct EtcdConditionTest { guard let cellNumber = row.number(of: condition.column) else { return nil } return Self.compare(cellNumber, number) case .text(let text): - guard let cell = row.text(of: condition.column) else { return nil } - guard isOrdering else { return cell.compare(text, options: textOptions) } - return Self.byteOrder(cell, text) + guard isOrdering else { + guard let cell = row.text(of: condition.column) else { return nil } + return cell.compare(text, options: textOptions) + } + guard let cell = row.bytes(of: condition.column) else { return nil } + return Self.byteOrder(cell, Data(text.utf8)) } } @@ -315,9 +353,9 @@ private struct EtcdConditionTest { return lhs < rhs ? .orderedAscending : .orderedDescending } - private static func byteOrder(_ lhs: String, _ rhs: String) -> ComparisonResult { - if lhs.utf8.elementsEqual(rhs.utf8) { return .orderedSame } - return lhs.utf8.lexicographicallyPrecedes(rhs.utf8) ? .orderedAscending : .orderedDescending + private static func byteOrder(_ lhs: Data, _ rhs: Data) -> ComparisonResult { + if lhs == rhs { return .orderedSame } + return lhs.lexicographicallyPrecedes(rhs) ? .orderedAscending : .orderedDescending } private static func operand(_ text: String, for column: EtcdColumn) throws(EtcdFilterRefusal) -> Operand { diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdStatementGenerator.swift b/apple/Plugins/EtcdDriverPlugin/EtcdStatementGenerator.swift index 952511c50f..d014660c31 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdStatementGenerator.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdStatementGenerator.swift @@ -2,10 +2,9 @@ // EtcdStatementGenerator.swift // EtcdDriverPlugin // -// Generates etcdctl commands from tracked cell changes. -// import Foundation +import TableProEtcdCore import TableProPluginKit struct EtcdStatementGenerator { @@ -16,13 +15,11 @@ struct EtcdStatementGenerator { private var valueColumnIndex: Int? { columns.firstIndex(of: "Value") } private var leaseColumnIndex: Int? { columns.firstIndex(of: "Lease") } - /// The columns a `put` can set. The rest, the version and the two revisions, are etcd's own. - /// etcd stores no NULL, so a NULL written to Value means an empty value, and to Lease, no lease. + // etcd stores no NULL, so a NULL written to Value means an empty value, and to Lease, no lease. private static let writableColumns: Set = ["Key", "Value", "Lease"] - /// The commands that write the grid's changes, each naming the change it writes. A change - /// carrying a value these commands cannot express is refused whole, because writing the rest - /// of it would let the save succeed and clear the value it left out. + // A change carrying a value these commands cannot express is refused whole: writing the rest of it + // would let the save succeed and clear the value it left out. func generateRowWrites( from changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], @@ -44,7 +41,7 @@ struct EtcdStatementGenerator { guard let key = extractKey(from: change) else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.unaddressableKeyReason) } - commands = ["del \(EtcdCommandArgument.quoted(key))"] + commands = [EtcdCommandText.command("del", arguments: [key])] } writes += commands.map { PluginRowWrite(statement: $0, rowIndices: [change.rowIndex]) } } @@ -60,19 +57,19 @@ struct EtcdStatementGenerator { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.serverOwnedReason(column)) } - var key: String? - var value: String? + var key: Data? + var value: Data? var leaseId: String? if let values = insertedRowData[change.rowIndex] { - if let ki = keyColumnIndex, ki < values.count { key = values[ki].asText } - if let vi = valueColumnIndex, vi < values.count { value = values[vi].asText } + if let ki = keyColumnIndex, ki < values.count { key = values[ki].etcdBytes } + if let vi = valueColumnIndex, vi < values.count { value = values[vi].etcdBytes } if let li = leaseColumnIndex, li < values.count { leaseId = values[li].asText } } else { for cellChange in change.cellChanges { switch cellChange.columnName { - case "Key": key = cellChange.newValue.asText - case "Value": value = cellChange.newValue.asText + case "Key": key = cellChange.newValue.etcdBytes + case "Value": value = cellChange.newValue.etcdBytes case "Lease": leaseId = cellChange.newValue.asText default: break } @@ -83,14 +80,9 @@ struct EtcdStatementGenerator { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String(localized: "A new key needs a name.")) } - // Prepend the current browse prefix if the key doesn't already include it - let fullKey: String - if !prefix.isEmpty && !k.hasPrefix("/") { - fullKey = prefix + k - } else { - fullKey = k - } - return [Self.putCommand(key: fullKey, value: value ?? "", lease: leaseId)] + // A key typed without a leading slash lands under the table's prefix. + let fullKey = !prefix.isEmpty && k.first != UInt8(ascii: "/") ? Data(prefix.utf8) + k : k + return [Self.putCommand(key: fullKey, value: value ?? Data(), lease: leaseId)] } private func updateCommands(for change: PluginRowChange) throws -> [String] { @@ -108,37 +100,36 @@ struct EtcdStatementGenerator { let valueChange = change.cellChanges.first { $0.columnName == "Value" } let leaseChange = change.cellChanges.first { $0.columnName == "Lease" } - let newKey = keyChange.map { $0.newValue.asText ?? "" } ?? originalKey + let newKey = keyChange.map { $0.newValue.etcdBytes ?? Data() } ?? originalKey guard !newKey.isEmpty else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String(localized: "A key needs a name.")) } let shouldDeleteOriginalKey = newKey != originalKey let editedLease = leaseChange.map { $0.newValue.asText ?? "" } - let lease = editedLease ?? originalText(at: leaseColumnIndex, of: change) + let lease = editedLease ?? originalCell(at: leaseColumnIndex, of: change)?.asText - if valueChange != nil || newKey != originalKey { - let newValue = valueChange.map { $0.newValue.asText ?? "" } ?? originalText(at: valueColumnIndex, of: change) ?? "" + if valueChange != nil || shouldDeleteOriginalKey { + let newValue = valueChange.map { $0.newValue.etcdBytes ?? Data() } ?? originalValue(of: change) commands.append(Self.putCommand(key: newKey, value: newValue, lease: lease)) if shouldDeleteOriginalKey { - commands.append("del \(EtcdCommandArgument.quoted(originalKey))") + commands.append(EtcdCommandText.command("del", arguments: [originalKey])) } } else if editedLease != nil { - let currentValue = originalText(at: valueColumnIndex, of: change) ?? "" - commands.append(Self.putCommand(key: newKey, value: currentValue, lease: lease)) + commands.append(Self.putCommand(key: newKey, value: originalValue(of: change), lease: lease)) } return commands } - private static func putCommand(key: String, value: String, lease: String?) -> String { - let put = "put \(EtcdCommandArgument.quoted(key)) \(EtcdCommandArgument.quoted(value))" - guard let lease, !lease.isEmpty, lease != "0" else { return put } - return put + " --lease=\(lease)" + private static func putCommand(key: Data, value: Data, lease: String?) -> String { + guard let lease, !lease.isEmpty, lease != "0" else { + return EtcdCommandText.command("put", arguments: [key, value]) + } + return EtcdCommandText.command("put", arguments: [key, value], flags: ["--lease=" + EtcdCommandText.quoted(lease)]) } - /// An edit to a column no `put` can set, which the save would otherwise drop. A new row's NULL - /// there leaves the value to etcd, so only a value set in one counts. + // A new row's NULL in a server-owned column leaves the value to etcd, so only a value set in one counts. private func serverOwnedColumn(in change: PluginRowChange) -> String? { change.cellChanges .first { cell in @@ -150,7 +141,7 @@ struct EtcdStatementGenerator { // MARK: - Refusals private static var unaddressableKeyReason: String { - String(localized: "This key's name is not text, so it cannot be addressed from the grid.") + String(localized: "This row has no key, so it cannot be addressed from the grid.") } private static func serverOwnedReason(_ column: String) -> String { @@ -159,14 +150,18 @@ struct EtcdStatementGenerator { // MARK: - Helpers - private func extractKey(from change: PluginRowChange) -> String? { - originalText(at: keyColumnIndex, of: change) + private func extractKey(from change: PluginRowChange) -> Data? { + originalCell(at: keyColumnIndex, of: change)?.etcdBytes + } + + private func originalValue(of change: PluginRowChange) -> Data { + originalCell(at: valueColumnIndex, of: change)?.etcdBytes ?? Data() } - private func originalText(at columnIndex: Int?, of change: PluginRowChange) -> String? { + private func originalCell(at columnIndex: Int?, of change: PluginRowChange) -> PluginCellValue? { guard let columnIndex, let originalRow = change.originalRow, columnIndex < originalRow.count else { return nil } - return originalRow[columnIndex].asText + return originalRow[columnIndex] } } diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Etcd.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Etcd.swift index e665828a15..88881fa10a 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Etcd.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Etcd.swift @@ -9,7 +9,7 @@ import TableProSQLGrammar extension QueryClassifier { static func etcdClassification(_ trimmed: String) -> QueryClassification { guard let operation = try? EtcdCommandParser.parse(trimmed) else { - return etcdUnparsedClassification(command: EtcdCommandParser.tokenize(trimmed).first?.lowercased() ?? "") + return etcdUnparsedClassification(command: EtcdCommandParser.commandName(trimmed) ?? "") } return etcdClassification(of: operation) } @@ -21,16 +21,16 @@ extension QueryClassifier { private static let etcdCommandsReachingFilesystem: Set = ["snapshot", "defrag"] private static func etcdClassification(of operation: EtcdOperation) -> QueryClassification { - switch operation { - case .get, .watch, .leaseTimetolive, .leaseList, .memberList, .endpointStatus, .endpointHealth, - .userList, .roleList: + if case .unknown(let command, _) = operation { + return etcdUnparsedClassification(command: command) + } + switch operation.tier { + case .read: return .safe - case .put, .leaseGrant, .leaseKeepAlive, .authEnable, .userAdd, .roleAdd, .userGrantRole, .userRevokeRole: + case .write, nil: return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) - case .del, .compaction, .leaseRevoke, .authDisable, .userDelete, .roleDelete: + case .destructive: return QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) - case .unknown(let command, _): - return etcdUnparsedClassification(command: command) } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 9f9cc2ef61..3209fdd836 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -200316,6 +200316,10 @@ } } }, + "This row has no key, so it cannot be addressed from the grid." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This row has no point id, so Qdrant cannot tell which point to change." : { "extractionState" : "manual", "generatesSymbol" : false diff --git a/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift b/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift index c3a6d6456a..4a9d37721f 100644 --- a/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift @@ -5,7 +5,9 @@ import Foundation @testable import TablePro +import TableProEtcdCore import TableProPluginKit +import TableProSQLGrammar import Testing // MARK: - KV Commands @@ -18,7 +20,7 @@ struct EtcdCommandParserGetTests { Issue.record("Expected .get, got \(op)") return } - #expect(key == "mykey") + #expect(key == Data("mykey".utf8)) #expect(prefix == false) #expect(limit == nil) #expect(keysOnly == false) @@ -33,7 +35,7 @@ struct EtcdCommandParserGetTests { Issue.record("Expected .get, got \(op)") return } - #expect(key == "/app") + #expect(key == Data("/app".utf8)) #expect(prefix == true) #expect(keysOnly == true) } @@ -45,7 +47,7 @@ struct EtcdCommandParserGetTests { Issue.record("Expected .get, got \(op)") return } - #expect(key == "/app") + #expect(key == Data("/app".utf8)) #expect(prefix == true) #expect(limit == 5) } @@ -63,7 +65,7 @@ struct EtcdCommandParserGetTests { Issue.record("Expected .get, got \(op)") return } - #expect(key == "/app") + #expect(key == Data("/app".utf8)) #expect(prefix == true) } @@ -74,7 +76,7 @@ struct EtcdCommandParserGetTests { Issue.record("Expected .get") return } - #expect(key == "/app/") + #expect(key == Data("/app/".utf8)) #expect(prefix == true) } @@ -135,7 +137,7 @@ struct EtcdCommandParserGetTests { Issue.record("Expected .get") return } - #expect(key == "/prefix/") + #expect(key == Data("/prefix/".utf8)) #expect(prefix == true) #expect(limit == 100) #expect(keysOnly == true) @@ -180,8 +182,8 @@ struct EtcdCommandParserPutTests { Issue.record("Expected .put, got \(op)") return } - #expect(key == "mykey") - #expect(value == "myvalue") + #expect(key == Data("mykey".utf8)) + #expect(value == Data("myvalue".utf8)) #expect(leaseId == nil) } @@ -212,8 +214,8 @@ struct EtcdCommandParserPutTests { Issue.record("Expected .put") return } - #expect(key == "my key") - #expect(value == "my value") + #expect(key == Data("my key".utf8)) + #expect(value == Data("my value".utf8)) } @Test("Put with single-quoted args") @@ -223,8 +225,8 @@ struct EtcdCommandParserPutTests { Issue.record("Expected .put") return } - #expect(key == "key") - #expect(value == "value") + #expect(key == Data("key".utf8)) + #expect(value == Data("value".utf8)) } @Test("Put with empty quoted key") @@ -234,8 +236,8 @@ struct EtcdCommandParserPutTests { Issue.record("Expected .put") return } - #expect(key == "") - #expect(value == "value") + #expect(key.isEmpty) + #expect(value == Data("value".utf8)) } @Test("Put with escape sequences in quotes") @@ -245,7 +247,7 @@ struct EtcdCommandParserPutTests { Issue.record("Expected .put") return } - #expect(value == "value\nwith\nnewlines") + #expect(value == Data("value\nwith\nnewlines".utf8)) } @Test("Put missing arguments throws") @@ -271,7 +273,7 @@ struct EtcdCommandParserDelTests { Issue.record("Expected .del, got \(op)") return } - #expect(key == "mykey") + #expect(key == Data("mykey".utf8)) #expect(prefix == false) } @@ -282,7 +284,7 @@ struct EtcdCommandParserDelTests { Issue.record("Expected .del") return } - #expect(key == "/app/") + #expect(key == Data("/app/".utf8)) #expect(prefix == true) } @@ -293,7 +295,7 @@ struct EtcdCommandParserDelTests { Issue.record("Expected .del, got \(op)") return } - #expect(key == "/app") + #expect(key == Data("/app".utf8)) #expect(prefix == true) } @@ -312,7 +314,7 @@ struct EtcdCommandParserDelTests { Issue.record("Expected .del, got \(op)") return } - #expect(key == "/app") + #expect(key == Data("/app".utf8)) #expect(prefix == false) } @@ -323,7 +325,7 @@ struct EtcdCommandParserDelTests { Issue.record("Expected .del") return } - #expect(key == "mykey") + #expect(key == Data("mykey".utf8)) } @Test("Del missing key throws") @@ -342,7 +344,7 @@ struct EtcdCommandParserWatchTests { Issue.record("Expected .watch, got \(op)") return } - #expect(key == "mykey") + #expect(key == Data("mykey".utf8)) #expect(prefix == false) #expect(timeout == 30) } @@ -873,7 +875,7 @@ struct EtcdCommandParserTokenizerTests { Issue.record("Expected .get") return } - #expect(key == "mykey") + #expect(key == Data("mykey".utf8)) } @Test("Leading and trailing whitespace is trimmed") @@ -883,7 +885,7 @@ struct EtcdCommandParserTokenizerTests { Issue.record("Expected .get") return } - #expect(key == "mykey") + #expect(key == Data("mykey".utf8)) } @Test("Multiple spaces between all tokens") @@ -893,8 +895,8 @@ struct EtcdCommandParserTokenizerTests { Issue.record("Expected .put") return } - #expect(key == "key") - #expect(value == "value") + #expect(key == Data("key".utf8)) + #expect(value == Data("value".utf8)) } @Test("Backslash outside quotes preserved") @@ -904,7 +906,7 @@ struct EtcdCommandParserTokenizerTests { Issue.record("Expected .put") return } - #expect(key == "C:\\path") + #expect(key == Data("C:\\path".utf8)) } @Test("Tab and return escape sequences inside double quotes") @@ -914,7 +916,7 @@ struct EtcdCommandParserTokenizerTests { Issue.record("Expected .put") return } - #expect(value == "a\tb\rc") + #expect(value == Data("a\tb\rc".utf8)) } @Test("Escaped backslash inside quotes") @@ -924,7 +926,7 @@ struct EtcdCommandParserTokenizerTests { Issue.record("Expected .put") return } - #expect(value == "a\\b") + #expect(value == Data("a\\b".utf8)) } @Test("Escaped double quote inside double quotes") @@ -934,7 +936,7 @@ struct EtcdCommandParserTokenizerTests { Issue.record("Expected .put") return } - #expect(value == "say \"hi\"") + #expect(value == Data("say \"hi\"".utf8)) } @Test("Case insensitivity for commands") @@ -944,7 +946,7 @@ struct EtcdCommandParserTokenizerTests { Issue.record("Expected .get") return } - #expect(key == "mykey") + #expect(key == Data("mykey".utf8)) } @Test("Mixed case commands") @@ -954,7 +956,210 @@ struct EtcdCommandParserTokenizerTests { Issue.record("Expected .get") return } - #expect(key == "mykey") + #expect(key == Data("mykey".utf8)) + } +} + +// MARK: - Generated Commands + +struct EtcdCommandParserRoundTripTests { + private static func deletedKey(_ line: String) throws -> Data? { + guard case .del(let key, let prefix) = try EtcdCommandParser.parse(line), !prefix else { return nil } + return key + } + + @Test( + "A key with a space or format character at an edge survives generate then parse", + arguments: [" ", "\t", "\n", "\u{00A0}", "\u{3000}", "\u{200B}", "\u{200C}", "\u{200D}", "\u{2060}", "\u{FEFF}"] + ) + func edgeCharacterKey(edge: String) throws { + for key in [edge + "foo", "foo" + edge, edge + "foo" + edge] { + let bytes = Data(key.utf8) + #expect(try Self.deletedKey(EtcdCommandText.command("del", arguments: [bytes])) == bytes) + #expect(try Self.deletedKey(EtcdCommandText.command("del", arguments: [bytes]) + "\n") == bytes) + } + } + + @Test("A typed zero width space stays part of the key") + func typedZeroWidthSpace() throws { + #expect(try Self.deletedKey("del foo\u{200B}") == Data("foo\u{200B}".utf8)) + } + + @Test("A key that is not UTF-8 survives generate then parse") + func nonUTF8Key() throws { + let key = Data([0x2F, 0x61, 0x2F, 0xFF, 0xC3]) + #expect(try Self.deletedKey(EtcdCommandText.command("del", arguments: [key])) == key) + + let value = Data([0x00, 0x80, 0xFE]) + let op = try EtcdCommandParser.parse(EtcdCommandText.command("put", arguments: [key, value])) + guard case .put(let parsedKey, let parsedValue, _) = op else { + Issue.record("Expected .put, got \(op)") + return + } + #expect(parsedKey == key) + #expect(parsedValue == value) + } + + @Test("A hex escape in a typed command is one byte of the key") + func typedHexEscape() throws { + let op = try EtcdCommandParser.parse(#"get "/a\xff" --prefix"#) + guard case .get(let key, let prefix, _, _, _, _) = op else { + Issue.record("Expected .get, got \(op)") + return + } + #expect(key == Data([0x2F, 0x61, 0xFF])) + #expect(prefix) + } + + @Test("A key that starts with -- is an argument, not a flag", arguments: ["--prefix", "--", "--lease=7"]) + func dashedKey(key: String) throws { + let bytes = Data(key.utf8) + #expect(try Self.deletedKey(EtcdCommandText.command("del", arguments: [bytes])) == bytes) + + let line = EtcdCommandText.command("put", arguments: [bytes, Data("v".utf8)], flags: ["--lease=0x7b"]) + guard case .put(let parsedKey, let value, let leaseId) = try EtcdCommandParser.parse(line) else { + Issue.record("Expected .put from \(line)") + return + } + #expect(parsedKey == bytes) + #expect(value == Data("v".utf8)) + #expect(leaseId == 0x7B) + } + + @Test("Flags before a bare -- still apply") + func flagsBeforeEndOfFlags() throws { + let op = try EtcdCommandParser.parse("del --prefix -- --cache/") + guard case .del(let key, let prefix) = op else { + Issue.record("Expected .del, got \(op)") + return + } + #expect(key == Data("--cache/".utf8)) + #expect(prefix) + } +} + +// MARK: - Hosts That Grade Commands Themselves + +struct EtcdCommandParserCommandWordTests { + @Test("A hex escape in the command word leaves it unknown, as a released host grades it") + func escapedCommandWord() throws { + let line = #""d\x65l" "" --prefix"# + let op = try EtcdCommandParser.parse(line) + guard case .unknown(let command, _) = op else { + Issue.record("Expected .unknown, got \(op)") + return + } + #expect(command == #""d\x65l""#) + // The released classifier treats this unrecognized verb as a plain write. + let releasedVerb = ReleasedEtcdTokenizer.tokenize(line).first?.lowercased() + #expect(releasedVerb == #"d\x65l"#) + let releasedDestructiveVerbs: Set = [ + "del", "delete", "compaction", "compact", "lease", "member", "auth", "user", "role", + ] + #expect(!releasedDestructiveVerbs.contains(releasedVerb ?? "")) + var releasedContext = PluginStatementContext() + releasedContext.allowsDestructive = false + #expect(op.refusal( + readOnly: releasedContext.readOnly, allowsDestructive: releasedContext.allowsDestructive + ) == nil) + #expect(op.tier == nil) + #expect(QueryClassifier.classifyTier(line, databaseType: .etcd) == .write) + } + + @Test("A command name is bare and literal", arguments: [ + #""del" "" --prefix"#, #"'del' "" --prefix"#, #"d"e"l "" --prefix"#, + #"d\x65l "" --prefix"#, #""d\x65l" "" --prefix"#, + ]) + func literalCommand(line: String) throws { + guard case .unknown = try EtcdCommandParser.parse(line) else { + Issue.record("Expected a literal unknown command") + return + } + } + + @Test( + "A hex escape in a subcommand leaves it unknown", + arguments: [#"lease "r\x65voke" 1"#, #"user "d\x65lete" bob"#, #"role 'd\x65lete' readers"#, #"auth "disabl\x65""#] + ) + func escapedSubcommand(line: String) { + #expect(throws: EtcdParseError.self) { try EtcdCommandParser.parse(line) } + #expect(QueryClassifier.classifyTier(line, databaseType: .etcd) == .destructive) + } + + @Test("A hex escape in a flag leaves it a different flag") + func escapedFlag() throws { + let op = try EtcdCommandParser.parse(#"get /a "--pr\x65fix""#) + guard case .get(let key, let prefix, _, _, _, _) = op else { + Issue.record("Expected .get, got \(op)") + return + } + #expect(key == Data("/a".utf8)) + #expect(!prefix) + } +} + +private let etcdReadLines = [ + "get /a --prefix", "watch /a", "lease timetolive 1", "lease list", "member list", "endpoint status", + "endpoint health", "user list", "role list", +] +private let etcdWriteLines = [ + "put /a v", "lease grant 60", "lease keep-alive 1", "auth enable", "user add bob pw", "role add readers", + "user grant-role bob readers", "user revoke-role bob readers", +] +private let etcdDeleteLines = [ + "del /a", #"del "" --prefix"#, "del /app/ --prefix", "compaction 5", "lease revoke 1", "auth disable", + "user delete bob", "role delete readers", +] + +struct EtcdCommandParserStatementPolicyTests { + @Test("A delete runs only when the approval covered deleting data", arguments: etcdDeleteLines) + func deleteNeedsApproval(line: String) throws { + let op = try EtcdCommandParser.parse(line) + #expect(op.tier == .destructive) + guard case .destructiveNotApproved? = op.refusal(readOnly: false, allowsDestructive: false) else { + Issue.record("\(line) ran without approval to delete") + return + } + guard case .writeWhileReadOnly? = op.refusal(readOnly: true, allowsDestructive: true) else { + Issue.record("\(line) ran as a read") + return + } + #expect(op.refusal(readOnly: false, allowsDestructive: true) == nil) + } + + @Test("A write runs unless the statement ran as a read", arguments: etcdWriteLines) + func writeNeedsWriteAccess(line: String) throws { + let op = try EtcdCommandParser.parse(line) + #expect(op.tier == .write) + guard case .writeWhileReadOnly? = op.refusal(readOnly: true, allowsDestructive: true) else { + Issue.record("\(line) ran as a read") + return + } + #expect(op.refusal(readOnly: false, allowsDestructive: false) == nil) + } + + @Test("A read always runs", arguments: etcdReadLines) + func readAlwaysRuns(line: String) throws { + let op = try EtcdCommandParser.parse(line) + #expect(op.tier == .read) + #expect(op.refusal(readOnly: true, allowsDestructive: false) == nil) + } + + @Test( + "The classifier grades each command at the tier the driver checks", + arguments: etcdReadLines + etcdWriteLines + etcdDeleteLines + ) + func classifierMatchesDriver(line: String) throws { + let expected: QueryTier + switch try EtcdCommandParser.parse(line).tier { + case .read: expected = .safe + case .write: expected = .write + case .destructive: expected = .destructive + case nil: + Issue.record("\(line) did not parse to a command") + return + } + #expect(QueryClassifier.classifyTier(line, databaseType: .etcd) == expected) } } diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index 64c953d466..b514758363 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -681,11 +681,9 @@ struct QueryClassifierNonSqlTests { let destructive = [ "lease revoke 7b", "LEASE REVOKE 0x7b", - "\"lease\" 'revoke' 7b", "user delete alice", "role delete readers", "auth disable", - "\"del\" /keys --prefix", "delete /keys", "compaction 5 --physical" ] @@ -706,8 +704,7 @@ struct QueryClassifierNonSqlTests { "endpoint status", "endpoint health", "user list", - "role list", - "'get' /keys --prefix" + "role list" ] for statement in reads { #expect( @@ -717,6 +714,17 @@ struct QueryClassifierNonSqlTests { } } + @Test("etcd quoted verbs stay unknown", arguments: [ + #""lease" 'revoke' 7b"#, #""del" /keys --prefix"#, #"'get' /keys --prefix"#, + ]) + func etcdQuotedVerbs(statement: String) throws { + #expect(QueryClassifier.classifyTier(statement, databaseType: .etcd) == .write) + guard case .unknown = try EtcdCommandParser.parse(statement) else { + Issue.record("Quoted verbs must not execute") + return + } + } + @Test("etcd grants, keep-alives and account additions are plain writes") func etcdWriteSubcommands() { let writes = [ diff --git a/apple/TableProTests/Core/Utilities/SQL/ReleasedEtcdTokenizer.swift b/apple/TableProTests/Core/Utilities/SQL/ReleasedEtcdTokenizer.swift new file mode 100644 index 0000000000..57ea007b4f --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/ReleasedEtcdTokenizer.swift @@ -0,0 +1,89 @@ +import Foundation + +// The tokenizer the 0.84.0 and 0.85.0-beta.1 hosts compile into their etcd classifier, kept as it shipped. An +// updated registry driver runs on those hosts, so its commands, flags and names must read the same words. +enum ReleasedEtcdTokenizer { + static func tokenize(_ input: String) -> [String] { + var tokens: [String] = [] + var current = String.UnicodeScalarView() + var inQuote = false + var quoteScalar: Unicode.Scalar = "\"" + var escapeNext = false + var tokenStarted = false + + for scalar in input.unicodeScalars { + if escapeNext { + tokenStarted = true + if inQuote { + switch scalar { + case "n": current.append("\n") + case "r": current.append("\r") + case "t": current.append("\t") + case "\\": current.append("\\") + case "\"": current.append("\"") + case "'": current.append("'") + default: + current.append("\\") + current.append(scalar) + } + } else { + current.append("\\") + current.append(scalar) + } + escapeNext = false + continue + } + + if scalar == "\\" { + if inQuote { + escapeNext = true + } else { + current.append(scalar) + tokenStarted = true + } + continue + } + + if inQuote { + if scalar == quoteScalar { + inQuote = false + tokenStarted = true + } else { + current.append(scalar) + tokenStarted = true + } + continue + } + + if scalar == "\"" || scalar == "'" { + inQuote = true + quoteScalar = scalar + tokenStarted = true + continue + } + + if scalar.properties.isWhitespace { + if tokenStarted { + tokens.append(String(current)) + current = String.UnicodeScalarView() + tokenStarted = false + } + continue + } + + current.append(scalar) + tokenStarted = true + } + + if escapeNext { + current.append("\\") + tokenStarted = true + } + + if tokenStarted { + tokens.append(String(current)) + } + + return tokens + } +} diff --git a/apple/changelog.d/4023.changed.md b/apple/changelog.d/4023.changed.md new file mode 100644 index 0000000000..0a60b6f256 --- /dev/null +++ b/apple/changelog.d/4023.changed.md @@ -0,0 +1 @@ +etcd keys and values that are not UTF-8 show as hex, and a quoted `\xHH` in an etcd command writes that byte. diff --git a/apple/changelog.d/4023.fixed.1.md b/apple/changelog.d/4023.fixed.1.md new file mode 100644 index 0000000000..095e01e4c5 --- /dev/null +++ b/apple/changelog.d/4023.fixed.1.md @@ -0,0 +1 @@ +Editing or deleting a non-UTF-8 etcd key changed a key named ``, and a rename saved a binary value as text. diff --git a/apple/changelog.d/4023.fixed.2.md b/apple/changelog.d/4023.fixed.2.md new file mode 100644 index 0000000000..6076fc6b3d --- /dev/null +++ b/apple/changelog.d/4023.fixed.2.md @@ -0,0 +1 @@ +Saving an etcd key that ends in a zero width space or starts with a byte order mark changed a different key. diff --git a/apple/changelog.d/4023.fixed.3.md b/apple/changelog.d/4023.fixed.3.md new file mode 100644 index 0000000000..cdc0dd8955 --- /dev/null +++ b/apple/changelog.d/4023.fixed.3.md @@ -0,0 +1 @@ +etcd keys starting with `--` could not be saved or deleted from the grid. diff --git a/apple/changelog.d/4023.fixed.4.md b/apple/changelog.d/4023.fixed.4.md new file mode 100644 index 0000000000..fc2d14d955 --- /dev/null +++ b/apple/changelog.d/4023.fixed.4.md @@ -0,0 +1 @@ +etcd `get "" --prefix`, and exporting the root of a connection with no key prefix, failed with "key is not provided". diff --git a/apple/changelog.d/4023.fixed.5.md b/apple/changelog.d/4023.fixed.5.md new file mode 100644 index 0000000000..92181d56c0 --- /dev/null +++ b/apple/changelog.d/4023.fixed.5.md @@ -0,0 +1 @@ +etcd commands could run without the read-only or delete limits set by the app. diff --git a/apple/changelog.d/4023.fixed.6.md b/apple/changelog.d/4023.fixed.6.md new file mode 100644 index 0000000000..c24be7e790 --- /dev/null +++ b/apple/changelog.d/4023.fixed.6.md @@ -0,0 +1 @@ +An etcd range query with invalid page bounds could crash the app. diff --git a/apple/changelog.d/4023.fixed.md b/apple/changelog.d/4023.fixed.md new file mode 100644 index 0000000000..dfabea43ce --- /dev/null +++ b/apple/changelog.d/4023.fixed.md @@ -0,0 +1 @@ +etcd `get`, `del` and `watch` with `--prefix` could reach keys outside the prefix when it ended in some non-ASCII text. diff --git a/apple/project.yml b/apple/project.yml index cb69af7315..ff2c390172 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -407,7 +407,7 @@ targets: - package: Sparkle - package: Yams - package: TableProCore - products: [TableProAnalytics, TableProAutocomplete, TableProConnectionLibrary, TableProCoreTypes, TableProGeometry, TableProGoogleCloud, TableProGzip, TableProHTTP, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProNumberFormatting, TableProProcess, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLQuoting, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTabular, TableProTabularIO, TableProTheme, TableProWeaviateCore] + products: [TableProAnalytics, TableProAutocomplete, TableProConnectionLibrary, TableProCoreTypes, TableProEtcdCore, TableProGeometry, TableProGoogleCloud, TableProGzip, TableProHTTP, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProNumberFormatting, TableProProcess, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLQuoting, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTabular, TableProTabularIO, TableProTheme, TableProWeaviateCore] - package: TableProMacCore products: [TableProMCP, TableProSSHConfig, TableProUtilities] # Writes nothing for a version CHANGELOG.md has no section for, so a 0.0.0-dev build ships no notes. @@ -1062,6 +1062,8 @@ targets: folder: EtcdDriverPlugin principalClass: EtcdPlugin dependencies: + - package: TableProCore + product: TableProEtcdCore - package: TableProCore product: TableProTLSClientIdentity diff --git a/apple/scripts/probes/check-etcd-page-bounds.sh b/apple/scripts/probes/check-etcd-page-bounds.sh new file mode 100755 index 0000000000..cdfcbe7989 --- /dev/null +++ b/apple/scripts/probes/check-etcd-page-bounds.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Registry drivers have no unit-test target, so compile the real query parser to check page bounds. +# Usage: lockf -k -t 540 /tmp/tablepro-heavy.lock apple/scripts/probes/check-etcd-page-bounds.sh +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORK="$(mktemp -d "$ROOT/../.analysis/etcd-page-bounds.XXXXXX")" +trap 'rm -rf "$WORK"' EXIT +PLUGIN="$ROOT/Plugins/EtcdDriverPlugin" +KIT_SOURCES=() +while IFS= read -r file; do KIT_SOURCES+=("$file"); done < <(rg --files "$ROOT/Plugins/TableProPluginKit" -g '*.swift') +xcrun swiftc -swift-version 6 -parse-as-library -emit-library -emit-module -module-name TableProPluginKit \ + -emit-module-path "$WORK/TableProPluginKit.swiftmodule" -o "$WORK/libTableProPluginKit.dylib" "${KIT_SOURCES[@]}" +xcrun swiftc -swift-version 6 -parse-as-library -emit-library -emit-module -module-name TableProEtcdCore \ + -emit-module-path "$WORK/TableProEtcdCore.swiftmodule" -o "$WORK/libTableProEtcdCore.dylib" \ + "$ROOT"/Packages/TableProCore/Sources/TableProEtcdCore/*.swift +cat > "$WORK/main.swift" <<'SWIFT' +import Foundation + +let builder = EtcdQueryBuilder() +for (limit, offset) in [(-1, 0), (1, -1), (Int.max, 1), (1, Int.max), (Int.min, Int.min)] { + let query = builder.buildBrowseQuery(prefix: "", sortColumns: [], limit: limit, offset: offset) + precondition(EtcdQueryBuilder.parseRangeQuery(query) == nil, "Accepted invalid page bounds") +} +for (limit, offset) in [(0, 0), (100, 0), (100, 100), (Int.max, 0), (0, Int.max)] { + let query = builder.buildBrowseQuery(prefix: "/app/", sortColumns: [], limit: limit, offset: offset) + guard let parsed = EtcdQueryBuilder.parseRangeQuery(query) else { fatalError("Rejected valid page bounds") } + precondition(parsed.limit == limit && parsed.offset == offset && parsed.prefix == "/app/") +} +print("PASS: 10 page-bound cases") +SWIFT +xcrun swiftc -swift-version 6 -I "$WORK" -L "$WORK" -lTableProPluginKit -lTableProEtcdCore \ + -Xlinker -rpath -Xlinker "$WORK" \ + "$PLUGIN/EtcdCommandParser.swift" "$PLUGIN/EtcdRowFilter.swift" "$PLUGIN/EtcdQueryBuilder.swift" \ + "$PLUGIN/EtcdLeaseID.swift" \ + "$WORK/main.swift" -o "$WORK/probe" +"$WORK/probe"