feat(kernel): complete SQL volume file transfers

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Honor `stagingAllowedLocalPath` for SQL PUT, GET, and REMOVE through the native backend and report transfer errors as `StagingError`. Requires matching native packages. (#582)
 - CloudFetch retry-exhaustion errors now identify the download and the last failure: the `RetryError` message reads `CloudFetch result download failed: <original message> (last error: <cause>)`. `<cause>` is `HTTP <status>: <statusText>` (or `HTTP <status>` when the status text is empty) for HTTP failures, and `network error <code>`, `network error (<type>)` (e.g. `aborted`, when there is no `code`) or a bare `network error` for network failures (the raw network-error message, which can contain the presigned URL, is not copied). If no cause can be determined, the ` (last error: …)` suffix is omitted. The `RetryError` instance, `errorCode` and `payload` (including the node-fetch `Request` / `Response`) are unchanged. (#471)
 - Report untyped NULL columns as STRING in kernel result metadata, matching Thrift. (#470)
 - Reject non-scalar query parameter values at bind time with a `ParameterError` on both the Thrift and kernel backends, instead of stringifying them (which could leave a Thrift operation stuck before FINISHED). Newly rejected on the Thrift backend: arrays, plain objects, boxed primitives (e.g. `new String('x')`, `new Number(1)`), functions, symbols, and a `DBSQLParameter` that wraps another `DBSQLParameter`. Primitives, `Date`, `Int64`, and `DBSQLParameter` are still accepted.

CONNECTION_PARAMETERS.md

@@ -156,19 +156,19 @@ telemetry implementation, so only the options explicitly forwarded by
 
 ## Per-statement options (`session.executeStatement(sql, options)`)
 
-| Option                    | Type                                                    | Thrift | Kernel | Default Value | Note                                                                                                                                                                      |
-| ------------------------- | ------------------------------------------------------- | :----: | :----: | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `queryTimeout`            | `number \| bigint \| Int64`                             |   ✅   |   ❌   | —             | Thrift sends this server-side timeout for Compute clusters. Kernel ignores it; SQL Warehouses use `STATEMENT_TIMEOUT`.                                                    |
-| `runAsync`                | `boolean`                                               |   ❌   |   ✅   | — / `false`   | Thrift always sends asynchronously and ignores the option. Kernel `false`/omitted uses direct execution; `true` submits and polls. This execute option is not deprecated. |
-| `maxRows`                 | `number \| bigint \| Int64 \| null`                     |   ✅   |   ❌   | `100000` / —  | Thrift uses it for the initial Direct Results row count; `null` disables Direct Results. Kernel does not retain or forward the supplied value.                            |
-| `useCloudFetch`           | `boolean`                                               |   ✅   |   ❌   | `true` / —    | Kernel logs and ignores this per-statement hint; Kernel result fetching owns its CloudFetch behavior.                                                                     |
-| `useLZ4Compression`       | `boolean`                                               |   ✅   |   ❌   | `true` / —    | Thrift uses it when supported and when the result is not CloudFetch. Kernel owns and auto-detects result compression.                                                     |
-| `stagingAllowedLocalPath` | `string \| string[]`                                    |   ✅   |   ❌   | —             | Local allowlist for Thrift volume/staging operations. Kernel volume operations are unsupported.                                                                           |
-| `namedParameters`         | `Record<string, DBSQLParameter \| DBSQLParameterValue>` |   ✅   |   ✅   | —             | Named SQL parameters. Cannot be combined with non-empty `ordinalParameters`. Thrift requires a protocol that supports parameterized queries.                              |
-| `ordinalParameters`       | `Array<DBSQLParameter \| DBSQLParameterValue>`          |   ✅   |   ✅   | —             | Positional SQL parameters. Cannot be combined with non-empty `namedParameters`.                                                                                           |
-| `queryTags`               | `Record<string, string \| null \| undefined>`           |   ✅   |   ✅   | —             | Serialized into the per-statement `query_tags` conf overlay on both.                                                                                                      |
-| `rowLimit`                | `number`                                                |   ❌   |   ✅   | —             | Kernel-only server-side row cap. Thrift logs and ignores it.                                                                                                              |
-| `statementConf`           | `Record<string, string>`                                |   ❌   |   ✅   | —             | Kernel-only per-statement Spark conf overlay. Structured `queryTags` overwrite its `query_tags` key when non-empty.                                                       |
+| Option                    | Type                                                    | Thrift | Kernel | Default Value | Note                                                                                                                                                                                                                              |
+| ------------------------- | ------------------------------------------------------- | :----: | :----: | ------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `queryTimeout`            | `number \| bigint \| Int64`                             |   ✅   |   ❌   | —             | Thrift sends this server-side timeout for Compute clusters. Kernel ignores it; SQL Warehouses use `STATEMENT_TIMEOUT`.                                                                                                            |
+| `runAsync`                | `boolean`                                               |   ❌   |   ✅   | — / `false`   | Thrift always sends asynchronously and ignores the option. Kernel `false`/omitted uses direct execution; `true` submits and polls. This execute option is not deprecated.                                                         |
+| `maxRows`                 | `number \| bigint \| Int64 \| null`                     |   ✅   |   ❌   | `100000` / —  | Thrift uses it for the initial Direct Results row count; `null` disables Direct Results. Kernel does not retain or forward the supplied value.                                                                                    |
+| `useCloudFetch`           | `boolean`                                               |   ✅   |   ❌   | `true` / —    | Kernel logs and ignores this per-statement hint; Kernel result fetching owns its CloudFetch behavior.                                                                                                                             |
+| `useLZ4Compression`       | `boolean`                                               |   ✅   |   ❌   | `true` / —    | Thrift uses it when supported and when the result is not CloudFetch. Kernel owns and auto-detects result compression.                                                                                                             |
+| `stagingAllowedLocalPath` | `string \| string[]`                                    |   ✅   |   ✅   | —             | Local allowlist for SQL PUT/GET; also enables SQL REMOVE without a local path. Native transfers require the kernel revision in `KERNEL_REV` and matching native packages; leaving the option unset returns transfer instructions. |
+| `namedParameters`         | `Record<string, DBSQLParameter \| DBSQLParameterValue>` |   ✅   |   ✅   | —             | Named SQL parameters. Cannot be combined with non-empty `ordinalParameters`. Thrift requires a protocol that supports parameterized queries.                                                                                      |
+| `ordinalParameters`       | `Array<DBSQLParameter \| DBSQLParameterValue>`          |   ✅   |   ✅   | —             | Positional SQL parameters. Cannot be combined with non-empty `namedParameters`.                                                                                                                                                   |
+| `queryTags`               | `Record<string, string \| null \| undefined>`           |   ✅   |   ✅   | —             | Serialized into the per-statement `query_tags` conf overlay on both.                                                                                                                                                              |
+| `rowLimit`                | `number`                                                |   ❌   |   ✅   | —             | Kernel-only server-side row cap. Thrift logs and ignores it.                                                                                                                                                                      |
+| `statementConf`           | `Record<string, string>`                                |   ❌   |   ✅   | —             | Kernel-only per-statement Spark conf overlay. Structured `queryTags` overwrite its `query_tags` key when non-empty.                                                                                                               |
 
 ## Metadata-operation request parameters
 

KERNEL_REV

@@ -1 +1 @@
-6f4b6d607a61bdde61dcda2cf82bc8e80fd32cb9
+37c7a3ec1b54f4690d9325d65f2220e0bb64b83e

lib/kernel/KernelErrorMapping.ts

@@ -2,6 +2,7 @@ import HiveDriverError from '../errors/HiveDriverError';
 import AuthenticationError from '../errors/AuthenticationError';
 import OperationStateError, { OperationStateErrorCode } from '../errors/OperationStateError';
 import ParameterError from '../errors/ParameterError';
+import StagingError from '../errors/StagingError';
 
 /**
  * Sentinel prefix the napi binding's `napi_err_from_kernel` puts on
@@ -27,6 +28,8 @@ export interface KernelErrorShape {
   message: string;
   /** Optional SQLSTATE — five-char alphanumeric, when the kernel was able to surface it. */
   sqlstate?: string;
+  /** Distinguishes transfer failures from broader kernel error categories. */
+  errorCode?: string;
 }
 
 /**
@@ -125,6 +128,9 @@ function defineErrorMetadata<K extends string, V>(error: Error, key: K, value: V
  *   DataLoss, Internal,
  *   InvalidStatementHandle, SqlError   → HiveDriverError
  *
+ * File-transfer diagnostics map to StagingError except Cancelled and Timeout,
+ * which retain their operation-state errors for lifecycle handling.
+ *
  * Unknown `code` values (e.g. if the kernel adds a new variant) fall through
  * to HiveDriverError so the driver never silently drops an error. The kernel's
  * `ErrorCode` is `#[non_exhaustive]` so this can legitimately happen.
@@ -137,58 +143,65 @@ export function mapKernelErrorToJsError(kErr: KernelErrorShape): ErrorWithSqlSta
 
   let error: ErrorWithSqlState;
 
-  switch (code as KernelErrorCode) {
-    case 'Unauthenticated':
-    case 'PermissionDenied':
-      error = new AuthenticationError(message);
-      break;
+  if (
+    code !== 'Cancelled' &&
+    code !== 'Timeout' &&
+    (kErr.errorCode === 'FILE_TRANSFER_ERROR' || kErr.errorCode === 'FILE_TRANSFER_PERMISSION_ERROR')
+  ) {
+    error = new StagingError(message);
+  } else {
+    switch (code as KernelErrorCode) {
+      case 'Unauthenticated':
+      case 'PermissionDenied':
+        error = new AuthenticationError(message);
+        break;
 
-    case 'Cancelled':
-      // OperationStateError with the Canceled code carries the kernel message
-      // through the response.displayMessage fallback path.
-      error = new OperationStateError(OperationStateErrorCode.Canceled);
-      error.message = message;
-      break;
+      case 'Cancelled':
+        // OperationStateError with the Canceled code carries the kernel message
+        // through the response.displayMessage fallback path.
+        error = new OperationStateError(OperationStateErrorCode.Canceled);
+        error.message = message;
+        break;
 
-    case 'Timeout':
-      error = new OperationStateError(OperationStateErrorCode.Timeout);
-      error.message = message;
-      break;
+      case 'Timeout':
+        error = new OperationStateError(OperationStateErrorCode.Timeout);
+        error.message = message;
+        break;
 
-    case 'InvalidArgument':
-      error = new ParameterError(message);
-      break;
+      case 'InvalidArgument':
+        error = new ParameterError(message);
+        break;
 
-    case 'SqlError': {
-      // A server-reported SQL execution failure (kernel `SqlError`, e.g. a
-      // bad query, missing table, divide-by-zero, invalid cast). The Thrift
-      // backend surfaces the same situation as `OperationStateError(Error)`
-      // when the operation reaches ERROR_STATE (see DBSQLOperation), so map
-      // SqlError to the same class for backend parity. OperationStateError
-      // extends HiveDriverError, so existing `instanceof HiveDriverError`
-      // catches are unaffected.
-      const stateError = new OperationStateError(OperationStateErrorCode.Error);
-      stateError.message = message;
-      error = stateError;
-      break;
-    }
+      case 'SqlError': {
+        // A server-reported SQL execution failure (kernel `SqlError`, e.g. a
+        // bad query, missing table, divide-by-zero, invalid cast). The Thrift
+        // backend surfaces the same situation as `OperationStateError(Error)`
+        // when the operation reaches ERROR_STATE (see DBSQLOperation), so map
+        // SqlError to the same class for backend parity. OperationStateError
+        // extends HiveDriverError, so existing `instanceof HiveDriverError`
+        // catches are unaffected.
+        const stateError = new OperationStateError(OperationStateErrorCode.Error);
+        stateError.message = message;
+        error = stateError;
+        break;
+      }
 
-    // All remaining kernel ErrorCode variants map to the base driver error class.
-    // M0 intentionally does not introduce new error classes; M1 may add nuance.
-    case 'NotFound':
-    case 'ResourceExhausted':
-    case 'Unavailable':
-    case 'DataLoss':
-    case 'Internal':
-    case 'InvalidStatementHandle':
-    case 'NetworkError':
-      error = new HiveDriverError(message);
-      break;
+      // All remaining kernel ErrorCode variants map to the base driver error class.
+      case 'NotFound':
+      case 'ResourceExhausted':
+      case 'Unavailable':
+      case 'DataLoss':
+      case 'Internal':
+      case 'InvalidStatementHandle':
+      case 'NetworkError':
+        error = new HiveDriverError(message);
+        break;
 
-    default:
-      // Unknown/future kernel variant — never drop the error, surface as base class.
-      error = new HiveDriverError(message);
-      break;
+      default:
+        // Unknown/future kernel variant — never drop the error, surface as base class.
+        error = new HiveDriverError(message);
+        break;
+    }
   }
 
   if (sqlstate !== undefined) {
@@ -305,7 +318,12 @@ export function decodeNapiKernelError(err: unknown): Error {
   const msg = envelope.message as string;
   const sqlState = typeof envelope.sqlState === 'string' ? envelope.sqlState : undefined;
 
-  const jsErr = mapKernelErrorToJsError({ code, message: msg, sqlstate: sqlState });
+  const jsErr = mapKernelErrorToJsError({
+    code,
+    message: msg,
+    sqlstate: sqlState,
+    errorCode: typeof envelope.errorCode === 'string' ? envelope.errorCode : undefined,
+  });
 
   const meta = buildKernelMetadata(envelope);
   // Skip the namespace attachment entirely when no fields validated

lib/kernel/KernelSessionBackend.ts

@@ -148,10 +148,8 @@ export default class KernelSessionBackend implements ISessionBackend {
    *     `query_tags` key (the same wire shape Thrift's `serializeQueryTags`
    *     produces), merged with any explicit `statementConf`.
    *
-   * Still rejected (genuinely unsupported on kernel, rather than silently
-   * dropped): `useCloudFetch` (governed by the kernel `ResultConfig`, not a
-   * per-statement knob), `useLZ4Compression` (kernel owns result compression),
-   * and `stagingAllowedLocalPath` (volume operations). `maxRows` is applied by
+   * `stagingAllowedLocalPath` configures kernel file transfers. `useCloudFetch`
+   * and `useLZ4Compression` remain kernel-owned settings. `maxRows` is applied by
    * the facade at fetch time, so it is intentionally not handled here.
    */
   public async executeStatement(statement: string, options: ExecuteStatementOptions): Promise<IOperationBackend> {
@@ -160,8 +158,7 @@ export default class KernelSessionBackend implements ISessionBackend {
     // Per-statement options the kernel backend doesn't honour are treated as
     // NO-OPs (logged at warn), not errors — so call sites written for the Thrift
     // backend are drop-in on the kernel path. These are perf/format hints the
-    // kernel governs internally (useCloudFetch / useLZ4Compression) or a feature
-    // it doesn't expose yet (staging); ignoring them cannot change query results.
+    // kernel governs internally (useCloudFetch / useLZ4Compression) and do not change query results.
     // NB: parameter binding (compound/BINARY) is deliberately NOT no-op'd — a
     // dropped param would silently change results, so it still throws.
     if (options.useCloudFetch !== undefined) {
@@ -180,15 +177,6 @@ export default class KernelSessionBackend implements ISessionBackend {
           'kernel executeStatement: ignoring per-statement `useLZ4Compression` — result compression is governed by the kernel (no-op on kernel).',
         );
     }
-    if (options.stagingAllowedLocalPath !== undefined) {
-      this.context
-        .getLogger()
-        .log(
-          LogLevel.warn,
-          'kernel executeStatement: ignoring `stagingAllowedLocalPath` — volume/staging operations are not yet supported on the kernel backend (no-op).',
-        );
-    }
-
     // `runAsync` selects the kernel execution path. NOTE: this is a kernel-
     // specific use of the option — the Thrift backend hardcodes `runAsync: true`
     // on the wire and never reads `options.runAsync`, so the field is a no-op
@@ -299,6 +287,11 @@ export default class KernelSessionBackend implements ISessionBackend {
     }
 
     const execOptions: KernelNativeExecuteOptions = {};
+    if (options.stagingAllowedLocalPath !== undefined) {
+      execOptions.stagingAllowedLocalPaths = Array.isArray(options.stagingAllowedLocalPath)
+        ? options.stagingAllowedLocalPath
+        : [options.stagingAllowedLocalPath];
+    }
     // Raw binding preserves qualified SQL types such as INTERVAL MONTH.
     const rawParams = positionalParams ?? namedParams;
     if (rawParams !== undefined) {