diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 3805300a8e7..86b3c5fdb67 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -190,6 +190,7 @@ func setReplsForTestEnvVars(t *testing.T, repls *testdiff.ReplacementsContext) { "TEST_DEFAULT_WAREHOUSE_ID", "TEST_DEFAULT_CLUSTER_ID", "TEST_INSTANCE_POOL_ID", + "TEST_SANDBOX_ID", } for _, envVar := range envVars { if value := os.Getenv(envVar); value != "" { @@ -449,6 +450,9 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo if os.Getenv("TEST_INSTANCE_POOL_ID") == "" { t.Setenv("TEST_INSTANCE_POOL_ID", testserver.TestDefaultInstancePoolId) } + if os.Getenv("TEST_SANDBOX_ID") == "" { + t.Setenv("TEST_SANDBOX_ID", testserver.TestDefaultSandboxId) + } } const sharedProxyHint = "; re-run with -debugsandbox to see which test caused this" diff --git a/acceptance/cmd/sandbox/fs/cat/hello.txt b/acceptance/cmd/sandbox/fs/cat/hello.txt new file mode 100644 index 00000000000..a0423896973 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cat/hello.txt @@ -0,0 +1 @@ +hello world! diff --git a/acceptance/cmd/sandbox/fs/cat/out.test.toml b/acceptance/cmd/sandbox/fs/cat/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cat/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/sandbox/fs/cat/output.txt b/acceptance/cmd/sandbox/fs/cat/output.txt new file mode 100644 index 00000000000..41cc3c7b508 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cat/output.txt @@ -0,0 +1,34 @@ + +=== Help +>>> [CLI] sandbox fs cat --help +Show file content. Use SANDBOX:/absolute/path for sandbox paths. + +Usage: + databricks sandbox fs cat FILE_PATH [flags] + +Examples: + databricks sandbox fs cat my-sandbox:/tmp/file + +Flags: + -h, --help help for cat + +Global Flags: + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + +=== cat a file +>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt +hello world! + +=== cat a directory, then a missing path +>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME] +Error: not a file: /tmp/sandbox-fs-[UNIQUE_NAME] + +Exit code: 1 + +>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/missing +Error: file does not exist: /tmp/sandbox-fs-[UNIQUE_NAME]/missing + +Exit code: 1 diff --git a/acceptance/cmd/sandbox/fs/cat/script b/acceptance/cmd/sandbox/fs/cat/script new file mode 100644 index 00000000000..ba5392b54be --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cat/script @@ -0,0 +1,11 @@ +title "Help" +trace $CLI sandbox fs cat --help + +$CLI sandbox fs cp hello.txt $SANDBOX_ROOT/hello.txt > /dev/null + +title "cat a file" +trace $CLI sandbox fs cat $SANDBOX_ROOT/hello.txt + +title "cat a directory, then a missing path" +errcode trace $CLI sandbox fs cat $SANDBOX_ROOT +errcode trace $CLI sandbox fs cat $SANDBOX_ROOT/missing diff --git a/acceptance/cmd/sandbox/fs/cp/local.txt b/acceptance/cmd/sandbox/fs/cp/local.txt new file mode 100644 index 00000000000..a0423896973 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cp/local.txt @@ -0,0 +1 @@ +hello world! diff --git a/acceptance/cmd/sandbox/fs/cp/localdir/file1.txt b/acceptance/cmd/sandbox/fs/cp/localdir/file1.txt new file mode 100644 index 00000000000..d9039017ab6 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cp/localdir/file1.txt @@ -0,0 +1 @@ +file1 content diff --git a/acceptance/cmd/sandbox/fs/cp/localdir/file2.txt b/acceptance/cmd/sandbox/fs/cp/localdir/file2.txt new file mode 100644 index 00000000000..f3c77b12c6c --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cp/localdir/file2.txt @@ -0,0 +1 @@ +file2 content diff --git a/acceptance/cmd/sandbox/fs/cp/out.test.toml b/acceptance/cmd/sandbox/fs/cp/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cp/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/sandbox/fs/cp/output.txt b/acceptance/cmd/sandbox/fs/cp/output.txt new file mode 100644 index 00000000000..71293b978ad --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cp/output.txt @@ -0,0 +1,77 @@ + +=== Help +>>> [CLI] sandbox fs cp --help +Copy files and directories between your local filesystem and sandboxes. + + Use SANDBOX:/absolute/path for sandbox paths. + + Recursively copying a directory will copy all files inside directory + at SOURCE_PATH to the directory at TARGET_PATH. + + When copying a file, if TARGET_PATH is a directory, the file will be created + inside the directory, otherwise the file is created at TARGET_PATH. + + Files that already exist at the target are skipped unless --overwrite is set. + +Usage: + databricks sandbox fs cp SOURCE_PATH TARGET_PATH [flags] + +Examples: + databricks sandbox fs cp ./file my-sandbox:/tmp/file + databricks sandbox fs cp --overwrite ./file my-sandbox:/tmp/file + databricks sandbox fs cp -r my-sandbox:/tmp/dir ./local/dir + +Flags: + --concurrency int number of parallel copy operations (default 8) + -h, --help help for cp + --overwrite overwrite existing files + -r, --recursive recursively copy files from directory + +Global Flags: + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + +=== Upload a file +>>> [CLI] sandbox fs cp local.txt [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt +local.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt + +>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt +hello world! + +=== Writing over an existing file needs --overwrite +>>> [CLI] sandbox fs cp local.txt [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt +local.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt (skipped; already exists) + +>>> [CLI] sandbox fs cp --overwrite updated.txt [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt +updated.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt + +>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/file.txt +hello again! + +=== Copying a file to a directory puts it inside +>>> [CLI] sandbox fs cp local.txt [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/target +local.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/target/local.txt + +>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/target +local.txt + +=== Writing into a directory that does not exist fails +>>> [CLI] sandbox fs cp local.txt [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/missing/file.txt +Error: no such directory: /tmp/sandbox-fs-[UNIQUE_NAME]/missing + +Exit code: 1 + +=== Copy a directory recursively, and back +>>> [CLI] sandbox fs cp -r localdir [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir +localdir/file1.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir/file1.txt +localdir/file2.txt -> [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir/file2.txt + +>>> [CLI] sandbox fs cp -r [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir downloaded +[TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir/file1.txt -> downloaded/file1.txt +[TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir/file2.txt -> downloaded/file2.txt + +>>> cat downloaded/file1.txt downloaded/file2.txt +file1 content +file2 content diff --git a/acceptance/cmd/sandbox/fs/cp/script b/acceptance/cmd/sandbox/fs/cp/script new file mode 100644 index 00000000000..3a152df8524 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cp/script @@ -0,0 +1,25 @@ +title "Help" +trace $CLI sandbox fs cp --help + +title "Upload a file" +trace $CLI sandbox fs cp local.txt $SANDBOX_ROOT/file.txt +trace $CLI sandbox fs cat $SANDBOX_ROOT/file.txt + +title "Writing over an existing file needs --overwrite" +trace $CLI sandbox fs cp local.txt $SANDBOX_ROOT/file.txt +trace $CLI sandbox fs cp --overwrite updated.txt $SANDBOX_ROOT/file.txt +trace $CLI sandbox fs cat $SANDBOX_ROOT/file.txt + +title "Copying a file to a directory puts it inside" +$CLI sandbox fs mkdir $SANDBOX_ROOT/target +trace $CLI sandbox fs cp local.txt $SANDBOX_ROOT/target +trace $CLI sandbox fs ls $SANDBOX_ROOT/target + +title "Writing into a directory that does not exist fails" +errcode trace $CLI sandbox fs cp local.txt $SANDBOX_ROOT/missing/file.txt + +title "Copy a directory recursively, and back" +trace $CLI sandbox fs cp -r localdir $SANDBOX_ROOT/dir 2>&1 | sort +trace $CLI sandbox fs cp -r $SANDBOX_ROOT/dir downloaded 2>&1 | sort +trace cat downloaded/file1.txt downloaded/file2.txt +rm -r downloaded diff --git a/acceptance/cmd/sandbox/fs/cp/test.toml b/acceptance/cmd/sandbox/fs/cp/test.toml new file mode 100644 index 00000000000..ef5d1bb5092 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cp/test.toml @@ -0,0 +1 @@ +Ignore = ["downloaded"] diff --git a/acceptance/cmd/sandbox/fs/cp/updated.txt b/acceptance/cmd/sandbox/fs/cp/updated.txt new file mode 100644 index 00000000000..69d2c08e682 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/cp/updated.txt @@ -0,0 +1 @@ +hello again! diff --git a/acceptance/cmd/sandbox/fs/ls/hello.txt b/acceptance/cmd/sandbox/fs/ls/hello.txt new file mode 100644 index 00000000000..a0423896973 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/ls/hello.txt @@ -0,0 +1 @@ +hello world! diff --git a/acceptance/cmd/sandbox/fs/ls/out.test.toml b/acceptance/cmd/sandbox/fs/ls/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/sandbox/fs/ls/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/sandbox/fs/ls/output.txt b/acceptance/cmd/sandbox/fs/ls/output.txt new file mode 100644 index 00000000000..c5afff6a72c --- /dev/null +++ b/acceptance/cmd/sandbox/fs/ls/output.txt @@ -0,0 +1,41 @@ + +=== Help +>>> [CLI] sandbox fs ls --help +Lists files. Use SANDBOX:/absolute/path for sandbox paths. + +Usage: + databricks sandbox fs ls DIR_PATH [flags] + +Examples: + databricks sandbox fs ls my-sandbox:/tmp + +Flags: + --absolute Displays absolute paths. + -h, --help help for ls + -l, --long Displays full information including size, file type and modification time since Epoch in milliseconds. + +Global Flags: + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + +=== List a directory +>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME] +dir +hello.txt + +>>> [CLI] sandbox fs ls --absolute [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME] +[TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir +[TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt + +=== List a file, then a missing path +>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt +Error: not a directory: /tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt + +Exit code: 1 + +>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/missing +Error: no such directory: /tmp/sandbox-fs-[UNIQUE_NAME]/missing + +Exit code: 1 diff --git a/acceptance/cmd/sandbox/fs/ls/script b/acceptance/cmd/sandbox/fs/ls/script new file mode 100644 index 00000000000..09647c11cc5 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/ls/script @@ -0,0 +1,13 @@ +title "Help" +trace $CLI sandbox fs ls --help + +$CLI sandbox fs cp hello.txt $SANDBOX_ROOT/hello.txt > /dev/null +$CLI sandbox fs mkdir $SANDBOX_ROOT/dir + +title "List a directory" +trace $CLI sandbox fs ls $SANDBOX_ROOT +trace $CLI sandbox fs ls --absolute $SANDBOX_ROOT + +title "List a file, then a missing path" +errcode trace $CLI sandbox fs ls $SANDBOX_ROOT/hello.txt +errcode trace $CLI sandbox fs ls $SANDBOX_ROOT/missing diff --git a/acceptance/cmd/sandbox/fs/mkdir/hello.txt b/acceptance/cmd/sandbox/fs/mkdir/hello.txt new file mode 100644 index 00000000000..a0423896973 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/mkdir/hello.txt @@ -0,0 +1 @@ +hello world! diff --git a/acceptance/cmd/sandbox/fs/mkdir/out.test.toml b/acceptance/cmd/sandbox/fs/mkdir/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/sandbox/fs/mkdir/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/sandbox/fs/mkdir/output.txt b/acceptance/cmd/sandbox/fs/mkdir/output.txt new file mode 100644 index 00000000000..13b5ea85dac --- /dev/null +++ b/acceptance/cmd/sandbox/fs/mkdir/output.txt @@ -0,0 +1,37 @@ + +=== Help +>>> [CLI] sandbox fs mkdir --help +Make directories. Use SANDBOX:/absolute/path for sandbox paths. + +Usage: + databricks sandbox fs mkdir DIR_PATH [flags] + +Aliases: + mkdir, mkdirs + +Examples: + databricks sandbox fs mkdir my-sandbox:/tmp/a/b + +Flags: + -h, --help help for mkdir + +Global Flags: + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + +=== Create nested directories +>>> [CLI] sandbox fs mkdir [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/a/b/c + +>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/a/b +c + +=== Creating an existing directory succeeds +>>> [CLI] sandbox fs mkdir [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/a/b/c + +=== Creating a directory over a file fails +>>> [CLI] sandbox fs mkdir [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt +Error: file already exists: /tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt + +Exit code: 1 diff --git a/acceptance/cmd/sandbox/fs/mkdir/script b/acceptance/cmd/sandbox/fs/mkdir/script new file mode 100644 index 00000000000..eed4f679f2c --- /dev/null +++ b/acceptance/cmd/sandbox/fs/mkdir/script @@ -0,0 +1,13 @@ +title "Help" +trace $CLI sandbox fs mkdir --help + +title "Create nested directories" +trace $CLI sandbox fs mkdir $SANDBOX_ROOT/a/b/c +trace $CLI sandbox fs ls $SANDBOX_ROOT/a/b + +title "Creating an existing directory succeeds" +trace $CLI sandbox fs mkdir $SANDBOX_ROOT/a/b/c + +title "Creating a directory over a file fails" +$CLI sandbox fs cp hello.txt $SANDBOX_ROOT/hello.txt > /dev/null +errcode trace $CLI sandbox fs mkdir $SANDBOX_ROOT/hello.txt diff --git a/acceptance/cmd/sandbox/fs/rm/hello.txt b/acceptance/cmd/sandbox/fs/rm/hello.txt new file mode 100644 index 00000000000..a0423896973 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/rm/hello.txt @@ -0,0 +1 @@ +hello world! diff --git a/acceptance/cmd/sandbox/fs/rm/out.test.toml b/acceptance/cmd/sandbox/fs/rm/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/sandbox/fs/rm/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/sandbox/fs/rm/output.txt b/acceptance/cmd/sandbox/fs/rm/output.txt new file mode 100644 index 00000000000..c896e4298af --- /dev/null +++ b/acceptance/cmd/sandbox/fs/rm/output.txt @@ -0,0 +1,47 @@ + +=== Help +>>> [CLI] sandbox fs rm --help +Remove files and directories. Use SANDBOX:/absolute/path for sandbox paths. + +Usage: + databricks sandbox fs rm PATH [flags] + +Examples: + databricks sandbox fs rm -r my-sandbox:/tmp/dir + +Flags: + -h, --help help for rm + -r, --recursive Recursively delete a non-empty directory. + +Global Flags: + --debug enable debug logging + -o, --output type output type: text or json (default text) + -p, --profile string ~/.databrickscfg profile + -t, --target string bundle target to use (if applicable) + +=== Remove a file +>>> [CLI] sandbox fs rm [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt + +>>> [CLI] sandbox fs cat [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt +Error: file does not exist: /tmp/sandbox-fs-[UNIQUE_NAME]/hello.txt + +Exit code: 1 + +=== Removing a non-empty directory needs -r +>>> [CLI] sandbox fs rm [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir +Error: directory not empty: /tmp/sandbox-fs-[UNIQUE_NAME]/dir + +Exit code: 1 + +>>> [CLI] sandbox fs rm -r [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir + +>>> [CLI] sandbox fs ls [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/dir +Error: no such directory: /tmp/sandbox-fs-[UNIQUE_NAME]/dir + +Exit code: 1 + +=== Removing a missing path fails +>>> [CLI] sandbox fs rm [TEST_SANDBOX_ID]:/tmp/sandbox-fs-[UNIQUE_NAME]/missing +Error: file does not exist: /tmp/sandbox-fs-[UNIQUE_NAME]/missing + +Exit code: 1 diff --git a/acceptance/cmd/sandbox/fs/rm/script b/acceptance/cmd/sandbox/fs/rm/script new file mode 100644 index 00000000000..cf61c16256d --- /dev/null +++ b/acceptance/cmd/sandbox/fs/rm/script @@ -0,0 +1,18 @@ +title "Help" +trace $CLI sandbox fs rm --help + +$CLI sandbox fs cp hello.txt $SANDBOX_ROOT/hello.txt > /dev/null +$CLI sandbox fs mkdir $SANDBOX_ROOT/dir +$CLI sandbox fs cp hello.txt $SANDBOX_ROOT/dir/hello.txt > /dev/null + +title "Remove a file" +trace $CLI sandbox fs rm $SANDBOX_ROOT/hello.txt +errcode trace $CLI sandbox fs cat $SANDBOX_ROOT/hello.txt + +title "Removing a non-empty directory needs -r" +errcode trace $CLI sandbox fs rm $SANDBOX_ROOT/dir +trace $CLI sandbox fs rm -r $SANDBOX_ROOT/dir +errcode trace $CLI sandbox fs ls $SANDBOX_ROOT/dir + +title "Removing a missing path fails" +errcode trace $CLI sandbox fs rm $SANDBOX_ROOT/missing diff --git a/acceptance/cmd/sandbox/fs/script.prepare b/acceptance/cmd/sandbox/fs/script.prepare new file mode 100644 index 00000000000..54bf397d678 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/script.prepare @@ -0,0 +1,14 @@ +# Cloud environments without a sandbox skip these tests. Locally, the harness sets +# TEST_SANDBOX_ID to the testserver's default sandbox. +if [ -z "${TEST_SANDBOX_ID:-}" ]; then + echo "SKIP_TEST TEST_SANDBOX_ID is not set" + exit 0 +fi + +# Each test works in its own directory under /tmp and removes it on exit. +SANDBOX_ROOT="$TEST_SANDBOX_ID:/tmp/sandbox-fs-$UNIQUE_NAME" +sandbox_fs_cleanup() { + $CLI sandbox fs rm -r "$SANDBOX_ROOT" 2>/dev/null || true +} +trap sandbox_fs_cleanup EXIT +$CLI sandbox fs mkdir "$SANDBOX_ROOT" diff --git a/acceptance/cmd/sandbox/fs/test.toml b/acceptance/cmd/sandbox/fs/test.toml new file mode 100644 index 00000000000..63fa9c9d381 --- /dev/null +++ b/acceptance/cmd/sandbox/fs/test.toml @@ -0,0 +1,3 @@ +# Runs against the testserver fake of the sandbox file APIs locally, and against the existing +# sandbox named by TEST_SANDBOX_ID on cloud, so a run doesn't create a sandbox per test. +Cloud = true diff --git a/cmd/fs/cat.go b/cmd/fs/cat.go index c9d23c99d57..be0b578c665 100644 --- a/cmd/fs/cat.go +++ b/cmd/fs/cat.go @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" ) -func newCatCommand() *cobra.Command { +func newCatCommand(filerForPath PathResolver) *cobra.Command { cmd := &cobra.Command{ Use: "cat FILE_PATH", Short: "Show file content.", @@ -27,10 +27,12 @@ func newCatCommand() *cobra.Command { if err != nil { return err } + defer r.Close() return cmdio.Render(ctx, r) } v := newValidArgs() + v.filerForPathFunc = filerForPath cmd.ValidArgsFunction = v.Validate return cmd diff --git a/cmd/fs/cp.go b/cmd/fs/cp.go index 734faaeb996..f1bd69ccf02 100644 --- a/cmd/fs/cp.go +++ b/cmd/fs/cp.go @@ -206,7 +206,7 @@ func trimTrailingDirSeparators(path string) string { return strings.TrimRight(path, string(os.PathSeparator)) } -func newCpCommand() *cobra.Command { +func newCpCommand(filerForPath PathResolver) *cobra.Command { cmd := &cobra.Command{ Use: "cp SOURCE_PATH TARGET_PATH", Short: "Copy files and directories.", @@ -253,14 +253,8 @@ func newCpCommand() *cobra.Command { return err } - c.sourceScheme = "" - if isDbfsPath(fullSourcePath) { - c.sourceScheme = "dbfs" - } - c.targetScheme = "" - if isDbfsPath(fullTargetPath) { - c.targetScheme = "dbfs" - } + c.sourceScheme, _ = SplitPath(fullSourcePath) + c.targetScheme, _ = SplitPath(fullTargetPath) c.sourceFiler = sourceFiler c.targetFiler = targetFiler @@ -296,6 +290,7 @@ func newCpCommand() *cobra.Command { } v := newValidArgs() + v.filerForPathFunc = filerForPath // The copy command has two paths that can be completed (SOURCE_PATH & TARGET_PATH) v.pathArgCount = 2 cmd.ValidArgsFunction = v.Validate diff --git a/cmd/fs/fs.go b/cmd/fs/fs.go index 074bd31a64c..1f320b19019 100644 --- a/cmd/fs/fs.go +++ b/cmd/fs/fs.go @@ -1,10 +1,21 @@ package fs import ( + "context" + + "github.com/databricks/cli/libs/filer" "github.com/spf13/cobra" ) +// PathResolver selects a filesystem and removes its prefix from a path. +type PathResolver func(context.Context, string) (filer.Filer, string, error) + func New() *cobra.Command { + return NewWithPathResolver(filerForPath) +} + +// NewWithPathResolver returns filesystem commands using the supplied path resolver. +func NewWithPathResolver(resolvePath PathResolver) *cobra.Command { cmd := &cobra.Command{ Use: "fs", Short: "Filesystem related commands", @@ -13,11 +24,11 @@ func New() *cobra.Command { } cmd.AddCommand( - newCatCommand(), - newCpCommand(), - newLsCommand(), - newMkdirCommand(), - newRmCommand(), + newCatCommand(resolvePath), + newCpCommand(resolvePath), + newLsCommand(resolvePath), + newMkdirCommand(resolvePath), + newRmCommand(resolvePath), ) return cmd diff --git a/cmd/fs/helpers.go b/cmd/fs/helpers.go index d2e98563314..911d59c82dc 100644 --- a/cmd/fs/helpers.go +++ b/cmd/fs/helpers.go @@ -13,29 +13,36 @@ import ( "github.com/spf13/cobra" ) -func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, error) { +// SplitPath separates a remote prefix from a path, preserving local Windows drive paths. +func SplitPath(fullPath string) (scheme, name string) { // Split path at : to detect any file schemes parts := strings.SplitN(fullPath, ":", 2) // If no scheme is specified, then local path if len(parts) < 2 { - f, err := filer.NewLocalClient("") - return f, fullPath, err + return "", fullPath } // On windows systems, paths start with a drive letter. If the scheme // is a single letter and the OS is windows, then we conclude the path // is meant to be a local path. if runtime.GOOS == "windows" && len(parts[0]) == 1 { + return "", fullPath + } + return parts[0], parts[1] +} + +func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, error) { + scheme, path := SplitPath(fullPath) + if scheme == "" && path == fullPath { f, err := filer.NewLocalClient("") - return f, fullPath, err + return f, path, err } - if parts[0] != "dbfs" { - return nil, "", fmt.Errorf("invalid scheme: %s", parts[0]) + if scheme != "dbfs" { + return nil, "", fmt.Errorf("invalid scheme: %s", scheme) } - path := parts[1] w := cmdctx.WorkspaceClient(ctx) // If the specified path has the "Volumes" or "Skills" prefix, use the Files API. @@ -49,12 +56,6 @@ func filerForPath(ctx context.Context, fullPath string) (filer.Filer, string, er return f, path, err } -const dbfsPrefix string = "dbfs:" - -func isDbfsPath(path string) bool { - return strings.HasPrefix(path, dbfsPrefix) -} - type validArgs struct { mustWorkspaceClientFunc func(cmd *cobra.Command, args []string) error filerForPathFunc func(ctx context.Context, fullPath string) (filer.Filer, string, error) @@ -91,9 +92,9 @@ func (v *validArgs) Validate(cmd *cobra.Command, args []string, toComplete strin completer := completer.New(cmd.Context(), filer, v.onlyDirs) // Dbfs should have a prefix and always use the "/" separator - isDbfsPath := isDbfsPath(toComplete) - if isDbfsPath { - completer.SetPrefix(dbfsPrefix) + scheme, _ := SplitPath(toComplete) + if scheme != "" { + completer.SetPrefix(scheme + ":") completer.SetIsLocalPath(false) } diff --git a/cmd/fs/helpers_test.go b/cmd/fs/helpers_test.go index ee1a88e23fb..862d2f59a70 100644 --- a/cmd/fs/helpers_test.go +++ b/cmd/fs/helpers_test.go @@ -176,3 +176,19 @@ func TestGetValidArgsFunctionNotCompletedArgument(t *testing.T) { assert.Nil(t, completions) assert.Equal(t, cobra.ShellCompDirectiveNoFileComp, directive) } + +func TestSplitPath(t *testing.T) { + cases := []struct{ input, scheme, name string }{ + {"./local/file", "", "./local/file"}, + {"dbfs:/foo", "dbfs", "/foo"}, + {"my-sandbox:/foo:bar", "my-sandbox", "/foo:bar"}, + {":/foo", "", "/foo"}, + } + for _, tc := range cases { + t.Run(tc.input, func(t *testing.T) { + scheme, name := SplitPath(tc.input) + assert.Equal(t, tc.scheme, scheme) + assert.Equal(t, tc.name, name) + }) + } +} diff --git a/cmd/fs/ls.go b/cmd/fs/ls.go index c6151b8a27a..d9a7967d8d0 100644 --- a/cmd/fs/ls.go +++ b/cmd/fs/ls.go @@ -38,7 +38,7 @@ func toJsonDirEntry(f fs.DirEntry, baseDir string, isAbsolute bool) (*jsonDirEnt }, nil } -func newLsCommand() *cobra.Command { +func newLsCommand(filerForPath PathResolver) *cobra.Command { cmd := &cobra.Command{ Use: "ls DIR_PATH", Short: "Lists files.", @@ -91,6 +91,7 @@ func newLsCommand() *cobra.Command { } v := newValidArgs() + v.filerForPathFunc = filerForPath v.onlyDirs = true cmd.ValidArgsFunction = v.Validate diff --git a/cmd/fs/mkdir.go b/cmd/fs/mkdir.go index 15faaad1569..fdcb36ba62f 100644 --- a/cmd/fs/mkdir.go +++ b/cmd/fs/mkdir.go @@ -5,7 +5,7 @@ import ( "github.com/spf13/cobra" ) -func newMkdirCommand() *cobra.Command { +func newMkdirCommand(filerForPath PathResolver) *cobra.Command { cmd := &cobra.Command{ Use: "mkdir DIR_PATH", // Alias `mkdirs` for this command exists for legacy purposes. This command @@ -29,6 +29,7 @@ func newMkdirCommand() *cobra.Command { } v := newValidArgs() + v.filerForPathFunc = filerForPath v.onlyDirs = true cmd.ValidArgsFunction = v.Validate diff --git a/cmd/fs/rm.go b/cmd/fs/rm.go index 8855e528804..e0b468bcf95 100644 --- a/cmd/fs/rm.go +++ b/cmd/fs/rm.go @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" ) -func newRmCommand() *cobra.Command { +func newRmCommand(filerForPath PathResolver) *cobra.Command { cmd := &cobra.Command{ Use: "rm PATH", Short: "Remove files and directories.", @@ -33,6 +33,7 @@ func newRmCommand() *cobra.Command { } v := newValidArgs() + v.filerForPathFunc = filerForPath cmd.ValidArgsFunction = v.Validate return cmd diff --git a/cmd/sandbox/completion.go b/cmd/sandbox/completion.go index e8c48535a3e..e7a74ab2472 100644 --- a/cmd/sandbox/completion.go +++ b/cmd/sandbox/completion.go @@ -58,15 +58,11 @@ func completeSSHKeyHashes(cmd *cobra.Command, args []string, toComplete string) } // completionProfile returns the profile name the cache is keyed under, -// matching the resolution used by the runtime commands (Profile if set, -// else Host). Returns "" if the workspace client can't be bootstrapped. +// matching the resolution used by the runtime commands (see cacheProfile). +// Returns "" if the workspace client can't be bootstrapped. func completionProfile(cmd *cobra.Command, args []string) string { if err := root.MustWorkspaceClient(cmd, args); err != nil { return "" } - w := cmdctx.WorkspaceClient(cmd.Context()) - if w.Config.Profile != "" { - return w.Config.Profile - } - return w.Config.Host + return cacheProfile(cmdctx.WorkspaceClient(cmd.Context())) } diff --git a/cmd/sandbox/fs.go b/cmd/sandbox/fs.go new file mode 100644 index 00000000000..c299e604ce3 --- /dev/null +++ b/cmd/sandbox/fs.go @@ -0,0 +1,104 @@ +package sandbox + +import ( + "context" + "errors" + "fmt" + "path" + "slices" + + filesystem "github.com/databricks/cli/cmd/fs" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/filer" + "github.com/spf13/cobra" +) + +// sandboxCpLong replaces the help of cp, which describes DBFS and UC paths, +// and keeps its description of how directories are copied. +const sandboxCpLong = `Copy files and directories between your local filesystem and sandboxes. + + Use SANDBOX:/absolute/path for sandbox paths. + + Recursively copying a directory will copy all files inside directory + at SOURCE_PATH to the directory at TARGET_PATH. + + When copying a file, if TARGET_PATH is a directory, the file will be created + inside the directory, otherwise the file is created at TARGET_PATH. + + Files that already exist at the target are skipped unless --overwrite is set. +` + +// sandboxFsExamples are shown in each command's help. +var sandboxFsExamples = map[string]string{ + "cp": ` databricks sandbox fs cp ./file my-sandbox:/tmp/file + databricks sandbox fs cp --overwrite ./file my-sandbox:/tmp/file + databricks sandbox fs cp -r my-sandbox:/tmp/dir ./local/dir`, + "ls": ` databricks sandbox fs ls my-sandbox:/tmp`, + "cat": ` databricks sandbox fs cat my-sandbox:/tmp/file`, + "mkdir": ` databricks sandbox fs mkdir my-sandbox:/tmp/a/b`, + "rm": ` databricks sandbox fs rm -r my-sandbox:/tmp/dir`, +} + +func newFsCommand() *cobra.Command { + cmd := filesystem.NewWithPathResolver(sandboxFilerForPath) + // The top-level fs command belongs to a help group that the sandbox command + // doesn't define, and cobra panics on an undefined group. + cmd.GroupID = "" + cmd.Long = `Transfer files between your local filesystem and sandboxes. + +Use SANDBOX:/absolute/path for sandbox paths. SANDBOX can be an ID or a +name from the local cache populated by sandbox list. Paths without a +sandbox prefix refer to the local filesystem.` + for _, sub := range cmd.Commands() { + sub.Long = sub.Short + " Use SANDBOX:/absolute/path for sandbox paths." + if sub.Name() == "cp" { + sub.Long = sandboxCpLong + } + sub.Example = sandboxFsExamples[sub.Name()] + sub.ValidArgsFunction = completeSandboxPaths(sub.ValidArgsFunction) + } + cmd.Example = ` databricks sandbox fs cp -r my-sandbox:/foo/bar ./local/output + databricks sandbox fs cp ./file my-sandbox:/tmp/file --overwrite + databricks sandbox fs ls my-sandbox:/tmp` + return cmd +} + +func sandboxFilerForPath(ctx context.Context, fullPath string) (filer.Filer, string, error) { + identifier, name := filesystem.SplitPath(fullPath) + if identifier == "" && name == fullPath { + f, err := filer.NewLocalClient("") + return f, name, err + } + if identifier == "" { + return nil, "", errors.New("empty sandbox identifier: use SANDBOX:/path") + } + if !path.IsAbs(name) { + return nil, "", fmt.Errorf("sandbox path must be absolute: use %s:/path", identifier) + } + w := cmdctx.WorkspaceClient(ctx) + id, err := resolveLocalID(ctx, cacheProfile(w), identifier) + if err != nil { + return nil, "", err + } + f, err := filer.NewSandboxClient(w, id, "/") + return f, path.Clean(name), err +} + +func completeSandboxPaths(complete cobra.CompletionFunc) cobra.CompletionFunc { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + completions, directive := complete(cmd, args, toComplete) + identifier, _ := filesystem.SplitPath(toComplete) + if identifier != "" || directive != cobra.ShellCompDirectiveNoSpace { + return completions, directive + } + completions = slices.DeleteFunc(completions, func(p string) bool { return p == "dbfs:/" }) + w := cmdctx.WorkspaceClient(cmd.Context()) + for _, s := range getSandboxes(cmd.Context(), cacheProfile(w)) { + completions = append(completions, s.ID+":/") + if s.Name != "" && s.Name != s.ID { + completions = append(completions, s.Name+":/") + } + } + return completions, directive + } +} diff --git a/cmd/sandbox/fs_test.go b/cmd/sandbox/fs_test.go new file mode 100644 index 00000000000..42b3b08615d --- /dev/null +++ b/cmd/sandbox/fs_test.go @@ -0,0 +1,28 @@ +package sandbox_test + +import ( + "testing" + + sandboxcmd "github.com/databricks/cli/cmd/sandbox" + "github.com/databricks/cli/libs/cmdio" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSandboxFsRejectsInvalidRemotePaths(t *testing.T) { + for _, tc := range []struct { + input string + err string + }{ + {"my-sandbox:relative", "sandbox path must be absolute: use my-sandbox:/path"}, + {":/tmp/file", "empty sandbox identifier: use SANDBOX:/path"}, + } { + t.Run(tc.input, func(t *testing.T) { + ctx, _ := cmdio.NewTestContextWithStdout(t.Context()) + cmd, _, err := sandboxcmd.New().Find([]string{"fs", "cat"}) + require.NoError(t, err) + cmd.SetContext(ctx) + assert.EqualError(t, cmd.RunE(cmd, []string{tc.input}), tc.err) + }) + } +} diff --git a/cmd/sandbox/list.go b/cmd/sandbox/list.go index ef0960d21cb..189d32fb1f5 100644 --- a/cmd/sandbox/list.go +++ b/cmd/sandbox/list.go @@ -39,10 +39,7 @@ Example: return fmt.Errorf("failed to list sandboxes: %w", err) } - profile := w.Config.Profile - if profile == "" { - profile = w.Config.Host - } + profile := cacheProfile(w) // `list` returns the full set (the API client loops through every // page), so it's the cheapest place to keep local state coherent: diff --git a/cmd/sandbox/resolve.go b/cmd/sandbox/resolve.go index 21bbcb841b6..3800cbf55de 100644 --- a/cmd/sandbox/resolve.go +++ b/cmd/sandbox/resolve.go @@ -5,8 +5,19 @@ import ( "errors" "fmt" "strings" + + "github.com/databricks/databricks-sdk-go" ) +// cacheProfile returns the key the local sandbox cache is stored under: the +// profile name if set, else the host. +func cacheProfile(w *databricks.WorkspaceClient) string { + if w.Config.Profile != "" { + return w.Config.Profile + } + return w.Config.Host +} + // resolveLocalID maps a user-typed argument to a sandbox ID using only // the local state cache — no API calls. Resolution order: // diff --git a/cmd/sandbox/sandbox.go b/cmd/sandbox/sandbox.go index 9b40c16d370..1af2a0e59fc 100644 --- a/cmd/sandbox/sandbox.go +++ b/cmd/sandbox/sandbox.go @@ -43,6 +43,7 @@ Common workflows: cmd.AddCommand(newStartCommand()) cmd.AddCommand(newStatusCommand()) cmd.AddCommand(newConfigCommand()) + cmd.AddCommand(newFsCommand()) return cmd } diff --git a/integration/libs/filer/filer_test.go b/integration/libs/filer/filer_test.go index 58618628b7e..7a8a4b427a0 100644 --- a/integration/libs/filer/filer_test.go +++ b/integration/libs/filer/filer_test.go @@ -4,10 +4,12 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "io/fs" "path" "strings" + "sync" "testing" "github.com/databricks/cli/internal/testutil" @@ -127,6 +129,7 @@ func TestFilerRecursiveDelete(t *testing.T) { {"dbfs", setupDbfsFiler}, {"files", setupUcVolumesFiler}, {"workspace files extensions", setupWsfsExtensionsFiler}, + {"sandbox", setupSandboxFiler}, } { t.Run(testCase.name, func(t *testing.T) { t.Parallel() @@ -229,6 +232,7 @@ func TestFilerReadWrite(t *testing.T) { {"dbfs", setupDbfsFiler}, {"files", setupUcVolumesFiler}, {"workspace files extensions", setupWsfsExtensionsFiler}, + {"sandbox", setupSandboxFiler}, } { t.Run(testCase.name, func(t *testing.T) { t.Parallel() @@ -335,6 +339,7 @@ func TestFilerReadDir(t *testing.T) { {"dbfs", setupDbfsFiler}, {"files", setupUcVolumesFiler}, {"workspace files extensions", setupWsfsExtensionsFiler}, + {"sandbox", setupSandboxFiler}, } { t.Run(testCase.name, func(t *testing.T) { t.Parallel() @@ -346,6 +351,60 @@ func TestFilerReadDir(t *testing.T) { } } +// Concurrent writes without OverwriteIfExists must let exactly one writer +// create the file. The lock file in libs/locker relies on this. +func commonFilerConcurrentExclusiveWriteTest(t *testing.T, ctx context.Context, f filer.Filer) { + const writers = 8 + + var wg sync.WaitGroup + errs := make([]error, writers) + for i := range writers { + wg.Go(func() { + errs[i] = f.Write(ctx, "file", strings.NewReader(fmt.Sprintf("writer-%d", i))) + }) + } + wg.Wait() + + winner := -1 + for i, err := range errs { + if err == nil { + assert.Equal(t, -1, winner, "more than one writer created the file") + winner = i + continue + } + assert.ErrorIs(t, err, fs.ErrExist) + } + require.NotEqual(t, -1, winner, "no writer created the file") + filerTest{t, f}.assertContents(ctx, "file", fmt.Sprintf("writer-%d", winner)) + + // Writers that lost must not leave anything behind. + entries, err := f.ReadDir(ctx, ".") + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "file", entries[0].Name()) +} + +func TestFilerConcurrentExclusiveWrite(t *testing.T) { + t.Parallel() + + // Only filers verified to make exclusive creates atomic are listed. + for _, testCase := range []struct { + name string + f func(t testutil.TestingT) (filer.Filer, string) + }{ + {"local", setupLocalFiler}, + {"sandbox", setupSandboxFiler}, + } { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + f, _ := testCase.f(t) + ctx := t.Context() + + commonFilerConcurrentExclusiveWriteTest(t, ctx, f) + }) + } +} + func TestFilerWorkspaceNotebook(t *testing.T) { t.Parallel() diff --git a/integration/libs/filer/helpers_test.go b/integration/libs/filer/helpers_test.go index baca230b791..35085869c17 100644 --- a/integration/libs/filer/helpers_test.go +++ b/integration/libs/filer/helpers_test.go @@ -1,6 +1,7 @@ package filer_test import ( + "context" "errors" "net/http" "os" @@ -12,6 +13,7 @@ import ( "github.com/databricks/cli/libs/filer" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/sandbox" "github.com/stretchr/testify/require" ) @@ -70,3 +72,36 @@ func setupUcVolumesFiler(t testutil.TestingT) (filer.Filer, string) { return f, path.Join("dbfs:/", tmpdir) } + +func setupSandboxFiler(t testutil.TestingT) (filer.Filer, string) { + ctx, wt := acc.WorkspaceTest(t) + + sandboxID := os.Getenv("TEST_SANDBOX_ID") + if sandboxID == "" { + t.Skip("Skipping tests that require a sandbox when TEST_SANDBOX_ID is not set.") + } + + parent := "sandboxes/" + sandboxID + tmpdir := "/tmp/" + testutil.RandomName("filer-it-") + _, err := wt.W.Sandbox.CreateDirectory(ctx, sandbox.CreateDirectoryRequest{ + Parent: parent, + Path: tmpdir, + Parents: true, + }) + require.NoError(t, err) + + t.Cleanup(func() { + _, err := wt.W.Sandbox.DeletePath(context.WithoutCancel(ctx), sandbox.DeletePathRequest{ + Parent: parent, + Path: tmpdir, + Recursive: true, + }) + if err != nil { + t.Logf("Unable to remove temporary sandbox directory %s: %#v", tmpdir, err) + } + }) + + f, err := filer.NewSandboxClient(wt.W, sandboxID, tmpdir) + require.NoError(t, err) + return f, tmpdir +} diff --git a/libs/filer/sandbox_client.go b/libs/filer/sandbox_client.go new file mode 100644 index 00000000000..677977e0e19 --- /dev/null +++ b/libs/filer/sandbox_client.go @@ -0,0 +1,342 @@ +package filer + +import ( + "cmp" + "context" + "errors" + "fmt" + "io" + "io/fs" + "maps" + "net/http" + "net/url" + "path" + "slices" + "strings" + "time" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/sandbox" +) + +// sandboxStatus returns the HTTP status code of err if it is (or wraps) an +// [apierr.APIError], and -1 otherwise. +// +// Sandbox filesystem errors carry the same reason and message ("sandbox +// filesystem operation failed") for unrelated conditions, and their error codes +// don't always match the status (a download of a directory is 409 with error +// code BAD_REQUEST). The status is the most specific signal, but it is still +// ambiguous, so callers stat the path to confirm a condition before mapping it. +func sandboxStatus(err error) int { + if aerr, ok := errors.AsType[*apierr.APIError](err); ok { + return aerr.StatusCode + } + return -1 +} + +// sandboxPathError maps the errors that mean the same for every operation. Callers map +// operation-specific statuses first, because those need a stat to disambiguate. +func sandboxPathError(err error, absPath string) error { + switch { + case errors.Is(err, apierr.ErrNotFound): + return fileDoesNotExistError{absPath} + case sandboxStatus(err) == http.StatusForbidden: + return permissionError{absPath, err} + default: + return err + } +} + +// sandboxFileInfo implements [fs.FileInfo] for an entry in a sandbox filesystem. +type sandboxFileInfo struct { + info sandbox.SandboxFileInfo +} + +func (i sandboxFileInfo) Name() string { + return i.info.Name +} + +func (i sandboxFileInfo) Size() int64 { + return i.info.Size +} + +// Mode reports symlinks as [fs.ModeSymlink] without following them, matching +// [os.Lstat]: the metadata and listing APIs describe the link itself, and +// reporting the link's own type keeps recursive walks from following links out +// of the tree. Read and Write still follow links, because the server does +// (per the API docs for download and upload). +// Only the permission bits of the Unix mode are kept; setuid, setgid and sticky +// use different bits in [fs.FileMode]. +func (i sandboxFileInfo) Mode() fs.FileMode { + mode := fs.FileMode(i.info.Mode) & fs.ModePerm + switch i.info.FileType { + case sandbox.SandboxFileTypeSandboxFileTypeFile: + case sandbox.SandboxFileTypeSandboxFileTypeDirectory: + mode |= fs.ModeDir + case sandbox.SandboxFileTypeSandboxFileTypeSymlink: + mode |= fs.ModeSymlink + case sandbox.SandboxFileTypeSandboxFileTypeOther: + mode |= fs.ModeIrregular + } + return mode +} + +func (i sandboxFileInfo) ModTime() time.Time { + return i.info.ModifyTime.AsTime() +} + +func (i sandboxFileInfo) IsDir() bool { + return i.info.FileType == sandbox.SandboxFileTypeSandboxFileTypeDirectory +} + +func (i sandboxFileInfo) Sys() any { + return i.info +} + +// SandboxClient implements the [Filer] interface for the filesystem of a +// single Databricks sandbox. +// +// Uploads create missing parents and deletes succeed for absent paths, so this +// client checks those paths to enforce the stricter Filer contract. +type SandboxClient struct { + api sandbox.SandboxInterface + uploadClient *client.DatabricksClient + + // parent is the sandbox resource name, "sandboxes/{sandbox_id}", that every + // file API call is scoped to. + parent string + + // File operations will be relative to this path. + root WorkspaceRootPath +} + +// NewSandboxClient returns a [Filer] for the filesystem of sandbox sandboxID, +// rooted at root, an absolute path inside the sandbox. Names are joined to root, +// and a name that would resolve outside it is rejected (see +// [WorkspaceRootPath.Join]). Per the API docs, a stopped sandbox starts +// automatically on the first call, which makes that call slow. +func NewSandboxClient(w *databricks.WorkspaceClient, sandboxID, root string) (Filer, error) { + // The ID becomes a path segment of every request URL, so it can't contain "/" or be a + // relative segment that the URL would resolve away. + if sandboxID == "" || sandboxID == "." || sandboxID == ".." || strings.Contains(sandboxID, "/") { + return nil, fmt.Errorf("invalid sandbox ID: %q", sandboxID) + } + if !path.IsAbs(root) { + return nil, fmt.Errorf("sandbox root must be absolute: %s", root) + } + uploadClient, err := client.New(w.Config) + if err != nil { + return nil, err + } + return &SandboxClient{ + api: w.Sandbox, + uploadClient: uploadClient, + parent: "sandboxes/" + url.PathEscape(sandboxID), + root: NewWorkspaceRootPath(root), + }, nil +} + +func (c *SandboxClient) Write(ctx context.Context, name string, reader io.Reader, mode ...WriteMode) error { + absPath, err := c.root.Join(name) + if err != nil { + return err + } + + // The server always creates missing parent directories, but the Filer + // contract requires fs.ErrNotExist without CreateParentDirectories. A parent + // created or removed after this check only changes which error is returned. + if !slices.Contains(mode, CreateParentDirectories) { + dir := path.Dir(absPath) + info, err := c.stat(ctx, dir) + if errors.Is(err, fs.ErrNotExist) { + return noSuchDirectoryError{dir} + } + if err != nil { + return err + } + // stat doesn't follow a final symlink, but upload does, so a symlink to a + // directory is a valid parent. A symlink to a file fails the upload instead. + if !info.IsDir() && info.Mode()&fs.ModeSymlink == 0 { + return notADirectory{dir} + } + } + + overwrite := slices.Contains(mode, OverwriteIfExists) + if !overwrite { + // Avoid streaming skipped files; overwrite=false still protects concurrent creates. + _, err = c.stat(ctx, absPath) + if err == nil { + return fileAlreadyExistsError{absPath} + } + if !errors.Is(err, fs.ErrNotExist) { + return err + } + } + headers := map[string]string{"Content-Type": "application/octet-stream"} + maps.Copy(headers, auth.WorkspaceIDHeaders(c.uploadClient.Config)) + // The pinned SDK lacks the overwrite field; its HTTP client still streams the body. + err = c.uploadClient.Do(ctx, http.MethodPut, "/api/2.0/sandbox-files/"+c.parent+"/content", + headers, map[string]any{"path": absPath, "overwrite": overwrite}, io.NopCloser(reader), nil) + if !overwrite && sandboxStatus(err) == http.StatusPreconditionFailed { + return fileAlreadyExistsError{absPath} + } + // A 409 can indicate a directory target or a parent component that is a file. + if sandboxStatus(err) == http.StatusConflict { + if info, serr := c.stat(ctx, absPath); serr == nil && info.IsDir() { + return notAFile{absPath} + } + } + return sandboxPathError(err, absPath) +} + +func (c *SandboxClient) Read(ctx context.Context, name string) (io.ReadCloser, error) { + absPath, err := c.root.Join(name) + if err != nil { + return nil, err + } + + resp, err := c.api.DownloadFile(ctx, sandbox.DownloadFileRequest{ + Parent: c.parent, + Path: absPath, + }) + if err == nil { + return resp.Contents, nil + } + + // The server returns 409 for a directory, and also when a parent path + // component is a file. Stat to tell them apart. Download follows symlinks + // but stat doesn't, so a 409 for a symlink means it points at a directory. + if sandboxStatus(err) == http.StatusConflict { + if info, serr := c.stat(ctx, absPath); serr == nil && (info.IsDir() || info.Mode()&fs.ModeSymlink != 0) { + return nil, notAFile{absPath} + } + } + + // The server returns 404 for a missing file or missing parent directory. It + // also returns 404 for an unknown sandbox, with the same message ("The + // sandbox or file was not found."), so that is reported as a missing file too. + return nil, sandboxPathError(err, absPath) +} + +func (c *SandboxClient) Delete(ctx context.Context, name string, mode ...DeleteMode) error { + absPath, err := c.root.Join(name) + if err != nil { + return err + } + + // Illegal to delete the root path. + if absPath == c.root.rootPath { + return cannotDeleteRootError{} + } + + // The server succeeds if the path is already absent, but the Filer contract + // requires fs.ErrNotExist. The stat also tells a directory from a file, which + // the error mapping below needs. + info, err := c.stat(ctx, absPath) + if err != nil { + return err + } + + // A recursive delete is a single server-side call, unlike the Files API + // filer, which walks the tree. + recursive := slices.Contains(mode, DeleteRecursively) + _, err = c.api.DeletePath(ctx, sandbox.DeletePathRequest{ + Parent: c.parent, + Path: absPath, + Recursive: recursive, + }) + + // The server returns a generic 400 for a non-empty directory without the + // recursive flag. The stat above confirmed that the path is a directory. + if !recursive && info.IsDir() && sandboxStatus(err) == http.StatusBadRequest { + return directoryNotEmptyError{absPath} + } + + return sandboxPathError(err, absPath) +} + +func (c *SandboxClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) { + absPath, err := c.root.Join(name) + if err != nil { + return nil, err + } + + // The listing is paginated (up to 1000 entries per page); this collects all pages. + infos, err := c.api.ListDirectoryContentsAll(ctx, sandbox.ListDirectoryContentsRequest{ + Parent: c.parent, + Path: absPath, + }) + if err != nil { + if errors.Is(err, apierr.ErrNotFound) { + return nil, noSuchDirectoryError{absPath} + } + + // The server returns a generic 400 when the path is a file. + if sandboxStatus(err) == http.StatusBadRequest { + if info, serr := c.stat(ctx, absPath); serr == nil && !info.IsDir() { + return nil, notADirectory{absPath} + } + } + + return nil, sandboxPathError(err, absPath) + } + + entries := make([]fs.DirEntry, len(infos)) + for i, info := range infos { + entries[i] = fs.FileInfoToDirEntry(sandboxFileInfo{info}) + } + // Sort for parity with os.ReadDir instead of relying on the server's + // documented name order. + slices.SortFunc(entries, func(a, b fs.DirEntry) int { return cmp.Compare(a.Name(), b.Name()) }) + return entries, nil +} + +func (c *SandboxClient) Mkdir(ctx context.Context, name string) error { + absPath, err := c.root.Join(name) + if err != nil { + return err + } + + _, err = c.api.CreateDirectory(ctx, sandbox.CreateDirectoryRequest{ + Parent: c.parent, + Path: absPath, + // Filer.Mkdir creates intermediate directories. With parents=false, + // the server also returns 409 if the directory already exists. + Parents: true, + }) + + // The server returns a generic 400 when a file exists at the path. + if sandboxStatus(err) == http.StatusBadRequest { + if info, serr := c.stat(ctx, absPath); serr == nil && !info.IsDir() { + return fileAlreadyExistsError{absPath} + } + } + + return sandboxPathError(err, absPath) +} + +func (c *SandboxClient) Stat(ctx context.Context, name string) (fs.FileInfo, error) { + absPath, err := c.root.Join(name) + if err != nil { + return nil, err + } + + return c.stat(ctx, absPath) +} + +// stat describes absPath without following a final symlink; see +// [sandboxFileInfo.Mode] for why that is kept. +func (c *SandboxClient) stat(ctx context.Context, absPath string) (fs.FileInfo, error) { + resp, err := c.api.GetMetadata(ctx, sandbox.GetMetadataRequest{ + Parent: c.parent, + Path: absPath, + }) + if err != nil { + return nil, sandboxPathError(err, absPath) + } + + return sandboxFileInfo{*resp.File}, nil +} diff --git a/libs/filer/sandbox_client_test.go b/libs/filer/sandbox_client_test.go new file mode 100644 index 00000000000..db0a697c2a0 --- /dev/null +++ b/libs/filer/sandbox_client_test.go @@ -0,0 +1,145 @@ +package filer_test + +import ( + "fmt" + "io" + "io/fs" + "strings" + "sync" + "testing" + + "github.com/databricks/cli/libs/filer" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newSandboxTestClient(t *testing.T, server *testserver.Server) *databricks.WorkspaceClient { + t.Helper() + w, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + // The SDK's default rate limit would throttle the concurrent writes below. + RateLimitPerSecond: 10000, + }) + require.NoError(t, err) + return w +} + +// newSandboxTestFiler returns a sandbox filer rooted at "/tmp", backed by the testserver +// fake of the sandbox file APIs, which reproduces the server semantics the filer works around. +func newSandboxTestFiler(t *testing.T) filer.Filer { + t.Helper() + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + f, err := filer.NewSandboxClient(newSandboxTestClient(t, server), testserver.TestDefaultSandboxId, "/tmp") + require.NoError(t, err) + return f +} + +func readString(t *testing.T, f filer.Filer, name string) string { + t.Helper() + r, err := f.Read(t.Context(), name) + require.NoError(t, err) + defer r.Close() + b, err := io.ReadAll(r) + require.NoError(t, err) + return string(b) +} + +func TestSandboxClientRejectsInvalidArguments(t *testing.T) { + w := &databricks.WorkspaceClient{} + for _, id := range []string{"", ".", "..", "a/b"} { + _, err := filer.NewSandboxClient(w, id, "/") + assert.ErrorContains(t, err, "invalid sandbox ID") + } + _, err := filer.NewSandboxClient(w, "id", "relative") + assert.ErrorContains(t, err, "sandbox root must be absolute") +} + +func TestSandboxClientWriteDoesNotOverwrite(t *testing.T) { + ctx := t.Context() + f := newSandboxTestFiler(t) + + require.NoError(t, f.Write(ctx, "file", strings.NewReader("one"))) + assert.ErrorIs(t, f.Write(ctx, "file", strings.NewReader("two")), fs.ErrExist) + assert.Equal(t, "one", readString(t, f, "file")) + + entries, err := f.ReadDir(ctx, ".") + require.NoError(t, err) + require.Len(t, entries, 1) + assert.Equal(t, "file", entries[0].Name()) +} + +func TestSandboxClientConcurrentExclusiveWrite(t *testing.T) { + ctx := t.Context() + f := newSandboxTestFiler(t) + + const writers = 8 + var wg sync.WaitGroup + errs := make([]error, writers) + for i := range writers { + wg.Go(func() { + errs[i] = f.Write(ctx, "file", strings.NewReader(fmt.Sprintf("writer-%d", i))) + }) + } + wg.Wait() + + winner := -1 + for i, err := range errs { + if err == nil { + assert.Equal(t, -1, winner, "more than one writer created the file") + winner = i + continue + } + assert.ErrorIs(t, err, fs.ErrExist) + } + require.NotEqual(t, -1, winner, "no writer created the file") + assert.Equal(t, fmt.Sprintf("writer-%d", winner), readString(t, f, "file")) +} + +func TestSandboxClientWriteMissingParent(t *testing.T) { + // The upload API always creates missing parents. + err := newSandboxTestFiler(t).Write(t.Context(), "missing/file", strings.NewReader("x")) + assert.ErrorIs(t, err, fs.ErrNotExist) +} + +func TestSandboxClientDeleteMissing(t *testing.T) { + // The delete API succeeds if the path is absent. + err := newSandboxTestFiler(t).Delete(t.Context(), "missing") + assert.ErrorIs(t, err, fs.ErrNotExist) +} + +func TestSandboxClientGenericErrors(t *testing.T) { + // The server reports each of these with a generic status, which the filer + // disambiguates with a stat. + ctx := t.Context() + f := newSandboxTestFiler(t) + require.NoError(t, f.Write(ctx, "dir/file", strings.NewReader("x"), filer.CreateParentDirectories)) + + _, err := f.Read(ctx, "dir") + assert.ErrorIs(t, err, fs.ErrInvalid, "read of a directory") + _, err = f.ReadDir(ctx, "dir/file") + assert.ErrorIs(t, err, fs.ErrInvalid, "list of a file") + assert.ErrorIs(t, f.Delete(ctx, "dir"), fs.ErrInvalid, "delete of a non-empty directory") + assert.ErrorIs(t, f.Mkdir(ctx, "dir/file"), fs.ErrExist, "mkdir onto a file") + err = f.Write(ctx, "dir", strings.NewReader("x"), filer.OverwriteIfExists) + assert.ErrorIs(t, err, fs.ErrInvalid, "overwrite of a directory") +} + +func TestSandboxClientPermissionDenied(t *testing.T) { + // The fake doesn't model filesystem permissions, so stub the response a real + // sandbox returns for a path the sandbox user can't access. + server := testserver.New(t) + server.Handle("GET", "/api/2.0/sandbox-files/sandboxes/{sandbox_id}/paths:describe", func(req testserver.Request) any { + return testserver.Response{StatusCode: 403, Body: map[string]string{ + "error_code": "PERMISSION_DENIED", "message": "Access to the requested operation is denied.", + }} + }) + f, err := filer.NewSandboxClient(newSandboxTestClient(t, server), testserver.TestDefaultSandboxId, "/") + require.NoError(t, err) + + _, err = f.Stat(t.Context(), "/root/x") + assert.ErrorIs(t, err, fs.ErrPermission) +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index c48ebb04df2..d937dec159b 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -213,6 +213,9 @@ type FakeWorkspace struct { // dbfs is the DBFS namespace, created on first use (see dbfs.go). dbfs *fakeDbfs + // sandboxFiles holds the filesystem of each sandbox, created on first use (see sandbox_files.go). + sandboxFiles map[string]*fakeSandboxFs + // snapshots records immutable-snapshot content paths. Breaking the glass on a // recorded path flips it dirty for good, as in the real API. snapshots map[string]*snapshotRecord diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index e5f4c37a16b..b018daa95d5 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -465,6 +465,30 @@ func AddDefaultHandlers(server *Server) { return data }) + // Sandbox file APIs: + sandboxFiles := "/api/2.0/sandbox-files/sandboxes/{sandbox_id}" + server.Handle("PUT", sandboxFiles+"/content", func(req Request) any { + return req.Workspace.SandboxFilesUpload(req, req.Vars["sandbox_id"]) + }) + server.Handle("GET", sandboxFiles+"/content", func(req Request) any { + return req.Workspace.SandboxFilesDownload(req, req.Vars["sandbox_id"]) + }) + server.Handle("GET", sandboxFiles+"/paths:describe", func(req Request) any { + return req.Workspace.SandboxFilesGetMetadata(req, req.Vars["sandbox_id"]) + }) + server.Handle("GET", sandboxFiles+"/directory-entries", func(req Request) any { + return req.Workspace.SandboxFilesList(req, req.Vars["sandbox_id"]) + }) + server.Handle("POST", sandboxFiles+"/directories:make", func(req Request) any { + return req.Workspace.SandboxFilesMkdir(req, req.Vars["sandbox_id"]) + }) + server.Handle("POST", sandboxFiles+"/paths:remove", func(req Request) any { + return req.Workspace.SandboxFilesDelete(req, req.Vars["sandbox_id"]) + }) + server.Handle("POST", sandboxFiles+"/paths:rename", func(req Request) any { + return req.Workspace.SandboxFilesRename(req, req.Vars["sandbox_id"]) + }) + // DBFS (dbfs:/ paths): server.Handle("GET", "/api/2.0/dbfs/get-status", func(req Request) any { return req.Workspace.DbfsGetStatus(req) diff --git a/libs/testserver/sandbox_files.go b/libs/testserver/sandbox_files.go new file mode 100644 index 00000000000..82ac475ca1a --- /dev/null +++ b/libs/testserver/sandbox_files.go @@ -0,0 +1,422 @@ +package testserver + +import ( + "encoding/json" + "maps" + "net/http" + "path" + "slices" + "strconv" + "strings" + "time" + + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/sandbox" +) + +// TestDefaultSandboxId is the sandbox that acceptance tests address locally (TEST_SANDBOX_ID). +const TestDefaultSandboxId = "test-sandbox-1234" + +// The sandbox file APIs report most filesystem failures with this one reason and message, +// whatever the condition; only the HTTP status differs. The download and upload endpoints +// use their own messages and no reason. +const ( + sandboxFilesFsReason = "SANDBOX_FILES_FILESYSTEM_ERROR" + sandboxFilesFsMessage = "sandbox filesystem operation failed" + sandboxFilesNotFound = "The sandbox or file was not found." + sandboxFilesWrongState = "The sandbox or file is not in the required state." + sandboxFilesBadUpload = "Invalid file request. Use an absolute UTF-8 path and supported headers." + + sandboxFilesMaxPageSize = 1000 + sandboxFilesDirSize = 4096 + sandboxFilesOwner = "sandbox-agent" +) + +// fakeSandboxFs is the filesystem of one sandbox, keyed by absolute path. It models the +// server's semantics, which are looser than the Filer contract: uploads create parents, +// and deletes of absent paths succeed. The filer must close those gaps itself, so +// the fake must not. +type fakeSandboxFs struct { + entries map[string]*sandboxFsEntry +} + +type sandboxFsEntry struct { + isDir bool + data []byte + mode int + permissions string + owner string + modTime time.Time +} + +// sandboxFs returns the filesystem of sandboxID, creating it on first use. The fake doesn't +// model sandbox lifecycle, so every sandbox ID exists; a real server returns 404 for unknown IDs. +func (s *FakeWorkspace) sandboxFs(sandboxID string) *fakeSandboxFs { + if s.sandboxFiles == nil { + s.sandboxFiles = map[string]*fakeSandboxFs{} + } + sfs, ok := s.sandboxFiles[sandboxID] + if !ok { + now := time.Now() + sfs = &fakeSandboxFs{entries: map[string]*sandboxFsEntry{ + "/": {isDir: true, mode: 0o755, permissions: "drwxr-xr-x", owner: "root", modTime: now}, + "/tmp": {isDir: true, mode: 0o1777, permissions: "drwxrwxrwt", owner: "root", modTime: now}, + }} + s.sandboxFiles[sandboxID] = sfs + } + return sfs +} + +func newSandboxFileEntry(data []byte) *sandboxFsEntry { + return &sandboxFsEntry{data: data, mode: 0o644, permissions: "-rw-r--r--", owner: sandboxFilesOwner, modTime: time.Now()} +} + +func newSandboxDirEntry() *sandboxFsEntry { + return &sandboxFsEntry{isDir: true, mode: 0o755, permissions: "drwxr-xr-x", owner: sandboxFilesOwner, modTime: time.Now()} +} + +func sandboxFilesError(statusCode int, errorCode, message string) Response { + return Response{ + StatusCode: statusCode, + Body: map[string]string{"error_code": errorCode, "message": message}, + } +} + +// sandboxFsError is the generic filesystem error, which carries an ErrorInfo reason. +func sandboxFsError(statusCode int, errorCode string) Response { + return Response{ + StatusCode: statusCode, + Body: map[string]any{ + "error_code": errorCode, + "message": sandboxFilesFsMessage, + "details": []map[string]any{{ + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": sandboxFilesFsReason, + "domain": "databricks.sandbox.files", + }}, + }, + } +} + +// sandboxFilesPath validates and cleans an absolute path argument named field. +func sandboxFilesPath(field, p string) (string, *Response) { + if !strings.HasPrefix(p, "/") { + resp := sandboxFilesError(400, "INVALID_PARAMETER_VALUE", field+" must be an absolute UTF-8 path") + return "", &resp + } + // A ".." segment is rejected before the request reaches the service, with a bare 400. + if slices.Contains(strings.Split(p, "/"), "..") { + resp := Response{StatusCode: 400, Body: "Bad Request"} + return "", &resp + } + return path.Clean(p), nil +} + +// lookup returns the entry at p and, if p is absent, whether a parent component is a file. +func (sfs *fakeSandboxFs) lookup(p string) (entry *sandboxFsEntry, underFile bool) { + if e, ok := sfs.entries[p]; ok { + return e, false + } + for dir := path.Dir(p); dir != "/"; dir = path.Dir(dir) { + if e, ok := sfs.entries[dir]; ok { + return nil, !e.isDir + } + } + return nil, false +} + +func (sfs *fakeSandboxFs) isEmptyDir(p string) bool { + prefix := strings.TrimSuffix(p, "/") + "/" + for name := range sfs.entries { + if strings.HasPrefix(name, prefix) { + return false + } + } + return true +} + +// mkdirAll creates p and its missing parents. It reports false if p or a parent is a file. +func (sfs *fakeSandboxFs) mkdirAll(p string) bool { + var missing []string + for dir := p; ; dir = path.Dir(dir) { + if e, ok := sfs.entries[dir]; ok { + if !e.isDir { + return false + } + break + } + missing = append(missing, dir) + // A recursive delete of "/" removes the root entry too. + if dir == "/" { + break + } + } + for _, dir := range missing { + sfs.entries[dir] = newSandboxDirEntry() + } + return true +} + +func (e *sandboxFsEntry) info(p string) sandbox.SandboxFileInfo { + info := sandbox.SandboxFileInfo{ + FileType: sandbox.SandboxFileTypeSandboxFileTypeFile, + Group: e.owner, + Mode: e.mode, + ModifyTime: sdktime.New(e.modTime), + Name: path.Base(p), + Owner: e.owner, + Path: p, + Permissions: e.permissions, + Size: int64(len(e.data)), + // The server sends these even when zero (an empty file has size 0). + ForceSendFields: []string{"FileType", "Group", "Mode", "Name", "Owner", "Path", "Permissions", "Size"}, + } + if e.isDir { + info.FileType = sandbox.SandboxFileTypeSandboxFileTypeDirectory + info.Size = sandboxFilesDirSize + } + return info +} + +// SandboxFilesUpload creates or replaces a file, creating missing parents. +func (s *FakeWorkspace) SandboxFilesUpload(req Request, sandboxID string) Response { + query := req.URL.Query() + for name := range query { + if name != "path" && name != "overwrite" { + return sandboxFilesError(400, "INVALID_PARAMETER_VALUE", sandboxFilesBadUpload) + } + } + overwrite := true + if values, ok := query["overwrite"]; ok { + if len(values) != 1 || (values[0] != "true" && values[0] != "false") { + return sandboxFilesError(400, "INVALID_PARAMETER_VALUE", sandboxFilesBadUpload) + } + overwrite = values[0] == "true" + } + p, errResp := sandboxFilesPath("path", query.Get("path")) + if errResp != nil { + return *errResp + } + + defer s.LockUnlock()() + sfs := s.sandboxFs(sandboxID) + existing, underFile := sfs.lookup(p) + if !overwrite && existing != nil { + return sandboxFilesError(http.StatusPreconditionFailed, "BAD_REQUEST", "The file does not satisfy the request preconditions.") + } + if underFile || (existing != nil && existing.isDir) { + return sandboxFilesError(409, "BAD_REQUEST", sandboxFilesWrongState) + } + if !sfs.mkdirAll(path.Dir(p)) { + return sandboxFilesError(409, "BAD_REQUEST", sandboxFilesWrongState) + } + sfs.entries[p] = newSandboxFileEntry(req.Body) + if existing != nil { + return Response{StatusCode: http.StatusNoContent} + } + return Response{StatusCode: http.StatusCreated} +} + +// SandboxFilesDownload returns the contents of a regular file. +func (s *FakeWorkspace) SandboxFilesDownload(req Request, sandboxID string) Response { + p, errResp := sandboxFilesPath("path", req.URL.Query().Get("path")) + if errResp != nil { + return *errResp + } + + defer s.LockUnlock()() + entry, underFile := s.sandboxFs(sandboxID).lookup(p) + if underFile || (entry != nil && entry.isDir) { + return sandboxFilesError(409, "BAD_REQUEST", sandboxFilesWrongState) + } + if entry == nil { + return sandboxFilesError(404, "NOT_FOUND", sandboxFilesNotFound) + } + return Response{ + Headers: http.Header{"Content-Type": {"application/octet-stream"}}, + Body: slices.Clone(entry.data), + } +} + +// SandboxFilesGetMetadata describes a path. +func (s *FakeWorkspace) SandboxFilesGetMetadata(req Request, sandboxID string) Response { + p, errResp := sandboxFilesPath("path", req.URL.Query().Get("path")) + if errResp != nil { + return *errResp + } + + defer s.LockUnlock()() + entry, underFile := s.sandboxFs(sandboxID).lookup(p) + if underFile { + return sandboxFsError(400, "BAD_REQUEST") + } + if entry == nil { + return sandboxFsError(404, "NOT_FOUND") + } + info := entry.info(p) + return Response{Body: sandbox.GetMetadataResponse{File: &info}} +} + +// SandboxFilesList lists a directory's children in name order, one page at a time. +func (s *FakeWorkspace) SandboxFilesList(req Request, sandboxID string) Response { + query := req.URL.Query() + p, errResp := sandboxFilesPath("path", query.Get("path")) + if errResp != nil { + return *errResp + } + pageSize, _ := strconv.Atoi(query.Get("page_size")) + if pageSize <= 0 || pageSize > sandboxFilesMaxPageSize { + pageSize = sandboxFilesMaxPageSize + } + // The page token is an offset into the sorted listing; real tokens are opaque. + offset, _ := strconv.Atoi(query.Get("page_token")) + + defer s.LockUnlock()() + sfs := s.sandboxFs(sandboxID) + entry, underFile := sfs.lookup(p) + if underFile || (entry != nil && !entry.isDir) { + return sandboxFsError(400, "BAD_REQUEST") + } + if entry == nil { + return sandboxFsError(404, "NOT_FOUND") + } + + var names []string + for name := range sfs.entries { + if name != "/" && path.Dir(name) == p { + names = append(names, name) + } + } + // Byte order of the paths is UTF-8 name order, since all share the same parent. + slices.Sort(names) + + resp := sandbox.ListDirectoryContentsResponse{} + end := min(offset+pageSize, len(names)) + for _, name := range names[min(offset, end):end] { + resp.Contents = append(resp.Contents, sfs.entries[name].info(name)) + } + if end < len(names) { + resp.NextPageToken = strconv.Itoa(end) + } + return Response{Body: resp} +} + +// SandboxFilesMkdir creates a directory, with missing parents unless parents is false. +func (s *FakeWorkspace) SandboxFilesMkdir(req Request, sandboxID string) Response { + var body struct { + Path string `json:"path"` + Parents *bool `json:"parents"` + } + if err := json.Unmarshal(req.Body, &body); err != nil { + return sandboxFilesError(400, "INVALID_PARAMETER_VALUE", err.Error()) + } + p, errResp := sandboxFilesPath("path", body.Path) + if errResp != nil { + return *errResp + } + + defer s.LockUnlock()() + sfs := s.sandboxFs(sandboxID) + entry, underFile := sfs.lookup(p) + // An omitted parents defaults to true, per the API docs; the SDK omits false + // unless it is in ForceSendFields. With parents, an existing directory is fine. + if body.Parents == nil || *body.Parents { + if !sfs.mkdirAll(p) { + return sandboxFsError(400, "BAD_REQUEST") + } + return Response{Body: sandbox.CreateDirectoryResponse{}} + } + + switch { + case entry != nil: + return sandboxFsError(409, "ALREADY_EXISTS") + case underFile: + return sandboxFsError(400, "BAD_REQUEST") + case sfs.entries[path.Dir(p)] == nil: + return sandboxFsError(404, "NOT_FOUND") + } + sfs.entries[p] = newSandboxDirEntry() + return Response{Body: sandbox.CreateDirectoryResponse{}} +} + +// SandboxFilesDelete deletes a path. It succeeds if the path is absent. +func (s *FakeWorkspace) SandboxFilesDelete(req Request, sandboxID string) Response { + var body sandbox.DeletePathRequest + if err := json.Unmarshal(req.Body, &body); err != nil { + return sandboxFilesError(400, "INVALID_PARAMETER_VALUE", err.Error()) + } + p, errResp := sandboxFilesPath("path", body.Path) + if errResp != nil { + return *errResp + } + + defer s.LockUnlock()() + sfs := s.sandboxFs(sandboxID) + entry, _ := sfs.lookup(p) + if entry == nil { + return Response{Body: sandbox.DeletePathResponse{}} + } + if entry.isDir && !body.Recursive && !sfs.isEmptyDir(p) { + return sandboxFsError(400, "BAD_REQUEST") + } + prefix := p + "/" + for name := range sfs.entries { + if name == p || strings.HasPrefix(name, prefix) { + delete(sfs.entries, name) + } + } + return Response{Body: sandbox.DeletePathResponse{}} +} + +// SandboxFilesRename moves a path. The destination must be absent: the check and the move +// happen under one lock, so of several concurrent renames onto the same destination exactly +// one succeeds, as on the real server. +func (s *FakeWorkspace) SandboxFilesRename(req Request, sandboxID string) Response { + var body sandbox.RenamePathRequest + if err := json.Unmarshal(req.Body, &body); err != nil { + return sandboxFilesError(400, "INVALID_PARAMETER_VALUE", err.Error()) + } + src, errResp := sandboxFilesPath("source", body.Source) + if errResp != nil { + return *errResp + } + // The server rejects a destination with a trailing slash instead of cleaning it. + if strings.HasSuffix(body.Destination, "/") && body.Destination != "/" { + return sandboxFsError(400, "BAD_REQUEST") + } + dst, errResp := sandboxFilesPath("destination", body.Destination) + if errResp != nil { + return *errResp + } + + defer s.LockUnlock()() + sfs := s.sandboxFs(sandboxID) + srcEntry, _ := sfs.lookup(src) + dstEntry, dstUnderFile := sfs.lookup(dst) + switch { + case srcEntry == nil: + return sandboxFsError(404, "NOT_FOUND") + case dstEntry != nil: + return sandboxFsError(409, "ALREADY_EXISTS") + case dstUnderFile, strings.HasPrefix(dst, src+"/"): + return sandboxFsError(400, "BAD_REQUEST") + case sfs.entries[path.Dir(dst)] == nil: + return sandboxFsError(404, "NOT_FOUND") + } + + moved := map[string]*sandboxFsEntry{} + prefix := src + "/" + for name, entry := range sfs.entries { + switch { + case name == src: + moved[dst] = entry + case strings.HasPrefix(name, prefix): + moved[dst+"/"+strings.TrimPrefix(name, prefix)] = entry + default: + continue + } + delete(sfs.entries, name) + } + maps.Copy(sfs.entries, moved) + return Response{Body: sandbox.RenamePathResponse{}} +} diff --git a/libs/testserver/sandbox_files_test.go b/libs/testserver/sandbox_files_test.go new file mode 100644 index 00000000000..8e5d69f6b21 --- /dev/null +++ b/libs/testserver/sandbox_files_test.go @@ -0,0 +1,121 @@ +package testserver_test + +import ( + "fmt" + "io" + "net/http" + "strings" + "sync" + "testing" + + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/config" + "github.com/databricks/databricks-sdk-go/service/sandbox" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const sandboxParent = "sandboxes/" + testserver.TestDefaultSandboxId + +func newSandboxAPI(t *testing.T) sandbox.SandboxInterface { + t.Helper() + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + w, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: server.URL, + Token: "testtoken", + RateLimitPerSecond: 10000, + }) + require.NoError(t, err) + return w.Sandbox +} + +func upload(t *testing.T, api sandbox.SandboxInterface, path, contents string) error { + t.Helper() + return api.UploadFile(t.Context(), sandbox.UploadFileRequest{ + Parent: sandboxParent, + Path: path, + Contents: io.NopCloser(strings.NewReader(contents)), + }) +} + +func TestSandboxFilesListPagination(t *testing.T) { + api := newSandboxAPI(t) + for _, name := range []string{"c", "a", "e", "b", "d"} { + require.NoError(t, upload(t, api, "/tmp/dir/"+name, "")) + } + + resp, err := api.ListDirectoryContents(t.Context(), sandbox.ListDirectoryContentsRequest{ + Parent: sandboxParent, + Path: "/tmp/dir", + PageSize: 2, + }).Next(t.Context()) + require.NoError(t, err) + assert.Equal(t, "a", resp.Name) + + all, err := api.ListDirectoryContentsAll(t.Context(), sandbox.ListDirectoryContentsRequest{ + Parent: sandboxParent, + Path: "/tmp/dir", + PageSize: 2, + }) + require.NoError(t, err) + var names []string + for _, info := range all { + names = append(names, info.Name) + } + assert.Equal(t, []string{"a", "b", "c", "d", "e"}, names) +} + +func TestSandboxFilesUploadRejectsUnknownParameters(t *testing.T) { + server := testserver.New(t) + testserver.AddDefaultHandlers(server) + c, err := client.New(&config.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + + err = c.Do(t.Context(), http.MethodPut, "/api/2.0/sandbox-files/"+sandboxParent+"/content", + map[string]string{"Content-Type": "application/octet-stream"}, + map[string]any{"path": "/tmp/file", "unknown": false}, + io.NopCloser(strings.NewReader("")), nil) + assert.ErrorIs(t, err, apierr.ErrInvalidParameterValue) +} + +func TestSandboxFilesUploadAfterDeletingRoot(t *testing.T) { + api := newSandboxAPI(t) + _, err := api.DeletePath(t.Context(), sandbox.DeletePathRequest{Parent: sandboxParent, Path: "/", Recursive: true}) + require.NoError(t, err) + require.NoError(t, upload(t, api, "/tmp/file", "")) +} + +func TestSandboxFilesRenameIsExclusive(t *testing.T) { + api := newSandboxAPI(t) + const n = 8 + for i := range n { + require.NoError(t, upload(t, api, fmt.Sprintf("/tmp/src%d", i), fmt.Sprintf("writer-%d", i))) + } + + var wg sync.WaitGroup + errs := make([]error, n) + for i := range n { + wg.Go(func() { + _, errs[i] = api.RenamePath(t.Context(), sandbox.RenamePathRequest{ + Parent: sandboxParent, + Source: fmt.Sprintf("/tmp/src%d", i), + Destination: "/tmp/dst", + }) + }) + } + wg.Wait() + + wins := 0 + for _, err := range errs { + if err == nil { + wins++ + continue + } + assert.ErrorIs(t, err, apierr.ErrAlreadyExists) + } + assert.Equal(t, 1, wins) +}