From 0235ad233259486806dada060cb03457fe3e4db7 Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Sun, 13 Sep 2026 08:14:38 +0000 Subject: [PATCH] feat(gitarchive): add Git archive support --- Makefile | 6 + cmd/pinner/main.go | 44 ++- go.mod | 21 +- go.sum | 40 +++ internal/cli/command_getter.go | 6 + internal/cli/command_registration_test.go | 21 ++ internal/cli/git.go | 42 +++ internal/cli/git_doctor.go | 131 +++++++ internal/cli/git_ls.go | 62 ++++ internal/cli/git_share.go | 124 +++++++ internal/cli/git_show.go | 134 ++++++++ internal/cli/git_status.go | 137 ++++++++ internal/cli/git_unwatch.go | 93 +++++ internal/cli/git_watch.go | 212 ++++++++++++ internal/cli/gitremote/dispatch.go | 62 ++++ internal/cli/gitremote/dispatch_test.go | 68 ++++ internal/cli/gitremote/gitops.go | 125 +++++++ internal/cli/gitremote/gitops_test.go | 66 ++++ internal/cli/gitremote/gogit_store.go | 74 ++++ internal/cli/gitremote/integration_test.go | 123 +++++++ internal/cli/gitremote/protocol.go | 319 ++++++++++++++++++ internal/cli/gitremote/protocol_test.go | 184 ++++++++++ internal/cli/gitremote/publish.go | 98 ++++++ internal/cli/gitremote/publish_test.go | 80 +++++ internal/cli/gitremote/replay.go | 40 +++ internal/cli/gitremote/session_store.go | 86 +++++ internal/cli/gitremote/session_store_sia.go | 255 ++++++++++++++ .../cli/gitremote/session_store_sia_test.go | 252 ++++++++++++++ internal/cli/gitremote/session_store_stub.go | 33 ++ internal/cli/gitremote/share_store.go | 271 +++++++++++++++ internal/cli/gitremote/share_store_test.go | 197 +++++++++++ internal/cli/gitremote/show.go | 231 +++++++++++++ internal/cli/gitremote/show_test.go | 105 ++++++ internal/cli/gitremote/store.go | 80 +++++ internal/cli/root.go | 1 + internal/cli/testutils.go | 42 ++- internal/core/gitarchive/db.go | 70 ++++ internal/core/gitarchive/dbg_test.go | 1 + internal/core/gitarchive/doctor.go | 88 +++++ internal/core/gitarchive/doctor_test.go | 74 ++++ internal/core/gitarchive/download.go | 76 +++++ internal/core/gitarchive/download_test.go | 73 ++++ internal/core/gitarchive/gitarchive_test.go | 147 ++++++++ internal/core/gitarchive/hook.go | 139 ++++++++ internal/core/gitarchive/hook_test.go | 85 +++++ internal/core/gitarchive/lineage.go | 104 ++++++ internal/core/gitarchive/lineage_test.go | 72 ++++ internal/core/gitarchive/local.go | 113 +++++++ internal/core/gitarchive/local_test.go | 151 +++++++++ internal/core/gitarchive/objectkey.go | 20 ++ internal/core/gitarchive/repo.go | 136 ++++++++ internal/core/gitarchive/repo_test.go | 91 +++++ internal/core/gitarchive/scan.go | 114 +++++++ internal/core/gitarchive/sdk.go | 48 +++ internal/core/gitarchive/sdk_fake_test.go | 119 +++++++ internal/core/gitarchive/session.go | 129 +++++++ internal/core/gitarchive/session_test.go | 90 +++++ internal/core/gitarchive/share.go | 241 +++++++++++++ internal/core/gitarchive/share_test.go | 103 ++++++ internal/core/gitarchive/upload.go | 95 ++++++ internal/core/gitarchive/upload_test.go | 92 +++++ internal/core/objmeta/objmeta.go | 206 +++++++++++ internal/core/objmeta/objmeta_test.go | 131 +++++++ 63 files changed, 6656 insertions(+), 17 deletions(-) create mode 100644 internal/cli/git.go create mode 100644 internal/cli/git_doctor.go create mode 100644 internal/cli/git_ls.go create mode 100644 internal/cli/git_share.go create mode 100644 internal/cli/git_show.go create mode 100644 internal/cli/git_status.go create mode 100644 internal/cli/git_unwatch.go create mode 100644 internal/cli/git_watch.go create mode 100644 internal/cli/gitremote/dispatch.go create mode 100644 internal/cli/gitremote/dispatch_test.go create mode 100644 internal/cli/gitremote/gitops.go create mode 100644 internal/cli/gitremote/gitops_test.go create mode 100644 internal/cli/gitremote/gogit_store.go create mode 100644 internal/cli/gitremote/integration_test.go create mode 100644 internal/cli/gitremote/protocol.go create mode 100644 internal/cli/gitremote/protocol_test.go create mode 100644 internal/cli/gitremote/publish.go create mode 100644 internal/cli/gitremote/publish_test.go create mode 100644 internal/cli/gitremote/replay.go create mode 100644 internal/cli/gitremote/session_store.go create mode 100644 internal/cli/gitremote/session_store_sia.go create mode 100644 internal/cli/gitremote/session_store_sia_test.go create mode 100644 internal/cli/gitremote/session_store_stub.go create mode 100644 internal/cli/gitremote/share_store.go create mode 100644 internal/cli/gitremote/share_store_test.go create mode 100644 internal/cli/gitremote/show.go create mode 100644 internal/cli/gitremote/show_test.go create mode 100644 internal/cli/gitremote/store.go create mode 100644 internal/core/gitarchive/db.go create mode 100644 internal/core/gitarchive/dbg_test.go create mode 100644 internal/core/gitarchive/doctor.go create mode 100644 internal/core/gitarchive/doctor_test.go create mode 100644 internal/core/gitarchive/download.go create mode 100644 internal/core/gitarchive/download_test.go create mode 100644 internal/core/gitarchive/gitarchive_test.go create mode 100644 internal/core/gitarchive/hook.go create mode 100644 internal/core/gitarchive/hook_test.go create mode 100644 internal/core/gitarchive/lineage.go create mode 100644 internal/core/gitarchive/lineage_test.go create mode 100644 internal/core/gitarchive/local.go create mode 100644 internal/core/gitarchive/local_test.go create mode 100644 internal/core/gitarchive/objectkey.go create mode 100644 internal/core/gitarchive/repo.go create mode 100644 internal/core/gitarchive/repo_test.go create mode 100644 internal/core/gitarchive/scan.go create mode 100644 internal/core/gitarchive/sdk.go create mode 100644 internal/core/gitarchive/sdk_fake_test.go create mode 100644 internal/core/gitarchive/session.go create mode 100644 internal/core/gitarchive/session_test.go create mode 100644 internal/core/gitarchive/share.go create mode 100644 internal/core/gitarchive/share_test.go create mode 100644 internal/core/gitarchive/upload.go create mode 100644 internal/core/gitarchive/upload_test.go create mode 100644 internal/core/objmeta/objmeta.go create mode 100644 internal/core/objmeta/objmeta_test.go diff --git a/Makefile b/Makefile index 1a7b0072..5f4acb92 100644 --- a/Makefile +++ b/Makefile @@ -75,6 +75,12 @@ build: assets install: assets CGO_ENABLED=1 go install -tags="$(TAGS)" -ldflags="$(LDFLAGS)" ./cmd/pinner + # Install the git-remote-pinner helper as a symlink to the same pinner + # binary (argv-0 identity dispatch — no second executable). + GOBIN_DIR="$$(go env GOBIN 2>/dev/null || printf '%s' "$$(go env GOPATH)/bin")"; \ + [ -n "$$GOBIN_DIR" ] || GOBIN_DIR="$$(go env GOPATH)/bin"; \ + ln -sf pinner "$$GOBIN_DIR/git-remote-pinner"; \ + echo "installed $$GOBIN_DIR/pinner and $$GOBIN_DIR/git-remote-pinner -> pinner" test: assets go test -tags "$(TAGS)" ./... diff --git a/cmd/pinner/main.go b/cmd/pinner/main.go index f97f059e..72b18c89 100644 --- a/cmd/pinner/main.go +++ b/cmd/pinner/main.go @@ -8,11 +8,49 @@ import ( "os" "go.lumeweb.com/pinner-cli/internal/cli" + "go.lumeweb.com/pinner-cli/internal/cli/gitremote" ) func main() { - if err := cli.Run(context.Background(), os.Args); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) + os.Exit(dispatch(context.Background(), os.Args, os.Stdin, os.Stdout, os.Stderr)) +} + +// dispatch routes the single `pinner` binary based on the basename it was +// invoked through (argv-0 identity dispatch — see gitremote.IsRemoteHelperInvocation). +// +// - invoked as `git-remote-pinner` (the symlink to this binary): run the Git +// remote-helper protocol, bypassing urfave CLI parsing entirely; +// - invoked as `pinner` (or anything else): run the normal CLI command tree. +// +// Both paths share the gitarchive Store/session packages, but protocol dispatch +// is isolated here at the entry point. The normal CLI path never reads Git helper +// stdin and the helper path never enters the command tree. +func dispatch(ctx context.Context, args []string, in, out, errw *os.File) int { + if gitremote.IsRemoteHelperInvocation(args[0]) { + // argv[1] is the remote URL (e.g. `pinner::` for the account + // archive, or `pinner::share/` for a profile-less shared clone). + // NewStoreFromRemote routes between the profile-backed SessionStore and + // the profile-less ShareStore accordingly. + remoteURL := "" + if len(args) >= 2 { + remoteURL = args[1] + } + store, err := gitremote.NewStoreFromRemote(ctx, remoteURL) + if err != nil { + fmt.Fprintf(errw, "git-remote-pinner: %v\n", err) + return 1 + } + defer store.Close() + if err := gitremote.Run(ctx, store, in, out, errw); err != nil { + fmt.Fprintf(errw, "git-remote-pinner: %v\n", err) + return 1 + } + return 0 + } + + if err := cli.Run(ctx, args); err != nil { + fmt.Fprintf(errw, "Error: %v\n", err) + return 1 } + return 0 } diff --git a/go.mod b/go.mod index c7bfe14a..fa78aed0 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/docker/go-units v0.5.0 github.com/gammazero/workerpool v1.2.1 github.com/ggwhite/go-masker v1.1.0 + github.com/go-git/go-git/v5 v5.19.2 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/invopop/jsonschema v0.14.0 @@ -50,9 +51,11 @@ require ( go.lumeweb.com/tunneler/cloudflare v0.0.0-20260907123602-56944ca7aeae go.lumeweb.com/tunneler/ngrok v0.0.0-20260907123602-56944ca7aeae go.sia.tech/core v0.21.7 + go.sia.tech/siastorage v0.2.1 go.uber.org/zap v1.28.0 golang.org/x/sys v0.47.0 gopkg.in/yaml.v3 v3.0.1 + gorm.io/datatypes v1.2.7 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.2 ) @@ -60,9 +63,12 @@ require ( require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/schedule v0.1.0 // indirect + dario.cat/mergo v1.0.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect github.com/Jorropo/jsync v1.0.1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Oudwins/zog v0.23.0 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/STARRY-S/zip v0.2.3 // indirect github.com/andybalholm/brotli v1.2.3 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect @@ -81,6 +87,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/cloudflare/circl v1.6.3 // indirect github.com/cloudflare/cloudflared v0.0.0-20260903222438-2253eeeb25a4 // indirect github.com/cloudwego/base64x v0.1.7 // indirect github.com/containerd/console v1.0.5 // indirect @@ -89,12 +96,14 @@ require ( github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf // indirect + github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/dunglas/httpsfv v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.2 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/gabriel-vasile/mimetype v1.4.15 // indirect @@ -105,6 +114,8 @@ require ( github.com/ghodss/yaml v1.0.0 // indirect github.com/go-chi/chi/v5 v5.3.2 // indirect github.com/go-chi/cors v1.2.2 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -117,6 +128,7 @@ require ( github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.4.0 // indirect github.com/goccy/go-json v0.10.6 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.2.0 // indirect github.com/google/gopacket v1.1.19 // indirect @@ -143,11 +155,13 @@ require ( github.com/ipfs/go-unixfsnode v1.10.6 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect github.com/ipld/go-ipld-prime v0.24.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.19.2 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect @@ -206,6 +220,7 @@ require ( github.com/pb33f/ordered-map/v2 v2.3.1 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pierrec/lz4/v4 v4.1.29 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/polydawn/refmt v0.90.0 // indirect github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect @@ -222,9 +237,11 @@ require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect github.com/segmentio/asm v1.2.1 // indirect github.com/segmentio/encoding v0.5.4 // indirect + github.com/sergi/go-diff v1.4.0 // indirect github.com/sethvargo/go-retry v0.4.0 // indirect github.com/shirou/gopsutil/v4 v4.26.7 // indirect github.com/shopspring/decimal v1.4.0 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect github.com/sorairolake/lzip-go v0.3.8 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/speakeasy-api/jsonpath v0.6.3 // indirect @@ -249,6 +266,7 @@ require ( github.com/whyrusleeping/cbor-gen v0.3.1 // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xo/terminfo v1.0.0 // indirect github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect @@ -277,7 +295,6 @@ require ( go.sia.tech/indexd v0.4.4 // indirect go.sia.tech/jape v0.14.2 // indirect go.sia.tech/mux v1.5.3 // indirect - go.sia.tech/siastorage v0.2.1 // indirect go.uber.org/dig v1.19.0 // indirect go.uber.org/fx v1.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect @@ -304,8 +321,8 @@ require ( google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gorm.io/datatypes v1.2.7 // indirect gorm.io/driver/mysql v1.6.0 // indirect lukechampine.com/blake3 v1.4.1 // indirect lukechampine.com/frand v1.5.1 // indirect diff --git a/go.sum b/go.sum index 2ba8062a..d9c78b32 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ atomicgo.dev/keyboard v0.2.10 h1:v7mvUKUZLHIggxULEIuWbT+WkkyQSgdbA201EziAhHU= atomicgo.dev/keyboard v0.2.10/go.mod h1:ap/z5ilnhLqYq852m6kPeTq5Z6aESGWu5mzRpJlC6aI= atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs= atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5 h1:JA0fFr+kxpqTdxR9LOBiTWpGNchqmkcsgmdeJZRclZ0= filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= @@ -22,8 +24,13 @@ github.com/LumeWeb/cli-docs v0.0.0-20260613190900-4a7a99d4f66c h1:6AeeSPg2OuKhFq github.com/LumeWeb/cli-docs v0.0.0-20260613190900-4a7a99d4f66c/go.mod h1:nMRw/y3IBV+1l11cP7Le1vuihUBDOAMgbfrVBafGsSQ= github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4= github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Oudwins/zog v0.23.0 h1:aNjOcy7mE5NWOQVR9sFO1L0YCKfEjWMlaUTNquKGRQY= github.com/Oudwins/zog v0.23.0/go.mod h1:yznncrIHC5tQeOui28FMz67LEGZV3nAoCdWL5t8QsNM= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4= github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk= @@ -75,6 +82,8 @@ github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cloudflare/cloudflare-go v0.117.0 h1:y00E0XCvxuZGplL+gkoMRIhWpfNqIgyBFS6UUWC4s0c= github.com/cloudflare/cloudflare-go v0.117.0/go.mod h1:Ds6urDwn/TF2uIU24mu7H91xkKP8gSAHxQ44DSZgVmU= github.com/cloudflare/cloudflared v0.0.0-20260903222438-2253eeeb25a4 h1:Bk6SCwmBHfE61bHCpb+LEWZkhHzTf41h8LvkxbxcH0g= @@ -95,6 +104,8 @@ github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf h1:dwGgBWn8 github.com/crackcomm/go-gitignore v0.0.0-20241020182519-7843d2ba8fdf/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= +github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= +github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -121,6 +132,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9gd6MPfXbKVU= @@ -157,6 +170,12 @@ github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= +github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= +github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -195,6 +214,8 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -320,6 +341,8 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= @@ -335,6 +358,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= @@ -557,6 +582,8 @@ github.com/pion/turn/v4 v4.0.2 h1:ZqgQ3+MjP32ug30xAbD6Mn+/K4Sxi3SdNOTFf+7mpps= github.com/pion/turn/v4 v4.0.2/go.mod h1:pMMKP/ieNAG/fN5cZiN4SDuyKsXtNTr0ccN7IToA1zs= github.com/pion/webrtc/v4 v4.1.2 h1:mpuUo/EJ1zMNKGE79fAdYNFZBX790KE7kQQpLMjjR54= github.com/pion/webrtc/v4 v4.1.2/go.mod h1:xsCXiNAmMEjIdFxAYU0MbB3RwRieJsegSB2JZsGN+8U= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -613,6 +640,9 @@ github.com/shirou/gopsutil/v4 v4.26.7 h1:IXzpHz/dkMRYAhKkOXr1HB6SuzWU3eoyyeWe7g3 github.com/shirou/gopsutil/v4 v4.26.7/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= @@ -640,6 +670,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -707,6 +738,8 @@ github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdz github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xo/terminfo v1.0.0 h1:2ZpYzqWzyyytjk3TP6aJVDhkMAkc99/1xKQdA3TDTBY= github.com/xo/terminfo v1.0.0/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg= @@ -839,6 +872,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk= @@ -857,6 +891,7 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -878,6 +913,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -885,11 +921,13 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -954,6 +992,8 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/cli/command_getter.go b/internal/cli/command_getter.go index fdc267b7..2040bb65 100644 --- a/internal/cli/command_getter.go +++ b/internal/cli/command_getter.go @@ -31,6 +31,11 @@ type flagGetterWithDuration interface { Duration(name string) time.Duration } +type flagGetterWithTimestamp interface { + flagGetterWithIsSet + Timestamp(name string) time.Time +} + // commandGetter is the broadest interface satisfied by cliCommandWrapper. // It encompasses all flag, arg, and CID access methods used across handlers. type commandGetter interface { @@ -40,6 +45,7 @@ type commandGetter interface { stringSliceGetter Uint(name string) uint Duration(name string) time.Duration + Timestamp(name string) time.Time } type argsGetter interface { diff --git a/internal/cli/command_registration_test.go b/internal/cli/command_registration_test.go index 7635fe9a..39896256 100644 --- a/internal/cli/command_registration_test.go +++ b/internal/cli/command_registration_test.go @@ -132,6 +132,7 @@ func TestCommandRegistration_RootSubcommands(t *testing.T) { "admin", "generate-docs", "vault", + "git", "mcp", } @@ -209,6 +210,26 @@ func TestCommandRegistration_PinsSubcommands(t *testing.T) { len(expectedPinsSubs), len(pins.Commands), names) } +func TestCommandRegistration_GitSubcommands(t *testing.T) { + root := NewRootCommand() + git := findCommand(root.Commands, "git") + require.NotNil(t, git, "git command should exist") + + expectedGitSubs := []string{"watch", "status", "ls", "show", "share", "unwatch", "doctor"} + names := commandNames(git.Commands) + nameSet := make(map[string]bool, len(names)) + for _, n := range names { + nameSet[n] = true + } + + for _, expected := range expectedGitSubs { + assert.True(t, nameSet[expected], "git should have subcommand %q", expected) + } + assert.Len(t, git.Commands, len(expectedGitSubs), + "git should have exactly %d subcommands, got %d: %v", + len(expectedGitSubs), len(git.Commands), names) +} + func TestCommandRegistration_AuthSubcommands(t *testing.T) { root := NewRootCommand() auth := findCommand(root.Commands, "auth") diff --git a/internal/cli/git.go b/internal/cli/git.go new file mode 100644 index 00000000..22cbddd6 --- /dev/null +++ b/internal/cli/git.go @@ -0,0 +1,42 @@ +package cli + +import "github.com/urfave/cli/v3" + +// newGitCommand returns the `pinner git` parent command for Git archive hosting +// on Sia. Commands are registered incrementally across the MVP steps; Step 5 +// provides watch / status / ls. +func newGitCommand() *cli.Command { + return &cli.Command{ + Name: "git", + Usage: "Git archive hosting on Sia", + Description: `Quiet Git archive hosting reached through an ordinary Git remote. + +Bind a local repository to a named locker with 'watch', inspect its archive +state with 'status', list known lockers with 'ls', browse an archived locker +with 'show', mint share URLs with 'share', undo a binding with 'unwatch', and +repair archive state with 'doctor'. All Git operations run in-process (go-git); +the "archive" remote points at the Sia-backed locker. 'watch --hook' installs a +git post-push hook that automatically republishes the repo to the archive after +every push (off by default). + +Examples: + pinner git watch + pinner git watch --locker myproject + pinner git watch --hook + pinner git status + pinner git ls + pinner git show + pinner git share + pinner git unwatch + pinner git doctor`, + Commands: []*cli.Command{ + newGitWatchCommand(), + newGitStatusCommand(), + newGitLsCommand(), + newGitShowCommand(), + newGitShareCommand(), + newGitUnwatchCommand(), + newGitDoctorCommand(), + }, + } +} diff --git a/internal/cli/git_doctor.go b/internal/cli/git_doctor.go new file mode 100644 index 00000000..60b01792 --- /dev/null +++ b/internal/cli/git_doctor.go @@ -0,0 +1,131 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "path/filepath" + + "github.com/urfave/cli/v3" + + "go.lumeweb.com/pinner-cli/internal/cli/gitremote" + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +func newGitDoctorCommand() *cli.Command { + return &cli.Command{ + Name: "doctor", + Usage: "Diagnose and repair a bound repository's git archive state", + ArgsUsage: "[path]", + Description: `Check the health of the repository at [path] (default: current +directory) against its git archive locker and repair what can be fixed locally: + + - verify the repo is bound to a locker; + - repair the "archive" remote so it points at the locker's canonical URL; + - verify the locker registration and report the archived object health + (cold vs has tip/pack objects); + - attempt an account scan + tip republish through the Sia-backed store when + reachable (reports the not-wired sentinel until the backend is live). + +Examples: + pinner git doctor + pinner git doctor ~/code/myproject`, + Action: withContext(func(ctx context.Context, cc *commandContext) error { + return gitDoctor(ctx, cc) + }), + } +} + +func gitDoctor(ctx context.Context, cc *commandContext) error { + path := "." + if cc.Cmd.Args().Len() >= 1 { + path = cc.Cmd.Args().Get(0) + } + abs, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("git doctor: resolve path: %w", err) + } + + session, err := gitarchive.NewSession("", "") + if err != nil { + return err + } + defer session.Close() + + // Local (DB-only) health pass. + report, err := gitarchive.LocalDoctor(session.DB, abs) + if err != nil { + return err + } + + if cc.Output.IsJSON() { + if err := cc.Output.PrintJSON(report); err != nil { + return err + } + } + + if !report.Bound { + cc.Output.Printfln("repository %s is not bound to a git archive; run `pinner git watch` first", abs) + return nil + } + + cc.Output.Printfln("locker: %s", report.Locker) + + // Repair the archive remote URL if missing or mispointed. + if repaired, err := gitarchive.RepairArchiveRemote(abs, report.Locker); err != nil { + return err + } else if repaired { + cc.Output.Printfln("repaired: archive remote now -> %s", gitarchive.GitRemoteURL(report.Locker)) + } else { + cc.Output.Printfln("ok: archive remote -> %s", gitarchive.GitRemoteURL(report.Locker)) + } + + if !report.RepoRegistered { + // Registration row missing: re-ensure it from the bind so bookkeeping + // is consistent (lineage recomputed from the local repo). + lineage, err := gitarchive.ComputeLineage(abs) + if err != nil { + return err + } + if _, err := gitarchive.EnsureRepo(session.DB, report.Locker, gitarchive.DefaultLockerName(abs), gitarchive.LineageKey(lineage)); err != nil { + return err + } + cc.Output.Printfln("repaired: restored locker registration %s", report.Locker) + } else { + cc.Output.Printfln("ok: locker registration present (tip gen %d)", report.TipGen) + } + + if report.Cold { + cc.Output.Printfln("warning: locker %s is cold (no git objects archived yet); run `pinner git watch` to publish", report.Locker) + } else { + cc.Output.Printfln("objects: %d git objects (%s tip, %s pack)", report.ObjectCount, + boolStr(report.HasTipObject), boolStr(report.HasPackObject)) + if !report.HasTipObject { + cc.Output.Printfln("warning: no archived tip object found; an account scan + republish is needed") + } + if !report.HasPackObject { + cc.Output.Printfln("warning: no archived pack object found; re-publish to restore packs") + } + } + + // Sia-dependent passes via the store seam, scoped to the bound locker. Until + // the backend is wired the store reports its not-wired sentinel; surface it + // as informational rather than a hard failure so the local repairs above are + // still reported. + store := gitremote.NewSessionStoreFromSession(session).ForLocker(report.Locker) + if _, err := store.List(ctx, false); err != nil { + if errors.Is(err, gitremote.ErrSiaNotWired) { + cc.Output.Printfln("backend: Sia archive not wired (account scan + republish unavailable)") + } else { + return fmt.Errorf("git doctor: backend check: %w", err) + } + } + return nil +} + +func boolStr(b bool) string { + if b { + return "yes" + } + return "no" +} diff --git a/internal/cli/git_ls.go b/internal/cli/git_ls.go new file mode 100644 index 00000000..01d21c1e --- /dev/null +++ b/internal/cli/git_ls.go @@ -0,0 +1,62 @@ +package cli + +import ( + "context" + "strconv" + + "github.com/urfave/cli/v3" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +func newGitLsCommand() *cli.Command { + return &cli.Command{ + Name: "ls", + Usage: "List known git archive lockers (git_repos rows)", + Description: `List every locker tracked locally in the git archive registry (the +git_repos table), with its display name, lineage fingerprint and last known tip +generation. + +Examples: + pinner git ls + pinner git ls --json`, + Action: withContext(func(ctx context.Context, cc *commandContext) error { + return gitLs(ctx, cc) + }), + } +} + +func gitLs(ctx context.Context, cc *commandContext) error { + session, err := gitarchive.NewSession("", "") + if err != nil { + return err + } + defer session.Close() + + repos, err := gitarchive.ListRepos(session.DB) + if err != nil { + return err + } + + if cc.Output.IsJSON() { + return cc.Output.PrintJSON(repos) + } + + if len(repos) == 0 { + cc.Output.Printfln("no git archive lockers; run `pinner git watch` to create one") + return nil + } + + headers := []string{"LOCKER", "NAME", "LINEAGE", "TIP GEN"} + rows := make([][]string, 0, len(repos)) + for _, r := range repos { + rows = append(rows, []string{ + r.Locker, + r.Name, + shortHash(r.Lineage), + strconv.Itoa(r.TipGen), + }) + } + cc.Output.PrintTable(headers, rows) + return nil +} diff --git a/internal/cli/git_share.go b/internal/cli/git_share.go new file mode 100644 index 00000000..c32532c9 --- /dev/null +++ b/internal/cli/git_share.go @@ -0,0 +1,124 @@ +package cli + +import ( + "context" + "fmt" + "path/filepath" + "time" + + "github.com/urfave/cli/v3" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +func newGitShareCommand() *cli.Command { + return &cli.Command{ + Name: "share", + Usage: "Mint a share URL for an archived locker's tip and packs", + ArgsUsage: "[path]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "locker", + Usage: "Explicit locker name (overrides the bound locker of [path])", + Category: "archive", + }, + &cli.DurationFlag{ + Name: "ttl", + Value: gitarchive.DefaultShareTTL, + Usage: "How long the share URLs remain valid (default 30d)", + Category: "archive", + }, + &cli.TimestampFlag{ + Name: "until", + Usage: "Absolute expiry for the share URLs (RFC3339); overrides --ttl", + Config: cli.TimestampConfig{Layouts: []string{"2006-01-02T15:04:05Z07:00", time.RFC3339}}, + Category: "archive", + }, + }, + Description: `Mint self-contained bearer share URLs for the archived locker's current +tip and its packs, and print the resulting share URL plus its expiry. The +share URL can be cloned by anyone with ` + "`git clone pinner::share/`" + ` — +no vault profile or app key is required to read the shared archive. + +By default only the share URL and expiry are printed. Nested detail (the +individual tip and pack URLs, generation and display name) is shown only with +--verbose. + +The repository at [path] (default: current directory) must be bound first via +'pinner git watch', unless an explicit --locker is given. + +Examples: + pinner git share + pinner git share ~/code/myproject + pinner git share --locker widgets --ttl 168h + pinner git share --until 2030-01-01T00:00:00Z`, + Action: withContext(func(ctx context.Context, cc *commandContext) error { + return gitShare(ctx, cc) + }), + } +} + +func gitShare(ctx context.Context, cc *commandContext) error { + path := "." + if cc.Cmd.Args().Len() >= 1 { + path = cc.Cmd.Args().Get(0) + } + abs, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("git share: resolve path: %w", err) + } + + session, err := gitarchive.NewSession("", "") + if err != nil { + return err + } + defer session.Close() + + locker := cc.Cmd.String("locker") + if locker == "" { + bind, err := gitarchive.FindBindByRepo(session.DB, abs) + if err != nil { + return err + } + if bind == nil { + return fmt.Errorf("repository %s is not bound to a git archive; run `pinner git watch` first or pass --locker", abs) + } + locker = bind.Locker + } + + var until time.Time + if cc.Cmd.IsSet("until") { + until = cc.Cmd.Timestamp("until") + } else { + until = time.Now().Add(cc.Cmd.Duration("ttl")) + } + + res, err := gitarchive.MintShare(ctx, session, locker, until) + if err != nil { + return err + } + + if cc.Output.IsJSON() { + return cc.Output.PrintJSON(res) + } + + cc.Output.Printfln("share URL: %s", res.URL) + cc.Output.Printfln("expires: %s", res.Until.UTC().Format(time.RFC3339)) + cc.Output.Printfln("clone: git clone %s", shareRemoteURL(res.URL)) + + // Nested detail is only surfaced with --verbose. + cc.Output.PrintVerbosef("locker: %s", res.Locker) + cc.Output.PrintVerbosef("gen: %d", res.Gen) + cc.Output.PrintVerbosef("name: %s", res.Name) + cc.Output.PrintVerbosef("tip url: %s", res.TipURL) + for i, u := range res.PackURLs { + cc.Output.PrintVerbosef("pack url[%d]: %s", i, u) + } + return nil +} + +// shareRemoteURL renders the git remote URL for a pre-signed share URL, i.e. +// the value a consumer passes to `git clone` (the profile-less helper path). +func shareRemoteURL(shareURL string) string { + return "pinner::share/" + shareURL +} diff --git a/internal/cli/git_show.go b/internal/cli/git_show.go new file mode 100644 index 00000000..84c3e590 --- /dev/null +++ b/internal/cli/git_show.go @@ -0,0 +1,134 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strconv" + + "github.com/urfave/cli/v3" + + "go.lumeweb.com/pinner-cli/internal/cli/gitremote" + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +func newGitShowCommand() *cli.Command { + return &cli.Command{ + Name: "show", + Usage: "Browse an archived locker's refs, commit log and files", + ArgsUsage: "[path]", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "n", + Value: gitremote.DefaultCommitLimit, + Usage: "Maximum number of commits to show (git log -n style)", + Category: "archive", + }, + }, + Description: `Inspect the state of a bound locker as archived: refs, the commit log +(mirroring "git log -n 20") and the recursive file listing (mirroring +"git ls-tree -r --long"), built by ingesting archived packs into the profile's +private bare mirror via go-git (no git subprocess, no worktree checkout). + +The repository at [path] (default: current directory) must be bound first via +'pinner git watch'. + +Examples: + pinner git show + pinner git show ~/code/myproject + pinner git show --n 5`, + Action: withContext(func(ctx context.Context, cc *commandContext) error { + return gitShow(ctx, cc) + }), + } +} + +func gitShow(ctx context.Context, cc *commandContext) error { + path := "." + if cc.Cmd.Args().Len() >= 1 { + path = cc.Cmd.Args().Get(0) + } + abs, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("git show: resolve path: %w", err) + } + + session, err := gitarchive.NewSession("", "") + if err != nil { + return err + } + defer session.Close() + + bind, err := gitarchive.FindBindByRepo(session.DB, abs) + if err != nil { + return err + } + if bind == nil { + return fmt.Errorf("repository %s is not bound to a git archive; run `pinner git watch` first", abs) + } + + repoDir := gitarchive.PrivateRepoDir(session.Profile, bind.Locker) + mirror, err := gitarchive.OpenPrivateBare(repoDir) + if err != nil { + return err + } + + // Ensure archived packs are present in the mirror (List+Download+Ingest). + // When the Sia backend is not yet wired the data ops report the shared + // not-wired sentinel; in that case we fall back to rendering whatever is + // already in the mirror rather than failing the whole command. + store := gitremote.NewSessionStoreFromSession(session).ForLocker(bind.Locker) + ensured, err := gitremote.EnsureArchiveRefs(ctx, mirror.Storer, store) + if err != nil && !errors.Is(err, gitremote.ErrSiaNotWired) { + return fmt.Errorf("git show: ensure archive packs: %w", err) + } + + n := int(cc.Cmd.Int("n")) + report, err := gitremote.Render(mirror.Storer, n) + if err != nil { + return fmt.Errorf("git show: render: %w", err) + } + + if cc.Output.IsJSON() { + return cc.Output.PrintJSON(report) + } + + cc.Output.Printfln("locker: %s mirror: %s", bind.Locker, repoDir) + if len(ensured) > 0 { + cc.Output.Printfln("archived refs ensured: %d", len(ensured)) + } + + cc.Output.Printfln("") + cc.Output.Printfln("refs") + refRows := make([][]string, 0, len(report.Refs)) + for _, r := range report.Refs { + refRows = append(refRows, []string{r.Name, shortHash(r.Hash.String())}) + } + cc.Output.PrintTable([]string{"REF", "TIP"}, refRows) + + cc.Output.Printfln("") + cc.Output.Printfln("commits (git log -n %d style)", n) + commitRows := make([][]string, 0, len(report.Commits)) + for _, c := range report.Commits { + commitRows = append(commitRows, []string{shortHash(c.Hash), c.Author, c.Message}) + } + if len(report.Commits) == 0 { + cc.Output.Printfln(" (no commits archived)") + } else { + cc.Output.PrintTable([]string{"COMMIT", "AUTHOR", "MESSAGE"}, commitRows) + } + + cc.Output.Printfln("") + cc.Output.Printfln("files (git ls-tree -r --long style)") + if len(report.Tree) == 0 { + cc.Output.Printfln(" (no files archived)") + } else { + treeRows := make([][]string, 0, len(report.Tree)) + for _, f := range report.Tree { + treeRows = append(treeRows, []string{f.Mode, shortHash(f.Hash), strconv.FormatInt(f.Size, 10), f.Path}) + } + cc.Output.PrintTable([]string{"MODE", "HASH", "SIZE", "PATH"}, treeRows) + } + return nil +} diff --git a/internal/cli/git_status.go b/internal/cli/git_status.go new file mode 100644 index 00000000..55e215e9 --- /dev/null +++ b/internal/cli/git_status.go @@ -0,0 +1,137 @@ +package cli + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/urfave/cli/v3" + + "go.lumeweb.com/pinner-cli/internal/cli/gitremote" + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +func newGitStatusCommand() *cli.Command { + return &cli.Command{ + Name: "status", + Usage: "Show a bound repository's archive state vs its archived tip", + ArgsUsage: "[path]", + Description: `Report how a watched repository compares to its archived tip: each local +branch is checked against the matching archived ref and reported as up to date, +ahead/behind, diverged, or not yet present on the archive. + +The repository at [path] (default: current directory) must be bound first via +'pinner git watch'. + +Examples: + pinner git status + pinner git status ~/code/myproject`, + Action: withContext(func(ctx context.Context, cc *commandContext) error { + return gitStatus(ctx, cc) + }), + } +} + +func gitStatus(ctx context.Context, cc *commandContext) error { + path := "." + if cc.Cmd.Args().Len() >= 1 { + path = cc.Cmd.Args().Get(0) + } + abs, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("git status: resolve path: %w", err) + } + + session, err := gitarchive.NewSession("", "") + if err != nil { + return err + } + defer session.Close() + + bind, err := gitarchive.FindBindByRepo(session.DB, abs) + if err != nil { + return err + } + if bind == nil { + return fmt.Errorf("repository %s is not bound to a git archive; run `pinner git watch` first", abs) + } + + repo, err := git.PlainOpenWithOptions(abs, &git.PlainOpenOptions{DetectDotGit: true}) + if err != nil { + return fmt.Errorf("git status: open repository: %w", err) + } + + localRefs, err := gitremote.LocalBranches(repo.Storer) + if err != nil { + return err + } + + store := gitremote.NewSessionStoreFromSession(session).ForLocker(bind.Locker) + remoteRefs, err := store.List(ctx, false) + if err != nil { + return fmt.Errorf("git status: list archive refs: %w", err) + } + remote := make(map[string]string, len(remoteRefs)) + for _, r := range remoteRefs { + remote[r.Name] = r.Hash.String() + } + + headers := []string{"BRANCH", "LOCAL", "ARCHIVE", "DELTA"} + rows := make([][]string, 0, len(localRefs)) + for _, lr := range localRefs { + remoteHash, ok := remote[lr.Name] + localHash := lr.Hash.String() + delta := "-" + if !ok { + delta = "not on archive" + } else if remoteHash == localHash { + delta = "up to date" + } else if rh, ok2 := remoteCommit(repo, remoteHash); ok2 { + ahead, behind, derr := gitremote.Divergence(repo.Storer, lr.Hash, rh) + if derr != nil { + delta = "error" + } else { + delta = divergenceString(ahead, behind) + } + } else { + delta = "archive ahead (tip not local; git fetch to compare)" + } + rows = append(rows, []string{lr.Name, shortHash(localHash), shortHash(remoteHash), delta}) + } + + cc.Output.Printfln("locker: %s remote: %s", bind.Locker, gitarchive.GitRemoteURL(bind.Locker)) + cc.Output.PrintTable(headers, rows) + return nil +} + +// remoteCommit resolves a hex remote hash to a plumbing.Hash when the object is +// present in the local store (so divergence can be computed without a fetch). +func remoteCommit(repo *git.Repository, hash string) (plumbing.Hash, bool) { + h := plumbing.NewHash(hash) + if _, err := repo.Storer.EncodedObject(plumbing.CommitObject, h); err != nil { + return plumbing.ZeroHash, false + } + return h, true +} + +func divergenceString(ahead, behind int) string { + switch { + case ahead > 0 && behind > 0: + return fmt.Sprintf("diverged (+%d/-%d)", ahead, behind) + case ahead > 0: + return fmt.Sprintf("%d ahead", ahead) + case behind > 0: + return fmt.Sprintf("%d behind", behind) + default: + return "up to date" + } +} + +func shortHash(h string) string { + if len(h) > 12 { + return h[:12] + } + return h +} diff --git a/internal/cli/git_unwatch.go b/internal/cli/git_unwatch.go new file mode 100644 index 00000000..2838db17 --- /dev/null +++ b/internal/cli/git_unwatch.go @@ -0,0 +1,93 @@ +package cli + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/urfave/cli/v3" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +func newGitUnwatchCommand() *cli.Command { + return &cli.Command{ + Name: "unwatch", + Usage: "Unbind a repository from its git archive locker", + ArgsUsage: "[path]", + Description: `Remove the local git-archive binding for the repository at [path] +(default: current directory): the git_binds record, the git_repos locker row, +and the "archive" remote (only when that remote points at this locker's pinner +URL — a same-named remote the user configured themselves is left untouched). + +Examples: + pinner git unwatch + pinner git unwatch ~/code/myproject`, + Action: withContext(func(ctx context.Context, cc *commandContext) error { + return gitUnwatch(ctx, cc) + }), + } +} + +func gitUnwatch(ctx context.Context, cc *commandContext) error { + path := "." + if cc.Cmd.Args().Len() >= 1 { + path = cc.Cmd.Args().Get(0) + } + abs, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("git unwatch: resolve path: %w", err) + } + + session, err := gitarchive.NewSession("", "") + if err != nil { + return err + } + defer session.Close() + + bind, err := gitarchive.FindBindByRepo(session.DB, abs) + if err != nil { + return err + } + if bind == nil { + return fmt.Errorf("repository %s is not bound to a git archive; nothing to unwatch", abs) + } + + // Remove the archive remote first (only if it is ours). + removed, err := gitarchive.RemoveArchiveRemote(abs, bind.Locker) + if err != nil { + return err + } + + // Then clear the bind and the locker registration. + bindRemoved, err := gitarchive.DeleteBind(session.DB, abs) + if err != nil { + return err + } + repoRemoved, err := gitarchive.DeleteRepoByLocker(session.DB, bind.Locker) + if err != nil { + return err + } + + // Clean up the optional post-push hook we may have installed (only when it + // is one of ours). + hookRemoved, err := gitarchive.RemovePostPushHook(abs) + if err != nil { + return err + } + + cc.Output.Printfln("unwatched %s (locker %s)", abs, bind.Locker) + if removed { + cc.Output.Printfln(" removed archive remote %s", gitarchive.GitRemoteURL(bind.Locker)) + } + if bindRemoved { + cc.Output.Printfln(" removed git_archive binding") + } + if repoRemoved { + cc.Output.Printfln(" removed locker registration %s", bind.Locker) + } + if hookRemoved { + cc.Output.Printfln(" removed post-push hook") + } + return nil +} diff --git a/internal/cli/git_watch.go b/internal/cli/git_watch.go new file mode 100644 index 00000000..f7e817b4 --- /dev/null +++ b/internal/cli/git_watch.go @@ -0,0 +1,212 @@ +package cli + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/urfave/cli/v3" + + "go.lumeweb.com/pinner-cli/internal/cli/gitremote" + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +func newGitWatchCommand() *cli.Command { + return &cli.Command{ + Name: "watch", + Usage: "Bind a local repository to a git archive locker and publish it", + ArgsUsage: "[path]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "locker", + Usage: "Explicit locker name (disambiguates an ambiguous lineage match or names a new locker)", + Category: "archive", + }, + &cli.BoolFlag{ + Name: "new", + Usage: "Create a new locker even when a lineage match already exists", + Category: "archive", + }, + &cli.BoolFlag{ + Name: "hook", + Usage: "Install a git post-push hook so this repo republishes to the archive after every push (off by default)", + Category: "archive", + }, + }, + Description: `Bind the repository at [path] (default: current directory) to a git +archive locker, install the "archive" remote, and perform a first publish. + +The repository's lineage (its set of root commits) is matched against known +lockers: + - no match -> a new locker is created (name derived from the directory), + - one match -> that locker is reused, + - many -> the matching locker names are reported and you must pick with + --locker or force a new one with --new. + +With --hook (off by default) the repository also gets a git post-push hook so +every subsequent push republishes it to the archive automatically. + +Examples: + pinner git watch + pinner git watch --locker myproject ~/code/myproject + pinner git watch --new + pinner git watch --hook`, + Action: withContext(func(ctx context.Context, cc *commandContext) error { + return gitWatch(ctx, cc) + }), + } +} + +func gitWatch(ctx context.Context, cc *commandContext) error { + // Frame a proper repo path: an absolute path is required for a stable bind + // key and the go-git open. + path := "." + if cc.Cmd.Args().Len() >= 1 { + path = cc.Cmd.Args().Get(0) + } + abs, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("git watch: resolve path: %w", err) + } + + lineage, err := gitarchive.ComputeLineage(abs) + if err != nil { + return err + } + key := gitarchive.LineageKey(lineage) + + // Open the session (profile cache DB) once; it drives lineage matching, + // the bind record, and the first-publish Store. + session, err := gitarchive.NewSession("", "") + if err != nil { + return err + } + defer session.Close() + + chosen, err := resolveLocker(cc, session, abs, key) + if err != nil { + return err + } + + // Record the locker (git_repos) and the bind (git_binds) before touching + // the archive remote so a later failure still leaves a recoverable state. + + if _, err := gitarchive.EnsureRepo(session.DB, chosen, gitarchive.DefaultLockerName(abs), key); err != nil { + return err + } + bind, err := gitarchive.AddBind(session.DB, chosen, abs) + if err != nil { + return err + } + + repo, err := git.PlainOpenWithOptions(abs, &git.PlainOpenOptions{DetectDotGit: true}) + if err != nil { + return fmt.Errorf("git watch: open repository: %w", err) + } + if err := addArchiveRemote(repo, chosen); err != nil { + return err + } + + cc.Output.Printfln("bound %s -> %s (%s)", bind.RepoPath, chosen, gitarchive.GitRemoteURL(chosen)) + + // First publish of every local branch. The backend is the profile-backed + // session Store (Step 10 wires real Sia; until then a session-backed Store + // reports Sia-not-wired via the shared sentinel). + store := gitremote.NewSessionStoreFromSession(session).ForLocker(chosen) + published, err := gitremote.PublishAll(ctx, repo.Storer, store) + if err != nil { + return fmt.Errorf("git watch: first publish: %w", err) + } + if len(published) == 0 { + cc.Output.Printfln("no local branches to publish (repository has no commits yet)") + } else { + cc.Output.Printfln("published %d branch(es) to %s", len(published), chosen) + for _, r := range published { + cc.Output.Printfln(" %s -> %s", r.Name, r.Hash) + } + } + + // Optional post-push hook (off by default): install a git hook that + // republishes this repo to its locker after every push, keeping the archive + // in sync even when the user pushes to a non-archive remote. + if cc.Cmd.Bool("hook") { + if _, err := gitarchive.InstallPostPushHook(abs); err != nil { + return err + } + cc.Output.Printfln("installed post-push hook in %s", gitarchive.RepoHooksDir(abs)) + } + return nil +} + +// resolveLocker determines the locker to bind given the local repo's lineage +// fingerprint. It returns the explicit --locker when set; otherwise it matches +// against known lockers. On an ambiguous match it reports the candidate names +// and errors, directing the user to --locker or --new. +func resolveLocker(cc *commandContext, session *gitarchive.Session, abs, lineageKey string) (string, error) { + if locker := cc.Cmd.String("locker"); locker != "" { + return locker, nil + } + if cc.Cmd.Bool("new") { + return gitarchive.DefaultLockerName(abs), nil + } + + matches, err := gitarchive.FindReposByLineage(session.DB, lineageKey) + if err != nil { + return "", err + } + switch len(matches) { + case 0: + return gitarchive.DefaultLockerName(abs), nil + case 1: + return matches[0].Locker, nil + default: + var names []string + for _, m := range matches { + names = append(names, m.Locker) + } + return "", ambiguousLineageError(names) + } +} + +func ambiguousLineageError(names []string) error { + return fmt.Errorf( + "lineage matches multiple lockers (%s); pick one with --locker or create a new one with --new", + joinNames(names), + ) +} + +func joinNames(names []string) string { + out := "" + for i, n := range names { + if i > 0 { + out += ", " + } + out += n + } + return out +} + +// addArchiveRemote installs (or updates) the "archive" remote on repo pointing +// at the locker URL. It is idempotent. +func addArchiveRemote(repo *git.Repository, locker string) error { + url := gitarchive.GitRemoteURL(locker) + cfg, err := repo.Config() + if err != nil { + return fmt.Errorf("git watch: read config: %w", err) + } + if existing, ok := cfg.Remotes[gitarchive.RemoteName]; ok { + if len(existing.URLs) == 1 && existing.URLs[0] == url { + return nil + } + } + cfg.Remotes[gitarchive.RemoteName] = &config.RemoteConfig{ + Name: gitarchive.RemoteName, + URLs: []string{url}, + } + if err := repo.Storer.SetConfig(cfg); err != nil { + return fmt.Errorf("git watch: write config: %w", err) + } + return nil +} diff --git a/internal/cli/gitremote/dispatch.go b/internal/cli/gitremote/dispatch.go new file mode 100644 index 00000000..6efa0a92 --- /dev/null +++ b/internal/cli/gitremote/dispatch.go @@ -0,0 +1,62 @@ +package gitremote + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/storage" +) + +// IsRemoteHelperInvocation reports whether argv0 means this process was invoked +// as the Git remote helper (via the `git-remote-pinner` symlink) rather than as +// the normal `pinner` CLI. +// +// This is the identity-dispatch contract: the single binary looks at +// filepath.Base(os.Args[0]) and routes to the helper protocol exactly when it is +// `git-remote-pinner`. Any other basename falls through to the CLI path. There +// is no `git-remote-sia` alias and no provider-selection branch. +func IsRemoteHelperInvocation(argv0 string) bool { + return filepath.Base(argv0) == "git-remote-pinner" +} + +// localRepoPath returns the directory to open as the local git repository: the +// $GIT_DIR environment variable when set (Git sets it for the helper), otherwise +// the current working directory with dot-git detection. +func localRepoPath() string { + if d := os.Getenv("GIT_DIR"); d != "" { + return d + } + wd, err := os.Getwd() + if err == nil { + return wd + } + return "." +} + +// OpenLocalStorer opens the local git repository (GIT_DIR, or dot-git detected +// from cwd) and returns its object/reference store. Fetch writes into it and +// push reads from it; the helper never shells out to git. +func OpenLocalStorer() (storage.Storer, error) { + repo, err := git.PlainOpenWithOptions(localRepoPath(), &git.PlainOpenOptions{ + DetectDotGit: true, + }) + if err != nil { + return nil, fmt.Errorf("gitremote: open local repository: %w", err) + } + return repo.Storer, nil +} + +// Run is the process-level entry point for helper mode. It opens the local git +// store (GIT_DIR/cwd) and drives the protocol over in/out using store for remote +// persistence. Diagnostics go to errw. +func Run(ctx context.Context, store Store, in io.Reader, out io.Writer, errw io.Writer) error { + local, err := OpenLocalStorer() + if err != nil { + return err + } + return Handle(ctx, local, store, in, out, errw) +} diff --git a/internal/cli/gitremote/dispatch_test.go b/internal/cli/gitremote/dispatch_test.go new file mode 100644 index 00000000..aed16d1d --- /dev/null +++ b/internal/cli/gitremote/dispatch_test.go @@ -0,0 +1,68 @@ +package gitremote + +import ( + "bytes" + "context" + "path/filepath" + "strings" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIsRemoteHelperInvocation(t *testing.T) { + assert.True(t, IsRemoteHelperInvocation("/usr/local/bin/git-remote-pinner")) + assert.True(t, IsRemoteHelperInvocation("git-remote-pinner")) + assert.False(t, IsRemoteHelperInvocation("/usr/local/bin/pinner")) + assert.False(t, IsRemoteHelperInvocation("pinner")) + assert.False(t, IsRemoteHelperInvocation("")) +} + +// TestRunDispatchesHelperPath drives the full Run entry (open local repo via +// GIT_DIR + drive protocol) with a synthetic argv-0 of `git-remote-pinner`. It is +// the same seam cmd/pinner/main.go uses for helper mode, exercised without the +// CLI tree (which cannot build in this worktree due to a pre-existing, unrelated +// internal/mcp/core/handoff issue). +func TestRunDispatchesHelperPath(t *testing.T) { + ctx := context.Background() + + // A real local repo so Run's OpenLocalStorer (via GIT_DIR) succeeds. + localDir := t.TempDir() + repo, err := git.PlainInit(localDir, false) + require.NoError(t, err) + h := commitFile(t, repo, "x.txt", "x\n", "init") + + // Remote bare store. + remoteDir := t.TempDir() + remote, err := git.PlainInit(remoteDir, true) + require.NoError(t, err) + store := &GoGitStore{Repo: remote} + defer store.Close() + + // Point GIT_DIR at the local repo so Run can find it without a worktree. + t.Setenv("GIT_DIR", localDir) + + var out, errw bytes.Buffer + err = Run(ctx, store, strings.NewReader( + "capabilities\nlist for-push\npush "+h.String()+":refs/heads/master\n\n", + ), &out, &errw) + require.NoError(t, err) + + ref, err := remote.Storer.Reference(plumbing.ReferenceName("refs/heads/master")) + require.NoError(t, err) + assert.Equal(t, h, ref.Hash()) + assert.Contains(t, out.String(), "ok refs/heads/master\n") +} + +// TestDispatchRoutingSharedPath verifies argv-0 selection is purely name-based: +// the same directory holds both names, but only the helper name routes to the +// helper. This mirrors the locked correction — one executable, argv-0 identity +// dispatch, no provider polymorphism. +func TestDispatchRoutingSharedPath(t *testing.T) { + dir := t.TempDir() + require.True(t, IsRemoteHelperInvocation(filepath.Join(dir, "git-remote-pinner"))) + require.False(t, IsRemoteHelperInvocation(filepath.Join(dir, "pinner"))) +} diff --git a/internal/cli/gitremote/gitops.go b/internal/cli/gitremote/gitops.go new file mode 100644 index 00000000..2226b0bd --- /dev/null +++ b/internal/cli/gitremote/gitops.go @@ -0,0 +1,125 @@ +package gitremote + +import ( + "bytes" + "fmt" + "io" + "regexp" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/format/packfile" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/revlist" + "github.com/go-git/go-git/v5/storage" +) + +// This file holds the in-process go-git object plumbing for the remote helper: +// pack encoding (incremental set-difference via revlist + packfile.NewEncoder) +// and pack ingestion (packfile.UpdateObjectStorage), plus reference resolution. +// Everything runs against go-git; no git binary is ever spawned. + +var hexHashRE = regexp.MustCompile(`^[0-9a-fA-F]{40}$`) + +// resolveRevision resolves a local revision string from a store to a concrete +// object hash. It accepts a full sha1 or a reference name (refs/..., HEAD, or a +// short branch/tag name). +func resolveRevision(s storage.Storer, rev string) (plumbing.Hash, error) { + if hexHashRE.MatchString(rev) { + return plumbing.NewHash(rev), nil + } + + names := candidateRefNames(rev) + for _, n := range names { + ref, err := s.Reference(plumbing.ReferenceName(n)) + if err == nil { + // Follow symbolic references (e.g. HEAD -> refs/heads/master). + if ref.Type() == plumbing.SymbolicReference && ref.Target() != "" { + return resolveRevision(s, ref.Target().String()) + } + return ref.Hash(), nil + } + } + return plumbing.ZeroHash, fmt.Errorf("gitremote: cannot resolve revision %q", rev) +} + +// candidateRefNames expands a possibly-short revision into the full reference +// names go-git stores, in lookup order (explicit refs first, then HEAD, then the +// common heads/tags namespaces). +func candidateRefNames(rev string) []string { + if rev == "HEAD" || rev == "refs/" { + return []string{rev} + } + full := []string{rev} + if name := plumbing.ReferenceName(rev); !name.IsBranch() && !name.IsRemote() && + !name.IsTag() && rev != "HEAD" { + full = append(full, "refs/heads/"+rev, "refs/tags/"+rev) + } + return full +} + +// EncodeIncrementalPack builds a raw pack (v2, delta=false) containing exactly +// the objects reachable from wants that are NOT reachable from haves. Both sets +// are read from s (the local pushing/fetching repository's object store). +// +// Passing no haves yields a full pack of everything reachable from wants (used +// for a first push / a fresh clone download). The pack file is written to a +// memory buffer and returned as bytes. When there is nothing new to send the +// returned buffer is empty. +func EncodeIncrementalPack(s storage.Storer, wants, haves []plumbing.Hash) ([]byte, error) { + objs, err := revlist.Objects(s, wants, haves) + if err != nil { + return nil, fmt.Errorf("gitremote: compute object set: %w", err) + } + if len(objs) == 0 { + return nil, nil + } + + var buf bytes.Buffer + enc := packfile.NewEncoder(&buf, s, false) // useRefDeltas=false; non-thin packs + // packWindow 0 turns delta compression off entirely, producing a self + // contained pack of exactly the given objects. + if _, err := enc.Encode(objs, 0); err != nil { + return nil, fmt.Errorf("gitremote: encode pack: %w", err) + } + return buf.Bytes(), nil +} + +// IngestPack writes the objects from a raw pack into the object store. It is +// the fetch-side counterpart to EncodeIncrementalPack and accepts an empty pack +// (or nil reader) as a no-op — there was nothing new to ingest. +func IngestPack(s storage.Storer, r io.Reader) error { + if r == nil { + return nil + } + data, err := io.ReadAll(r) + if err != nil { + return fmt.Errorf("gitremote: read pack: %w", err) + } + if len(data) == 0 { + return nil + } + if err := packfile.UpdateObjectStorage(s, bytes.NewReader(data)); err != nil { + return fmt.Errorf("gitremote: ingest pack: %w", err) + } + return nil +} + +// SetLocalRef sets refName to hash in the local store (used by the go-git store +// to move refs after upload/delete). +func SetLocalRef(s storage.Storer, refName string, hash plumbing.Hash) error { + ref := plumbing.NewReferenceFromStrings(refName, hash.String()) + if err := s.SetReference(ref); err != nil { + return fmt.Errorf("gitremote: set ref %s: %w", refName, err) + } + return nil +} + +// readCommit loads an ingested commit from a store, used by tests to assert that +// a fetch transferred full history. +func readCommit(s storage.Storer, hash plumbing.Hash) (*object.Commit, error) { + obj, err := s.EncodedObject(plumbing.CommitObject, hash) + if err != nil { + return nil, err + } + return object.DecodeCommit(s, obj) +} diff --git a/internal/cli/gitremote/gitops_test.go b/internal/cli/gitremote/gitops_test.go new file mode 100644 index 00000000..fee61f75 --- /dev/null +++ b/internal/cli/gitremote/gitops_test.go @@ -0,0 +1,66 @@ +package gitremote + +import ( + "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEncodeIncrementalPackDifference(t *testing.T) { + srcDir := t.TempDir() + repo, err := git.PlainInit(srcDir, false) + require.NoError(t, err) + c1 := commitFile(t, repo, "a.txt", "a\n", "one") + c2 := commitFile(t, repo, "b.txt", "b\n", "two") + + // whole = everything reachable from c2; delta = c2 minus c1 (only c2's new + // objects). The delta must be strictly smaller than whole. + whole, err := EncodeIncrementalPack(repo.Storer, []plumbing.Hash{c2}, nil) + require.NoError(t, err) + require.NotEmpty(t, whole) + delta, err := EncodeIncrementalPack(repo.Storer, []plumbing.Hash{c2}, []plumbing.Hash{c1}) + require.NoError(t, err) + t.Logf("whole=%dB delta=%dB", len(whole), len(delta)) + // The set-difference is a real optimization: the incremental pack for a second + // commit is much smaller than the full history pack. + assert.Less(t, len(delta), len(whole)) + assert.NotEmpty(t, delta) + + // Nothing new → empty pack. + none, err := EncodeIncrementalPack(repo.Storer, []plumbing.Hash{c2}, []plumbing.Hash{c2}) + require.NoError(t, err) + assert.Empty(t, none) +} + +func TestResolveRevision(t *testing.T) { + srcDir := t.TempDir() + repo, err := git.PlainInit(srcDir, false) + require.NoError(t, err) + h := commitFile(t, repo, "a.txt", "a\n", "one") + + // Full ref name and short branch name. + got, err := resolveRevision(repo.Storer, "refs/heads/master") + require.NoError(t, err) + require.Equal(t, h, got) + + got, err = resolveRevision(repo.Storer, "master") + require.NoError(t, err) + require.Equal(t, h, got) + + // HEAD is a symbolic reference and must be followed. + got, err = resolveRevision(repo.Storer, "HEAD") + require.NoError(t, err) + require.Equal(t, h, got) + + // Raw sha1. + got, err = resolveRevision(repo.Storer, h.String()) + require.NoError(t, err) + require.Equal(t, h, got) + + // Unknown revision errors. + _, err = resolveRevision(repo.Storer, "does-not-exist") + require.Error(t, err) +} diff --git a/internal/cli/gitremote/gogit_store.go b/internal/cli/gitremote/gogit_store.go new file mode 100644 index 00000000..3a16087c --- /dev/null +++ b/internal/cli/gitremote/gogit_store.go @@ -0,0 +1,74 @@ +package gitremote + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/format/packfile" + "github.com/go-git/go-git/v5/plumbing/revlist" +) + +// GoGitStore is a Store backed by an in-process go-git repository (typically a +// bare repository acting as the archive twin). It exists so the full fetch/push +// protocol can be exercised end-to-end with only go-git — no Sia, no network — +// and doubles as a local mirror backend for git-only development/testing. +// +// Upload ingests the incoming pack into the repo's object store and advances the +// reference; Download re-encodes a full pack of the history reachable from the +// requested tip; List/Delete read/remove references directly. +type GoGitStore struct { + // Repo is the bare/archive repository that acts as the remote. + Repo *git.Repository +} + +var _ Store = (*GoGitStore)(nil) + +func (s *GoGitStore) List(ctx context.Context, forPush bool) ([]Ref, error) { + iter, err := s.Repo.Storer.IterReferences() + if err != nil { + return nil, fmt.Errorf("gogit store: list refs: %w", err) + } + defer iter.Close() + var refs []Ref + _ = iter.ForEach(func(r *plumbing.Reference) error { + if r.Type() == plumbing.SymbolicReference { + return nil + } + refs = append(refs, Ref{Name: r.Name().String(), Hash: r.Hash()}) + return nil + }) + return refs, nil +} + +// Download re-encodes a full pack of every object reachable from targetHash. +func (s *GoGitStore) Download(ctx context.Context, refName, targetHash string) (io.ReadCloser, error) { + hash := plumbing.NewHash(targetHash) + objs, err := revlist.Objects(s.Repo.Storer, []plumbing.Hash{hash}, nil) + if err != nil { + return nil, fmt.Errorf("gogit store: resolve download set: %w", err) + } + var buf bytes.Buffer + enc := packfile.NewEncoder(&buf, s.Repo.Storer, false) // delta=false + if _, err := enc.Encode(objs, 0); err != nil { + return nil, fmt.Errorf("gogit store: encode download pack: %w", err) + } + return io.NopCloser(bytes.NewReader(buf.Bytes())), nil +} + +func (s *GoGitStore) Upload(ctx context.Context, dstRef, srcHash string, pack io.Reader) error { + st := s.Repo.Storer + if err := IngestPack(st, pack); err != nil { + return err + } + return SetLocalRef(st, dstRef, plumbing.NewHash(srcHash)) +} + +func (s *GoGitStore) Delete(ctx context.Context, dstRef string) error { + return s.Repo.Storer.RemoveReference(plumbing.ReferenceName(dstRef)) +} + +func (s *GoGitStore) Close() error { return nil } diff --git a/internal/cli/gitremote/integration_test.go b/internal/cli/gitremote/integration_test.go new file mode 100644 index 00000000..34e2c782 --- /dev/null +++ b/internal/cli/gitremote/integration_test.go @@ -0,0 +1,123 @@ +package gitremote + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// This test exercises the full fetch/push remote-helper protocol through go-git +// only — no Sia, no network, no git subprocess. It: +// +// 1. builds a source repo with two commits, +// 2. pushes both commits (in two separate pushes so the second one is an +// incremental pack against the archived tip), +// 3. verifies the remote bare-repo store now serves the ref + full history, and +// 4. fetches into a brand new repo and confirms the objects arrived intact. +func TestGoGitPushThenFetch(t *testing.T) { + ctx := context.Background() + + // 1. Source repo with two commits on refs/heads/master. + srcDir := t.TempDir() + src, err := git.PlainInit(srcDir, false) + require.NoError(t, err) + c1 := commitFile(t, src, "a.txt", "hello from a\n", "first") + c2 := commitFile(t, src, "b.txt", "hello from b\n", "second") + + // 2. Remote is a bare go-git repo wrapped in GoGitStore. + remoteDir := t.TempDir() + remote, err := git.PlainInit(remoteDir, true) + require.NoError(t, err) + store := &GoGitStore{Repo: remote} + defer store.Close() + + // First push: full pack of everything reachable from c2. + out1 := runHandle(t, ctx, src.Storer, store, + "capabilities\nlist for-push\npush refs/heads/master:refs/heads/master\n\n") + assert.Contains(t, out1, "ok refs/heads/master\n") + + ref, err := remote.Storer.Reference(plumbing.ReferenceName("refs/heads/master")) + require.NoError(t, err) + require.Equal(t, c2, ref.Hash()) + + // Second push: add a third commit; the incremental pack must only carry the + // new objects not already archived. + c3 := commitFile(t, src, "c.txt", "hello from c\n", "third") + out2 := runHandle(t, ctx, src.Storer, store, + "capabilities\nlist for-push\npush refs/heads/master:refs/heads/master\n\n") + assert.Contains(t, out2, "ok refs/heads/master\n") + ref, err = remote.Storer.Reference(plumbing.ReferenceName("refs/heads/master")) + require.NoError(t, err) + require.Equal(t, c3, ref.Hash()) + + // 3. Fresh empty repo fetches the advertised tip and gets full history. + dstDir := t.TempDir() + dst, err := git.PlainInit(dstDir, false) + require.NoError(t, err) + + outList := runHandle(t, ctx, dst.Storer, store, "capabilities\nlist\n") + assert.Contains(t, outList, c3.String()+" refs/heads/master\n") + + fetchIn := "capabilities\nlist\nfetch " + c3.String() + " refs/heads/master\n\n" + runHandle(t, ctx, dst.Storer, store, fetchIn) + + // 4. Objects are present in the fresh store with full lineage. + c3got, err := readCommit(dst.Storer, c3) + require.NoError(t, err) + require.Equal(t, c2, c3got.ParentHashes[0]) + // c2 (and transitively c1) transferred because Download sent a full pack. + _, err = readCommit(dst.Storer, c2) + require.NoError(t, err) + _, err = readCommit(dst.Storer, c1) + require.NoError(t, err) + + tree, err := c3got.Tree() + require.NoError(t, err) + names := map[string]bool{} + _ = tree.Files().ForEach(func(f *object.File) error { + names[f.Name] = true + return nil + }) + assert.True(t, names["a.txt"]) + assert.True(t, names["b.txt"]) + assert.True(t, names["c.txt"]) +} + +// runHandle runs the protocol engine against local + store with the given stdin +// and returns the stdout text. +func runHandle(t *testing.T, ctx context.Context, local storage.Storer, store Store, in string) string { + t.Helper() + var out, errw bytes.Buffer + err := Handle(ctx, local, store, strings.NewReader(in), &out, &errw) + require.NoError(t, err, "Handle failed: %v", err) + return out.String() +} + +// commitFile writes a file into the worktree, stages it, and commits it on +// refs/heads/master (go-git sets that ref for a non-bare repo), returning the +// new commit hash. +func commitFile(t *testing.T, repo *git.Repository, name, content, msg string) plumbing.Hash { + t.Helper() + wt, err := repo.Worktree() + require.NoError(t, err) + f, err := wt.Filesystem.Create(name) + require.NoError(t, err) + _, werr := f.Write([]byte(content)) + require.NoError(t, werr) + require.NoError(t, f.Close()) + _, err = wt.Add(name) + require.NoError(t, err) + h, err := wt.Commit(msg, &git.CommitOptions{ + Author: &object.Signature{Name: "t", Email: "t@example.com"}, + }) + require.NoError(t, err) + return h +} diff --git a/internal/cli/gitremote/protocol.go b/internal/cli/gitremote/protocol.go new file mode 100644 index 00000000..cf53b1f3 --- /dev/null +++ b/internal/cli/gitremote/protocol.go @@ -0,0 +1,319 @@ +package gitremote + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "strings" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage" +) + +// supportedCaps are the capabilities the helper advertises (option / fetch / +// push). The helper serves only the dumb fetch/push protocol — it does not +// implement connect/import/export or wire-protocol v2. +var supportedCaps = []string{"option", "fetch", "push"} + +// okOptions are transport options we accept and simply acknowledge; they only +// expose knobs the engine does not need to act on (verbosity, progress, depth +// stubs, force/cloning hints...). Everything else is reported unsupported, and +// object-format in particular is rejected so Git falls back to sha1 (go-git's +// hash width). +var okOptions = map[string]bool{ + "verbosity": true, + "progress": true, + "depth": true, + "deepen-since": true, + "deepen-not": true, + "deepen-relative": true, + "followtags": true, + "dry-run": true, + "servpath": true, + "check-connectivity": true, + "force": true, + "cloning": true, + "update-shallow": true, + "pushcert": true, + "push-option": true, + "from-promisor": true, + "no-dependents": true, + "atomic": true, +} + +// Handle runs the remote-helper protocol over in/out against store, transferring +// objects through the local go-git object store (the repository Git pointed the +// helper at via GIT_DIR/cwd). Normal protocol output goes to out; diagnostics go +// to errw. It returns a non-nil error only for fatal conditions (a failed fetch, +// malformed input); push failures are reported per-ref on the protocol stream +// rather than as stream-wide errors, per gitremote-helpers(7). +func Handle(ctx context.Context, local storage.Storer, store Store, in io.Reader, out, errw io.Writer) error { + if store == nil { + return fmt.Errorf("gitremote: nil store") + } + h := &handler{local: local, store: store, out: out, errw: errw} + if errw == nil { + h.errw = io.Discard + } + + reader := bufio.NewReader(in) + for { + line, err := readLine(reader) + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if strings.TrimSpace(line) == "" { + // A blank line at the top level terminates the command stream. + return nil + } + fields := strings.Fields(line) + switch fields[0] { + case "capabilities": + h.writeCapabilities() + case "list": + forPush := len(fields) > 1 && fields[1] == "for-push" + if err := h.handleList(ctx, forPush); err != nil { + return err + } + case "option": + name := "" + value := "" + if len(fields) > 1 { + name = fields[1] + } + if len(fields) > 2 { + value = fields[2] + } + h.handleOption(name, value) + case "fetch": + cmds := [][2]string{{fields[1], fields[2]}} + // A fetch batch continues until a blank line. + for { + l2, err := readLine(reader) + if err == io.EOF { + break + } + if err != nil { + return err + } + if strings.TrimSpace(l2) == "" { + break + } + f2 := strings.Fields(l2) + if len(f2) < 3 || f2[0] != "fetch" { + return fmt.Errorf("gitremote: malformed fetch command %q", l2) + } + cmds = append(cmds, [2]string{f2[1], f2[2]}) + } + if err := h.handleFetch(ctx, cmds); err != nil { + return err + } + case "push": + // A push command is `push ` — one token like +a:b or :b. + cmds := [][1]string{{fields[1]}} + for { + l2, err := readLine(reader) + if err == io.EOF { + break + } + if err != nil { + return err + } + if strings.TrimSpace(l2) == "" { + break + } + f2 := strings.Fields(l2) + if f2[0] != "push" { + return fmt.Errorf("gitremote: malformed push command %q", l2) + } + cmds = append(cmds, [1]string{f2[1]}) + } + if err := h.handlePush(ctx, cmds); err != nil { + return err + } + default: + return fmt.Errorf("gitremote: unknown command %q", fields[0]) + } + } +} + +// handler carries the protocol engine's per-invocation state. +type handler struct { + local storage.Storer + store Store + out io.Writer + errw io.Writer + + // refsForPush is the most recent `list for-push` result. It records the + // remote's current refs so a push batch can compute the incremental pack + // against the archived state. + refsForPush []Ref +} + +func (h *handler) writeCapabilities() { + for _, c := range supportedCaps { + fmt.Fprintf(h.out, "%s\n", c) + } + fmt.Fprintf(h.out, "\n") +} + +func (h *handler) handleList(ctx context.Context, forPush bool) error { + refs, err := h.store.List(ctx, forPush) + if err != nil { + return fmt.Errorf("gitremote: list: %w", err) + } + if forPush { + h.refsForPush = refs + } + for _, r := range refs { + val := r.Hash.String() + if r.Hash == plumbing.ZeroHash { + val = "?" + } + fmt.Fprintf(h.out, "%s %s\n", val, r.Name) + } + fmt.Fprintf(h.out, "\n") + return nil +} + +func (h *handler) handleOption(name, value string) { + if name == "object-format" { + // We only speak sha1; asking for object-format (sha256) is unsupported, + // which makes Git fall back to sha1. + fmt.Fprintf(h.out, "unsupported\n") + return + } + if okOptions[name] { + fmt.Fprintf(h.out, "ok\n") + return + } + fmt.Fprintf(h.out, "unsupported\n") +} + +func (h *handler) handleFetch(ctx context.Context, cmds [][2]string) error { + for _, cmd := range cmds { + sha1, name := cmd[0], cmd[1] + if !hexHashRE.MatchString(sha1) { + return fmt.Errorf("gitremote: fetch: invalid object %q", sha1) + } + rc, err := h.store.Download(ctx, name, sha1) + if err != nil { + return fmt.Errorf("gitremote: fetch %s: %w", name, err) + } + if err := IngestPack(h.local, rc); err != nil { + if rc != nil { + rc.Close() + } + return fmt.Errorf("gitremote: fetch %s: %w", name, err) + } + if rc != nil { + rc.Close() + } + } + // Single blank line marks the whole batch complete. + fmt.Fprintf(h.out, "\n") + return nil +} + +func (h *handler) handlePush(ctx context.Context, cmds [][1]string) error { + // Remote state before this batch (from `list for-push`), keyed by name. + remote := map[string]plumbing.Hash{} + for _, r := range h.refsForPush { + if r.Hash != plumbing.ZeroHash { + remote[r.Name] = r.Hash + } + } + + for _, cmd := range cmds { + refspec := cmd[0] + colon := strings.IndexByte(refspec, ':') + if colon < 0 { + fmt.Fprintf(h.out, "error %s missing destination\n", refspec) + continue + } + src := strings.TrimPrefix(refspec[:colon], "+") + dst := refspec[colon+1:] + + // Delete ref push: empty source. + if src == "" { + if err := h.store.Delete(ctx, dst); err != nil { + fmt.Fprintf(h.out, "error %s %s\n", dst, quoteWhy(err)) + } else { + fmt.Fprintf(h.out, "ok %s\n", dst) + } + continue + } + + srcHash, err := resolveRevision(h.local, src) + if err != nil { + fmt.Fprintf(h.out, "error %s %s\n", dst, quoteWhy(err)) + continue + } + + // Incremental pack: new objects minus what the remote already had. + var haves []plumbing.Hash + if old, ok := remote[dst]; ok { + haves = append(haves, old) + } + pack, err := EncodeIncrementalPack(h.local, []plumbing.Hash{srcHash}, haves) + if err != nil { + fmt.Fprintf(h.out, "error %s %s\n", dst, quoteWhy(err)) + continue + } + if err := h.store.Upload(ctx, dst, srcHash.String(), bytes.NewReader(pack)); err != nil { + fmt.Fprintf(h.out, "error %s %s\n", dst, quoteWhy(err)) + continue + } + fmt.Fprintf(h.out, "ok %s\n", dst) + } + // Blank line terminates the status report. + fmt.Fprintf(h.out, "\n") + return nil +} + +// quoteWhy renders an error for the `error ` line, quoting it in C +// style when it contains a newline (per the protocol spec). +func quoteWhy(err error) string { + msg := err.Error() + if strings.ContainsAny(msg, "\n\r\"") { + return strconvQuote(msg) + } + return msg +} + +func strconvQuote(s string) string { + var b strings.Builder + b.Grow(len(s) + 2) + b.WriteByte('"') + for _, r := range s { + switch r { + case '"': + b.WriteString(`\"`) + case '\\': + b.WriteString(`\\`) + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + default: + b.WriteRune(r) + } + } + b.WriteByte('"') + return b.String() +} + +// readLine reads one line from r without the trailing newline; io.EOF is +// returned when the stream ends cleanly. +func readLine(r *bufio.Reader) (string, error) { + line, err := r.ReadString('\n') + if err != nil && len(line) == 0 { + return "", err + } + return strings.TrimSuffix(line, "\n"), nil +} diff --git a/internal/cli/gitremote/protocol_test.go b/internal/cli/gitremote/protocol_test.go new file mode 100644 index 00000000..acd910b4 --- /dev/null +++ b/internal/cli/gitremote/protocol_test.go @@ -0,0 +1,184 @@ +package gitremote + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage/memory" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeStore is a bytes-only Store stub that records the calls the protocol +// engine makes, so the engine can be tested in isolation from any real backend. +type fakeStore struct { + refs []Ref + listErr error + downloads []string // refName:targetHash seen + download io.ReadCloser + downloadErr error + uploads []uploadCall + uploadErr error + deletes []string + deleteErr error +} + +type uploadCall struct { + dstRef string + srcHash string + pack []byte +} + +func (f *fakeStore) List(_ context.Context, _ bool) ([]Ref, error) { + if f.listErr != nil { + return nil, f.listErr + } + return f.refs, nil +} + +func (f *fakeStore) Download(_ context.Context, refName, targetHash string) (io.ReadCloser, error) { + f.downloads = append(f.downloads, refName+":"+targetHash) + if f.downloadErr != nil { + return nil, f.downloadErr + } + if f.download != nil { + return f.download, nil + } + return io.NopCloser(bytes.NewReader(nil)), nil +} + +func (f *fakeStore) Upload(_ context.Context, dstRef, srcHash string, pack io.Reader) error { + data, _ := io.ReadAll(pack) + f.uploads = append(f.uploads, uploadCall{dstRef: dstRef, srcHash: srcHash, pack: data}) + return f.uploadErr +} + +func (f *fakeStore) Delete(_ context.Context, dstRef string) error { + f.deletes = append(f.deletes, dstRef) + return f.deleteErr +} + +func (f *fakeStore) Close() error { return nil } + +// handleText runs Handle against an empty in-memory local store with a synthetic +// stdin string and returns the stdout text. Protocol tests only assert the +// command/output wiring; object movement is covered by the go-git integration +// test. +func handleText(t *testing.T, store Store, in string) string { + t.Helper() + var out, errw bytes.Buffer + err := Handle(context.Background(), memory.NewStorage(), store, strings.NewReader(in), &out, &errw) + require.NoError(t, err, "Handle failed: %v", err) + return out.String() +} + +func TestCapabilities(t *testing.T) { + out := handleText(t, &fakeStore{}, "capabilities\n") + lines := strings.Split(strings.TrimSuffix(out, "\n"), "\n") + require.Equal(t, []string{"option", "fetch", "push", ""}, lines) +} + +func TestListAdvertisesRefs(t *testing.T) { + store := &fakeStore{refs: []Ref{ + {Name: "refs/heads/master", Hash: plumbing.NewHash("aaaabbbbccccddddeeeeffff0000111122223333")}, + {Name: "refs/tags/v1", Hash: plumbing.NewHash("1111222233334444555566667777888899990000")}, + }} + out := handleText(t, store, "capabilities\nlist\n") + assert.Contains(t, out, "aaaabbbbccccddddeeeeffff0000111122223333 refs/heads/master\n") + assert.Contains(t, out, "1111222233334444555566667777888899990000 refs/tags/v1\n") + assert.True(t, strings.HasSuffix(out, "\n\n")) +} + +// TestListForPush advertises refs in `list for-push` mode (the preparer for a +// push batch). The old-tip-as-haves incremental path is exercised with real +// objects in TestGoGitPushThenFetch (integration test). +func TestListForPush(t *testing.T) { + store := &fakeStore{refs: []Ref{ + {Name: "refs/heads/master", Hash: plumbing.NewHash("bbb0000000000000000000000000000000000000")}, + }} + out := handleText(t, store, "capabilities\nlist for-push\n") + assert.Contains(t, out, "bbb0000000000000000000000000000000000000 refs/heads/master\n") +} + +func TestPushOkAndDelete(t *testing.T) { + store := &fakeStore{} + const in = "" + + "capabilities\n" + + "list for-push\n" + + "push :refs/heads/gone\n" + + "\n" + out := handleText(t, store, in) + require.Equal(t, []string{"refs/heads/gone"}, store.deletes) + assert.Contains(t, out, "ok refs/heads/gone\n") + assert.True(t, strings.HasSuffix(out, "\n\n")) +} + +func TestOptionHandling(t *testing.T) { + store := &fakeStore{} + const in = "" + + "capabilities\n" + + "option verbosity 3\n" + + "option object-format true\n" + + "option not-a-real-option 1\n" + + "list\n" + out := handleText(t, store, in) + assert.Contains(t, out, "ok\n") + assert.Contains(t, out, "unsupported\n") +} + +func TestFetchBatchOutputsBlankLine(t *testing.T) { + store := &fakeStore{download: io.NopCloser(bytes.NewReader(nil))} + const in = "" + + "capabilities\n" + + "list\n" + + "fetch aaaabbbbccccddddeeeeffff0000111122223333 refs/heads/master\n" + + "fetch 1111222233334444555566667777888899990000 refs/heads/dev\n" + + "\n" + out := handleText(t, store, in) + require.Equal(t, []string{ + "refs/heads/master:aaaabbbbccccddddeeeeffff0000111122223333", + "refs/heads/dev:1111222233334444555566667777888899990000", + }, store.downloads) + // Output: capabilities response, empty list, then exactly one blank line + // terminating the fetch batch. + assert.Equal(t, "option\nfetch\npush\n\n\n\n", out) +} + +func TestFetchFails(t *testing.T) { + store := &fakeStore{downloadErr: io.ErrUnexpectedEOF} + const in = "" + + "capabilities\n" + + "list\n" + + "fetch aaaabbbbccccddddeeeeffff0000111122223333 refs/heads/master\n" + + "\n" + var out, errw bytes.Buffer + err := Handle(context.Background(), memory.NewStorage(), store, strings.NewReader(in), &out, &errw) + require.Error(t, err) + assert.Contains(t, err.Error(), "fetch") +} + +func TestPushFailureReportedPerRef(t *testing.T) { + store := &fakeStore{uploadErr: io.ErrClosedPipe} + const in = "" + + "capabilities\n" + + "list for-push\n" + + "push ccc1111111111111111111111111111111111111:refs/heads/master\n" + + "push :refs/heads/other\n" + + "\n" + out := handleText(t, store, in) + // Push errors are per-ref status lines, not fatal stream errors. + assert.Contains(t, out, "error refs/heads/master ") + assert.Contains(t, out, "ok refs/heads/other\n") + assert.True(t, strings.HasSuffix(out, "\n\n")) +} + +func TestUnknownCommandFatal(t *testing.T) { + var out bytes.Buffer + err := Handle(context.Background(), memory.NewStorage(), &fakeStore{}, strings.NewReader("capabilities\nbogus\n"), &out, io.Discard) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown command") +} diff --git a/internal/cli/gitremote/publish.go b/internal/cli/gitremote/publish.go new file mode 100644 index 00000000..fc11adc9 --- /dev/null +++ b/internal/cli/gitremote/publish.go @@ -0,0 +1,98 @@ +package gitremote + +import ( + "bytes" + "context" + "fmt" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/revlist" + "github.com/go-git/go-git/v5/storage" +) + +// LocalBranches lists the local branch refs (refs/heads/*) with their concrete +// tip hashes, skipping symbolic references. +func LocalBranches(st storage.Storer) ([]Ref, error) { + iter, err := st.IterReferences() + if err != nil { + return nil, fmt.Errorf("gitremote: list local branches: %w", err) + } + defer iter.Close() + var refs []Ref + _ = iter.ForEach(func(r *plumbing.Reference) error { + if r.Type() == plumbing.SymbolicReference || !r.Name().IsBranch() { + return nil + } + refs = append(refs, Ref{Name: r.Name().String(), Hash: r.Hash()}) + return nil + }) + return refs, nil +} + +// PublishAll performs a first publish (or a refresh) of every local branch tip +// to the archive: for each branch it encodes a full pack of everything +// reachable from the tip (the archive is treated as initially empty) and +// Uploads it under the same branch ref name. It returns the refs published. The +// pack encode and the Store upload both run in-process via go-git — no git +// binary is spawned. +func PublishAll(ctx context.Context, local storage.Storer, store Store) ([]Ref, error) { + refs, err := LocalBranches(local) + if err != nil { + return nil, err + } + for _, r := range refs { + pack, err := EncodeIncrementalPack(local, []plumbing.Hash{r.Hash}, nil) + if err != nil { + return nil, fmt.Errorf("gitremote: encode first pack for %s: %w", r.Name, err) + } + if err := store.Upload(ctx, r.Name, r.Hash.String(), bytes.NewReader(pack)); err != nil { + return nil, fmt.Errorf("gitremote: publish %s: %w", r.Name, err) + } + } + return refs, nil +} + +// Divergence computes how far ahead (commits present only on the local side) +// and behind (commits present only on the remote/archived side) localHash is +// relative to remoteHash. It counts commit objects only, ignoring trees, blobs +// and tags reachable from either tip. +func Divergence(st storage.Storer, localHash, remoteHash plumbing.Hash) (ahead, behind int, err error) { + loc, err := commitSet(st, localHash) + if err != nil { + return 0, 0, fmt.Errorf("gitremote: walk local commits: %w", err) + } + rem, err := commitSet(st, remoteHash) + if err != nil { + return 0, 0, fmt.Errorf("gitremote: walk remote commits: %w", err) + } + for h := range loc { + if !rem[h] { + ahead++ + } + } + for h := range rem { + if !loc[h] { + behind++ + } + } + return ahead, behind, nil +} + +// commitSet returns the set of commit hashes reachable from h in st. +func commitSet(st storage.Storer, h plumbing.Hash) (map[plumbing.Hash]bool, error) { + objs, err := revlist.Objects(st, []plumbing.Hash{h}, nil) + if err != nil { + return nil, err + } + set := make(map[plumbing.Hash]bool, len(objs)) + for _, oh := range objs { + o, err := st.EncodedObject(plumbing.AnyObject, oh) + if err != nil { + continue + } + if o.Type() == plumbing.CommitObject { + set[oh] = true + } + } + return set, nil +} diff --git a/internal/cli/gitremote/publish_test.go b/internal/cli/gitremote/publish_test.go new file mode 100644 index 00000000..20810bb0 --- /dev/null +++ b/internal/cli/gitremote/publish_test.go @@ -0,0 +1,80 @@ +package gitremote + +import ( + "context" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestPublishAllFirstPublish pushes every local branch as a full pack into an +// empty archive (a bare go-git repo) so full history is available remotely. +func TestPublishAllFirstPublish(t *testing.T) { + ctx := context.Background() + + srcDir := t.TempDir() + src, err := git.PlainInit(srcDir, false) + require.NoError(t, err) + c1 := commitFile(t, src, "a.txt", "a\n", "first") + c2 := commitFile(t, src, "b.txt", "b\n", "second") + + remoteDir := t.TempDir() + remote, err := git.PlainInit(remoteDir, true) + require.NoError(t, err) + store := &GoGitStore{Repo: remote} + defer store.Close() + + refs, err := PublishAll(ctx, src.Storer, store) + require.NoError(t, err) + require.Len(t, refs, 1) + require.Equal(t, "refs/heads/master", refs[0].Name) + require.Equal(t, c2, refs[0].Hash) + + // The archive now serves the branch tip and full history (both commits). + ref, err := remote.Storer.Reference(plumbing.ReferenceName("refs/heads/master")) + require.NoError(t, err) + require.Equal(t, c2, ref.Hash()) + _, err = readCommit(remote.Storer, c2) + require.NoError(t, err) + _, err = readCommit(remote.Storer, c1) + require.NoError(t, err) +} + +func TestLocalBranches(t *testing.T) { + dir := t.TempDir() + repo, err := git.PlainInit(dir, false) + require.NoError(t, err) + commitFile(t, repo, "x.txt", "x\n", "init") + + refs, err := LocalBranches(repo.Storer) + require.NoError(t, err) + // Only branches are reported (HEAD symbolic + refs/heads/*). + require.Len(t, refs, 1) + require.Equal(t, "refs/heads/master", refs[0].Name) +} + +func TestDivergence(t *testing.T) { + dir := t.TempDir() + repo, err := git.PlainInit(dir, false) + require.NoError(t, err) + c1 := commitFile(t, repo, "a.txt", "a\n", "first") + c2 := commitFile(t, repo, "b.txt", "b\n", "second") + + ahead, behind, err := Divergence(repo.Storer, c2, c1) + require.NoError(t, err) + assert.Equal(t, 1, ahead) + assert.Equal(t, 0, behind) + + ahead, behind, err = Divergence(repo.Storer, c1, c2) + require.NoError(t, err) + assert.Equal(t, 0, ahead) + assert.Equal(t, 1, behind) + + ahead, behind, err = Divergence(repo.Storer, c2, c2) + require.NoError(t, err) + assert.Equal(t, 0, ahead) + assert.Equal(t, 0, behind) +} diff --git a/internal/cli/gitremote/replay.go b/internal/cli/gitremote/replay.go new file mode 100644 index 00000000..ffc903fc --- /dev/null +++ b/internal/cli/gitremote/replay.go @@ -0,0 +1,40 @@ +package gitremote + +import ( + "fmt" + "io" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage/memory" +) + +// replayPacksAndEncode reconstructs a full pack of everything reachable from +// target by playing every archived pack (in replay order) into an in-memory +// object store and then re-encoding a single full pack. It is the single DRY +// implementation shared by the account-scoped SessionStore.Download (which +// opens packs by object key) and the profile-less ShareStore.Download (which +// opens packs over pre-signed share URLs): each caller only supplies the +// ordered list of pack-openers, so the replay/encode logic lives in one place. +// +// opening is called lazily per pack and returns a reader over the raw pack +// bytes; every reader is closed after ingest. All object work runs in-process +// via go-git — no git subprocess. +func replayPacksAndEncode(openers []func() (io.ReadCloser, error), target plumbing.Hash) ([]byte, error) { + mem := memory.NewStorage() + for i, open := range openers { + rc, err := open() + if err != nil { + return nil, fmt.Errorf("gitremote: open pack %d: %w", i, err) + } + ingestErr := IngestPack(mem, rc) + rc.Close() + if ingestErr != nil { + return nil, fmt.Errorf("gitremote: ingest pack %d: %w", i, ingestErr) + } + } + pack, err := EncodeIncrementalPack(mem, []plumbing.Hash{target}, nil) + if err != nil { + return nil, err + } + return pack, nil +} diff --git a/internal/cli/gitremote/session_store.go b/internal/cli/gitremote/session_store.go new file mode 100644 index 00000000..bea8d84c --- /dev/null +++ b/internal/cli/gitremote/session_store.go @@ -0,0 +1,86 @@ +package gitremote + +import ( + "context" + "errors" + "os" + "strings" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +// ErrSiaNotWired is returned by the production Store's data operations while the +// Sia-backed archive backend is not compiled in (the default build). The +// remote-helper protocol engine, the go-git pack encode/ingest path, and +// reference handling are always complete; publishing a pack + tip to Sia and +// downloading tip packs back is the dedicated Sia integration enabled by the +// `gitarchive_sia` build tag (see session_store_sia.go). Without that tag the +// data operations report this sentinel; with it, they exercise the real SDK. +var ErrSiaNotWired = errors.New("git remote helper: Sia archive backend not wired yet (build without gitarchive_sia)") + +// SessionStore is the production Store backed by a resolved Pinner profile +// session (see internal/core/gitarchive.Session). It owns the session so Close +// releases the SDK/DB. +type SessionStore struct { + session *gitarchive.Session + locker string // account-scoped locker this store reads/writes (pinner::) +} + +var _ Store = (*SessionStore)(nil) + +// NewSessionStoreFromEnv resolves the active profile (flag/PINNER_PROFILE/ +// default, via the vault registry) and returns a session-backed Store. It also +// surfaces gitarchive.ErrNoProfile intact so the helper prints the exact +// "no vault profile; run pinner vault create or pinner vault restore" message. +func NewSessionStoreFromEnv(ctx context.Context) (*SessionStore, error) { + return NewSessionStore(os.Getenv("PINNER_PROFILE"), "") +} + +// NewSessionStore builds a session-backed Store for the given profile and Sia +// indexer URL. An empty indexerURL is acceptable until real SDK calls are made +// (the session builds its SDK lazily on first use). +func NewSessionStore(profile, indexerURL string) (*SessionStore, error) { + s, err := gitarchive.NewSession(profile, indexerURL) + if err != nil { + return nil, err + } + return &SessionStore{session: s}, nil +} + +// NewSessionStoreFromSession wraps an existing session so CLI git commands +// (watch/status) can share the session they already opened for DB access. The +// returned store does NOT own the session — the caller manages Close on the +// underlying session. +func NewSessionStoreFromSession(s *gitarchive.Session) *SessionStore { + return &SessionStore{session: s} +} + +// ForLocker scopes this store to the account locker named locker (the locker +// portion of a `pinner::` remote URL) and returns the store for +// chaining. Data operations against a locker are meaningless without it. +func (s *SessionStore) ForLocker(locker string) *SessionStore { + s.locker = locker + return s +} + +// Close releases SDK-held resources. +func (s *SessionStore) Close() error { + if s.session == nil { + return nil + } + return s.session.Close() +} + +// ParseLockedRemote reports whether url is an account-scoped `pinner::` +// remote (as opposed to the profile-less `pinner::share/...` path) and, if so, +// returns the locker name embedded in it. +func ParseLockedRemote(url string) (string, bool) { + if !strings.HasPrefix(url, "pinner::") { + return "", false + } + locker := strings.TrimPrefix(url, "pinner::") + if locker == "" || strings.HasPrefix(locker, "share/") { + return "", false + } + return locker, true +} diff --git a/internal/cli/gitremote/session_store_sia.go b/internal/cli/gitremote/session_store_sia.go new file mode 100644 index 00000000..079a61b2 --- /dev/null +++ b/internal/cli/gitremote/session_store_sia.go @@ -0,0 +1,255 @@ +//go:build gitarchive_sia + +package gitremote + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + + "github.com/go-git/go-git/v5/plumbing" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +// This file implements the REAL Sia-backed SessionStore, compiled in only with +// the `gitarchive_sia` build tag (opt-in, since it requires a live Sia indexer +// configuration to be meaningful). It drives the actual Sia SDK through the +// gitarchive session: pack/tip upload via gitarchive.UploadPack/UploadTip, +// download over gitarchive.FetchObjectBytes (which verifies each card digest), +// and generation CAS on publish. Every Git operation still runs in-process via +// go-git — no git subprocess, no shell-outs. + +// loadTip returns the current archived tip for the store's locker and its +// on-disk object key, or (nil, nil) when the locker has no tip yet. It +// downloads and digest-verifies the tip payload each time. +func (s *SessionStore) loadTip(ctx context.Context) (*gitarchive.TipBytes, *gitarchive.GitObject, error) { + if s.session == nil { + return nil, nil, errors.New("gitremote: session store has no session") + } + tipObj, _, err := gitarchive.FindTipForLocker(s.session.DB, s.locker) + if err != nil { + return nil, nil, err + } + if tipObj == nil { + return nil, nil, nil + } + key, err := gitarchive.ParseObjectKey(tipObj.ObjectKey) + if err != nil { + return nil, nil, err + } + _, data, err := gitarchive.FetchObjectBytes(ctx, s.session, key) + if err != nil { + return nil, nil, err + } + tip, err := gitarchive.ParseTipBytes(data) + if err != nil { + return nil, nil, err + } + return tip, tipObj, nil +} + +// List advertises the archived refs by loading the current tip's refs map, +// sorted by name (deterministic protocol output). +func (s *SessionStore) List(ctx context.Context, forPush bool) ([]Ref, error) { + tip, _, err := s.loadTip(ctx) + if err != nil { + return nil, err + } + if tip == nil { + return nil, nil + } + refs := make([]Ref, 0, len(tip.Refs)) + for name, hash := range tip.Refs { + refs = append(refs, Ref{Name: name, Hash: plumbing.NewHash(hash)}) + } + sort.Slice(refs, func(i, j int) bool { return refs[i].Name < refs[j].Name }) + return refs, nil +} + +// Download reconstructs a full pack of everything reachable from targetHash by +// downloading every archived pack (in replay order) over the Sia SDK and +// re-encoding a single full pack in-process. Each pack's payload is +// digest-verified against its card by FetchObjectBytes before replay, so a +// corrupted/tampered pack surfaces as an error here. +func (s *SessionStore) Download(ctx context.Context, refName, targetHash string) (io.ReadCloser, error) { + if s.locker == "" { + return nil, errors.New("gitremote: session store has no locker") + } + tip, _, err := s.loadTip(ctx) + if err != nil { + return nil, err + } + if tip == nil { + return nil, fmt.Errorf("gitremote: locker %q has no archived tips to download", s.locker) + } + openers := make([]func() (io.ReadCloser, error), 0, len(tip.Packs)) + for _, p := range tip.Packs { + p := p + openers = append(openers, func() (io.ReadCloser, error) { + key, err := gitarchive.ParseObjectKey(p.Key) + if err != nil { + return nil, err + } + _, data, err := gitarchive.FetchObjectBytes(ctx, s.session, key) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(data)), nil + }) + } + pack, err := replayPacksAndEncode(openers, plumbing.NewHash(targetHash)) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(pack)), nil +} + +// Upload persists the incoming pack as a new git.pack card and publishes a new +// git.tip card (gen+1, prev=old tip) carrying the updated refs and the appended +// pack. It refuses to publish if the generation moved since it read the old tip +// (gen CAS), so a concurrent upload cannot silently clobber a newer archive. +func (s *SessionStore) Upload(ctx context.Context, dstRef, srcHash string, pack io.Reader) error { + if s.locker == "" { + return errors.New("gitremote: session store has no locker") + } + tip, tipObj, err := s.loadTip(ctx) + if err != nil { + return err + } + + oldRefs := map[string]string{} + var oldPacks []gitarchive.PackRef + oldKey, gen := "", 0 + prevName, prevLineage := "", []string{} + if tip != nil { + oldRefs = tip.Refs + oldPacks = tip.Packs + oldKey = tipObj.ObjectKey + gen = tip.Gen + prevName, prevLineage = tip.Name, tip.Lineage + } + + newRefs := copyStringMap(oldRefs) + newRefs[dstRef] = srcHash + newPacks := append([]gitarchive.PackRef{}, oldPacks...) + + data, err := io.ReadAll(pack) + if err != nil { + return fmt.Errorf("gitremote: read incoming pack: %w", err) + } + // An empty pack (incremental encode found nothing new) adds no object — the + // ref move alone is a new tip over the existing pack set. + if len(data) > 0 { + uo, err := gitarchive.UploadPack(ctx, s.session, s.locker, + fmt.Sprintf("%s:%s:%d", dstRef, srcHash, gen), false, bytes.NewReader(data)) + if err != nil { + return err + } + newPacks = append(newPacks, gitarchive.PackRef{Key: uo.Key.String(), Full: false, Digest: uo.Digest}) + } + + // Gen CAS: publish must not move a generation that changed after we read it. + if err := s.casPublish(ctx, oldKey); err != nil { + return err + } + + newGen := gen + 1 + newTip := &gitarchive.TipBytes{ + Locker: s.locker, + Gen: newGen, + Name: prevName, + Lineage: prevLineage, + Refs: newRefs, + Packs: newPacks, + Prev: oldKey, + } + return s.publishTip(ctx, newTip, newGen, oldKey) +} + +// Delete publishes a new tip with dstRef removed from the refs map (the +// protocol's `:dst` delete-ref push). No pack object is produced; only the +// reference set changes. +func (s *SessionStore) Delete(ctx context.Context, dstRef string) error { + if s.locker == "" { + return errors.New("gitremote: session store has no locker") + } + tip, tipObj, err := s.loadTip(ctx) + if err != nil { + return err + } + if tip == nil { + return nil // nothing archived to delete + } + newRefs := copyStringMap(tip.Refs) + delete(newRefs, dstRef) + oldKey := tipObj.ObjectKey + + if err := s.casPublish(ctx, oldKey); err != nil { + return err + } + + newGen := tip.Gen + 1 + newTip := &gitarchive.TipBytes{ + Locker: s.locker, + Gen: newGen, + Name: tip.Name, + Lineage: tip.Lineage, + Refs: newRefs, + Packs: tip.Packs, + Prev: oldKey, + } + return s.publishTip(ctx, newTip, newGen, oldKey) +} + +// casPublish verifies the archived tip has not moved since the caller read it +// ("" meaning no tip existed before). It is the generation compare-and-swap: if +// the current tip key differs from the one the caller based its new tip on, the +// publish is refused. This is a local best-effort CAS over the profile cache DB +// (the coordination point for single-profile writers); it surfaces a clear +// conflict instead of silently overwriting a newer generation. +func (s *SessionStore) casPublish(ctx context.Context, expectKey string) error { + _, tipObj, err := s.loadTip(ctx) + if err != nil { + return err + } + currentKey := "" + if tipObj != nil { + currentKey = tipObj.ObjectKey + } + if currentKey != expectKey { + return fmt.Errorf( + "gitremote: publish refused: archive generation moved (concurrent publish); re-fetch and retry") + } + return nil +} + +// publishTip uploads the new tip card and bumps the tracked TipGen in git_repos. +func (s *SessionStore) publishTip(ctx context.Context, tip *gitarchive.TipBytes, gen int, prev string) error { + raw, err := json.Marshal(tip) + if err != nil { + return fmt.Errorf("gitremote: marshal tip document: %w", err) + } + if _, err := gitarchive.UploadTip(ctx, s.session, s.locker, gen, prev, bytes.NewReader(raw)); err != nil { + return err + } + if err := s.session.DB.Model(&gitarchive.GitRepo{}). + Where("locker = ?", s.locker). + Update("tip_gen", gen).Error; err != nil { + return fmt.Errorf("gitremote: record tip gen for %q: %w", s.locker, err) + } + return nil +} + +// copyStringMap returns a shallow copy of m (nil-safe). +func copyStringMap(m map[string]string) map[string]string { + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = v + } + return out +} diff --git a/internal/cli/gitremote/session_store_sia_test.go b/internal/cli/gitremote/session_store_sia_test.go new file mode 100644 index 00000000..a0153d70 --- /dev/null +++ b/internal/cli/gitremote/session_store_sia_test.go @@ -0,0 +1,252 @@ +//go:build gitarchive_sia + +package gitremote + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "strings" + "sync" + "testing" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/stretchr/testify/require" + "go.sia.tech/core/types" + "go.sia.tech/siastorage" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" + "go.lumeweb.com/pinner-cli/internal/core/objmeta" +) + +// siaFakeSDK is a minimal SDKClient for exercising the Sia-backed SessionStore +// publish/list/delete flow without a live indexer. Uploads are stored by +// content address. +// +// NOTE ON SCOPE: like the shared fakeSDK in the gitarchive package, this fake +// keys objects on siastorage.Object.ID(), which is a hash of the object's slabs. +// An in-memory Upload cannot fabricate real slabs, so every object collapses to +// the same empty-slab ID and "last write wins". That is sufficient to exercise +// the store's publish → list → delete path (the tip is always the object last +// written, so the tip reads back correctly), but NOT a multi-pack download +// reconstruction (which would need per-object keys). Multi-write download +// coverage would require a keyed fake the shared SDKClient seam does not provide +// — the same limitation the existing gitarchive fakeSDK has. Compile + vet + +// single-object publish/download digest coverage (download_test.go) plus this +// publish/list/delete coverage are the feasible gate for the tagged build. +type siaFakeSDK struct { + mu sync.Mutex + objects map[types.Hash256]siaStored +} + +type siaStored struct { + meta json.RawMessage + data []byte +} + +func newSiaFakeSDK() *siaFakeSDK { + return &siaFakeSDK{objects: map[types.Hash256]siaStored{}} +} + +func (f *siaFakeSDK) Upload(ctx context.Context, obj *siastorage.Object, r io.Reader, _ ...siastorage.UploadOption) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + f.mu.Lock() + defer f.mu.Unlock() + f.objects[obj.ID()] = siaStored{meta: obj.Metadata(), data: data} + return nil +} +func (f *siaFakeSDK) PinObject(ctx context.Context, obj siastorage.Object) error { return nil } +func (f *siaFakeSDK) Object(ctx context.Context, key types.Hash256) (siastorage.Object, error) { + f.mu.Lock() + defer f.mu.Unlock() + fs, ok := f.objects[key] + if !ok { + return siastorage.Object{}, fmt.Errorf("fake: object %s not found", key) + } + o := siastorage.NewEmptyObject() + o.UpdateMetadata(fs.meta) + return o, nil +} +func (f *siaFakeSDK) Download(obj siastorage.Object, _ ...siastorage.DownloadOption) (io.ReadCloser, error) { + f.mu.Lock() + defer f.mu.Unlock() + fs, ok := f.objects[obj.ID()] + if !ok { + return nil, fmt.Errorf("fake: no data for object %s", obj.ID()) + } + return io.NopCloser(bytes.NewReader(fs.data)), nil +} +func (f *siaFakeSDK) CreateSharedObjectURL(ctx context.Context, key types.Hash256, until time.Time) (string, error) { + return "_shared_" + key.String(), nil +} +func (f *siaFakeSDK) DownloadSharedObject(ctx context.Context, url string, _ ...siastorage.DownloadOption) (io.ReadCloser, error) { + return nil, fmt.Errorf("fake: no shared download in this test") +} +func (f *siaFakeSDK) Close() error { return nil } + +// siaTestStore builds a Session wired to a fresh migrated DB + fake SDK and a +// SessionStore scoped to locker. +func siaTestStore(t *testing.T, locker string) (*SessionStore, *siaFakeSDK, *gorm.DB) { + t.Helper() + db, err := gorm.Open(sqlite.Open(t.TempDir()+"/test.db"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + require.NoError(t, err) + require.NoError(t, gitarchive.AutoMigrate(db)) + sdk := newSiaFakeSDK() + s := &gitarchive.Session{Profile: "work", IndexerURL: "http://localhost:9980", DB: db, SDK: sdk} + return NewSessionStoreFromSession(s).ForLocker(locker), sdk, db +} + +// makeRepoWithCommit builds a repo containing a single commit with the given +// file content and returns its storer and the commit hash. +func makeRepoWithCommit(t *testing.T, name, content string) (*git.Repository, plumbing.Hash) { + t.Helper() + dir := t.TempDir() + repo, err := git.PlainInit(dir, false) + require.NoError(t, err) + wt, err := repo.Worktree() + require.NoError(t, err) + fname := name + ".txt" + f, err := wt.Filesystem.Create(fname) + require.NoError(t, err) + _, err = f.Write([]byte(content)) + require.NoError(t, err) + require.NoError(t, f.Close()) + _, err = wt.Add(fname) + require.NoError(t, err) + _, err = wt.Commit("add "+fname, &git.CommitOptions{Author: &object.Signature{Name: "t", Email: "t@t", When: time.Now()}}) + require.NoError(t, err) + head, err := repo.Head() + require.NoError(t, err) + return repo, head.Hash() +} + +func TestSiaStorePublishList(t *testing.T) { + ctx := context.Background() + + repo, tip := makeRepoWithCommit(t, "a", "hello") + pack, err := EncodeIncrementalPack(repo.Storer, []plumbing.Hash{tip}, nil) + require.NoError(t, err) + require.NotEmpty(t, pack) + + store, _, db := siaTestStore(t, "widgets") + + // Mirror `pinner git watch`: the git_repos row is registered before the + // store publishes (publish records the tip generation onto it). + require.NoError(t, db.Create(&gitarchive.GitRepo{Locker: "widgets", Name: "widgets"}).Error) + + // First publish of refs/heads/master with a full pack. + require.NoError(t, store.Upload(ctx, "refs/heads/master", tip.String(), bytes.NewReader(pack))) + + // The tip generation was recorded in git_repos. + var repoRow gitarchive.GitRepo + require.NoError(t, db.Where("locker = ?", "widgets").First(&repoRow).Error) + require.Equal(t, 1, repoRow.TipGen) + + // List advertises the ref from the current tip. + refs, err := store.List(ctx, false) + require.NoError(t, err) + require.Len(t, refs, 1) + require.Equal(t, "refs/heads/master", refs[0].Name) + require.Equal(t, tip, refs[0].Hash) +} + +func TestSiaStoreDelete(t *testing.T) { + ctx := context.Background() + store, _, _ := siaTestStore(t, "widgets") + + repo, tip := makeRepoWithCommit(t, "one", "v1") + pack, err := EncodeIncrementalPack(repo.Storer, []plumbing.Hash{tip}, nil) + require.NoError(t, err) + require.NoError(t, store.Upload(ctx, "refs/heads/master", tip.String(), bytes.NewReader(pack))) + + refs, err := store.List(ctx, true) + require.NoError(t, err) + require.Len(t, refs, 1) + require.Equal(t, "refs/heads/master", refs[0].Name) + + // Delete publishes a new tip without the ref. + require.NoError(t, store.Delete(ctx, "refs/heads/master")) + refs, err = store.List(ctx, false) + require.NoError(t, err) + require.Len(t, refs, 0, "deleted ref must no longer be advertised") +} + +func TestSiaStoreDispatchRemote(t *testing.T) { + locker, ok := ParseLockedRemote("pinner::widgets") + require.True(t, ok) + require.Equal(t, "widgets", locker) + + _, ok = ParseLockedRemote("pinner::share/abc") + require.False(t, ok, "a share remote is not an account locker remote") + share, ok := ParseShareRemote("pinner::share/abc") + require.True(t, ok) + require.Equal(t, "abc", share) +} + +// TestSiaStorePublishRefusesMovedGeneration exercises the generation CAS on the +// publish path: casPublish must refuse to publish when the archived tip moved +// after the caller read it (a concurrent publish), and must accept the current +// tip key. +func TestSiaStorePublishRefusesMovedGeneration(t *testing.T) { + ctx := context.Background() + store, sdk, db := siaTestStore(t, "widgets") + + // Two distinct valid 32-byte keys: keyA is the "current" archived tip key + // the store's caller read; keyB is a stale/foreign key the caller based its + // new tip on (i.e. a generation that has since moved). + keyA, err := gitarchive.ParseObjectKey("01" + strings.Repeat("0", 62)) + require.NoError(t, err) + keyB := "02" + strings.Repeat("0", 62) + require.NotEqual(t, keyA.String(), keyB) + + // The tip payload loadTip must be able to fetch and parse. + tipData, err := json.Marshal(&gitarchive.TipBytes{ + Locker: "widgets", Gen: 5, + Refs: map[string]string{"refs/heads/master": "abc"}, + Packs: []gitarchive.PackRef{}, + }) + require.NoError(t, err) + + // Seed the fake SDK with the tip object at keyA. An empty card digest skips + // the download integrity check so the fixture need not self-verify. + card := objmeta.New(objmeta.KindGitTip, int64(len(tipData)), "", time.Now().Unix(), nil) + raw, err := objmeta.Encode(card) + require.NoError(t, err) + // The fake's Upload stores by empty-slab obj.ID() (a single collapsed key), + // so store the payload under both that ID (for Download) and keyA (for + // Object-by-key lookup) — mirroring how generalized-object stores behave. + emptyObj := siastorage.NewEmptyObject() + sdk.mu.Lock() + fixture := siaStored{meta: raw, data: tipData} + sdk.objects[emptyObj.ID()] = fixture + sdk.objects[keyA] = fixture + sdk.mu.Unlock() + + // Record the tip row in the local cache DB (the CAS coordination point). + tipBody, err := json.Marshal(objmeta.TipBody{Locker: "widgets", Gen: 5}) + require.NoError(t, err) + require.NoError(t, db.Create(&gitarchive.GitObject{ + ObjectKey: keyA.String(), Locker: "widgets", Kind: string(objmeta.KindGitTip), Body: tipBody, + }).Error) + + // CAS against a stale key must refuse: the archive generation moved after + // the caller read it. + err = store.casPublish(ctx, keyB) + require.Error(t, err) + require.Contains(t, err.Error(), "moved") + require.Contains(t, err.Error(), "concurrent publish") + + // CAS against the current tip key passes. + require.NoError(t, store.casPublish(ctx, keyA.String())) +} diff --git a/internal/cli/gitremote/session_store_stub.go b/internal/cli/gitremote/session_store_stub.go new file mode 100644 index 00000000..9d830018 --- /dev/null +++ b/internal/cli/gitremote/session_store_stub.go @@ -0,0 +1,33 @@ +//go:build !gitarchive_sia + +package gitremote + +import ( + "context" + "fmt" + "io" +) + +// These are the default-build implementations of the SessionStore data +// operations. The real Sia-backed implementations live in session_store_sia.go +// guarded by the `gitarchive_sia` build tag; without that tag we cannot know we +// are wired to a live Sia indexer, so the data operations report the +// ErrSiaNotWired sentinel rather than pretending to work. Only List/Download/ +// Upload/Delete differ; the struct, constructors and Close (session_store.go) +// are shared. + +func (s *SessionStore) List(ctx context.Context, forPush bool) ([]Ref, error) { + return nil, fmt.Errorf("%w: list", ErrSiaNotWired) +} + +func (s *SessionStore) Download(ctx context.Context, refName, targetHash string) (io.ReadCloser, error) { + return nil, fmt.Errorf("%w: download", ErrSiaNotWired) +} + +func (s *SessionStore) Upload(ctx context.Context, dstRef, srcHash string, pack io.Reader) error { + return fmt.Errorf("%w: upload", ErrSiaNotWired) +} + +func (s *SessionStore) Delete(ctx context.Context, dstRef string) error { + return fmt.Errorf("%w: delete", ErrSiaNotWired) +} diff --git a/internal/cli/gitremote/share_store.go b/internal/cli/gitremote/share_store.go new file mode 100644 index 00000000..25448728 --- /dev/null +++ b/internal/cli/gitremote/share_store.go @@ -0,0 +1,271 @@ +package gitremote + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/go-git/go-git/v5/plumbing" + "go.sia.tech/core/types" + "go.sia.tech/siastorage" + + "go.lumeweb.com/pinner/core/vault" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +// This file implements the profile-less helper share path. A share URL is a +// self-contained bearer credential (see internal/core/gitarchive.SDKClient): +// `pinner git share` mints pre-signed URLs for the tip + packs and records them +// in a git.share ShareDocument object. Anyone holding the top-level share URL +// can `git clone pinner::share/` WITHOUT a vault profile/app key — the +// Git remote-helper routes to a ShareStore that reads everything over the +// pre-signed URLs via DownloadSharedObject. + +// shareRemotePrefix is the remote URL prefix that selects the profile-less +// share path in the helper (as opposed to the account-scoped `pinner::` +// path routed to SessionStore). +const shareRemotePrefix = "pinner::share/" + +// ErrShareReadonly is returned when a push/delete is attempted against a share +// remote. Share archives are read-only by construction (their URLs carry the +// embedded key but no account write scope). +var ErrShareReadonly = errors.New("gitremote: share archive is read-only") + +// ShareDownloader downloads self-contained bearer share objects given a +// pre-signed URL. *siastorage.SDK satisfies this interface directly; tests +// substitute an in-memory fake. +type ShareDownloader interface { + // DownloadSharedObject streams an object's plaintext from a pre-signed + // share URL (no profile/app key needed — the URL embeds the encryption key). + DownloadSharedObject(ctx context.Context, sharedURL string, opts ...siastorage.DownloadOption) (io.ReadCloser, error) + // Close releases SDK-held resources. + Close() error +} + +// ParseShareRemote reports whether url is a `pinner::share/...` remote and, if +// so, returns the pre-signed share-URL embedded in it. The profile-less helper +// path uses this to decide it needs no vault profile. +func ParseShareRemote(url string) (string, bool) { + if !strings.HasPrefix(url, shareRemotePrefix) { + return "", false + } + rest := url[len(shareRemotePrefix):] + if rest == "" { + return "", false + } + return rest, true +} + +// ShareStore is a read-only Store backed by a ShareDocument reached over +// self-contained bearer URLs. List advertises the archived refs from the +// current tip; Download reconstructs a full pack of the requested tip by +// downloading every archived pack (in replay order) and re-encoding it +// in-process via go-git. It never touches a vault profile or app key. +type ShareStore struct { + downloader ShareDownloader + shareURL string // pre-signed URL of the git.share document + + doc *gitarchive.ShareDocument // cached after the first List/Download + tip *gitarchive.TipBytes // cached current tip parsed from doc.TipURL +} + +var _ Store = (*ShareStore)(nil) + +// NewShareStore builds a profile-less store over a pre-signed share document +// URL. The downloader does the actual bearer reads. +func NewShareStore(d ShareDownloader, shareURL string) *ShareStore { + return &ShareStore{downloader: d, shareURL: shareURL} +} + +// NewShareStoreForRemote parses a `pinner::share/...` remote URL and returns a +// profile-less ShareStore for it. It returns an error when url is not a share +// remote. +func NewShareStoreForRemote(d ShareDownloader, remoteURL string) (*ShareStore, error) { + shareURL, ok := ParseShareRemote(remoteURL) + if !ok { + return nil, fmt.Errorf("gitremote: %q is not a pinner share remote", remoteURL) + } + return NewShareStore(d, shareURL), nil +} + +// Close releases the downloader. +func (s *ShareStore) Close() error { + if s.downloader != nil { + return s.downloader.Close() + } + return nil +} + +// load downloads the share document and its tip once and caches them. All +// reads go through DownloadSharedObject (bearer URLs) — no profile is resolved. +func (s *ShareStore) load(ctx context.Context) error { + if s.doc != nil && s.tip != nil { + return nil + } + if s.downloader == nil { + return fmt.Errorf("gitremote: share store has no downloader") + } + + docRaw, err := readShared(ctx, s.downloader, s.shareURL) + if err != nil { + return fmt.Errorf("gitremote: read share document: %w", err) + } + doc, err := gitarchive.ParseShareDocument(docRaw) + if err != nil { + return err + } + tipRaw, err := readShared(ctx, s.downloader, doc.TipURL) + if err != nil { + return fmt.Errorf("gitremote: read share tip: %w", err) + } + tip, err := gitarchive.ParseTipBytes(tipRaw) + if err != nil { + return err + } + s.doc = doc + s.tip = tip + return nil +} + +func readShared(ctx context.Context, d ShareDownloader, url string) ([]byte, error) { + rc, err := d.DownloadSharedObject(ctx, url) + if err != nil { + return nil, err + } + defer rc.Close() + return io.ReadAll(rc) +} + +// List advertises the archived refs from the share tip, sorted by name. +func (s *ShareStore) List(ctx context.Context, forPush bool) ([]Ref, error) { + if err := s.load(ctx); err != nil { + return nil, err + } + refs := make([]Ref, 0, len(s.tip.Refs)) + for name, hash := range s.tip.Refs { + refs = append(refs, Ref{Name: name, Hash: plumbing.NewHash(hash)}) + } + sort.Slice(refs, func(i, j int) bool { return refs[i].Name < refs[j].Name }) + return refs, nil +} + +// Download reconstructs a full pack of everything reachable from targetHash by +// downloading every archived pack over its pre-signed URL (in the share +// document's pack order), ingesting them into an in-memory store, and +// re-encoding a combined pack in-process via the shared replay helper. +// refName is not used beyond validating the ref exists in the tip. +func (s *ShareStore) Download(ctx context.Context, refName, targetHash string) (io.ReadCloser, error) { + if err := s.load(ctx); err != nil { + return nil, err + } + // Share fetches are whole-history: the archive only ever sends what the + // consumer lacks, so a full pack of the requested tip is always sufficient. + // Download every pack (replay order), verify its SHA-256 digest against the + // digest recorded in the tip document (when present), then re-encode a + // single full pack. + openers := make([]func() (io.ReadCloser, error), 0, len(s.doc.PackURLs)) + digests := make([]string, 0, len(s.tip.Packs)) + for _, p := range s.tip.Packs { + digests = append(digests, p.Digest) + } + for i, u := range s.doc.PackURLs { + u := u + want := "" + if i < len(digests) { + want = digests[i] + } + openers = append(openers, func() (io.ReadCloser, error) { + rc, err := s.downloader.DownloadSharedObject(ctx, u) + if err != nil { + return nil, fmt.Errorf("gitremote: download shared pack: %w", err) + } + data, err := io.ReadAll(rc) + rc.Close() + if err != nil { + return nil, fmt.Errorf("gitremote: read shared pack: %w", err) + } + if want != "" { + if got := fmt.Sprintf("%x", sha256.Sum256(data)); got != want { + return nil, fmt.Errorf("gitremote: shared pack digest mismatch: tip declares %s, payload computes %s", want, got) + } + } + return io.NopCloser(bytes.NewReader(data)), nil + }) + } + pack, err := replayPacksAndEncode(openers, plumbing.NewHash(targetHash)) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(pack)), nil +} + +// Upload rejects writes — share archives are read-only. +func (s *ShareStore) Upload(ctx context.Context, dstRef, srcHash string, pack io.Reader) error { + return fmt.Errorf("%w: upload to a share", ErrShareReadonly) +} + +// Delete rejects writes — share archives are read-only. +func (s *ShareStore) Delete(ctx context.Context, dstRef string) error { + return fmt.Errorf("%w: delete from a share", ErrShareReadonly) +} + +// NewStoreFromRemote routes a helper remote URL to the right Store: +// +// - `pinner::share/...` → a profile-less ShareStore (no vault profile); +// - anything else (e.g. `pinner::`) → the profile-backed SessionStore. +// +// main() calls this with os.Args[1] so the helper serves both the account-scoped +// archive (`pinner::`) and the profile-less share clone +// (`pinner::share/`). +func NewStoreFromRemote(ctx context.Context, remoteURL string) (Store, error) { + if shareURL, ok := ParseShareRemote(remoteURL); ok { + d, err := newSharedDownloader(os.Getenv("PINNER_SIA_INDEXER")) + if err != nil { + return nil, err + } + return NewShareStore(d, shareURL), nil + } + // Account-scoped path (`pinner::`): a profile-backed SessionStore + // scoped to the locker embedded in the remote URL. + if locker, ok := ParseLockedRemote(remoteURL); ok { + s, err := NewSessionStoreFromEnv(ctx) + if err != nil { + return nil, err + } + return s.ForLocker(locker), nil + } + // Fallback for a bare/malformed remote: an unscoped session store (its data + // operations will report a missing-locker/not-wired error as appropriate). + return NewSessionStoreFromEnv(ctx) +} + +// newSharedDownloader builds a Sia SDK used only for self-contained shared +// reads. It has no vault profile/app key: the shared object's encryption key is +// embedded in the pre-signed URL, so the SDK app key merely authenticates the +// (throwaway) indexer session. Requires the indexer origin via +// PINNER_SIA_INDEXER until Step 10 wires a configured default. +func newSharedDownloader(indexerURL string) (ShareDownloader, error) { + if indexerURL == "" { + return nil, fmt.Errorf("gitremote: shared download needs a Sia indexer URL (set PINNER_SIA_INDEXER)") + } + key := types.GeneratePrivateKey() + metadata := siastorage.AppMetadata{ + ID: vault.AppID(), + Name: "Pinner CLI Git Archive (shared)", + Description: "Read-only shared git archive access", + ServiceURL: indexerURL, + } + builder := siastorage.NewBuilder(indexerURL, metadata) + sdk, err := builder.SDK(key) + if err != nil { + return nil, fmt.Errorf("gitremote: build shared SDK: %w", err) + } + return sdk, nil +} diff --git a/internal/cli/gitremote/share_store_test.go b/internal/cli/gitremote/share_store_test.go new file mode 100644 index 00000000..6e3fd1e2 --- /dev/null +++ b/internal/cli/gitremote/share_store_test.go @@ -0,0 +1,197 @@ +package gitremote + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "testing" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.sia.tech/siastorage" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +// fakeShareDownloader serves pre-signed URL payloads in memory, standing in for +// the bearer DownloadSharedObject path (no profile/app key). +type fakeShareDownloader struct { + byURL map[string][]byte + err error +} + +func (f *fakeShareDownloader) DownloadSharedObject(_ context.Context, url string, _ ...siastorage.DownloadOption) (io.ReadCloser, error) { + if f.err != nil { + return nil, f.err + } + data, ok := f.byURL[url] + if !ok { + return nil, fmt.Errorf("fake: no data for shared url %q", url) + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (f *fakeShareDownloader) Close() error { return nil } + +// buildShareFixture builds a source repo with one commit, encodes a full pack of +// it, and returns the commit hash plus a fake downloader serving a coherent +// share document (doc -> tip -> one pack) plus the tip/share doc payloads and +// the pack bytes. +func buildShareFixture(t *testing.T) (plumbing.Hash, *fakeShareDownloader) { + t.Helper() + + srcDir := t.TempDir() + src, err := git.PlainInit(srcDir, false) + require.NoError(t, err) + c := commitFile(t, src, "a.txt", "hello from a\n", "first") + + pack, err := EncodeIncrementalPack(src.Storer, []plumbing.Hash{c}, nil) + require.NoError(t, err) + require.NotEmpty(t, pack, "share fixture pack must not be empty") + + doc := gitarchive.ShareDocument{ + Locker: "widgets", + Gen: 1, + Name: "widgets", + TipURL: "u://tip", + PackURLs: []string{"u://pack"}, + Until: time.Now().Add(time.Hour).Unix(), + } + docRaw, err := json.Marshal(doc) + require.NoError(t, err) + + tip := gitarchive.TipBytes{ + Locker: "widgets", + Gen: 1, + Name: "widgets", + Refs: map[string]string{"refs/heads/master": c.String()}, + } + tipRaw, err := json.Marshal(tip) + require.NoError(t, err) + + dl := &fakeShareDownloader{byURL: map[string][]byte{ + "u://doc": docRaw, + "u://tip": tipRaw, + "u://pack": pack, + }} + return c, dl +} + +func TestParseShareRemote(t *testing.T) { + u, ok := ParseShareRemote("pinner::share/xyzabc") + require.True(t, ok) + require.Equal(t, "xyzabc", u) + + _, ok = ParseShareRemote("pinner::widgets") + require.False(t, ok, "account locker remote is not a share remote") + + _, ok = ParseShareRemote("pinner::share/") + require.False(t, ok, "share remote with empty URL is rejected") + + _, ok = ParseShareRemote("https://example.com/x") + require.False(t, ok) +} + +func TestNewShareStoreForRemote(t *testing.T) { + dl := &fakeShareDownloader{} + store, err := NewShareStoreForRemote(dl, "pinner::share/someurl") + require.NoError(t, err) + require.NotNil(t, store) + require.Equal(t, "someurl", store.shareURL) + + _, err = NewShareStoreForRemote(dl, "pinner::widgets") + require.Error(t, err) +} + +func TestShareStoreListAdvertisesTipRefs(t *testing.T) { + ctx := context.Background() + want, dl := buildShareFixture(t) + store := NewShareStore(dl, "u://doc") + + refs, err := store.List(ctx, false) + require.NoError(t, err) + require.Len(t, refs, 1) + require.Equal(t, "refs/heads/master", refs[0].Name) + require.Equal(t, want, refs[0].Hash) +} + +func TestShareStoreIsReadOnly(t *testing.T) { + ctx := context.Background() + _, dl := buildShareFixture(t) + store := NewShareStore(dl, "u://doc") + + err := store.Upload(ctx, "refs/heads/x", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil) + require.ErrorIs(t, err, ErrShareReadonly) + err = store.Delete(ctx, "refs/heads/x") + require.ErrorIs(t, err, ErrShareReadonly) +} + +func TestShareStoreProtocolClone(t *testing.T) { + ctx := context.Background() + want, dl := buildShareFixture(t) + store := NewShareStore(dl, "u://doc") + defer store.Close() + + // A fresh bare repo acts as the local clone target. + dstDir := t.TempDir() + dst, err := git.PlainInit(dstDir, true) + require.NoError(t, err) + + // capabilities + list + fetch the advertised tip. + out := runHandle(t, ctx, dst.Storer, store, + "capabilities\nlist\nfetch "+want.String()+" refs/heads/master\n\n") + assert.Contains(t, out, want.String()+" refs/heads/master\n") + assert.Contains(t, out, "option\n") + assert.Contains(t, out, "fetch\n") + assert.Contains(t, out, "push\n") + + // The fetched history landed in the local store. + cm, err := readCommit(dst.Storer, want) + require.NoError(t, err, "fetched commit must be reachable") + require.Equal(t, "first", cm.Message) +} + +func TestShareStoreDownloadVerifiesPackDigest(t *testing.T) { + ctx := context.Background() + + // Build a valid pack and its sha256 digest. + c, _ := buildShareFixture(t) + _ = c + + // Craft a share doc + tip referencing the pack with a REAL digest, but serve + // tampered pack bytes so verification must fail. + pack := []byte("PACK\x00tampered-payload") + wrongDigest := fmt.Sprintf("%x", sha256.Sum256(pack)) + + doc := gitarchive.ShareDocument{ + Locker: "widgets", Gen: 1, Name: "widgets", + TipURL: "u://tip", PackURLs: []string{"u://pack"}, + Until: time.Now().Add(time.Hour).Unix(), + } + docRaw, _ := json.Marshal(doc) + tip := gitarchive.TipBytes{ + Locker: "widgets", Gen: 1, Name: "widgets", + Refs: map[string]string{"refs/heads/master": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + Packs: []gitarchive.PackRef{{Key: "k", Full: true, Digest: "0000000000000000000000000000000000000000000000000000000000000000"}}, + } + tipRaw, _ := json.Marshal(tip) + + dl := &fakeShareDownloader{byURL: map[string][]byte{ + "u://doc": docRaw, + "u://tip": tipRaw, + "u://pack": pack, + }} + store := NewShareStore(dl, "u://doc") + + _, err := store.Download(ctx, "refs/heads/master", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + require.Error(t, err) + require.Contains(t, err.Error(), "pack digest mismatch") + // Sanity: the digest string we compared must equal the live computation. + require.Equal(t, fmt.Sprintf("%x", sha256.Sum256(pack)), wrongDigest) +} diff --git a/internal/cli/gitremote/show.go b/internal/cli/gitremote/show.go new file mode 100644 index 00000000..ef7ffe62 --- /dev/null +++ b/internal/cli/gitremote/show.go @@ -0,0 +1,231 @@ +package gitremote + +import ( + "context" + "fmt" + "sort" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage" +) + +// This file implements `pinner git show`: it turns the archive into a local, +// renderable object graph using only go-git (no git subprocess). It is split +// into two independent halves so each can be tested in isolation: +// +// - EnsureArchiveRefs pulls every advertised archive ref's pack into a local +// store and advances the corresponding reference, so the local store is a +// ready mirror of the archive; +// - Render reads refs, the commit log (git log -n 20 style) and the recursive +// file listing (git ls-tree -r --long style) out of an already-populated +// store. +// +// The byte-level Store seam keeps this backend-agnostic: production binds the +// Sia-backed session, while tests use a go-git bare-repo store. + +// CommitLine is one rendered commit for `pinner git show` (git log -n 20 +// style). It carries only the git-side metadata surfaced by the log template — +// never Sia object/slab/pin keys. +type CommitLine struct { + Hash string `json:"hash"` + Author string `json:"author"` + When int64 `json:"when"` // unix seconds + Message string `json:"message"` +} + +// TreeLine is one file entry for `pinner git show` (git ls-tree -r --long +// style). Only blob files are listed (via object.Tree.Files), mirroring the +// recursive file view; mode/hash/size/path are the long-format columns. +type TreeLine struct { + Mode string `json:"mode"` + Hash string `json:"hash"` + Size int64 `json:"size"` + Path string `json:"path"` +} + +// ShowReport is the fully rendered archive view produced by Render. +type ShowReport struct { + Refs []Ref `json:"refs"` + Commits []CommitLine `json:"commits"` + Tree []TreeLine `json:"tree"` +} + +// DefaultCommitLimit is the number of commits surfaced by `pinner git show` +// (mirrors `git log -n 20`). +const DefaultCommitLimit = 20 + +// EnsureArchiveRefs pulls every non-symbolic ref advertised by store into local +// (downloading its pack and ingesting it, then pointing the ref at the tip) so +// local becomes a ready mirror. It returns the refs that were ensured. A store +// that reports zero refs (empty archive) leaves local untouched. +func EnsureArchiveRefs(ctx context.Context, local storage.Storer, store Store) ([]Ref, error) { + refs, err := store.List(ctx, false) + if err != nil { + return nil, fmt.Errorf("gitremote: list archive refs: %w", err) + } + for _, r := range refs { + if r.Hash == plumbing.ZeroHash { + continue + } + rc, err := store.Download(ctx, r.Name, r.Hash.String()) + if err != nil { + return nil, fmt.Errorf("gitremote: download pack for %s: %w", r.Name, err) + } + ingestErr := IngestPack(local, rc) + if rc != nil { + rc.Close() + } + if ingestErr != nil { + return nil, fmt.Errorf("gitremote: ingest pack for %s: %w", r.Name, ingestErr) + } + if err := SetLocalRef(local, r.Name, r.Hash); err != nil { + return nil, fmt.Errorf("gitremote: set ref %s: %w", r.Name, err) + } + } + return refs, nil +} + +// Render produces the show report from an already-populated store: the refs, +// the most recent maxCommits commits across all ref tips (deduplicated by hash, +// ordered newest-first like a combined git log), and the recursive file listing +// across every ref tip tree. maxCommits <= 0 falls back to DefaultCommitLimit. +func Render(local storage.Storer, maxCommits int) (*ShowReport, error) { + if maxCommits <= 0 { + maxCommits = DefaultCommitLimit + } + refs, err := allRefs(local) + if err != nil { + return nil, err + } + + // Collect commits reachable from every ref tip, deduped by hash. + commits, err := collectCommits(local, refs, maxCommits) + if err != nil { + return nil, err + } + + // Collect files from every ref tip tree, deduped by path. + tree, err := collectFiles(local, refs) + if err != nil { + return nil, err + } + + return &ShowReport{Refs: refs, Commits: commits, Tree: tree}, nil +} + +// allRefs lists every non-symbolic reference in the store, sorted by name. +func allRefs(st storage.Storer) ([]Ref, error) { + iter, err := st.IterReferences() + if err != nil { + return nil, fmt.Errorf("gitremote: list refs: %w", err) + } + defer iter.Close() + var refs []Ref + _ = iter.ForEach(func(r *plumbing.Reference) error { + if r.Type() == plumbing.SymbolicReference { + return nil + } + refs = append(refs, Ref{Name: r.Name().String(), Hash: r.Hash()}) + return nil + }) + sort.Slice(refs, func(i, j int) bool { return refs[i].Name < refs[j].Name }) + return refs, nil +} + +// collectCommits gathers up to maxCommits distinct commits reachable from the +// given ref tips, ordered newest-first (by author time, ties by hash) to mirror +// a combined `git log -n 20`. +func collectCommits(st storage.Storer, refs []Ref, maxCommits int) ([]CommitLine, error) { + seen := map[plumbing.Hash]bool{} + lines := []CommitLine{} + for _, r := range refs { + if r.Hash == plumbing.ZeroHash { + continue + } + if err := walkCommits(st, r.Hash, seen, &lines); err != nil { + return nil, err + } + } + sort.Slice(lines, func(i, j int) bool { + if lines[i].When != lines[j].When { + return lines[i].When > lines[j].When // newest first + } + return lines[i].Hash < lines[j].Hash + }) + if len(lines) > maxCommits { + lines = lines[:maxCommits] + } + return lines, nil +} + +// walkCommits recursively appends commit lines for the history reachable from h. +func walkCommits(st storage.Storer, h plumbing.Hash, seen map[plumbing.Hash]bool, out *[]CommitLine) error { + if seen[h] { + return nil + } + seen[h] = true + c, err := object.GetCommit(st, h) + if err != nil { + // Not a commit (e.g. an annotated tag object or a missing object) — + // skip rather than fail the whole render. + return nil + } + t := c.Author.When + if t.IsZero() { + t = c.Committer.When + } + *out = append(*out, CommitLine{ + Hash: h.String(), + Author: c.Author.Name, + When: t.Unix(), + Message: c.Message, + }) + for _, ph := range c.ParentHashes { + if err := walkCommits(st, ph, seen, out); err != nil { + return err + } + } + return nil +} + +// collectFiles gathers the recursive file listing (git ls-tree -r --long style) +// across every ref tip tree, deduped by path. +func collectFiles(st storage.Storer, refs []Ref) ([]TreeLine, error) { + seen := map[string]bool{} + lines := []TreeLine{} + for _, r := range refs { + if r.Hash == plumbing.ZeroHash { + continue + } + c, err := object.GetCommit(st, r.Hash) + if err != nil { + continue + } + tree, err := c.Tree() + if err != nil { + return nil, fmt.Errorf("gitremote: read tree for %s: %w", r.Name, err) + } + if err := tree.Files().ForEach(func(f *object.File) error { + if seen[f.Name] { + return nil + } + seen[f.Name] = true + size := int64(0) + if blob := f.Blob; blob != (object.Blob{}) { + size = blob.Size + } + lines = append(lines, TreeLine{ + Mode: f.Mode.String(), + Hash: f.Hash.String(), + Size: size, + Path: f.Name, + }) + return nil + }); err != nil { + return nil, fmt.Errorf("gitremote: list files for %s: %w", r.Name, err) + } + } + sort.Slice(lines, func(i, j int) bool { return lines[i].Path < lines[j].Path }) + return lines, nil +} diff --git a/internal/cli/gitremote/show_test.go b/internal/cli/gitremote/show_test.go new file mode 100644 index 00000000..acc7b4a9 --- /dev/null +++ b/internal/cli/gitremote/show_test.go @@ -0,0 +1,105 @@ +package gitremote + +import ( + "context" + "reflect" + "strings" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/stretchr/testify/require" +) + +// TestEnsureArchiveRefsAndRender drives `git show` end-to-end with only go-git: +// it pushes two commits to a bare GoGitStore, ensures the archive refs into a +// fresh bare mirror, and renders refs + commit log + tree files. No Sia, no git +// subprocess. +func TestEnsureArchiveRefsAndRender(t *testing.T) { + ctx := context.Background() + + // Push two commits into an archive backed by a bare go-git store. + srcDir := t.TempDir() + src, err := git.PlainInit(srcDir, false) + require.NoError(t, err) + c1 := commitFile(t, src, "a.txt", "hello a\n", "first") + c2 := commitFile(t, src, "b.txt", "hello b\n", "second") + + remoteDir := t.TempDir() + remote, err := git.PlainInit(remoteDir, true) + require.NoError(t, err) + store := &GoGitStore{Repo: remote} + defer func() { _ = store.Close() }() + + out := runHandle(t, ctx, src.Storer, store, + "capabilities\nlist for-push\npush refs/heads/master:refs/heads/master\n\n") + require.Contains(t, out, "ok refs/heads/master\n") + + // Fresh bare mirror becomes the private show view. + mirrorDir := t.TempDir() + mirror, err := git.PlainInit(mirrorDir, true) + require.NoError(t, err) + + refs, err := EnsureArchiveRefs(ctx, mirror.Storer, store) + require.NoError(t, err) + require.Len(t, refs, 1) + require.Equal(t, "refs/heads/master", refs[0].Name) + require.Equal(t, c2, refs[0].Hash) + + report, err := Render(mirror.Storer, 0) + require.NoError(t, err) + + // Refs surfaced. + require.Len(t, report.Refs, 1) + require.Equal(t, "refs/heads/master", report.Refs[0].Name) + + // Commit log is newest-first and covers both commits. + require.Len(t, report.Commits, 2) + require.Equal(t, c2.String(), report.Commits[0].Hash) + require.Equal(t, c1.String(), report.Commits[1].Hash) + + // Tree listing covers both files, sorted by path. + require.Len(t, report.Tree, 2) + require.Equal(t, "a.txt", report.Tree[0].Path) + require.Equal(t, "b.txt", report.Tree[1].Path) + require.Greater(t, report.Tree[0].Size, int64(0)) + require.NotEmpty(t, report.Tree[0].Mode) + require.NotEmpty(t, report.Tree[0].Hash) +} + +// TestRenderEmptyMirror confirms an empty (never ensured) mirror renders with no +// refs, no commits and no files rather than erroring. +func TestRenderEmptyMirror(t *testing.T) { + dir := t.TempDir() + repo, err := git.PlainInit(dir, true) + require.NoError(t, err) + + report, err := Render(repo.Storer, 0) + require.NoError(t, err) + require.Empty(t, report.Refs) + require.Empty(t, report.Commits) + require.Empty(t, report.Tree) +} + +// TestShowReportOmitsObjectKeys guards the default-output contract: the show +// report may carry only git-side identifiers (sha1 hashes, paths, sizes), never +// Sia object/slab/pin keys. It reflects over the exported JSON field names of +// every type rendered by `git show` and asserts none is named like a backend +// key. This is a structural guard so a future refactor cannot quietly add a +// slab/object key to the default output. +func TestShowReportOmitsObjectKeys(t *testing.T) { + for _, typ := range []reflect.Type{ + reflect.TypeOf(ShowReport{}), + reflect.TypeOf(CommitLine{}), + reflect.TypeOf(TreeLine{}), + } { + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + name := strings.ToLower(field.Name) + for _, banned := range []string{"objectkey", "slab", "pin", "key"} { + require.False(t, strings.Contains(name, banned), + "%s.%s must not expose a backend %q key in default output", + typ.Name(), field.Name, banned) + } + } + } +} diff --git a/internal/cli/gitremote/store.go b/internal/cli/gitremote/store.go new file mode 100644 index 00000000..cb5c8da7 --- /dev/null +++ b/internal/cli/gitremote/store.go @@ -0,0 +1,80 @@ +// Package gitremote implements the shared Git remote-helper protocol engine +// for the single `pinner` executable. +// +// A Git remote helper is a program named `git-remote-` that Git +// spawns and talks to over stdin/stdout. Exactly one binary is built: `cmd/pinner`. +// When that binary is invoked through a `git-remote-pinner` symlink, main detects +// the argv-0 basename and runs THIS package's protocol engine instead of the +// normal CLI tree. The helper never enters urfave command parsing and the normal +// CLI never reads Git helper stdin — dispatch is isolated at the entry point. +// +// The protocol engine is deliberately decoupled from any particular backend. The +// remote-helper protocol (see gitremote-helpers(7)) reduces to four concerns: +// +// - capabilities: what the helper supports (option / fetch / push); +// - list [for-push]: advertise the remote refs; +// - fetch : write objects reachable from sha1 into the local DB; +// - push +:: read objects from the local repo, persist them and move +// the remote ref. +// +// The Go object transfer — pack encode / pack ingest / reference handling — all +// happens in-process via go-git (no git subprocess, no shell-outs). The Store +// interface is the injected seam between the protocol engine and whatever backs +// an archive: production binds the Sia-backed session (Steps 5+ wire the actual +// pack/tip publish + download), while tests and the git-only integration mode use +// a go-git bare repository (see GoGitStore). +package gitremote + +import ( + "context" + "io" + + "github.com/go-git/go-git/v5/plumbing" +) + +// Ref is one advertised remote reference produced by Store.List. +type Ref struct { + // Name is the full reference name, e.g. "refs/heads/master". + Name string + // Hash is the 40-hex sha1 of the ref tip. ZeroHash advertises an unknown + // value ("?" in protocol terms) — used sparingly; most helpers advertise a + // concrete sha1. + Hash plumbing.Hash +} + +// Store is the persistence seam for the remote-helper protocol engine. +// +// Fetch/push are described in terms of raw packfiles and references: +// +// - Download returns a reader over the pack that, when ingested into a local +// object store, provides all objects reachable from targetHash for refName. +// - Upload persists a raw pack (the set of new objects for srcHash) and moves +// dstRef to srcHash. If the pack is empty the store only has to move the ref. +// - Delete removes dstRef (the empty-source delete case: `push :dst`). +// +// The signature deliberately stays at the byte/ref level: all go-git encode and +// ingest (see gitops.go) happens on the helper side against the *local* repo, +// while the Store is the remote that consumes/produces packs. This is what keeps +// the engine backend-agnostic and lets identical tests run against a go-git +// bare-repo store and, later, the Sia-backed store. +type Store interface { + // List advertises the remote refs. forPush=true is the `list for-push` + // form used to prepare a push batch. + List(ctx context.Context, forPush bool) ([]Ref, error) + + // Download returns the raw pack of objects reachable from targetHash for + // refName. For a non-incremental archive this is the full history pack; the + // caller ingests it into the local object store via IngestPack. + Download(ctx context.Context, refName, targetHash string) (io.ReadCloser, error) + + // Upload persists the given raw pack (objects needed for srcHash) so that + // the archive can serve them, then moves dstRef to srcHash. An empty pack + // means no new objects were needed and only the reference is updated. + Upload(ctx context.Context, dstRef, srcHash string, pack io.Reader) error + + // Delete removes dstRef from the remote (delete ref push). + Delete(ctx context.Context, dstRef string) error + + // Close releases any backend-held resources (e.g. a Sia SDK, an open DB). + Close() error +} diff --git a/internal/cli/root.go b/internal/cli/root.go index a27e5c44..103a16b0 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -109,6 +109,7 @@ For more help on any command: pinner --help`, newAdminCommand(), newDocsCommand(), newVaultCommand(), + newGitCommand(), }, Flags: GlobalFlags(), Action: func(ctx context.Context, cmd *cli.Command) error { diff --git a/internal/cli/testutils.go b/internal/cli/testutils.go index 8ed836fc..e5a22e21 100644 --- a/internal/cli/testutils.go +++ b/internal/cli/testutils.go @@ -56,18 +56,19 @@ func newMockCommand() *mockCommand { // mockCommand is a map-backed test double for commandGetter interfaces. // It replaces hand-written mock command structs across test files. type mockCommand struct { - stringFields map[string]string - intFields map[string]int - int64Fields map[string]int64 - uint64Fields map[string]uint64 - uintFields map[string]uint - durationFields map[string]time.Duration - floatFields map[string]float64 - boolFields map[string]bool - stringSlices map[string][]string - isSetFields map[string]bool - args cli.Args - cid string + stringFields map[string]string + intFields map[string]int + int64Fields map[string]int64 + uint64Fields map[string]uint64 + uintFields map[string]uint + durationFields map[string]time.Duration + timestampFields map[string]time.Time + floatFields map[string]float64 + boolFields map[string]bool + stringSlices map[string][]string + isSetFields map[string]bool + args cli.Args + cid string } func (m *mockCommand) withString(name, value string) *mockCommand { @@ -118,6 +119,14 @@ func (m *mockCommand) withDuration(name string, value time.Duration) *mockComman return m } +func (m *mockCommand) withTimestamp(name string, value time.Time) *mockCommand { + if m.timestampFields == nil { + m.timestampFields = make(map[string]time.Time) + } + m.timestampFields[name] = value + return m +} + func (m *mockCommand) withFloat(name string, value float64) *mockCommand { if m.floatFields == nil { m.floatFields = make(map[string]float64) @@ -261,6 +270,15 @@ func (m *mockCommand) GetCID() string { return m.cid } +func (m *mockCommand) Timestamp(name string) time.Time { + if m.timestampFields != nil { + if v, ok := m.timestampFields[name]; ok { + return v + } + } + return time.Time{} +} + // Compile-time interface satisfaction checks var ( _ flagGetter = (*mockCommand)(nil) diff --git a/internal/core/gitarchive/db.go b/internal/core/gitarchive/db.go new file mode 100644 index 00000000..5bbda5c9 --- /dev/null +++ b/internal/core/gitarchive/db.go @@ -0,0 +1,70 @@ +// Package gitarchive implements the git-archive domain on top of the profile's +// Sia account and its local cache database. +// +// Unlike vault files, git archive objects are NOT vault.File rows. They are +// pinner object cards (see go.lumeweb.com/pinner-cli/internal/core/objmeta) +// stored in the same Sia object store but tracked locally in dedicated tables +// (git_repos / git_objects / git_binds). They are created/exactly by the +// library's vaultService.Sync, which rejects git cards at ParseFileMetadata +// (the integer `created_at` guard) and so never writes File rows for them. +// This package performs its own account-scan / ingest instead. +package gitarchive + +import ( + "time" + + "gorm.io/datatypes" + "gorm.io/gorm" +) + +// GitRepo is one named locker (a git archive repository) tracked locally. +type GitRepo struct { + ID uint `gorm:"primaryKey"` + Locker string `gorm:"uniqueIndex:idx_git_repos_locker"` // canonical locker name + Name string // display name + Lineage string // root-commit fingerprint used to match local clones to this locker + TipGen int // last known generation of the tip card + CreatedAt time.Time + UpdatedAt time.Time +} + +// GitObject is one Sia object that git archive owns (a pack, tip, or share +// card), keyed by its content-addressed object key (hex of types.Hash256). +// Ingesting a git card creates exactly one GitObject row and never a vault +// File row. +type GitObject struct { + ID uint `gorm:"primaryKey"` + ObjectKey string `gorm:"uniqueIndex:idx_git_objects_objectkey"` // hex object key + Locker string `gorm:"index:idx_git_objects_locker"` + Kind string // objmeta.KindGitPack / KindGitTip / KindGitShare + Size int64 // payload size in bytes (card.Size) + Digest string // sha256 hex (card.Digest) + Created int64 // unix seconds (card.Created) + Body datatypes.JSON // raw card body + CreatedAt time.Time // local ingest time + UpdatedAt time.Time +} + +// GitBind binds a local repo path to a remote locker name. +type GitBind struct { + ID uint `gorm:"primaryKey"` + Locker string `gorm:"uniqueIndex:idx_git_binds_locker"` + Remote string // remote name (e.g. "archive") + RepoPath string // canonical local repo path + CreatedAt time.Time + UpdatedAt time.Time +} + +// autoMigrateModels is the git archive schema. AutoMigrate is additive and +// idempotent; it runs on the same cache.db as the vault schema without +// disturbing the vault tables. +var autoMigrateModels = []any{ + &GitRepo{}, + &GitObject{}, + &GitBind{}, +} + +// AutoMigrate creates (or updates) the git archive tables on db. +func AutoMigrate(db *gorm.DB) error { + return db.AutoMigrate(autoMigrateModels...) +} diff --git a/internal/core/gitarchive/dbg_test.go b/internal/core/gitarchive/dbg_test.go new file mode 100644 index 00000000..d7e595b3 --- /dev/null +++ b/internal/core/gitarchive/dbg_test.go @@ -0,0 +1 @@ +package gitarchive_test diff --git a/internal/core/gitarchive/doctor.go b/internal/core/gitarchive/doctor.go new file mode 100644 index 00000000..10cbae9c --- /dev/null +++ b/internal/core/gitarchive/doctor.go @@ -0,0 +1,88 @@ +package gitarchive + +import ( + "fmt" + + "gorm.io/gorm" +) + +// This file holds the local, DB-only health checks powering `pinner git doctor`. +// Sia-dependent passes (account scan, pack download, republish) live behind the +// Store/session seam and are wired at the CLI layer; here we verify the local +// bookkeeping invariants that doctor can always check without a network round +// trip: the bind, the locker registration, the archived tip generation, and +// whether the git_objects table is cold for the locker. + +// DoctorReport is the result of a local git archive health check. +type DoctorReport struct { + // Bound reports whether the repo is bound to a locker. + Bound bool + // Locker is the bound locker name (empty when not bound). + Locker string + // RepoRegistered reports whether a git_repos row exists for the locker. + RepoRegistered bool + // TipGen is the registered tip generation for the locker. + TipGen int + // ObjectCount is the number of git_objects rows for the locker. + ObjectCount int64 + // Cold reports whether no git_objects rows exist for the locker (a fresh, + // never-scanned or never-published locker). + Cold bool + // HasTipObject reports whether at least one git.tip object row exists. + HasTipObject bool + // HasPackObject reports whether at least one git.pack object row exists. + HasPackObject bool +} + +// LocalDoctor runs the local (DB-only) portion of `pinner git doctor` for the +// repo at repoPath. It reports whether the repo is bound and the health of the +// locker registration and git_objects rows, without touching the network. The +// Sia-dependent passes are the CLI layer's responsibility (via the Store). +func LocalDoctor(db *gorm.DB, repoPath string) (*DoctorReport, error) { + report := &DoctorReport{} + + bind, err := FindBindByRepo(db, repoPath) + if err != nil { + return nil, err + } + if bind == nil { + return report, nil // not bound: nothing else to check locally + } + report.Bound = true + report.Locker = bind.Locker + + var repo GitRepo + err = db.Where("locker = ?", bind.Locker).First(&repo).Error + switch { + case err == gorm.ErrRecordNotFound: + report.RepoRegistered = false + case err != nil: + return nil, fmt.Errorf("gitarchive: find repo %q: %w", bind.Locker, err) + default: + report.RepoRegistered = true + report.TipGen = repo.TipGen + } + + if err := db.Model(&GitObject{}). + Where("locker = ?", bind.Locker). + Count(&report.ObjectCount).Error; err != nil { + return nil, fmt.Errorf("gitarchive: count objects %q: %w", bind.Locker, err) + } + report.Cold = report.ObjectCount == 0 + + var tipN, packN int64 + if err := db.Model(&GitObject{}). + Where("locker = ? AND kind = ?", bind.Locker, "git.tip"). + Count(&tipN).Error; err != nil { + return nil, fmt.Errorf("gitarchive: count tip objects %q: %w", bind.Locker, err) + } + if err := db.Model(&GitObject{}). + Where("locker = ? AND kind = ?", bind.Locker, "git.pack"). + Count(&packN).Error; err != nil { + return nil, fmt.Errorf("gitarchive: count pack objects %q: %w", bind.Locker, err) + } + report.HasTipObject = tipN > 0 + report.HasPackObject = packN > 0 + + return report, nil +} diff --git a/internal/core/gitarchive/doctor_test.go b/internal/core/gitarchive/doctor_test.go new file mode 100644 index 00000000..317e4a74 --- /dev/null +++ b/internal/core/gitarchive/doctor_test.go @@ -0,0 +1,74 @@ +package gitarchive_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" + "go.lumeweb.com/pinner-cli/internal/core/objmeta" +) + +func TestLocalDoctorNotBound(t *testing.T) { + db := openDB(t) + require.NoError(t, gitarchive.AutoMigrate(db)) + + report, err := gitarchive.LocalDoctor(db, "/home/u/unbound") + require.NoError(t, err) + require.False(t, report.Bound) + require.Empty(t, report.Locker) +} + +func TestLocalDoctorHealthyAndCold(t *testing.T) { + db := openDB(t) + require.NoError(t, gitarchive.AutoMigrate(db)) + + _, err := gitarchive.AddBind(db, "widgets", "/home/u/widgets") + require.NoError(t, err) + _, err = gitarchive.EnsureRepo(db, "widgets", "widgets", "lin") + require.NoError(t, err) + + report, err := gitarchive.LocalDoctor(db, "/home/u/widgets") + require.NoError(t, err) + require.True(t, report.Bound) + require.Equal(t, "widgets", report.Locker) + require.True(t, report.RepoRegistered) + require.True(t, report.Cold, "no objects ingested yet -> cold") + require.False(t, report.HasTipObject) + require.False(t, report.HasPackObject) +} + +func TestLocalDoctorWithObjects(t *testing.T) { + s, _, _ := testSession(t) + _, err := gitarchive.AddBind(s.DB, "widgets", "/home/u/widgets") + require.NoError(t, err) + + // Ingest one tip and one pack card (distinct object keys) so doctor sees + // both kinds recorded. + for _, tc := range []struct { + key string + kind objmeta.Kind + body string + }{{ + key: "aa00tipkey", + kind: objmeta.KindGitTip, + body: `{"locker":"widgets","gen":1}`, + }, { + key: "bb00packkey", + kind: objmeta.KindGitPack, + body: `{"locker":"widgets","pack":"pack-1","full":true}`, + }} { + raw := card(t, tc.kind, tc.body) + ok, err := gitarchive.IngestCard(s.DB, tc.key, raw) + require.NoError(t, err) + require.True(t, ok) + } + + report, err := gitarchive.LocalDoctor(s.DB, "/home/u/widgets") + require.NoError(t, err) + require.True(t, report.Bound) + require.False(t, report.Cold) + require.True(t, report.HasTipObject) + require.True(t, report.HasPackObject) + require.EqualValues(t, 2, report.ObjectCount) +} diff --git a/internal/core/gitarchive/download.go b/internal/core/gitarchive/download.go new file mode 100644 index 00000000..ad91fcd8 --- /dev/null +++ b/internal/core/gitarchive/download.go @@ -0,0 +1,76 @@ +package gitarchive + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + + "go.sia.tech/core/types" + + "go.lumeweb.com/pinner-cli/internal/core/objmeta" +) + +// DownloadedObject is a fetched git archive object: its card metadata plus a +// reader over the plaintext payload. The caller owns the reader and must Close +// it. +type DownloadedObject struct { + // Key is the object's content-addressed object key. + Key types.Hash256 + // Card is the card metadata that routed/validated this object. + Card objmeta.Card + // Reader streams the object's plaintext payload. + Reader io.ReadCloser +} + +// FetchObject retrieves a git archive object (pack, tip, or share card) by its +// content-addressed key. It decrypts/validates the object through the SDK, +// routes its card metadata, and returns a reader over the plaintext payload. +func FetchObject(ctx context.Context, s *Session, key types.Hash256) (*DownloadedObject, error) { + sdk, err := s.ensureSDK() + if err != nil { + return nil, err + } + obj, err := sdk.Object(ctx, key) + if err != nil { + return nil, fmt.Errorf("gitarchive: get object %s: %w", key, err) + } + card, err := objmeta.Decode(obj.Metadata()) + if err != nil { + return nil, fmt.Errorf("gitarchive: decode object %s metadata: %w", key, err) + } + rc, err := sdk.Download(obj) + if err != nil { + return nil, fmt.Errorf("gitarchive: download object %s: %w", key, err) + } + return &DownloadedObject{Key: key, Card: *card, Reader: rc}, nil +} + +// FetchObjectBytes is FetchObject followed by reading the whole payload into +// memory. The payload is the raw packfile (for git.pack) or the tip/share +// document (for git.tip / git.share), so it is fully read to be parsed. +// +// It verifies the payload against the card's SHA-256 digest, so every fully +// downloaded object (packs, tips, share documents) is integrity-checked +// against the digest recorded in its sealed card at publish time. A digest +// mismatch is an error — a corrupted/tampered object is never returned. +func FetchObjectBytes(ctx context.Context, s *Session, key types.Hash256) (*DownloadedObject, []byte, error) { + downloaded, err := FetchObject(ctx, s, key) + if err != nil { + return nil, nil, err + } + defer downloaded.Reader.Close() + data, err := io.ReadAll(downloaded.Reader) + if err != nil { + return nil, nil, fmt.Errorf("gitarchive: read object %s: %w", key, err) + } + if downloaded.Card.Digest != "" { + if got := fmt.Sprintf("%x", sha256.Sum256(data)); got != downloaded.Card.Digest { + return nil, nil, fmt.Errorf( + "gitarchive: object %s digest mismatch: card declares %s, payload computes %s", + key, downloaded.Card.Digest, got) + } + } + downloaded.Reader = io.NopCloser(nil) + return downloaded, data, nil +} diff --git a/internal/core/gitarchive/download_test.go b/internal/core/gitarchive/download_test.go new file mode 100644 index 00000000..656fe224 --- /dev/null +++ b/internal/core/gitarchive/download_test.go @@ -0,0 +1,73 @@ +package gitarchive_test + +import ( + "bytes" + "context" + "testing" + + "github.com/stretchr/testify/require" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +func TestFetchObjectBytesRoundTrip(t *testing.T) { + ctx := context.Background() + s, _, _ := testSession(t) + + payload := []byte(`{"locker":"widgets","gen":3,"name":"widgets","lineage":[],"refs":{},"prev":"p"}`) + uo, err := gitarchive.UploadTip(ctx, s, "widgets", 3, "prevkey", bytes.NewReader(payload)) + require.NoError(t, err) + + downloaded, data, err := gitarchive.FetchObjectBytes(ctx, s, uo.Key) + require.NoError(t, err) + require.Equal(t, payload, []byte(data)) + require.Equal(t, uo.Key, downloaded.Key) + require.Equal(t, "git.tip", string(downloaded.Card.Kind)) +} + +func TestFetchObjectReader(t *testing.T) { + ctx := context.Background() + s, _, _ := testSession(t) + + payload := []byte("PACK\x00reader-payload") + uo, err := gitarchive.UploadPack(ctx, s, "widgets", "pack-1", false, bytes.NewReader(payload)) + require.NoError(t, err) + + downloaded, err := gitarchive.FetchObject(ctx, s, uo.Key) + require.NoError(t, err) + defer downloaded.Reader.Close() + + buf := new(bytes.Buffer) + _, err = buf.ReadFrom(downloaded.Reader) + require.NoError(t, err) + require.Equal(t, payload, buf.Bytes()) +} + +func TestFetchObjectNotFound(t *testing.T) { + ctx := context.Background() + s, _, _ := testSession(t) + + // No object uploaded: fetching a key the fake has never seen must error. + _, err := gitarchive.FetchObject(ctx, s, objectKey(t)) + require.Error(t, err) +} + +func TestFetchObjectBytesVerifiesDigest(t *testing.T) { + ctx := context.Background() + s, sdk, _ := testSession(t) + + payload := []byte("PACK\x00integrity-payload") + uo, err := gitarchive.UploadPack(ctx, s, "widgets", "pack-1", false, bytes.NewReader(payload)) + require.NoError(t, err) + + // Corrupt the stored payload so it no longer matches the card digest. + sdk.mu.Lock() + fs := sdk.objects[uo.Key] + fs.data = append(append([]byte{}, fs.data...), []byte("TAMPER")...) + sdk.objects[uo.Key] = fs + sdk.mu.Unlock() + + _, _, err = gitarchive.FetchObjectBytes(ctx, s, uo.Key) + require.Error(t, err) + require.Contains(t, err.Error(), "digest mismatch") +} diff --git a/internal/core/gitarchive/gitarchive_test.go b/internal/core/gitarchive/gitarchive_test.go new file mode 100644 index 00000000..9d5f8725 --- /dev/null +++ b/internal/core/gitarchive/gitarchive_test.go @@ -0,0 +1,147 @@ +package gitarchive_test + +import ( + "context" + "crypto/sha256" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "go.sia.tech/core/types" + "go.sia.tech/siastorage" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + "go.lumeweb.com/pinner/core/vault" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" + "go.lumeweb.com/pinner-cli/internal/core/objmeta" +) + +// openDB opens an in-memory-ish SQLite test DB in a temp file so AutoMigrate +// (which writes DDL) behaves exactly as on the real cache.db. +func openDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(t.TempDir()+"/test.db"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + require.NoError(t, err) + return db +} + +func card(t *testing.T, kind objmeta.Kind, body string) []byte { + t.Helper() + raw, err := objmeta.Encode(objmeta.New(kind, 2048, "sha256digest", 1700000000, json.RawMessage(body))) + require.NoError(t, err) + return raw +} + +func objectKey(t *testing.T) types.Hash256 { + t.Helper() + sum := sha256.Sum256([]byte("git-object-key")) + return types.Hash256(sum) +} + +func TestAutoMigrateCreatesTables(t *testing.T) { + db := openDB(t) + require.NoError(t, gitarchive.AutoMigrate(db)) + + require.True(t, db.Migrator().HasTable("git_repos")) + require.True(t, db.Migrator().HasTable("git_objects")) + require.True(t, db.Migrator().HasTable("git_binds")) + + // AutoMigrate must run on the same DB as the vault schema without + // clobbering it. Creating the vault File table then re-migrating git tables + // keeps both intact. + require.NoError(t, db.AutoMigrate(&vault.File{})) + require.True(t, db.Migrator().HasTable("files")) + require.NoError(t, gitarchive.AutoMigrate(db)) + require.True(t, db.Migrator().HasTable("git_objects")) + require.True(t, db.Migrator().HasTable("files")) +} + +func TestIngestCardWritesGitObjectNoFileRows(t *testing.T) { + db := openDB(t) + // Migrate BOTH the vault schema (so a File table exists) and gitarchive. + require.NoError(t, db.AutoMigrate(&vault.File{})) + require.NoError(t, gitarchive.AutoMigrate(db)) + + raw := card(t, objmeta.KindGitPack, `{"locker":"widgets","pack":"k","full":true}`) + key := objectKey(t) + + handled, err := gitarchive.IngestCard(db, key.String(), raw) + require.NoError(t, err) + require.True(t, handled) + + // Exactly one git_objects row. + var gobjs []gitarchive.GitObject + require.NoError(t, db.Find(&gobjs).Error) + require.Len(t, gobjs, 1) + require.Equal(t, key.String(), gobjs[0].ObjectKey) + require.Equal(t, "git.pack", gobjs[0].Kind) + require.Equal(t, "widgets", gobjs[0].Locker) + require.Equal(t, int64(2048), gobjs[0].Size) + require.Equal(t, "sha256digest", gobjs[0].Digest) + require.Equal(t, int64(1700000000), gobjs[0].Created) + + // And ZERO vault File rows: ingesting a git card must never create one. + var files int64 + require.NoError(t, db.Model(&vault.File{}).Count(&files).Error) + require.Zero(t, files) + + // Ingesting the same object key again is idempotent. + handled, err = gitarchive.IngestCard(db, key.String(), raw) + require.NoError(t, err) + require.True(t, handled) + require.NoError(t, db.Find(&gobjs).Error) + require.Len(t, gobjs, 1) +} + +func TestIngestCardIgnoresLegacyVaultFile(t *testing.T) { + db := openDB(t) + require.NoError(t, db.AutoMigrate(&vault.File{})) + require.NoError(t, gitarchive.AutoMigrate(db)) + + // A legacy vault FileMetadata must NOT be ingested as a git object. + legacy := `{"id":"11111111-2222-3333-4444-555555555555","name":"report.pdf","media_type":"application/pdf","size":2048,"created_at":"2024-01-01T00:00:00Z"}` + handled, err := gitarchive.IngestCard(db, "legacykey", []byte(legacy)) + require.NoError(t, err) + require.False(t, handled) + + var gobjs []gitarchive.GitObject + require.NoError(t, db.Find(&gobjs).Error) + require.Len(t, gobjs, 0) +} + +func TestScanIngestsGitCardsAndSkipsVaultFiles(t *testing.T) { + db := openDB(t) + require.NoError(t, db.AutoMigrate(&vault.File{})) + require.NoError(t, gitarchive.AutoMigrate(db)) + + // Fixture events: one git card, one vault file, one deleted event. + gitObj := siastorage.NewEmptyObject() + gitObj.UpdateMetadata(card(t, objmeta.KindGitTip, `{"locker":"widgets","gen":2,"prev":"p"}`)) + + vaultObj := siastorage.NewEmptyObject() + vaultObj.UpdateMetadata([]byte(`{"id":"22222222-2222-3333-4444-555555555555","name":"notes.txt","created_at":"2024-01-01T00:00:00Z"}`)) + + events := []siastorage.ObjectEvent{ + {Key: objectKey(t), Object: &gitObj}, + {Key: types.Hash256(sha256.Sum256([]byte("vault"))), Object: &vaultObj}, + {Key: types.Hash256(sha256.Sum256([]byte("del"))), Deleted: true}, + } + + ingested, err := gitarchive.Scan(context.Background(), db, events) + require.NoError(t, err) + require.Equal(t, 1, ingested) + + var gitObjs []gitarchive.GitObject + require.NoError(t, db.Find(&gitObjs).Error) + require.Len(t, gitObjs, 1) + require.Equal(t, "git.tip", gitObjs[0].Kind) + + var files int64 + require.NoError(t, db.Model(&vault.File{}).Count(&files).Error) + require.Zero(t, files) +} diff --git a/internal/core/gitarchive/hook.go b/internal/core/gitarchive/hook.go new file mode 100644 index 00000000..a65c61ac --- /dev/null +++ b/internal/core/gitarchive/hook.go @@ -0,0 +1,139 @@ +package gitarchive + +import ( + "fmt" + "os" + "path/filepath" + "runtime" +) + +// This file implements the optional git post-push hook that `pinner git watch +// --hook` installs (Step 8, off by default). The hook is a small executable +// script Git itself runs after a successful push; it re-invokes the pinner CLI +// to republish the repository to its archive locker. Git runs the hook, so +// this does not shell out to the git binary from pinner — pinner only writes a +// hook script file into the repository's hooks dir via the OS filesystem. + +// postPushHookName is the git hook file we own. We only ever touch this exact +// file and never a user's own post-push hook: install is explicit (--hook) and +// remove is scoped to a file we wrote (verified by our marker comment). +const postPushHookName = "post-push" + +// postPushMarker is a distinctive first line we write into the hook so remove +// can tell our generated hook from a user-authored one. +const postPushMarker = "# pinner git archive post-push hook (generated)" + +// RepoHooksDir returns the hooks directory for the repository at repoPath. +// Git layout assumptions are intentionally minimal (matching the go-git +// DetectDotGit open used everywhere else): a standard /.git directory. +// Worktree/submodule discovery is out of scope for the MVP. +func RepoHooksDir(repoPath string) string { + return filepath.Join(repoPath, ".git", "hooks") +} + +// InstallPostPushHook writes an executable post-push hook into the repository +// at repoPath that runs `pinner git watch ` after every push, so the +// archive locker stays in sync even when the user pushes to a non-archive +// remote. It refuses to overwrite an existing post-push hook that is not one of +// ours (i.e. does not carry the marker). The returned path is the hook file +// written. +func InstallPostPushHook(repoPath string) (string, error) { + dir := RepoHooksDir(repoPath) + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", fmt.Errorf("gitarchive: create hooks dir: %w", err) + } + + pinnerBin, err := os.Executable() + if err != nil { + return "", fmt.Errorf("gitarchive: resolve pinner executable: %w", err) + } + + path := filepath.Join(dir, postPushHookName) + if existing, err := os.ReadFile(path); err == nil { + if !isOurHook(string(existing)) { + return "", fmt.Errorf("gitarchive: refusing to overwrite existing %s hook (not installed by pinner)", path) + } + } else if !os.IsNotExist(err) { + return "", fmt.Errorf("gitarchive: read existing hook: %w", err) + } + + script := hookScript(pinnerBin, repoPath) + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + return "", fmt.Errorf("gitarchive: write post-push hook: %w", err) + } + return path, nil +} + +// RemovePostPushHook removes the post-push hook from the repository at +// repoPath, but only when it is one of ours (carries the marker). It returns +// (removed true, nil) when our hook was present and deleted, and +// (false, nil) when there is no hook or the hook belongs to the user (left +// untouched). +func RemovePostPushHook(repoPath string) (bool, error) { + path := filepath.Join(RepoHooksDir(repoPath), postPushHookName) + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("gitarchive: read hook for removal: %w", err) + } + if !isOurHook(string(data)) { + return false, nil + } + if err := os.Remove(path); err != nil { + return false, fmt.Errorf("gitarchive: remove post-push hook: %w", err) + } + return true, nil +} + +// HasPostPushHook reports whether the repository at repoPath has a post-push +// hook that pinner installed. +func HasPostPushHook(repoPath string) (bool, error) { + data, err := os.ReadFile(filepath.Join(RepoHooksDir(repoPath), postPushHookName)) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("gitarchive: read hook: %w", err) + } + return isOurHook(string(data)), nil +} + +// isOurHook reports whether hook content carries the pinner marker line. +func isOurHook(content string) bool { + return len(content) >= len(postPushMarker) && content[:len(postPushMarker)] == postPushMarker +} + +// hookScript renders the executable shell script Git runs post-push. It quotes +// binary and repo paths for POSIX sh so paths with spaces survive. +func hookScript(pinnerBin, repoPath string) string { + return fmt.Sprintf( + "%s\n"+ + "# Re-publishes this repository to its pinner git-archive locker after\n"+ + "# every successful push. Managed by `pinner git watch --hook` / `pinner git unwatch`.\n"+ + "\n"+ + "%s git watch %s >/dev/null 2>&1 || exit 0\n", + postPushMarker, shellQuote(pinnerBin), shellQuote(repoPath)) +} + +// shellQuote wraps s in single quotes for POSIX sh, escaping embedded single +// quotes the standard way ('\”). +func shellQuote(s string) string { + if runtime.GOOS == "windows" { + return `"` + s + `"` + } + return "'" + shellEsc(s) + "'" +} + +func shellEsc(s string) string { + out := "" + for _, r := range s { + if r == '\'' { + out += `'\''` + } else { + out += string(r) + } + } + return out +} diff --git a/internal/core/gitarchive/hook_test.go b/internal/core/gitarchive/hook_test.go new file mode 100644 index 00000000..37df1d06 --- /dev/null +++ b/internal/core/gitarchive/hook_test.go @@ -0,0 +1,85 @@ +package gitarchive + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestInstallPostPushHookRoundTrip confirms install writes an executable hook +// that is reported present, and remove only deletes a hook we own. +func TestInstallPostPushHookRoundTrip(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".git"), 0o755)) + + has, err := HasPostPushHook(dir) + require.NoError(t, err) + require.False(t, has, "no hook before install") + + path, err := InstallPostPushHook(dir) + require.NoError(t, err) + require.Equal(t, filepath.Join(dir, ".git", "hooks", "post-push"), path) + + // Executable bit set. + info, err := os.Stat(path) + require.NoError(t, err) + require.NotZero(t, info.Mode()&0o111, "hook must be executable") + + data, err := os.ReadFile(path) + require.NoError(t, err) + require.Contains(t, string(data), postPushMarker) + require.Contains(t, string(data), "git watch") + + has, err = HasPostPushHook(dir) + require.NoError(t, err) + require.True(t, has) + + // Remove clears it and reports true. + removed, err := RemovePostPushHook(dir) + require.NoError(t, err) + require.True(t, removed) + + // Re-remove is a clean no-op. + removed, err = RemovePostPushHook(dir) + require.NoError(t, err) + require.False(t, removed) + + has, err = HasPostPushHook(dir) + require.NoError(t, err) + require.False(t, has) +} + +// TestRemovePostPushHookLeavesForeignHook: a post-push hook the user wrote +// themselves (no marker) is never deleted and install refuses to overwrite it. +func TestRemovePostPushHookLeavesForeignHook(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, ".git", "hooks"), 0o755)) + + path := filepath.Join(dir, ".git", "hooks", "post-push") + userHook := "#!/bin/sh\necho user hook\n" + require.NoError(t, os.WriteFile(path, []byte(userHook), 0o755)) + + // Install must refuse to clobber the user's hook. + _, err := InstallPostPushHook(dir) + require.Error(t, err) + require.Contains(t, err.Error(), "refusing to overwrite") + + // Remove must leave it. + removed, err := RemovePostPushHook(dir) + require.NoError(t, err) + require.False(t, removed) + + data, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, userHook, string(data)) +} + +// TestHookScriptQuoting: paths with spaces are safely single-quoted for POSIX +// sh. +func TestHookScriptQuoting(t *testing.T) { + script := hookScript("/opt/pinner bin/pinner", "/home/u/my repo") + require.Contains(t, script, "'/opt/pinner bin/pinner'") + require.Contains(t, script, "'/home/u/my repo'") +} diff --git a/internal/core/gitarchive/lineage.go b/internal/core/gitarchive/lineage.go new file mode 100644 index 00000000..252dd6b5 --- /dev/null +++ b/internal/core/gitarchive/lineage.go @@ -0,0 +1,104 @@ +package gitarchive + +import ( + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage" +) + +// Lineage is the identity fingerprint of a git repository for archive +// matching: two clones that share history have identical root-commit sets, so +// the (sorted, deduplicated) root commit hashes form a stable key that lets +// `pinner git watch` match a local repo to an existing archive locker. +// +// All Git work here runs in-process via go-git — no git binary, no shell-outs. + +// ComputeLineage opens the repository at path (detecting a nested .git) and +// returns its root-commit lineage. path may be the repository root, a +// subdirectory, or any enclosing directory. +func ComputeLineage(path string) ([]plumbing.Hash, error) { + repo, err := git.PlainOpenWithOptions(path, &git.PlainOpenOptions{DetectDotGit: true}) + if err != nil { + return nil, fmt.Errorf("gitarchive: open repo: %w", err) + } + return RepoLineage(repo.Storer) +} + +// RepoLineage computes the root-commit lineage from a repository's object +// store (see ComputeLineage). +func RepoLineage(st storage.Storer) ([]plumbing.Hash, error) { + roots := map[plumbing.Hash]bool{} + iter, err := st.IterReferences() + if err != nil { + return nil, fmt.Errorf("gitarchive: list references: %w", err) + } + defer iter.Close() + if err := iter.ForEach(func(r *plumbing.Reference) error { + if r.Type() == plumbing.SymbolicReference { + return nil + } + if err := collectRoots(st, r.Hash(), roots); err != nil { + return err + } + return nil + }); err != nil { + return nil, fmt.Errorf("gitarchive: walk lineage: %w", err) + } + hs := make([]plumbing.Hash, 0, len(roots)) + for h := range roots { + hs = append(hs, h) + } + sort.Slice(hs, func(i, j int) bool { return hs[i].String() < hs[j].String() }) + return hs, nil +} + +// collectRoots records every root commit (zero-parent commit) reachable from h, +// walking history recursively. Non-commit objects (e.g. a presenting tag object +// under refs/tags/...) are skipped rather than treated as roots. +func collectRoots(st storage.Storer, h plumbing.Hash, roots map[plumbing.Hash]bool) error { + if roots[h] { + return nil + } + c, err := object.GetCommit(st, h) + if err != nil { + // Not a commit (or not present locally) — skip; branches still yield + // the full root set through their tip history. + return nil + } + if c.NumParents() == 0 { + roots[h] = true + return nil + } + for _, ph := range c.ParentHashes { + if err := collectRoots(st, ph, roots); err != nil { + return err + } + } + return nil +} + +// LineageKey renders a lineage as its canonical string form — the sorted root +// hashes joined by commas. Sorting internally makes the key order-independent +// (the same archive fingerprint whichever order the roots were discovered in). +// This is the value stored in git_repos.Lineage and compared during lineage +// matching. +func LineageKey(hashes []plumbing.Hash) string { + sorted := make([]string, len(hashes)) + for i, h := range hashes { + sorted[i] = h.String() + } + sort.Strings(sorted) + return strings.Join(sorted, ",") +} + +// DefaultLockerName derives a default locker name from a local repo path (its +// base directory name). Callers override it with an explicit --locker. +func DefaultLockerName(path string) string { + return filepath.Base(path) +} diff --git a/internal/core/gitarchive/lineage_test.go b/internal/core/gitarchive/lineage_test.go new file mode 100644 index 00000000..07cfa7ea --- /dev/null +++ b/internal/core/gitarchive/lineage_test.go @@ -0,0 +1,72 @@ +package gitarchive_test + +import ( + "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/stretchr/testify/require" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +// initCommit creates a fresh non-bare repo and commits one file, returning the +// repo and the new commit hash. +func initCommit(t *testing.T, name, content string) (*git.Repository, plumbing.Hash) { + t.Helper() + repo, err := git.PlainInit(t.TempDir(), false) + require.NoError(t, err) + wt, err := repo.Worktree() + require.NoError(t, err) + f, err := wt.Filesystem.Create(name) + require.NoError(t, err) + _, werr := f.Write([]byte(content)) + require.NoError(t, werr) + require.NoError(t, f.Close()) + _, err = wt.Add(name) + require.NoError(t, err) + h, err := wt.Commit("init", &git.CommitOptions{ + Author: &object.Signature{Name: "t", Email: "t@example.com"}, + }) + require.NoError(t, err) + return repo, h +} + +func TestComputeLineageSingleRoot(t *testing.T) { + repo, h := initCommit(t, "a.txt", "hello") + + wt, err := repo.Worktree() + require.NoError(t, err) + lineage, err := gitarchive.ComputeLineage(wt.Filesystem.Root()) + require.NoError(t, err) + require.Len(t, lineage, 1) + require.Equal(t, h, lineage[0]) +} + +func TestLineageKeyIsSortedFingerprint(t *testing.T) { + a := initCommitT(t, "a.txt") + b := initCommitT(t, "b.txt") + + key := gitarchive.LineageKey([]plumbing.Hash{b, a}) + key2 := gitarchive.LineageKey([]plumbing.Hash{a, b}) + require.Equal(t, key, key2, "LineageKey must be order-independent") + // The canonical form is the lexicographically sorted join of both hashes. + expected := a.String() + "," + b.String() + if b.String() < a.String() { + expected = b.String() + "," + a.String() + } + require.Equal(t, expected, key) +} + +// initCommitT is a thin wrapper returning only the commit hash for tests that +// just need distinct root commits. +func initCommitT(t *testing.T, name string) plumbing.Hash { + _, h := initCommit(t, name, "content") + return h +} + +func TestDefaultLockerName(t *testing.T) { + require.Equal(t, "myproject", gitarchive.DefaultLockerName("/home/u/code/myproject")) + require.Equal(t, ".", gitarchive.DefaultLockerName(".")) +} diff --git a/internal/core/gitarchive/local.go b/internal/core/gitarchive/local.go new file mode 100644 index 00000000..b510fd70 --- /dev/null +++ b/internal/core/gitarchive/local.go @@ -0,0 +1,113 @@ +package gitarchive + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + + "go.lumeweb.com/pinner/core/vault" +) + +// This file holds the local-only helpers shared by `pinner git show`, `pinner +// git unwatch`, and `pinner git doctor` that operate on the profile's private +// bare mirror and the watched worktree config — all through go-git (no git +// subprocess). Sia object read/write stays in the Store/session layer; these +// helpers only manage the local filesystem bookkeeping. + +// PrivateRepoDir returns the on-disk directory holding the private bare git +// mirror for a locker, under the profile dir: +// +// /git/.git +// +// This mirror is where `pinner git show` ingests archived packs so the user can +// browse refs, the commit log, and the tree with go-git rendered locally. +func PrivateRepoDir(profile, locker string) string { + return filepath.Join(vault.ProfileDir(profile), "git", locker+".git") +} + +// OpenPrivateBare opens (creating if needed) the private bare mirror repository +// for a locker in the given profile dir. The returned repository's Storer holds +// whatever packs have been ingested and the refs that were set from archived +// tips; it is not a worktree. +func OpenPrivateBare(repoDir string) (*git.Repository, error) { + if _, err := os.Stat(filepath.Join(repoDir, "HEAD")); os.IsNotExist(err) { + // The mirror does not exist yet: initialize an empty bare repo so show + // can later ingest packs into a valid object database. + if err := os.MkdirAll(repoDir, 0o700); err != nil { + return nil, fmt.Errorf("gitarchive: create private mirror dir: %w", err) + } + repo, err := git.PlainInit(repoDir, true) + if err != nil { + return nil, fmt.Errorf("gitarchive: init private mirror: %w", err) + } + return repo, nil + } + repo, err := git.PlainOpen(repoDir) + if err != nil { + return nil, fmt.Errorf("gitarchive: open private mirror: %w", err) + } + return repo, nil +} + +// RemoveArchiveRemote removes the "archive" remote from the work repo at repoPath +// IF that remote points at this locker's pinner URL (i.e. it is ours to remove). +// It returns (removed true, nil) when the remote was present and pointed at the +// locker URL, (removed false, nil) when the remote was absent or pointed +// elsewhere (left untouched), and an error only on a config read/write failure. +// Unwatch only removes a remote it can attribute to us — never a user's own +// remote that happens to share the name. +func RemoveArchiveRemote(repoPath, locker string) (bool, error) { + want := GitRemoteURL(locker) + repo, err := git.PlainOpenWithOptions(repoPath, &git.PlainOpenOptions{DetectDotGit: true}) + if err != nil { + return false, fmt.Errorf("gitarchive: open repo for unwatch: %w", err) + } + cfg, err := repo.Config() + if err != nil { + return false, fmt.Errorf("gitarchive: read config: %w", err) + } + remote, ok := cfg.Remotes[RemoteName] + if !ok { + return false, nil + } + // Only remove when the remote is unambiguously ours (single URL equal to + // the canonical pinner URL for this locker). + if len(remote.URLs) == 1 && remote.URLs[0] == want { + delete(cfg.Remotes, RemoteName) + if err := repo.Storer.SetConfig(cfg); err != nil { + return false, fmt.Errorf("gitarchive: write config during unwatch: %w", err) + } + return true, nil + } + return false, nil +} + +// RepairArchiveRemote ensures the work repo at repoPath carries the canonical +// "archive" remote pointing at locker's URL, fixing a missing or mispointed one. +// It returns (repaired bool, nil) where repaired reports whether a change was +// actually written. This backs `pinner git doctor`'s URL-repair pass. +func RepairArchiveRemote(repoPath, locker string) (bool, error) { + want := GitRemoteURL(locker) + repo, err := git.PlainOpenWithOptions(repoPath, &git.PlainOpenOptions{DetectDotGit: true}) + if err != nil { + return false, fmt.Errorf("gitarchive: open repo for repair: %w", err) + } + cfg, err := repo.Config() + if err != nil { + return false, fmt.Errorf("gitarchive: read config: %w", err) + } + if existing, ok := cfg.Remotes[RemoteName]; ok && len(existing.URLs) == 1 && existing.URLs[0] == want { + return false, nil + } + cfg.Remotes[RemoteName] = &config.RemoteConfig{ + Name: RemoteName, + URLs: []string{want}, + } + if err := repo.Storer.SetConfig(cfg); err != nil { + return false, fmt.Errorf("gitarchive: write config during repair: %w", err) + } + return true, nil +} diff --git a/internal/core/gitarchive/local_test.go b/internal/core/gitarchive/local_test.go new file mode 100644 index 00000000..1ffdd01a --- /dev/null +++ b/internal/core/gitarchive/local_test.go @@ -0,0 +1,151 @@ +package gitarchive_test + +import ( + "path/filepath" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/stretchr/testify/require" + + "go.lumeweb.com/pinner/core/vault" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +// TestDeleteBindAndRepo confirms unwatch bookkeeping: a bound path is removed +// (and re-removing is a no-op), and the git_repos row for the locker gone. +func TestDeleteBindAndRepo(t *testing.T) { + db := openDB(t) + require.NoError(t, gitarchive.AutoMigrate(db)) + + _, err := gitarchive.AddBind(db, "widgets", "/home/u/widgets") + require.NoError(t, err) + _, err = gitarchive.EnsureRepo(db, "widgets", "widgets", "lin") + require.NoError(t, err) + + removed, err := gitarchive.DeleteBind(db, "/home/u/widgets") + require.NoError(t, err) + require.True(t, removed) + + // Re-delete is a clean no-op. + removed, err = gitarchive.DeleteBind(db, "/home/u/widgets") + require.NoError(t, err) + require.False(t, removed) + + removed, err = gitarchive.DeleteRepoByLocker(db, "widgets") + require.NoError(t, err) + require.True(t, removed) + + removed, err = gitarchive.DeleteRepoByLocker(db, "widgets") + require.NoError(t, err) + require.False(t, removed) + + repos, err := gitarchive.ListRepos(db) + require.NoError(t, err) + require.Empty(t, repos) +} + +// TestRemoveArchiveRemoteOnlyWhenOurs: an archive remote pointing at the locker +// URL is removed; a same-named remote pointing elsewhere is left untouched. +func TestRemoveArchiveRemoteOnlyWhenOurs(t *testing.T) { + dir := t.TempDir() + repo, err := git.PlainInit(dir, false) + require.NoError(t, err) + + setRemote := func(url string) { + cfg, err := repo.Config() + require.NoError(t, err) + cfg.Remotes[gitarchive.RemoteName] = &config.RemoteConfig{ + Name: gitarchive.RemoteName, + URLs: []string{url}, + } + require.NoError(t, repo.Storer.SetConfig(cfg)) + } + + // Not ours -> untouched. + setRemote("https://example.com/other") + removed, err := gitarchive.RemoveArchiveRemote(dir, "widgets") + require.NoError(t, err) + require.False(t, removed) + + cfg, err := repo.Config() + require.NoError(t, err) + _, ok := cfg.Remotes[gitarchive.RemoteName] + require.True(t, ok, "foreign remote must remain") + + // Ours -> removed. + setRemote(gitarchive.GitRemoteURL("widgets")) + removed, err = gitarchive.RemoveArchiveRemote(dir, "widgets") + require.NoError(t, err) + require.True(t, removed) + + cfg, err = repo.Config() + require.NoError(t, err) + _, ok = cfg.Remotes[gitarchive.RemoteName] + require.False(t, ok, "our remote must be removed") + + // Removing again is a no-op (remote already gone). + removed, err = gitarchive.RemoveArchiveRemote(dir, "widgets") + require.NoError(t, err) + require.False(t, removed) +} + +// TestRepairArchiveRemote installs/corrects the archive remote to the canonical +// URL, and reports no repair when it already matches. +func TestRepairArchiveRemote(t *testing.T) { + dir := t.TempDir() + repo, err := git.PlainInit(dir, false) + require.NoError(t, err) + + // Missing remote -> repaired. + repaired, err := gitarchive.RepairArchiveRemote(dir, "widgets") + require.NoError(t, err) + require.True(t, repaired) + + cfg, err := repo.Config() + require.NoError(t, err) + require.Equal(t, gitarchive.GitRemoteURL("widgets"), cfg.Remotes[gitarchive.RemoteName].URLs[0]) + + // Already correct -> no repair. + repaired, err = gitarchive.RepairArchiveRemote(dir, "widgets") + require.NoError(t, err) + require.False(t, repaired) + + // Wrong URL -> repaired. + cfg.Remotes[gitarchive.RemoteName] = &config.RemoteConfig{ + Name: gitarchive.RemoteName, + URLs: []string{"https://bad.example/x"}, + } + require.NoError(t, repo.Storer.SetConfig(cfg)) + repaired, err = gitarchive.RepairArchiveRemote(dir, "widgets") + require.NoError(t, err) + require.True(t, repaired) + + cfg, err = repo.Config() + require.NoError(t, err) + require.Equal(t, gitarchive.GitRemoteURL("widgets"), cfg.Remotes[gitarchive.RemoteName].URLs[0]) +} + +// TestPrivateBareOpenCreateAndReopen verifies the private mirror is created as a +// bare repo on first open and reopens cleanly, and that its path lives under +// the profile git dir. +func TestPrivateBareOpenCreateAndReopen(t *testing.T) { + overrideHome(t, t.TempDir()) + + // The profile git dir helpers resolve under the profile dir. + profileDir := vault.ProfileDir("work") + repoDir := gitarchive.PrivateRepoDir("work", "widgets") + require.Contains(t, repoDir, filepath.Join(profileDir, "git")) + require.Equal(t, "widgets.git", filepath.Base(repoDir)) + + dir := t.TempDir() + repo, err := gitarchive.OpenPrivateBare(dir) + require.NoError(t, err) + require.NotNil(t, repo) + + // Reopen an existing mirror. + repo2, err := gitarchive.OpenPrivateBare(dir) + require.NoError(t, err) + require.NotNil(t, repo2) +} diff --git a/internal/core/gitarchive/objectkey.go b/internal/core/gitarchive/objectkey.go new file mode 100644 index 00000000..cf2e23f1 --- /dev/null +++ b/internal/core/gitarchive/objectkey.go @@ -0,0 +1,20 @@ +package gitarchive + +import ( + "fmt" + + "go.sia.tech/core/types" +) + +// ParseObjectKey parses the 40-hex hex representation of a Sia object key (the +// string form stored in git_objects.ObjectKey and referenced by tip/share +// documents) into a types.Hash256. It is the single source of truth for +// converting between the on-disk hex form and the SDK's binary key, so all +// callers — helper store, share minting, downloads — agree on the format. +func ParseObjectKey(s string) (types.Hash256, error) { + var key types.Hash256 + if err := key.UnmarshalText([]byte(s)); err != nil { + return types.Hash256{}, fmt.Errorf("gitarchive: invalid object key %q: %w", s, err) + } + return key, nil +} diff --git a/internal/core/gitarchive/repo.go b/internal/core/gitarchive/repo.go new file mode 100644 index 00000000..83d94db8 --- /dev/null +++ b/internal/core/gitarchive/repo.go @@ -0,0 +1,136 @@ +package gitarchive + +import ( + "fmt" + + "gorm.io/gorm" +) + +// RemoteName is the fixed remote name `pinner git watch` installs pointing at +// the archive locker. +const RemoteName = "archive" + +// GitRemoteURL renders the pinner remote URL for a locker (the value installed +// as Git's "archive" remote and used by the git-remote-pinner helper). +func GitRemoteURL(locker string) string { + return "pinner::" + locker +} + +// EnsureRepo upserts the git_repos row for locker, recording its display name +// and lineage fingerprint. When a row already exists it is updated in place and +// returned. +func EnsureRepo(db *gorm.DB, locker, name, lineage string) (GitRepo, error) { + var repo GitRepo + err := db.Where("locker = ?", locker).First(&repo).Error + if err == nil { + repo.Name = name + repo.Lineage = lineage + if err := db.Save(&repo).Error; err != nil { + return GitRepo{}, fmt.Errorf("gitarchive: update repo %q: %w", locker, err) + } + return repo, nil + } + if err != gorm.ErrRecordNotFound { + return GitRepo{}, fmt.Errorf("gitarchive: find repo %q: %w", locker, err) + } + repo = GitRepo{Locker: locker, Name: name, Lineage: lineage} + if err := db.Create(&repo).Error; err != nil { + return GitRepo{}, fmt.Errorf("gitarchive: create repo %q: %w", locker, err) + } + return repo, nil +} + +// FindReposByLineage returns every locker whose recorded lineage matches key, +// ordered by locker name. Returning more than one entry signals an ambiguous +// lineage match — the caller should ask the user to disambiguate with --locker +// or force a new locker with --new. +func FindReposByLineage(db *gorm.DB, key string) ([]GitRepo, error) { + var repos []GitRepo + if err := db.Where("lineage = ?", key).Order("locker").Find(&repos).Error; err != nil { + return nil, fmt.Errorf("gitarchive: match lineage: %w", err) + } + return repos, nil +} + +// AddBind records that the repo at repoPath is bound to locker under +// RemoteName, upserting on repeated binds of the same locker. +func AddBind(db *gorm.DB, locker, repoPath string) (GitBind, error) { + bind := GitBind{Locker: locker, Remote: RemoteName, RepoPath: repoPath} + var existing GitBind + err := db.Where("locker = ?", locker).First(&existing).Error + if err == nil { + existing.RepoPath = repoPath + existing.Remote = RemoteName + if err := db.Save(&existing).Error; err != nil { + return GitBind{}, fmt.Errorf("gitarchive: update bind %q: %w", locker, err) + } + return existing, nil + } + if err != gorm.ErrRecordNotFound { + return GitBind{}, fmt.Errorf("gitarchive: find bind %q: %w", locker, err) + } + if err := db.Create(&bind).Error; err != nil { + return GitBind{}, fmt.Errorf("gitarchive: create bind %q: %w", locker, err) + } + return bind, nil +} + +// FindBindByRepo returns the bind (and thus the locker + remote) for a local +// repo path, or nil when the path is not bound to any archive. +func FindBindByRepo(db *gorm.DB, repoPath string) (*GitBind, error) { + var bind GitBind + err := db.Where("repo_path = ?", repoPath).First(&bind).Error + if err == gorm.ErrRecordNotFound { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("gitarchive: find bind %q: %w", repoPath, err) + } + return &bind, nil +} + +// ListRepos returns every tracked locker (git_repos rows) ordered by locker +// name. This backs `pinner git ls`. +func ListRepos(db *gorm.DB) ([]GitRepo, error) { + var repos []GitRepo + if err := db.Order("locker").Find(&repos).Error; err != nil { + return nil, fmt.Errorf("gitarchive: list repos: %w", err) + } + return repos, nil +} + +// DeleteBind removes the git_binds row for repoPath, returning (true, nil) when +// a bind existed and was removed, and (false, nil) when repoPath was not bound. +// It backs `pinner git unwatch` (the local bookkeeping half). +func DeleteBind(db *gorm.DB, repoPath string) (bool, error) { + var bind GitBind + err := db.Where("repo_path = ?", repoPath).First(&bind).Error + if err == gorm.ErrRecordNotFound { + return false, nil + } + if err != nil { + return false, fmt.Errorf("gitarchive: find bind %q: %w", repoPath, err) + } + if err := db.Delete(&bind).Error; err != nil { + return false, fmt.Errorf("gitarchive: delete bind %q: %w", repoPath, err) + } + return true, nil +} + +// DeleteRepoByLocker removes the git_repos row for locker, returning +// (true, nil) when a row existed and was removed. It backs `pinner git unwatch` +// cleanup so an unwatched locker no longer appears in `pinner git ls`. +func DeleteRepoByLocker(db *gorm.DB, locker string) (bool, error) { + var repo GitRepo + err := db.Where("locker = ?", locker).First(&repo).Error + if err == gorm.ErrRecordNotFound { + return false, nil + } + if err != nil { + return false, fmt.Errorf("gitarchive: find repo %q: %w", locker, err) + } + if err := db.Delete(&repo).Error; err != nil { + return false, fmt.Errorf("gitarchive: delete repo %q: %w", locker, err) + } + return true, nil +} diff --git a/internal/core/gitarchive/repo_test.go b/internal/core/gitarchive/repo_test.go new file mode 100644 index 00000000..3c6eba24 --- /dev/null +++ b/internal/core/gitarchive/repo_test.go @@ -0,0 +1,91 @@ +package gitarchive_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +func TestEnsureRepoUpsertAndMatch(t *testing.T) { + db := openDB(t) + require.NoError(t, gitarchive.AutoMigrate(db)) + + lineage := "aaa123,bbb456" + repo, err := gitarchive.EnsureRepo(db, "widgets", "widgets", lineage) + require.NoError(t, err) + require.Equal(t, "widgets", repo.Locker) + require.Equal(t, lineage, repo.Lineage) + + // Match by lineage returns the locker. + matches, err := gitarchive.FindReposByLineage(db, lineage) + require.NoError(t, err) + require.Len(t, matches, 1) + require.Equal(t, "widgets", matches[0].Locker) + + // Upserting the same locker updates the lineage without duplicating rows. + _, err = gitarchive.EnsureRepo(db, "widgets", "widgets", "newlineage") + require.NoError(t, err) + repos, err := gitarchive.ListRepos(db) + require.NoError(t, err) + require.Len(t, repos, 1) + require.Equal(t, "newlineage", repos[0].Lineage) + + // Matches for an unknown lineage are empty. + matches, err = gitarchive.FindReposByLineage(db, "zzz") + require.NoError(t, err) + require.Empty(t, matches) +} + +func TestFindReposByLineageAmbiguity(t *testing.T) { + db := openDB(t) + require.NoError(t, gitarchive.AutoMigrate(db)) + + lineage := "sharedroot" + _, err := gitarchive.EnsureRepo(db, "widgets-a", "a", lineage) + require.NoError(t, err) + _, err = gitarchive.EnsureRepo(db, "widgets-b", "b", lineage) + require.NoError(t, err) + + matches, err := gitarchive.FindReposByLineage(db, lineage) + require.NoError(t, err) + require.Len(t, matches, 2) + // Ordered by locker name. + require.Equal(t, "widgets-a", matches[0].Locker) + require.Equal(t, "widgets-b", matches[1].Locker) +} + +func TestAddBindAndFindByRepo(t *testing.T) { + db := openDB(t) + require.NoError(t, gitarchive.AutoMigrate(db)) + + _, err := gitarchive.AddBind(db, "widgets", "/home/u/widgets") + require.NoError(t, err) + + bind, err := gitarchive.FindBindByRepo(db, "/home/u/widgets") + require.NoError(t, err) + require.NotNil(t, bind) + require.Equal(t, "widgets", bind.Locker) + require.Equal(t, gitarchive.RemoteName, bind.Remote) + + // Not-bound path returns nil, not an error. + bind, err = gitarchive.FindBindByRepo(db, "/home/u/other") + require.NoError(t, err) + require.Nil(t, bind) + + // Rebinding the same locker updates the path without a duplicate row. + _, err = gitarchive.AddBind(db, "widgets", "/home/u/new-location") + require.NoError(t, err) + bind, err = gitarchive.FindBindByRepo(db, "/home/u/new-location") + require.NoError(t, err) + require.NotNil(t, bind) + + var count int64 + require.NoError(t, db.Model(&gitarchive.GitBind{}).Count(&count).Error) + require.Equal(t, int64(1), count) +} + +func TestGitRemoteURL(t *testing.T) { + require.Equal(t, "pinner::widgets", gitarchive.GitRemoteURL("widgets")) +} diff --git a/internal/core/gitarchive/scan.go b/internal/core/gitarchive/scan.go new file mode 100644 index 00000000..78f639e6 --- /dev/null +++ b/internal/core/gitarchive/scan.go @@ -0,0 +1,114 @@ +package gitarchive + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "go.sia.tech/siastorage" + "gorm.io/datatypes" + "gorm.io/gorm" + + "go.lumeweb.com/pinner-cli/internal/core/objmeta" +) + +// IngestCard records one git-archive card (already recognized at the objmeta +// layer) into the local git_objects table. It returns (true, nil) when the +// metadata was a git card that was ingested, (false, nil) when it was NOT a +// git card (a vault file or anything else) and so was ignored, and a non-nil +// error only on a local DB failure. +// +// IngestCard intentionally writes ONLY to git_objects/git_repos; it never +// touches the vault File table, so scanning git cards can never produce vault +// File rows. +func IngestCard(db *gorm.DB, objectKey string, raw []byte) (bool, error) { + card, err := objmeta.Decode(raw) + if err != nil { + // Not a pinner card at all (including legacy vault FileMetadata, or an + // unparseable/oversize payload): not ours to ingest. + if errors.Is(err, objmeta.ErrNotCard) || errors.Is(err, objmeta.ErrUnknownKind) { + return false, nil + } + return false, nil + } + + var body datatypes.JSON + if len(card.Body) > 0 { + b, jerr := json.Marshal(card.Body) + if jerr == nil { + body = datatypes.JSON(b) + } + } + + now := time.Now().UTC() + obj := GitObject{ + ObjectKey: objectKey, + Locker: lockerFromCard(card), + Kind: string(card.Kind), + Size: card.Size, + Digest: card.Digest, + Created: card.Created, + Body: body, + CreatedAt: now, + UpdatedAt: now, + } + + // Idempotent upsert keyed by object key. + if err := db.Where("object_key = ?", objectKey).Assign(map[string]any{ + "locker": obj.Locker, + "kind": obj.Kind, + "size": obj.Size, + "digest": obj.Digest, + "created": obj.Created, + "body": obj.Body, + "updated_at": now, + }).FirstOrCreate(&obj).Error; err != nil { + return true, fmt.Errorf("gitarchive: ingest card %s: %w", objectKey, err) + } + return true, nil +} + +// lockerFromCard extracts the locker/name from a card body for indexing. It is +// best-effort: bodies vary by kind (pack/tip/share all carry a locker). +func lockerFromCard(c *objmeta.Card) string { + var probe struct { + Locker string `json:"locker"` + } + if len(c.Body) > 0 { + _ = json.Unmarshal(c.Body, &probe) + } + return probe.Locker +} + +// Scan walks object events and ingests any pinner git cards found into +// git_objects. Non-card events (including vault files) are skipped. It returns +// the number of git cards ingested. This is the gitarchive parallel to +// vault.Sync: it only owns git-kind objects and never writes File rows. +func Scan(ctx context.Context, db *gorm.DB, events []siastorage.ObjectEvent) (int, error) { + ingested := 0 + for _, ev := range events { + if err := ctx.Err(); err != nil { + return ingested, err + } + if ev.Deleted || ev.Object == nil { + continue + } + meta := ev.Object.Metadata() + if len(meta) == 0 { + continue + } + if !objmeta.IsCard(meta) { + continue // vault files and unknown metadata are not git cards + } + ok, err := IngestCard(db, ev.Key.String(), meta) + if err != nil { + return ingested, err + } + if ok { + ingested++ + } + } + return ingested, nil +} diff --git a/internal/core/gitarchive/sdk.go b/internal/core/gitarchive/sdk.go new file mode 100644 index 00000000..ba3212b6 --- /dev/null +++ b/internal/core/gitarchive/sdk.go @@ -0,0 +1,48 @@ +package gitarchive + +import ( + "context" + "io" + "time" + + "go.sia.tech/core/types" + "go.sia.tech/siastorage" +) + +// SDKClient is the subset of *siastorage.SDK that git archive needs. It exists +// so the git archive domain can be exercised against a fake SDK in tests +// without a live indexer or a network round-trip. The concrete realSDK wraps +// *siastorage.SDK and satisfies it directly. +// +// Share URLs are self-contained bearer credentials: CreateSharedObjectURL +// returns a pre-signed URL that embeds the object's encryption key, so anyone +// holding the URL can read the object without any profile/app key. The +// profile-less helper share path relies on this — it will carry the full +// pre-signed URL rather than an account-scoped key. +type SDKClient interface { + // Upload streams r into obj (which already carries its card metadata via + // UpdateMetadata) and records the uploaded slabs on obj. + Upload(ctx context.Context, obj *siastorage.Object, r io.Reader, opts ...siastorage.UploadOption) error + // PinObject persists obj's sealed metadata/indexer record after Upload. + PinObject(ctx context.Context, obj siastorage.Object) error + // Object retrieves an account object by its content-addressed key. + Object(ctx context.Context, objectKey types.Hash256) (siastorage.Object, error) + // Download streams an account object's plaintext. + Download(obj siastorage.Object, opts ...siastorage.DownloadOption) (io.ReadCloser, error) + // CreateSharedObjectURL builds a self-contained bearer share URL for an + // object valid until the given time. + CreateSharedObjectURL(ctx context.Context, objectKey types.Hash256, validUntil time.Time) (string, error) + // DownloadSharedObject streams a shared object's plaintext from a + // self-contained bearer share URL (no app key needed for the read). + DownloadSharedObject(ctx context.Context, sharedURL string, opts ...siastorage.DownloadOption) (io.ReadCloser, error) + // Close releases SDK-held resources (host connections, thread group). + Close() error +} + +// realSDK adapts the concrete *siastorage.SDK to the SDKClient interface. The +// embedded *siastorage.SDK provides every method in SDKClient unchanged, so +// the adapter is a thin type that makes the production client satisfy the +// mockable interface. +type realSDK struct { + *siastorage.SDK +} diff --git a/internal/core/gitarchive/sdk_fake_test.go b/internal/core/gitarchive/sdk_fake_test.go new file mode 100644 index 00000000..3fb73b6d --- /dev/null +++ b/internal/core/gitarchive/sdk_fake_test.go @@ -0,0 +1,119 @@ +package gitarchive_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "sync" + "time" + + "go.sia.tech/core/types" + "go.sia.tech/siastorage" +) + +// fakeSDK is an in-memory SDKClient used to exercise the git archive +// upload/download plumbing without a live indexer. Uploads are stored by their +// content-addressed object key; Object/Download read them back. It also +// records the sequence of pins for assertions. +type fakeSDK struct { + mu sync.Mutex + objects map[types.Hash256]fakeStored + pinned []types.Hash256 + + // shared-object stubs (used by the share-URL path). + shareURL string + shareErr error + sharedData []byte + // mintURLs, when set, overrides the returned pre-signed URL per object key. + mintURLs map[types.Hash256]string + // mintLog records, in order, every key handed to CreateSharedObjectURL. + mintLog []types.Hash256 + // sharedByURL, when set, serves DownloadSharedObject payloads per URL. + sharedByURL map[string][]byte +} + +type fakeStored struct { + meta json.RawMessage + data []byte +} + +func newFakeSDK() *fakeSDK { + return &fakeSDK{objects: map[types.Hash256]fakeStored{}} +} + +func (f *fakeSDK) Upload(ctx context.Context, obj *siastorage.Object, r io.Reader, _ ...siastorage.UploadOption) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + f.mu.Lock() + defer f.mu.Unlock() + f.objects[obj.ID()] = fakeStored{meta: obj.Metadata(), data: data} + return nil +} + +func (f *fakeSDK) PinObject(ctx context.Context, obj siastorage.Object) error { + f.mu.Lock() + defer f.mu.Unlock() + f.pinned = append(f.pinned, obj.ID()) + return nil +} + +func (f *fakeSDK) Object(ctx context.Context, key types.Hash256) (siastorage.Object, error) { + f.mu.Lock() + defer f.mu.Unlock() + fs, ok := f.objects[key] + if !ok { + return siastorage.Object{}, fmt.Errorf("fake: object %s not found", key) + } + obj := siastorage.NewEmptyObject() + obj.UpdateMetadata(fs.meta) + return obj, nil +} + +func (f *fakeSDK) Download(obj siastorage.Object, _ ...siastorage.DownloadOption) (io.ReadCloser, error) { + f.mu.Lock() + defer f.mu.Unlock() + fs, ok := f.objects[obj.ID()] + if !ok { + return nil, fmt.Errorf("fake: no data for object %s", obj.ID()) + } + return io.NopCloser(bytes.NewReader(fs.data)), nil +} + +func (f *fakeSDK) CreateSharedObjectURL(ctx context.Context, objectKey types.Hash256, validUntil time.Time) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.mintLog = append(f.mintLog, objectKey) + if f.shareErr != nil { + return "", f.shareErr + } + if u, ok := f.mintURLs[objectKey]; ok { + return u, nil + } + if f.shareURL != "" { + return f.shareURL, nil + } + // No explicit URL set: derive a stable, key-specific URL so multiple objects + // mint distinct URLs without the caller pre-populating the map. + return "_fake_share_" + objectKey.String(), nil +} + +func (f *fakeSDK) DownloadSharedObject(ctx context.Context, sharedURL string, _ ...siastorage.DownloadOption) (io.ReadCloser, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.shareErr != nil { + return nil, f.shareErr + } + if data, ok := f.sharedByURL[sharedURL]; ok { + return io.NopCloser(bytes.NewReader(data)), nil + } + if f.sharedData == nil { + return nil, fmt.Errorf("fake: no shared object data") + } + return io.NopCloser(bytes.NewReader(f.sharedData)), nil +} + +func (f *fakeSDK) Close() error { return nil } diff --git a/internal/core/gitarchive/session.go b/internal/core/gitarchive/session.go new file mode 100644 index 00000000..5f607747 --- /dev/null +++ b/internal/core/gitarchive/session.go @@ -0,0 +1,129 @@ +package gitarchive + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "go.sia.tech/core/types" + "go.sia.tech/siastorage" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "go.lumeweb.com/pinner/core/vault" +) + +// ErrNoProfile is surfaced whenever git archive cannot resolve a usable vault +// profile. It is the canonical missing-profile error seen by the CLI and the +// helper (exact message: "no vault profile; run pinner vault create or pinner +// vault restore"). +var ErrNoProfile = errors.New("no vault profile; run pinner vault create or pinner vault restore") + +// Session carries the resolved, profile-scoped context for git archive +// operations: the profile's app key (which authenticates the Sia account), the +// Sia indexer URL, the open SQLite cache DB (with the git archive tables +// migrated), and the mockable Sia storage SDK. +type Session struct { + // Profile is the resolved vault profile name. + Profile string + // AppKey is the profile's app key (hex-decoded from state.json). + AppKey types.PrivateKey + // IndexerURL is the Sia indexer origin for this profile. + IndexerURL string + // DB is the profile SQLite cache with the gitarchive tables present. + DB *gorm.DB + + // SDK is the Sia storage client. nil until first use; callers may inject a + // fake for tests (see ensureSDK). + SDK SDKClient +} + +// NewSession resolves the active profile (flag → PINNER_PROFILE → default → +// single-profile, via the vault registry), loads and decodes its app key, +// opens its SQLite cache DB (running the git archive AutoMigrate), and returns +// a Session with the SDK still unbuilt. The SDK is built lazily on first use +// because constructing it hits the network (CheckAppAuth + refreshHosts). +func NewSession(profileValue, indexerURL string) (*Session, error) { + name, err := vault.ResolveProfile(profileValue) + if err != nil { + return nil, ErrNoProfile + } + if err := vault.ValidateProfileName(name); err != nil { + return nil, ErrNoProfile + } + state, err := vault.LoadProfileState(name) + if err != nil { + return nil, fmt.Errorf("gitarchive: load profile state: %w", err) + } + if state.AppKey == "" { + return nil, ErrNoProfile + } + appKey, err := vault.DecodeAppKey(state.AppKey) + if err != nil { + return nil, fmt.Errorf("gitarchive: decode app key: %w", err) + } + + db, err := openProfileDB(name) + if err != nil { + return nil, err + } + return &Session{ + Profile: name, + AppKey: appKey, + IndexerURL: indexerURL, + DB: db, + }, nil +} + +// openProfileDB opens the profile's SQLite cache and runs the git archive +// schema migration. Unlike the vault library it uses Gorm sqlite directly and +// only the additive gitarchive AutoMigrate; it never touches the vault File +// schema. +func openProfileDB(profile string) (*gorm.DB, error) { + path := vault.ProfileDBPath(profile) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, fmt.Errorf("gitarchive: create profile dir: %w", err) + } + db, err := gorm.Open(sqlite.Open(path), &gorm.Config{}) + if err != nil { + return nil, fmt.Errorf("gitarchive: open profile cache: %w", err) + } + if err := AutoMigrate(db); err != nil { + return nil, fmt.Errorf("gitarchive: migrate profile cache: %w", err) + } + return db, nil +} + +// ensureSDK returns the Sia SDK, building the real one lazily on first use +// (mirroring the vault service). When a fake SDK has been injected into the +// Session (e.g. by tests) it is returned unchanged and no network is touched. +func (s *Session) ensureSDK() (SDKClient, error) { + if s.SDK != nil { + return s.SDK, nil + } + if s.IndexerURL == "" { + return nil, fmt.Errorf("gitarchive: no Sia indexer URL for profile %q", s.Profile) + } + metadata := siastorage.AppMetadata{ + ID: vault.AppID(), + Name: "Pinner CLI Git Archive", + Description: "Git archive hosting via Sia", + ServiceURL: s.IndexerURL, + } + builder := siastorage.NewBuilder(s.IndexerURL, metadata) + real, err := builder.SDK(s.AppKey) + if err != nil { + return nil, fmt.Errorf("gitarchive: build SDK: %w", err) + } + s.SDK = &realSDK{SDK: real} + return s.SDK, nil +} + +// Close releases SDK-held resources (host connections, thread group). +func (s *Session) Close() error { + if s.SDK != nil { + return s.SDK.Close() + } + return nil +} diff --git a/internal/core/gitarchive/session_test.go b/internal/core/gitarchive/session_test.go new file mode 100644 index 00000000..69e581fa --- /dev/null +++ b/internal/core/gitarchive/session_test.go @@ -0,0 +1,90 @@ +package gitarchive_test + +import ( + "bytes" + "encoding/hex" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.sia.tech/core/types" + + "go.lumeweb.com/pinner/core/vault" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +// overrideHome isolates the vault registry/profile state from the real user +// home by pointing every config/data resolver at a temp dir (mirrors the CLI +// test helper). +func overrideHome(t *testing.T, home string) { + t.Helper() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache")) + t.Setenv("APPDATA", filepath.Join(home, "AppData", "Roaming")) + t.Setenv("LOCALAPPDATA", filepath.Join(home, "AppData", "Local")) + t.Setenv("PINNER_PROFILE", "") +} + +func seedProfile(t *testing.T, name string, appKey []byte) { + t.Helper() + require.NoError(t, vault.SaveRegistry(&vault.VaultRegistry{ + Default: name, + Profiles: map[string]vault.ProfileConfig{ + name: {VaultID: "vault:test"}, + }, + })) + require.NoError(t, vault.SaveProfileState(name, &vault.ProfileState{ + AppKey: hex.EncodeToString(appKey), + DeviceID: "dev-1", + CreatedAt: "2026-01-01T00:00:00Z", + })) +} + +func TestNewSessionResolvesProfile(t *testing.T) { + overrideHome(t, t.TempDir()) + appKey := bytes.Repeat([]byte{0x42}, 32) + seedProfile(t, "work", appKey) + + s, err := gitarchive.NewSession("", "http://localhost:9980") + require.NoError(t, err) + require.NotNil(t, s) + require.Equal(t, "work", s.Profile) + require.Equal(t, "http://localhost:9980", s.IndexerURL) + require.Equal(t, types.PrivateKey(appKey), s.AppKey) + require.NotNil(t, s.DB) + + // The git archive schema is migrated onto the profile cache. + require.True(t, s.DB.Migrator().HasTable("git_repos")) + require.True(t, s.DB.Migrator().HasTable("git_objects")) + require.True(t, s.DB.Migrator().HasTable("git_binds")) + + // SDK is lazy: construction must not touch the network. + require.Nil(t, s.SDK) +} + +func TestNewSessionNoProfile(t *testing.T) { + overrideHome(t, t.TempDir()) + // Empty registry: no profiles at all. + require.NoError(t, vault.SaveRegistry(&vault.VaultRegistry{Profiles: map[string]vault.ProfileConfig{}})) + + _, err := gitarchive.NewSession("", "http://localhost:9980") + require.ErrorIs(t, err, gitarchive.ErrNoProfile) +} + +func TestNewSessionMissingAppKey(t *testing.T) { + overrideHome(t, t.TempDir()) + // Registry present but state.json has no app key. + require.NoError(t, vault.SaveRegistry(&vault.VaultRegistry{ + Profiles: map[string]vault.ProfileConfig{"work": {VaultID: "vault:test"}}, + })) + require.NoError(t, vault.SaveProfileState("work", &vault.ProfileState{})) + + // The profile exists but cannot authenticate: LoadProfileState rejects it + // because state.json has no app key. This is a broken-profile error, not a + // missing-profile (ErrNoProfile) error. + _, err := gitarchive.NewSession("", "http://localhost:9980") + require.Error(t, err) + require.NotErrorIs(t, err, gitarchive.ErrNoProfile) +} diff --git a/internal/core/gitarchive/share.go b/internal/core/gitarchive/share.go new file mode 100644 index 00000000..6cf76331 --- /dev/null +++ b/internal/core/gitarchive/share.go @@ -0,0 +1,241 @@ +package gitarchive + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "time" + + "go.sia.tech/core/types" + "gorm.io/gorm" + + "go.lumeweb.com/pinner-cli/internal/core/objmeta" +) + +// This file implements `pinner git share`: it mints self-contained bearer share +// URLs for an archived locker's tip + packs and records them in a git.share +// ShareDocument object. The minted URLs are pre-signed (see SDKClient +// documentation): each embeds the object's encryption key, so anyone holding a +// URL can read the object WITHOUT a profile/app key. The profile-less helper +// share path (`pinner::share/...`) consumes exactly these URLs later. + +// DefaultShareTTL is the default validity window for a minted share URL +// (30 days, per the plan). Callers override it with --until. +const DefaultShareTTL = 30 * 24 * time.Hour + +// ShareDocument is the git.share object payload ("share bytes"). Order of +// PackURLs matches the tip's pack order so a consumer playing the packs back in +// order reconstructs the archived history. Until is a unix-seconds expiry. +type ShareDocument struct { + Locker string `json:"locker"` + Gen int `json:"gen"` + Name string `json:"name"` + TipURL string `json:"tip_url"` + PackURLs []string `json:"pack_urls"` + Until int64 `json:"until"` // unix seconds +} + +// PackRef is one archived pack referenced from a tip document. Key is the hex +// object key of the git.pack object; order within the tip's Packs slice is +// replay order. +type PackRef struct { + Key string `json:"key"` + Full bool `json:"full"` + Digest string `json:"digest"` +} + +// TipBytes is the git.tip object payload ("tip bytes"). Refs maps ref names to +// their tip sha1 hashes; Packs lists the archived packs in replay order. +type TipBytes struct { + Locker string `json:"locker"` + Gen int `json:"gen"` + Name string `json:"name"` + Lineage []string `json:"lineage"` + Refs map[string]string `json:"refs"` + Packs []PackRef `json:"packs"` + Prev string `json:"prev,omitempty"` +} + +// ParseShareDocument decodes a git.share object payload. +func ParseShareDocument(data []byte) (*ShareDocument, error) { + var doc ShareDocument + if err := json.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("gitarchive: parse share document: %w", err) + } + if doc.Locker == "" || doc.TipURL == "" { + return nil, fmt.Errorf("gitarchive: share document missing locker or tip_url") + } + return &doc, nil +} + +// ParseTipBytes decodes a git.tip object payload. +func ParseTipBytes(data []byte) (*TipBytes, error) { + var tip TipBytes + if err := json.Unmarshal(data, &tip); err != nil { + return nil, fmt.Errorf("gitarchive: parse tip document: %w", err) + } + if tip.Locker == "" { + return nil, fmt.Errorf("gitarchive: tip document missing locker") + } + return &tip, nil +} + +// hashFromHex parses a 40-hex object key string into a types.Hash256. It +// delegates to the exported ParseObjectKey — the single source of truth for +// hex↔Hash256 conversion shared by the helper store and share minting. +func hashFromHex(s string) (types.Hash256, error) { + return ParseObjectKey(s) +} + +// ShareResult is the outcome of MintShare: the top-level share URL of the share +// document object, the validity window, and the inner URLs so callers can show +// nested detail under --verbose. +type ShareResult struct { + // Locker is the archived locker that was shared. + Locker string + // Gen is the archived tip generation that was shared. + Gen int + // Name is the locker's display name. + Name string + // URL is the pre-signed share URL (of the git.share document object). This + // is the value a consumer feeds to `git clone pinner::share/`. + URL string + // Until is when the pre-signed URLs expire. + Until time.Time + // TipURL is the pre-signed URL of the tip object. + TipURL string + // PackURLs are the pre-signed URLs of the archived packs, in replay order. + PackURLs []string + // Document is the serialized share document (for --verbose / tests). + Document *ShareDocument +} + +// FindTipForLocker returns the current git.tip object row for locker (the one +// with the greatest generation in its card body) and its decoded card body, or +// (nil, nil) when the locker has no archived tip. +func FindTipForLocker(db *gorm.DB, locker string) (*GitObject, *objmeta.TipBody, error) { + var objs []GitObject + if err := db.Where("locker = ? AND kind = ?", locker, string(objmeta.KindGitTip)).Find(&objs).Error; err != nil { + return nil, nil, fmt.Errorf("gitarchive: find tip for %q: %w", locker, err) + } + var best *GitObject + var bestBody *objmeta.TipBody + for i := range objs { + var b objmeta.TipBody + if err := json.Unmarshal(objs[i].Body, &b); err != nil { + continue // legacy/foreign tip row without a TipBody; skip + } + if best == nil || b.Gen > bestBody.Gen { + cp := objs[i] + bb := b + best = &cp + bestBody = &bb + } + } + return best, bestBody, nil +} + +// MintShare mints a share for locker valid until validUntil. It: +// +// 1. locates the current tip object and downloads/parses its tip document to +// learn the archived packs and refs; +// 2. mints a pre-signed URL for the tip and one for every archived pack; +// 3. uploads a git.share ShareDocument object carrying those URLs; +// 4. mint a pre-signed URL for the share document itself (the share URL). +// +// All SDK calls go through the session's mockable SDKClient, so the whole flow +// is exercised against a fake SDK in tests. +func MintShare(ctx context.Context, s *Session, locker string, validUntil time.Time) (*ShareResult, error) { + tipObj, tipBody, err := FindTipForLocker(s.DB, locker) + if err != nil { + return nil, err + } + if tipObj == nil { + return nil, fmt.Errorf("gitarchive: locker %q has no archived tip; run `pinner git watch` to publish it first", locker) + } + + sdk, err := s.ensureSDK() + if err != nil { + return nil, err + } + + tipKey, err := hashFromHex(tipObj.ObjectKey) + if err != nil { + return nil, err + } + + // Download the tip payload to learn name + the archived pack list. + _, tipData, err := FetchObjectBytes(ctx, s, tipKey) + if err != nil { + return nil, err + } + tipDoc, err := ParseTipBytes(tipData) + if err != nil { + return nil, err + } + + tipURL, err := sdk.CreateSharedObjectURL(ctx, tipKey, validUntil) + if err != nil { + return nil, fmt.Errorf("gitarchive: mint tip share URL: %w", err) + } + + packURLs := make([]string, 0, len(tipDoc.Packs)) + for _, p := range tipDoc.Packs { + pk, err := hashFromHex(p.Key) + if err != nil { + return nil, err + } + u, err := sdk.CreateSharedObjectURL(ctx, pk, validUntil) + if err != nil { + return nil, fmt.Errorf("gitarchive: mint pack share URL: %w", err) + } + packURLs = append(packURLs, u) + } + + name := tipDoc.Name + if name == "" { + name = locker + } + gen := tipDoc.Gen + if gen == 0 && tipBody != nil { + gen = tipBody.Gen + } + + doc := &ShareDocument{ + Locker: locker, + Gen: gen, + Name: name, + TipURL: tipURL, + PackURLs: packURLs, + Until: validUntil.Unix(), + } + raw, err := json.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("gitarchive: marshal share document: %w", err) + } + shareBody, err := json.Marshal(objmeta.ShareBody{Locker: locker, Gen: gen, Until: validUntil.Unix()}) + if err != nil { + return nil, fmt.Errorf("gitarchive: marshal share body: %w", err) + } + uploaded, err := UploadObject(ctx, s, objmeta.KindGitShare, shareBody, bytes.NewReader(raw)) + if err != nil { + return nil, err + } + + shareURL, err := sdk.CreateSharedObjectURL(ctx, uploaded.Key, validUntil) + if err != nil { + return nil, fmt.Errorf("gitarchive: mint share document URL: %w", err) + } + + return &ShareResult{ + Locker: locker, + Gen: gen, + Name: name, + URL: shareURL, + Until: validUntil, + TipURL: tipURL, + PackURLs: packURLs, + Document: doc, + }, nil +} diff --git a/internal/core/gitarchive/share_test.go b/internal/core/gitarchive/share_test.go new file mode 100644 index 00000000..235d1e1b --- /dev/null +++ b/internal/core/gitarchive/share_test.go @@ -0,0 +1,103 @@ +package gitarchive_test + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" +) + +// seedArchivedLocker uploads one pack + one tip (whose payload references that +// pack) for "widgets" and returns the tip object key string and pack key string. +func seedArchivedLocker(t *testing.T, s *gitarchive.Session, gen int) (tipKey, packKey string) { + t.Helper() + ctx := context.Background() + + pack, err := gitarchive.UploadPack(ctx, s, "widgets", "pack-1", true, + bytes.NewReader([]byte("PACK\x00fake-pack-data"))) + require.NoError(t, err) + packKey = pack.Key.String() + + tipRefs := map[string]string{"refs/heads/master": strings.Repeat("a", 40)} + tipPayload, err := json.Marshal(gitarchive.TipBytes{ + Locker: "widgets", + Gen: gen, + Name: "widgets", + Lineage: []string{strings.Repeat("b", 40)}, + Refs: tipRefs, + Packs: []gitarchive.PackRef{{Key: packKey, Full: true, Digest: "abc"}}, + }) + require.NoError(t, err) + + tip, err := gitarchive.UploadTip(ctx, s, "widgets", gen, "prev", bytes.NewReader(tipPayload)) + require.NoError(t, err) + tipKey = tip.Key.String() + return tipKey, packKey +} + +func TestParseShareDocument(t *testing.T) { + doc, err := gitarchive.ParseShareDocument([]byte( + `{"locker":"w","gen":1,"name":"w","tip_url":"u1","pack_urls":["p1"],"until":999}`)) + require.NoError(t, err) + require.Equal(t, "w", doc.Locker) + require.Equal(t, 1, doc.Gen) + require.Equal(t, "u1", doc.TipURL) + require.Equal(t, []string{"p1"}, doc.PackURLs) + require.Equal(t, int64(999), doc.Until) +} + +func TestParseShareDocumentMissingTipURL(t *testing.T) { + _, err := gitarchive.ParseShareDocument([]byte(`{"locker":"w"}`)) + require.Error(t, err) + require.Contains(t, err.Error(), "missing locker or tip_url") +} + +func TestParseTipBytes(t *testing.T) { + tip, err := gitarchive.ParseTipBytes([]byte( + `{"locker":"w","gen":2,"name":"w","lineage":["x"],"refs":{"refs/heads/master":"aaa"},"packs":[{"key":"pk1","full":true}]}`)) + require.NoError(t, err) + require.Equal(t, 2, tip.Gen) + require.Len(t, tip.Packs, 1) + require.Equal(t, "pk1", tip.Packs[0].Key) +} + +func TestMintShareMintsTipAndPackURLs(t *testing.T) { + ctx := context.Background() + s, sdk, _ := testSession(t) + seedArchivedLocker(t, s, 1) + + until := time.Now().Add(30 * 24 * time.Hour).Truncate(time.Second) + res, err := gitarchive.MintShare(ctx, s, "widgets", until) + require.NoError(t, err) + require.Equal(t, "widgets", res.Locker) + require.Equal(t, 1, res.Gen) + require.NotEmpty(t, res.URL, "share document URL must be minted") + require.NotEmpty(t, res.TipURL) + require.Len(t, res.PackURLs, 1) + require.Equal(t, until.Unix(), res.Document.Until) + + // The share document itself was uploaded as a git.share object and pinned. + var shareRows int64 + require.NoError(t, s.DB.Table("git_objects").Where("kind = ?", "git.share").Count(&shareRows).Error) + require.Equal(t, int64(1), shareRows) + + // URLs were minted for tip + one pack + the share document (3 calls), + // and the minted URL is a self-contained bearer (not a profile-scoped key). + require.Len(t, sdk.mintLog, 3) + require.NotEqual(t, "", res.URL) + require.NotContains(t, res.URL, "profile=") +} + +func TestMintShareNoTip(t *testing.T) { + ctx := context.Background() + s, _, _ := testSession(t) + _, err := gitarchive.MintShare(ctx, s, "nothing", time.Now().Add(time.Hour)) + require.Error(t, err) + require.Contains(t, err.Error(), "no archived tip") +} diff --git a/internal/core/gitarchive/upload.go b/internal/core/gitarchive/upload.go new file mode 100644 index 00000000..24ecaa11 --- /dev/null +++ b/internal/core/gitarchive/upload.go @@ -0,0 +1,95 @@ +package gitarchive + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "time" + + "go.sia.tech/core/types" + "go.sia.tech/siastorage" + + "go.lumeweb.com/pinner-cli/internal/core/objmeta" +) + +// UploadedObject describes a successfully uploaded+pinned git archive object. +type UploadedObject struct { + // Key is the object's content-addressed object key (hex of types.Hash256). + Key types.Hash256 + // Size is the plaintext payload size in bytes. + Size int64 + // Digest is the SHA-256 hex digest of the plaintext payload. + Digest string + // Card is the sealed card metadata attached to the object. + Card objmeta.Card +} + +// UploadObject uploads a git-archive card. It reads the plaintext payload +// from r, computes its SHA-256 digest and size, seals a pinner.obj/v1 card +// (≤1 KiB, enforced by objmeta.Encode), uploads+pins the object through the +// mockable SDK, and records the resulting content-addressed object into the +// session's git_objects table. +// +// The card body is caller-supplied and kind-specific (PackBody for git.pack, +// TipBody for git.tip, ShareBody for git.share); it carries only the small +// routing/index fields, while the full payload (packfile bytes, tip/share +// document) is the object data. +func UploadObject(ctx context.Context, s *Session, kind objmeta.Kind, body json.RawMessage, r io.Reader) (*UploadedObject, error) { + if !kind.Valid() { + return nil, fmt.Errorf("gitarchive: invalid card kind %q", kind) + } + data, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("gitarchive: read %s payload: %w", kind, err) + } + digest := fmt.Sprintf("%x", sha256.Sum256(data)) + + card := objmeta.New(kind, int64(len(data)), digest, time.Now().Unix(), body) + raw, err := objmeta.Encode(card) + if err != nil { + return nil, fmt.Errorf("gitarchive: encode %s card: %w", kind, err) + } + + obj := siastorage.NewEmptyObject() + obj.UpdateMetadata(raw) + + sdk, err := s.ensureSDK() + if err != nil { + return nil, err + } + if err := sdk.Upload(ctx, &obj, bytes.NewReader(data)); err != nil { + return nil, fmt.Errorf("gitarchive: upload %s object: %w", kind, err) + } + if err := sdk.PinObject(ctx, obj); err != nil { + return nil, fmt.Errorf("gitarchive: pin %s object: %w", kind, err) + } + + key := obj.ID() + if _, err := IngestCard(s.DB, key.String(), raw); err != nil { + return nil, fmt.Errorf("gitarchive: record %s object: %w", kind, err) + } + return &UploadedObject{Key: key, Size: int64(len(data)), Digest: digest, Card: *card}, nil +} + +// UploadPack uploads a git packfile as a git.pack card. packID is the object's +// bridging identifier recorded in the card body; the authoritative object key +// for later download lives in the tip's packs list and the git_objects table. +func UploadPack(ctx context.Context, s *Session, locker, packID string, full bool, r io.Reader) (*UploadedObject, error) { + body, err := json.Marshal(objmeta.PackBody{Locker: locker, Pack: packID, Full: full}) + if err != nil { + return nil, fmt.Errorf("gitarchive: marshal pack body: %w", err) + } + return UploadObject(ctx, s, objmeta.KindGitPack, body, r) +} + +// UploadTip uploads a tip document as a git.tip card. +func UploadTip(ctx context.Context, s *Session, locker string, gen int, prev string, r io.Reader) (*UploadedObject, error) { + body, err := json.Marshal(objmeta.TipBody{Locker: locker, Gen: gen, Prev: prev}) + if err != nil { + return nil, fmt.Errorf("gitarchive: marshal tip body: %w", err) + } + return UploadObject(ctx, s, objmeta.KindGitTip, body, r) +} diff --git a/internal/core/gitarchive/upload_test.go b/internal/core/gitarchive/upload_test.go new file mode 100644 index 00000000..44ba5262 --- /dev/null +++ b/internal/core/gitarchive/upload_test.go @@ -0,0 +1,92 @@ +package gitarchive_test + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "gorm.io/gorm" + + "go.lumeweb.com/pinner-cli/internal/core/gitarchive" + "go.lumeweb.com/pinner-cli/internal/core/objmeta" +) + +// testSession returns a Session wired to a fresh migrated DB and a fake SDK. +func testSession(t *testing.T) (*gitarchive.Session, *fakeSDK, *gorm.DB) { + t.Helper() + db := openDB(t) + require.NoError(t, gitarchive.AutoMigrate(db)) + sdk := newFakeSDK() + return &gitarchive.Session{ + Profile: "work", + IndexerURL: "http://localhost:9980", + DB: db, + SDK: sdk, + }, sdk, db +} + +func TestUploadPackSDKRoundTrip(t *testing.T) { + ctx := context.Background() + s, sdk, db := testSession(t) + + payload := []byte("PACK\x00fixture-data") + uo, err := gitarchive.UploadPack(ctx, s, "widgets", "pack-1", true, bytes.NewReader(payload)) + require.NoError(t, err) + require.Equal(t, int64(len(payload)), uo.Size) + require.Equal(t, fmt.Sprintf("%x", sha256.Sum256(payload)), uo.Digest) + require.Equal(t, string(objmeta.KindGitPack), string(uo.Card.Kind)) + require.Equal(t, objmeta.KindGitPack, uo.Card.Kind) + + // The SDK was driven: one upload, one pin, no other SDK calls. + require.Len(t, sdk.pinned, 1) + require.Len(t, sdk.objects, 1) + + // The uploaded object's metadata is the sealed card (routable as a git card). + var keys []string + for k := range sdk.objects { + keys = append(keys, k.String()) + } + require.Len(t, keys, 1) + stored := sdk.objects[uo.Key] + require.True(t, objmeta.IsCard(stored.meta), "object metadata must be a pinner card") + + // Exactly one git_objects row, matching the card. + var gobjs []gitarchive.GitObject + require.NoError(t, db.Find(&gobjs).Error) + require.Len(t, gobjs, 1) + require.Equal(t, "git.pack", gobjs[0].Kind) + require.Equal(t, "widgets", gobjs[0].Locker) + require.Equal(t, int64(len(payload)), gobjs[0].Size) + require.Equal(t, uo.Digest, gobjs[0].Digest) + require.Equal(t, uo.Key.String(), gobjs[0].ObjectKey) +} + +func TestUploadTipSDKRoundTrip(t *testing.T) { + ctx := context.Background() + s, _, db := testSession(t) + + payload := []byte(`{"locker":"widgets","gen":3,"name":"widgets","lineage":[],"refs":{},"prev":"p"}`) + uo, err := gitarchive.UploadTip(ctx, s, "widgets", 3, "prevkey", bytes.NewReader(payload)) + require.NoError(t, err) + require.Equal(t, int64(len(payload)), uo.Size) + require.Equal(t, objmeta.KindGitTip, uo.Card.Kind) + + var gobjs []gitarchive.GitObject + require.NoError(t, db.Find(&gobjs).Error) + require.Len(t, gobjs, 1) + require.Equal(t, "git.tip", gobjs[0].Kind) + require.Equal(t, "widgets", gobjs[0].Locker) + require.Equal(t, uo.Digest, gobjs[0].Digest) +} + +func TestUploadRejectsInvalidKind(t *testing.T) { + ctx := context.Background() + s, _, _ := testSession(t) + + _, err := gitarchive.UploadObject(ctx, s, objmeta.KindVaultFile, nil, bytes.NewReader([]byte("x"))) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid card kind") +} diff --git a/internal/core/objmeta/objmeta.go b/internal/core/objmeta/objmeta.go new file mode 100644 index 00000000..35980e48 --- /dev/null +++ b/internal/core/objmeta/objmeta.go @@ -0,0 +1,206 @@ +// Package objmeta owns the pinner object-metadata wire format for objects that +// are NOT vault files. +// +// The vault domain stamps object metadata as a vault.FileMetadata (RFC3339 +// created_at etc.). Git archive hosting reuses the same Sia object store but +// for a different kind of object: git pack / tip / share cards. These live in +// the same object namespace, so they must be distinguishable from vault files +// by ANY consumer of object metadata — including the library's vaultService.Sync, +// which we must not modify. +// +// # Vault-compatibility guard (kind routing) +// +// vault.Sync calls vault.ParseFileMetadata on object metadata. Because Go's +// json.Unmarshal ignores unknown fields, a card containing only card-specific +// fields would parse "successfully" into an empty FileMetadata, and Sync would +// create empty-named vault File rows for git objects. To prevent that without +// touching the library, every card carries a creation timestamp as an INTEGER +// unix-seconds value serialized under the key `created_at` — the SAME key +// vault.FileMetadata expects an RFC3339 STRING for. ParseFileMetadata therefore +// returns a type error (json: cannot unmarshal number into ... created_at of +// type string), vault.Sync skips the object, and no File row is created. This +// integer-`created_at` collision is the kind-routing mechanism: git cards are +// rejected by the vault parser by construction. +// +// Card wire format (≤1024 bytes after marshal): +// +// {"schema":"pinner.obj/v1","kind":"git.pack|git.tip|git.share", +// "size":N,"digest":"","created_at":,"body":{...}} +// +// Legacy vault FileMetadata (no schema/kind, has id+name) routes to +// KindVaultFile. +package objmeta + +import ( + "encoding/json" + "errors" + "fmt" +) + +// Schema is the fixed schema identifier for pinner object cards. +const Schema = "pinner.obj/v1" + +// MaxCardSize is the hard upper bound on the serialized card (after marshal). +// Cards larger than this are rejected before upload so a git card can never +// silently exceed the sealed-metadata budget the vault stack is tuned for. +const MaxCardSize = 1024 + +// Kind identifies the object schema that a piece of metadata carries. +type Kind string + +const ( + // KindGitPack is a git packfile card. + KindGitPack Kind = "git.pack" + // KindGitTip is a git tip/refs card. + KindGitTip Kind = "git.tip" + // KindGitShare is a git share URL card. + KindGitShare Kind = "git.share" + // KindVaultFile is the legacy vault FileMetadata schema (no schema/kind). + KindVaultFile Kind = "vault.file" +) + +// Valid reports whether k is a card kind (a pinner.obj/v1 schema kind). It is +// false for KindVaultFile, which is not a card. +func (k Kind) Valid() bool { + switch k { + case KindGitPack, KindGitTip, KindGitShare: + return true + default: + return false + } +} + +// Sentinel errors for metadata routing. +var ( + // ErrNotCard is returned when the payload is not a pinner.obj/v1 card. + ErrNotCard = errors.New("objmeta: not a pinner.obj/v1 card") + // ErrOversize is returned when a card exceeds MaxCardSize. + ErrOversize = errors.New("objmeta: card exceeds maximum size") + // ErrUnknownKind is returned when a card carries an unknown kind. + ErrUnknownKind = errors.New("objmeta: unknown card kind") + // ErrNilCard is returned when encoding a nil card. + ErrNilCard = errors.New("objmeta: nil card") +) + +// Card is the pinner object card. Created is the unix-seconds creation +// timestamp; it marshals under the key `created_at` (an integer) which is the +// vault-compatibility guard described in the package comment. +type Card struct { + Schema string `json:"schema"` + Kind Kind `json:"kind"` + Size int64 `json:"size"` + Digest string `json:"digest"` + Created int64 `json:"created_at"` // unix seconds (int) — vault guard + Body json.RawMessage `json:"body"` +} + +// PackBody is the git.pack card body. +type PackBody struct { + Locker string `json:"locker"` + Pack string `json:"pack"` + Full bool `json:"full"` +} + +// TipBody is the git.tip card body. +type TipBody struct { + Locker string `json:"locker"` + Gen int `json:"gen"` + Prev string `json:"prev,omitempty"` +} + +// ShareBody is the git.share card body. +type ShareBody struct { + Locker string `json:"locker"` + Gen int `json:"gen"` + Until int64 `json:"until,omitempty"` +} + +// New builds a card with the fixed schema set. +func New(kind Kind, size int64, digest string, created int64, body json.RawMessage) *Card { + return &Card{ + Schema: Schema, + Kind: kind, + Size: size, + Digest: digest, + Created: created, + Body: body, + } +} + +// Encode serializes a card and enforces the 1024-byte cap. +func Encode(c *Card) (json.RawMessage, error) { + if c == nil { + return nil, ErrNilCard + } + raw, err := json.Marshal(c) + if err != nil { + return nil, fmt.Errorf("objmeta: encode card: %w", err) + } + if len(raw) > MaxCardSize { + return nil, fmt.Errorf("%w: %d bytes", ErrOversize, len(raw)) + } + return raw, nil +} + +// Decode parses and validates a card, enforcing the 1024-byte cap. +func Decode(raw []byte) (*Card, error) { + if len(raw) == 0 { + return nil, ErrNotCard + } + if len(raw) > MaxCardSize { + return nil, fmt.Errorf("%w: %d bytes", ErrOversize, len(raw)) + } + var c Card + if err := json.Unmarshal(raw, &c); err != nil { + return nil, fmt.Errorf("objmeta: decode card: %w", err) + } + if c.Schema != Schema { + return nil, fmt.Errorf("%w: schema %q", ErrNotCard, c.Schema) + } + if !c.Kind.Valid() { + return nil, fmt.Errorf("%w: kind %q", ErrUnknownKind, c.Kind) + } + return &c, nil +} + +// Route probes a piece of object metadata and returns its kind. It returns a +// git.* kind for a valid card, KindVaultFile for a legacy vault FileMetadata +// (no schema/kind, carries id+name), or an error otherwise. +func Route(raw []byte) (Kind, error) { + if len(raw) == 0 { + return "", ErrNotCard + } + var probe struct { + Schema string `json:"schema"` + Kind Kind `json:"kind"` + ID string `json:"id"` + Name string `json:"name"` + } + if err := json.Unmarshal(raw, &probe); err != nil { + return "", fmt.Errorf("objmeta: route: %w", err) + } + if probe.Schema == Schema { + if !probe.Kind.Valid() { + return "", fmt.Errorf("%w: kind %q", ErrUnknownKind, probe.Kind) + } + return probe.Kind, nil + } + // Legacy vault FileMetadata probe: no schema, carries a stable id + name. + if probe.ID != "" && probe.Name != "" { + return KindVaultFile, nil + } + return "", ErrNotCard +} + +// IsVaultFile reports whether raw is a legacy vault FileMetadata (routed to +// KindVaultFile). +func IsVaultFile(raw []byte) bool { + k, err := Route(raw) + return err == nil && k == KindVaultFile +} + +// IsCard reports whether raw is a valid pinner.obj/v1 card (any git kind). +func IsCard(raw []byte) bool { + k, err := Route(raw) + return err == nil && k.Valid() +} diff --git a/internal/core/objmeta/objmeta_test.go b/internal/core/objmeta/objmeta_test.go new file mode 100644 index 00000000..71c5ca92 --- /dev/null +++ b/internal/core/objmeta/objmeta_test.go @@ -0,0 +1,131 @@ +package objmeta_test + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "go.lumeweb.com/pinner/core/vault" + + "go.lumeweb.com/pinner-cli/internal/core/objmeta" +) + +func TestCardRoundtrip(t *testing.T) { + c := objmeta.New(objmeta.KindGitPack, 12345, "abc123", 1700000000, json.RawMessage(`{"locker":"widgets","pack":"k","full":true}`)) + raw, err := objmeta.Encode(c) + require.NoError(t, err) + + got, err := objmeta.Decode(raw) + require.NoError(t, err) + require.Equal(t, objmeta.Schema, got.Schema) + require.Equal(t, objmeta.KindGitPack, got.Kind) + require.Equal(t, int64(12345), got.Size) + require.Equal(t, "abc123", got.Digest) + require.Equal(t, int64(1700000000), got.Created) + require.JSONEq(t, `{"locker":"widgets","pack":"k","full":true}`, string(got.Body)) +} + +func TestCardCreatedIsIntegerUnixTimestamp(t *testing.T) { + c := objmeta.New(objmeta.KindGitTip, 10, "d", 1700000000, nil) + raw, err := objmeta.Encode(c) + require.NoError(t, err) + + // The wire format must carry the creation timestamp as an INTEGER under + // `created_at` (the vault compatibility guard). Assert the JSON shape so a + // future refactor cannot silently flip it back to a string. + s := string(raw) + require.Contains(t, s, `"created_at":1700000000`) + require.False(t, strings.Contains(s, `"created_at":"`), "created_at must be an integer, not a string") +} + +func TestCardCap(t *testing.T) { + // A card within budget is accepted. + small := objmeta.New(objmeta.KindGitShare, 5, "d", 1, json.RawMessage(`{"locker":"x","gen":0,"until":9}`)) + raw, err := objmeta.Encode(small) + require.NoError(t, err) + require.LessOrEqual(t, len(raw), objmeta.MaxCardSize) + + // Oversize body pushes the serialized card past the 1024-byte cap. + big := objmeta.New(objmeta.KindGitPack, 5, "d", 1, json.RawMessage(fmt.Sprintf(`{"locker":"%s"}`, strings.Repeat("x", objmeta.MaxCardSize)))) + _, err = objmeta.Encode(big) + require.ErrorIs(t, err, objmeta.ErrOversize) + + // Decode also rejects oversize payloads. + _, err = objmeta.Decode([]byte(strings.Repeat(" ", objmeta.MaxCardSize+1))) + require.ErrorIs(t, err, objmeta.ErrOversize) +} + +func TestRouteLegacyVaultFile(t *testing.T) { + // A legacy vault FileMetadata (it has id + name and no schema/kind) routes + // to KindVaultFile — the vault.file schema, not a git card. + legacy := `{"id":"11111111-2222-3333-4444-555555555555","version_id":"v1","name":"report.pdf","directory":"/reports","media_type":"application/pdf","size":2048,"created_at":"2024-01-01T00:00:00Z","content_digest":"dd","status":"ok"}` + k, err := objmeta.Route([]byte(legacy)) + require.NoError(t, err) + require.Equal(t, objmeta.KindVaultFile, k) + require.True(t, objmeta.IsVaultFile([]byte(legacy))) + require.False(t, objmeta.IsCard([]byte(legacy))) + + // Sanity: the same legacy metadata must parse fine as a vault FileMetadata. + _, err = vault.ParseFileMetadata(json.RawMessage(legacy)) + require.NoError(t, err) +} + +func TestRouteCard(t *testing.T) { + for _, tc := range []struct { + kind objmeta.Kind + want objmeta.Kind + }{ + {objmeta.KindGitPack, objmeta.KindGitPack}, + {objmeta.KindGitTip, objmeta.KindGitTip}, + {objmeta.KindGitShare, objmeta.KindGitShare}, + } { + raw, err := objmeta.Encode(objmeta.New(tc.kind, 1, "d", 1, nil)) + require.NoError(t, err) + k, err := objmeta.Route(raw) + require.NoError(t, err) + require.Equal(t, tc.want, k) + require.True(t, objmeta.IsCard(raw)) + } +} + +func TestRouteUnknown(t *testing.T) { + _, err := objmeta.Route(nil) + require.ErrorIs(t, err, objmeta.ErrNotCard) + + // JSON but not a pinner card and not a legacy vault file. + _, err = objmeta.Route([]byte(`{"foo":1}`)) + require.ErrorIs(t, err, objmeta.ErrNotCard) + + // Wrong schema. + _, err = objmeta.Route([]byte(`{"schema":"pinner.other/v1","kind":"git.pack"}`)) + require.ErrorIs(t, err, objmeta.ErrNotCard) + + // Unknown kind on a valid schema. + _, err = objmeta.Route([]byte(`{"schema":"pinner.obj/v1","kind":"git.bogus"}`)) + require.ErrorIs(t, err, objmeta.ErrUnknownKind) +} + +// TestGitCardDoesNotParseAsVaultFileMetadata is the compatibility guard that +// makes kind routing work: a git card, because its created_at is an INTEGER +// (unix seconds), must FAIL vault.ParseFileMetadata, so vault.Sync skips it and +// never creates an empty-named vault File row. +func TestGitCardDoesNotParseAsVaultFileMetadata(t *testing.T) { + raw, err := objmeta.Encode(objmeta.New(objmeta.KindGitPack, 1024, "sha256abcd", 1700000000, + json.RawMessage(`{"locker":"widgets","pack":"k","full":true}`))) + require.NoError(t, err) + + // Route identifies it as a git card... + k, err := objmeta.Route(raw) + require.NoError(t, err) + require.Equal(t, objmeta.KindGitPack, k) + + // ...but the vault parser must REJECT it (created_at is an int, not the + // RFC3339 string FileMetadata expects). Without this guard, vault.Sync + // would upsert an empty-named File row for every git object. + _, err = vault.ParseFileMetadata(raw) + require.Error(t, err, "git card must not parse as vault FileMetadata") + require.Contains(t, err.Error(), "created_at") +}