From 04d5a17a066300d17b16c72f7af11bc9167c17b5 Mon Sep 17 00:00:00 2001 From: Alexander North Date: Mon, 16 Nov 2026 14:39:14 +0100 Subject: [PATCH 1/4] fix: prevent deadlock when deleting a datastore Signed-off-by: Alexander North --- go.mod | 6 +-- go.sum | 12 ++--- pkg/cache/local.go | 41 +++++++-------- pkg/cache/local_test.go | 50 +++++++++++++++++++ pkg/datastore/datastore_rpc.go | 31 ++++++++---- .../netconf/driver/scrapligo/scrapligo.go | 9 +++- pkg/datastore/transaction_rpc.go | 3 ++ pkg/server/intent.go | 4 +- pkg/server/server.go | 8 ++- pkg/server/transaction.go | 2 +- 10 files changed, 120 insertions(+), 46 deletions(-) create mode 100644 pkg/cache/local_test.go diff --git a/go.mod b/go.mod index 34bea1db..a0ccd8c3 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/openconfig/ygot v0.36.0 github.com/prometheus/client_golang v1.25.0 github.com/scrapli/scrapligo v1.4.3 - github.com/sdcio/cache v0.0.38 + github.com/sdcio/cache v0.0.39-0.20261116125209-a39dae012695 github.com/sdcio/logger v0.0.3 github.com/sdcio/schema-server v0.0.34 github.com/sdcio/sdc-protos v0.0.56 @@ -45,8 +45,8 @@ require ( github.com/clipperhouse/uax29/v2 v2.6.0 // indirect github.com/creack/pty v1.1.24 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/dgraph-io/badger/v4 v4.9.0 // indirect - github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect + github.com/dgraph-io/badger/v4 v4.9.2 // indirect + github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/fatih/color v1.18.0 // indirect diff --git a/go.sum b/go.sum index fe7afb4f..b359e7ee 100644 --- a/go.sum +++ b/go.sum @@ -27,10 +27,10 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v4 v4.9.0 h1:tpqWb0NewSrCYqTvywbcXOhQdWcqephkVkbBmaaqHzc= -github.com/dgraph-io/badger/v4 v4.9.0/go.mod h1:5/MEx97uzdPUHR4KtkNt8asfI2T4JiEiQlV7kWUo8c0= -github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= -github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= +github.com/dgraph-io/badger/v4 v4.9.2 h1:Wb5qw8gElqwV1a8msHTeQKova9b1V10heFKMIiPd80E= +github.com/dgraph-io/badger/v4 v4.9.2/go.mod h1:nJjaJTUOSsQEBhsq209FmwCvMJzEA3e74RjZw6V2pQI= +github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU= +github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -183,8 +183,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/scrapli/scrapligo v1.4.3 h1:H1IkZrCqFHT+MaY/j4MgimjLPKFfPRymhxBGYeXUOWI= github.com/scrapli/scrapligo v1.4.3/go.mod h1:pOWxVyPsQRrWTrkoSSDg05tjOqtWfLffAZtAsCc0w3M= -github.com/sdcio/cache v0.0.38 h1:8meykZC/DAgzKI+R3GtZvTs0/OXWRCeNN5mxVKG0Ba8= -github.com/sdcio/cache v0.0.38/go.mod h1:PHJd4pTmfFZBLdPohtXJAVBNmAKhhKdZHTi3MejGrk0= +github.com/sdcio/cache v0.0.39-0.20261116125209-a39dae012695 h1:I0dYZw9PGvBF6iUvmhEsGE069LpdAMNMOx9YWAJiREE= +github.com/sdcio/cache v0.0.39-0.20261116125209-a39dae012695/go.mod h1:sOhPItnWwrdr3b8QwPTrXxEk3PtcnHkhL1KQ+ey3wzs= github.com/sdcio/goyang v1.6.2-2 h1:qfeUKBmoKpiKAruuEc3+V8wgHKP/n1jRDEnTy23knV8= github.com/sdcio/goyang v1.6.2-2/go.mod h1:5WolITjek1NF8yrNERyVZ7jqjOClJTpO8p/+OwmETM4= github.com/sdcio/logger v0.0.3 h1:IFUbObObGry+S8lHGwOQKKRxJSuOphgRU/hxVhOdMOM= diff --git a/pkg/cache/local.go b/pkg/cache/local.go index 5235dd92..e6083ddc 100644 --- a/pkg/cache/local.go +++ b/pkg/cache/local.go @@ -61,6 +61,10 @@ func (l *LocalCache) InstanceIntentGet(ctx context.Context, cacheName string, in } func (l *LocalCache) InstanceIntentGetAll(ctx context.Context, cacheName string, excludeIntentNames []string, intentChanOrig chan<- *tree_persist.Intent, errChanOrig chan<- error) { + // cancelling on return releases the store stream if we stop consuming early + ctx, cancel := context.WithCancel(ctx) + defer cancel() + // create new channels intentChan := make(chan *types.Intent, 5) errChan := make(chan error, 1) @@ -70,31 +74,24 @@ func (l *LocalCache) InstanceIntentGetAll(ctx context.Context, cacheName string, defer close(intentChanOrig) defer close(errChanOrig) - for { - select { - case <-ctx.Done(): // Or stop if context is canceled - return - case intent, ok := <-intentChan: // retieve intent - if !ok { - return - } - // unmarshall it into a tree_persit.Intent - tpIntent := &tree_persist.Intent{} - err := proto.Unmarshal(intent.Data(), tpIntent) - if err != nil { - errChanOrig <- err - return - } - // forward to caller - intentChanOrig <- tpIntent - case err, ok := <-errChan: // Handle errors after intents - if !ok { - errChan = nil // Mark errChan as nil so select ignores it - continue - } + for intent := range intentChan { + // unmarshall it into a tree_persit.Intent + tpIntent := &tree_persist.Intent{} + err := proto.Unmarshal(intent.Data(), tpIntent) + if err != nil { errChanOrig <- err return } + // forward to caller + select { + case intentChanOrig <- tpIntent: + case <-ctx.Done(): + return + } + } + // the store sends its error before closing either channel + if err, ok := <-errChan; ok { + errChanOrig <- err } } diff --git a/pkg/cache/local_test.go b/pkg/cache/local_test.go new file mode 100644 index 00000000..7560b49b --- /dev/null +++ b/pkg/cache/local_test.go @@ -0,0 +1,50 @@ +package cache + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/sdcio/cache/pkg/config" + "github.com/sdcio/sdc-protos/tree_persist" +) + +// An abandoned InstanceIntentGetAll must not leave the store locked, otherwise InstanceDelete deadlocks. +func TestLocalCache_InstanceIntentGetAll_AbandonedReleasesStore(t *testing.T) { + ctx := context.Background() + c, err := NewLocalCache(&config.CacheConfig{Dir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + const name = "ds" + if err := c.InstanceCreate(ctx, name); err != nil { + t.Fatal(err) + } + for i := range 20 { + if err := c.InstanceIntentModify(ctx, name, &tree_persist.Intent{IntentName: fmt.Sprintf("intent-%d", i)}); err != nil { + t.Fatal(err) + } + } + + getCtx, cancel := context.WithCancel(ctx) + intentChan := make(chan *tree_persist.Intent) + errChan := make(chan error, 1) + go c.InstanceIntentGetAll(getCtx, name, nil, intentChan, errChan) + + <-intentChan + // let the store producer fill its buffer and block before cancelling + time.Sleep(100 * time.Millisecond) + cancel() + + done := make(chan error, 1) + go func() { done <- c.InstanceDelete(ctx, name) }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("InstanceDelete blocked: store lock still held by abandoned InstanceIntentGetAll") + } +} diff --git a/pkg/datastore/datastore_rpc.go b/pkg/datastore/datastore_rpc.go index 561ef979..7e1357f5 100644 --- a/pkg/datastore/datastore_rpc.go +++ b/pkg/datastore/datastore_rpc.go @@ -21,6 +21,7 @@ import ( "sync" "time" + sdccache "github.com/sdcio/cache/pkg/cache" logf "github.com/sdcio/logger" sdcpb "github.com/sdcio/sdc-protos/sdcpb" "google.golang.org/grpc" @@ -120,7 +121,10 @@ func New(ctx context.Context, c *config.DatastoreConfig, sc schema.Client, cc ca // create cache instance if needed // this is a blocking call - ds.initCache(ctx) + if err := ds.initCache(ctx); err != nil { + cancel() + return nil, err + } go func() { // init sbi, this is a blocking call @@ -144,21 +148,26 @@ func (d *Datastore) IntentsList(ctx context.Context) ([]string, error) { return d.cacheClient.IntentsList(ctx) } -func (d *Datastore) initCache(ctx context.Context) { +func (d *Datastore) initCache(ctx context.Context) error { log := logf.FromContext(ctx) exists := d.cacheClient.InstanceExists(ctx) if exists { log.V(logf.VDebug).Info("cache already exists") - return + return nil } log.Info("creating cache instance") -CREATE: - err := d.cacheClient.InstanceCreate(ctx) - if err != nil { + for { + err := d.cacheClient.InstanceCreate(ctx) + if err == nil || errors.Is(err, sdccache.ErrorCacheInstanceAlreadyExists) { + return nil + } log.Error(err, "failed to create cache") - time.Sleep(time.Second) - goto CREATE + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } } } @@ -203,7 +212,11 @@ func (d *Datastore) Config() *config.DatastoreConfig { } func (d *Datastore) Delete(ctx context.Context) error { - return d.cacheClient.InstanceDelete(ctx) + err := d.cacheClient.InstanceDelete(ctx) + if errors.Is(err, sdccache.ErrorCacheInstanceNotFound) { + return nil + } + return err } func (d *Datastore) ConnectionState() *targettypes.TargetStatus { diff --git a/pkg/datastore/target/netconf/driver/scrapligo/scrapligo.go b/pkg/datastore/target/netconf/driver/scrapligo/scrapligo.go index fd0b1578..957b8b12 100644 --- a/pkg/datastore/target/netconf/driver/scrapligo/scrapligo.go +++ b/pkg/datastore/target/netconf/driver/scrapligo/scrapligo.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "io" + "sync" "github.com/beevik/etree" scraplinetconf "github.com/scrapli/scrapligo/driver/netconf" @@ -51,6 +52,9 @@ func normalizeTransportError(err error) error { type ScrapligoNetconfTarget struct { driver *scraplinetconf.Driver + // scrapligo's Driver.Close blocks forever when called a second time + closeOnce sync.Once + closeErr error } // NewScrapligoNetconfTarget inits a new ScrapligoNetconfTarget which is already connected to the target node @@ -97,7 +101,10 @@ func (snt *ScrapligoNetconfTarget) Close() error { if snt.driver == nil { return nil } - return snt.driver.Close() + snt.closeOnce.Do(func() { + snt.closeErr = snt.driver.Close() + }) + return snt.closeErr } func (snt *ScrapligoNetconfTarget) IsAlive() bool { diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index 03c4740b..785385ac 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -160,6 +160,9 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.RootEntry) ([]string, error) { log := logger.FromContext(ctx) + // releases the cache producer if we return before consuming all intents + ctx, cancel := context.WithCancel(ctx) + defer cancel() intentNames := []string{} IntentChan := make(chan *tree_persist.Intent) diff --git a/pkg/server/intent.go b/pkg/server/intent.go index 336234e7..b419abe1 100644 --- a/pkg/server/intent.go +++ b/pkg/server/intent.go @@ -26,7 +26,7 @@ func (s *Server) ListIntent(ctx context.Context, req *sdcpb.ListIntentRequest) ( return nil, status.Error(codes.InvalidArgument, "missing datastore name") } - ds, err := s.datastores.getDataStore(req.GetDatastoreName()) + ds, err := s.datastores.GetDataStore(req.GetDatastoreName()) if err != nil { return nil, status.Error(codes.NotFound, err.Error()) } @@ -62,7 +62,7 @@ func (s *Server) GetIntent(ctx context.Context, req *sdcpb.GetIntentRequest) (*s return nil, status.Error(codes.InvalidArgument, "missing intent name") } // retrieve the referenced datastore - ds, err := s.datastores.getDataStore(req.GetDatastoreName()) + ds, err := s.datastores.GetDataStore(req.GetDatastoreName()) if err != nil { return nil, status.Error(codes.NotFound, err.Error()) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 471b7173..76def6eb 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -77,7 +77,7 @@ func NewDatastoreMap() *DatastoreMap { } } func (d *DatastoreMap) StopAll(ctx context.Context) { - for _, ds := range d.datastores { + for _, ds := range d.GetDatastoreAll() { _ = ds.Stop(ctx) } } @@ -100,7 +100,10 @@ func (d *DatastoreMap) DeleteDatastore(ctx context.Context, name string) error { if err != nil { return err } - _ = ds.Delete(ctx) + // keep the entry on failure so a retried delete can clean up the cache instance + if err := ds.Delete(ctx); err != nil { + return err + } delete(d.datastores, name) return nil } @@ -291,6 +294,7 @@ func (s *Server) createInitialDatastores(ctx context.Context) { ds, err := datastore.New(ctx, dsCfg, s.schemaClient, s.cacheClient, s.gnmiOpts...) if err != nil { log.Error(err, "failed to create datastore") + return } err = s.datastores.AddDatastore(ds) if err != nil { diff --git a/pkg/server/transaction.go b/pkg/server/transaction.go index 390f39d2..728e4226 100644 --- a/pkg/server/transaction.go +++ b/pkg/server/transaction.go @@ -49,7 +49,7 @@ func (s *Server) TransactionSet(ctx context.Context, req *sdcpb.TransactionSetRe } // retrieve the referenced datastore - ds, err := s.datastores.getDataStore(req.DatastoreName) + ds, err := s.datastores.GetDataStore(req.DatastoreName) if err != nil { return nil, status.Error(codes.NotFound, err.Error()) } From 837bb174016622dcc2e602c023c32bbe27ba54b9 Mon Sep 17 00:00:00 2001 From: Alexander North Date: Mon, 16 Nov 2026 14:56:56 +0100 Subject: [PATCH 2/4] fix: report a cancelled intent stream as an error Signed-off-by: Alexander North --- pkg/cache/local.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/cache/local.go b/pkg/cache/local.go index e6083ddc..bb4d2338 100644 --- a/pkg/cache/local.go +++ b/pkg/cache/local.go @@ -86,6 +86,7 @@ func (l *LocalCache) InstanceIntentGetAll(ctx context.Context, cacheName string, select { case intentChanOrig <- tpIntent: case <-ctx.Done(): + errChanOrig <- ctx.Err() return } } From a180ac31c859202806f6db69fbbd964b1f468628 Mon Sep 17 00:00:00 2001 From: Alexander North Date: Mon, 16 Nov 2026 15:52:40 +0100 Subject: [PATCH 3/4] bump cache module to v0.0.39 Signed-off-by: Alexander North --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a0ccd8c3..666ecded 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/openconfig/ygot v0.36.0 github.com/prometheus/client_golang v1.25.0 github.com/scrapli/scrapligo v1.4.3 - github.com/sdcio/cache v0.0.39-0.20261116125209-a39dae012695 + github.com/sdcio/cache v0.0.39 github.com/sdcio/logger v0.0.3 github.com/sdcio/schema-server v0.0.34 github.com/sdcio/sdc-protos v0.0.56 diff --git a/go.sum b/go.sum index b359e7ee..b1ab28d4 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/scrapli/scrapligo v1.4.3 h1:H1IkZrCqFHT+MaY/j4MgimjLPKFfPRymhxBGYeXUOWI= github.com/scrapli/scrapligo v1.4.3/go.mod h1:pOWxVyPsQRrWTrkoSSDg05tjOqtWfLffAZtAsCc0w3M= -github.com/sdcio/cache v0.0.39-0.20261116125209-a39dae012695 h1:I0dYZw9PGvBF6iUvmhEsGE069LpdAMNMOx9YWAJiREE= -github.com/sdcio/cache v0.0.39-0.20261116125209-a39dae012695/go.mod h1:sOhPItnWwrdr3b8QwPTrXxEk3PtcnHkhL1KQ+ey3wzs= +github.com/sdcio/cache v0.0.39 h1:8Vk5pJueBrp4fK58aC+c9pkOtUcI2GNDp/9iuJA2Ko8= +github.com/sdcio/cache v0.0.39/go.mod h1:sOhPItnWwrdr3b8QwPTrXxEk3PtcnHkhL1KQ+ey3wzs= github.com/sdcio/goyang v1.6.2-2 h1:qfeUKBmoKpiKAruuEc3+V8wgHKP/n1jRDEnTy23knV8= github.com/sdcio/goyang v1.6.2-2/go.mod h1:5WolITjek1NF8yrNERyVZ7jqjOClJTpO8p/+OwmETM4= github.com/sdcio/logger v0.0.3 h1:IFUbObObGry+S8lHGwOQKKRxJSuOphgRU/hxVhOdMOM= From 3cd0611bfecd996761157fb24aa8f7ae88a3a3cb Mon Sep 17 00:00:00 2001 From: Markus Vahlenkamp Date: Tue, 17 Nov 2026 09:24:31 +0100 Subject: [PATCH 4/4] fix: avoid holding datastore map lock during delete and harden intent streams - Release DatastoreMap lock during Stop/Delete I/O using per-name pending serialization so slow deletes do not block unrelated datastores, while restoring the entry on delete failure. - Guard Datastore.sbi with sbiMutex and close late-connecting SBI targets if Stop already cancelled the datastore. - Guard LocalCache.InstanceIntentGetAll error delivery against unbuffered errChan callers and cancel the store stream before waiting on errChan. - Simplify LoadAllButRunningIntents to range over intentChan and read errChan afterwards. - Add unit tests for unbuffered/unmarshal stream cases, LoadAllButRunningIntents cancellation, and concurrent datastore delete/recreate/retry. Signed-off-by: Markus Vahlenkamp --- pkg/cache/cache.go | 3 + pkg/cache/local.go | 45 ++++++---- pkg/cache/local_test.go | 120 ++++++++++++++++++++++--- pkg/datastore/datastore_rpc.go | 48 +++++++--- pkg/datastore/intent_rpc.go | 5 +- pkg/datastore/transaction_rpc.go | 53 ++++------- pkg/datastore/transaction_rpc_test.go | 70 +++++++++++++++ pkg/server/datastore.go | 17 ++-- pkg/server/datastore_test.go | 119 +++++++++++++++++++++++++ pkg/server/server.go | 121 +++++++++++++++++++++++--- 10 files changed, 501 insertions(+), 100 deletions(-) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 8000da53..2a455a19 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -31,5 +31,8 @@ type Client interface { InstanceIntentModify(ctx context.Context, cacheName string, intent *tree_persist.Intent) error InstanceIntentDelete(ctx context.Context, cacheName string, intentName string, IgnoreNonExisting bool) error InstanceIntentExists(ctx context.Context, cacheName string, intentName string) (bool, error) + // InstanceIntentGetAll streams all intents not in excludeIntentNames to intentChan, sends at most + // one error on errChan, and closes errChan before intentChan. A buffered errChan always receives + // the error; an unbuffered errChan receives it unless ctx is cancelled with no reader waiting. InstanceIntentGetAll(ctx context.Context, cacheName string, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) } diff --git a/pkg/cache/local.go b/pkg/cache/local.go index bb4d2338..999b66bc 100644 --- a/pkg/cache/local.go +++ b/pkg/cache/local.go @@ -61,38 +61,51 @@ func (l *LocalCache) InstanceIntentGet(ctx context.Context, cacheName string, in } func (l *LocalCache) InstanceIntentGetAll(ctx context.Context, cacheName string, excludeIntentNames []string, intentChanOrig chan<- *tree_persist.Intent, errChanOrig chan<- error) { - // cancelling on return releases the store stream if we stop consuming early - ctx, cancel := context.WithCancel(ctx) + var streamErr error + defer func() { + finishIntentStream(ctx, streamErr, intentChanOrig, errChanOrig) + }() + + // Cancel before finishIntentStream so the store stream releases before any errChanOrig wait. + streamCtx, cancel := context.WithCancel(ctx) defer cancel() - // create new channels intentChan := make(chan *types.Intent, 5) errChan := make(chan error, 1) - go l.Cache.InstanceIntentGetAll(ctx, cacheName, excludeIntentNames, intentChan, errChan) - - defer close(intentChanOrig) - defer close(errChanOrig) + go l.Cache.InstanceIntentGetAll(streamCtx, cacheName, excludeIntentNames, intentChan, errChan) for intent := range intentChan { - // unmarshall it into a tree_persit.Intent tpIntent := &tree_persist.Intent{} - err := proto.Unmarshal(intent.Data(), tpIntent) - if err != nil { - errChanOrig <- err + if err := proto.Unmarshal(intent.Data(), tpIntent); err != nil { + streamErr = err return } - // forward to caller select { case intentChanOrig <- tpIntent: case <-ctx.Done(): - errChanOrig <- ctx.Err() + streamErr = ctx.Err() return } } - // the store sends its error before closing either channel - if err, ok := <-errChan; ok { - errChanOrig <- err + streamErr = <-errChan +} + +func finishIntentStream(ctx context.Context, err error, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + defer close(intentChan) + defer close(errChan) + if err == nil { + return + } + // Buffered or waiting errChan receives err even when ctx is already done. + select { + case errChan <- err: + return + default: + } + select { + case errChan <- err: + case <-ctx.Done(): } } diff --git a/pkg/cache/local_test.go b/pkg/cache/local_test.go index 7560b49b..96249493 100644 --- a/pkg/cache/local_test.go +++ b/pkg/cache/local_test.go @@ -2,6 +2,7 @@ package cache import ( "context" + "errors" "fmt" "testing" "time" @@ -10,41 +11,132 @@ import ( "github.com/sdcio/sdc-protos/tree_persist" ) -// An abandoned InstanceIntentGetAll must not leave the store locked, otherwise InstanceDelete deadlocks. +// An abandoned InstanceIntentGetAll must exit cleanly and release the store for both buffered and unbuffered errChan. func TestLocalCache_InstanceIntentGetAll_AbandonedReleasesStore(t *testing.T) { + for _, errCap := range []int{0, 1} { + t.Run(fmt.Sprintf("errChanCap=%d", errCap), func(t *testing.T) { + ctx := context.Background() + c, err := NewLocalCache(&config.CacheConfig{Dir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + const name = "ds" + if err := c.InstanceCreate(ctx, name); err != nil { + t.Fatal(err) + } + for i := range 20 { + if err := c.InstanceIntentModify(ctx, name, &tree_persist.Intent{IntentName: fmt.Sprintf("intent-%d", i)}); err != nil { + t.Fatal(err) + } + } + + getCtx, cancel := context.WithCancel(ctx) + intentChan := make(chan *tree_persist.Intent) + errChan := make(chan error, errCap) + streamDone := make(chan struct{}) + go func() { + defer close(streamDone) + c.InstanceIntentGetAll(getCtx, name, nil, intentChan, errChan) + }() + + <-intentChan + time.Sleep(50 * time.Millisecond) + cancel() + + select { + case <-streamDone: + case <-time.After(5 * time.Second): + t.Fatal("InstanceIntentGetAll blocked after ctx cancellation") + } + if errCap > 0 { + if err := <-errChan; !errors.Is(err, context.Canceled) { + t.Fatalf("got err %v, want %v", err, context.Canceled) + } + } + + done := make(chan error, 1) + go func() { done <- c.InstanceDelete(ctx, name) }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("InstanceDelete blocked: store lock still held by abandoned InstanceIntentGetAll") + } + }) + } +} + +func TestLocalCache_InstanceIntentGetAll_UnmarshalErrorReleasesStore(t *testing.T) { ctx := context.Background() - c, err := NewLocalCache(&config.CacheConfig{Dir: t.TempDir()}) + raw, err := NewLocalCache(&config.CacheConfig{Dir: t.TempDir()}) if err != nil { t.Fatal(err) } + lc := raw.(*LocalCache) + const name = "ds" - if err := c.InstanceCreate(ctx, name); err != nil { + if err := lc.InstanceCreate(ctx, name); err != nil { + t.Fatal(err) + } + if err := lc.Cache.InstanceIntentModify(ctx, name, "corrupt", []byte{0xff, 0xff, 0xff}); err != nil { t.Fatal(err) } for i := range 20 { - if err := c.InstanceIntentModify(ctx, name, &tree_persist.Intent{IntentName: fmt.Sprintf("intent-%d", i)}); err != nil { + if err := lc.InstanceIntentModify(ctx, name, &tree_persist.Intent{IntentName: fmt.Sprintf("intent-%d", i)}); err != nil { t.Fatal(err) } } + // Unbuffered errChan with no reader yet: the store stream must still be released immediately. getCtx, cancel := context.WithCancel(ctx) + defer cancel() intentChan := make(chan *tree_persist.Intent) - errChan := make(chan error, 1) - go c.InstanceIntentGetAll(getCtx, name, nil, intentChan, errChan) + errChan := make(chan error) + streamDone := make(chan struct{}) + go func() { + defer close(streamDone) + lc.InstanceIntentGetAll(getCtx, name, nil, intentChan, errChan) + }() - <-intentChan - // let the store producer fill its buffer and block before cancelling - time.Sleep(100 * time.Millisecond) - cancel() + // Drain any valid intents emitted before the corrupt one, then stop reading errChan briefly. + readLoop: + for { + select { + case _, ok := <-intentChan: + if !ok { + break readLoop + } + case <-time.After(50 * time.Millisecond): + break readLoop + } + } - done := make(chan error, 1) - go func() { done <- c.InstanceDelete(ctx, name) }() + // Even before errChan is read or getCtx is cancelled, InstanceDelete must not deadlock on the store. + deleted := make(chan error, 1) + go func() { deleted <- lc.InstanceDelete(ctx, name) }() select { - case err := <-done: + case err := <-deleted: if err != nil { t.Fatal(err) } case <-time.After(5 * time.Second): - t.Fatal("InstanceDelete blocked: store lock still held by abandoned InstanceIntentGetAll") + t.Fatal("InstanceDelete blocked on unmarshal error with unbuffered errChan") + } + + // Reading errChan receives the unmarshal error and lets InstanceIntentGetAll finish. + select { + case err := <-errChan: + if err == nil { + t.Fatal("expected unmarshal error, got nil") + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for unmarshal error") + } + select { + case <-streamDone: + case <-time.After(5 * time.Second): + t.Fatal("InstanceIntentGetAll did not return after delivering unmarshal error") } } diff --git a/pkg/datastore/datastore_rpc.go b/pkg/datastore/datastore_rpc.go index 7e1357f5..f3012d9f 100644 --- a/pkg/datastore/datastore_rpc.go +++ b/pkg/datastore/datastore_rpc.go @@ -45,7 +45,8 @@ type Datastore struct { cacheClient cache.CacheClientBound // SBI target of this datastore - sbi target.Target + sbiMutex sync.RWMutex + sbi target.Target // schema server client // schemaClient sdcpb.SchemaServerClient @@ -171,13 +172,38 @@ func (d *Datastore) initCache(ctx context.Context) error { } } +// installSBI publishes sbi onto the datastore unless ctx was cancelled while connecting, +// in which case sbi is closed immediately so Stop cannot miss it. +func (d *Datastore) installSBI(ctx context.Context, sbi target.Target) error { + if !d.setSBI(ctx, sbi) { + _ = sbi.Close(ctx) + return ctx.Err() + } + return nil +} + +func (d *Datastore) setSBI(ctx context.Context, sbi target.Target) bool { + d.sbiMutex.Lock() + defer d.sbiMutex.Unlock() + if ctx.Err() != nil { + return false + } + d.sbi = sbi + return true +} + +func (d *Datastore) getSBI() target.Target { + d.sbiMutex.RLock() + defer d.sbiMutex.RUnlock() + return d.sbi +} + func (d *Datastore) connectSBI(ctx context.Context, opts ...grpc.DialOption) error { log := logf.FromContext(ctx) - var err error - d.sbi, err = target.New(ctx, d.config.Name, d.config.SBI, d.schemaClient, d, d.config.Sync.Config, d.taskPool, opts...) + sbi, err := target.New(ctx, d.config.Name, d.config.SBI, d.schemaClient, d, d.config.Sync.Config, d.taskPool, opts...) if err == nil { - return nil + return d.installSBI(ctx, sbi) } log.Error(err, "failed to create DS target") @@ -189,12 +215,12 @@ func (d *Datastore) connectSBI(ctx context.Context, opts ...grpc.DialOption) err case <-ctx.Done(): return ctx.Err() case <-ticker.C: - d.sbi, err = target.New(ctx, d.config.Name, d.config.SBI, d.schemaClient, d, d.config.Sync.Config, d.taskPool, opts...) + sbi, err = target.New(ctx, d.config.Name, d.config.SBI, d.schemaClient, d, d.config.Sync.Config, d.taskPool, opts...) if err != nil { log.Error(err, "failed to create DS target") continue } - return nil + return d.installSBI(ctx, sbi) } } } @@ -220,10 +246,11 @@ func (d *Datastore) Delete(ctx context.Context) error { } func (d *Datastore) ConnectionState() *targettypes.TargetStatus { - if d.sbi == nil { + sbi := d.getSBI() + if sbi == nil { return targettypes.NewTargetStatus(sdcpb.TargetStatus_NOT_CONNECTED) } - return d.sbi.Status() + return sbi.Status() } func (d *Datastore) Stop(ctx context.Context) error { @@ -231,10 +258,11 @@ func (d *Datastore) Stop(ctx context.Context) error { return nil } d.cfn() - if d.sbi == nil { + sbi := d.getSBI() + if sbi == nil { return nil } - err := d.sbi.Close(ctx) + err := sbi.Close(ctx) if err != nil { logf.DefaultLogger.Error(err, "datastore failed to close the target connection", "datastore-name", d.Name()) } diff --git a/pkg/datastore/intent_rpc.go b/pkg/datastore/intent_rpc.go index 62e34bb3..e4efc8c6 100644 --- a/pkg/datastore/intent_rpc.go +++ b/pkg/datastore/intent_rpc.go @@ -45,11 +45,12 @@ func (d *Datastore) applyIntent(ctx context.Context, source targettypes.TargetSo return &sdcpb.SetDataResponse{}, nil } - if d.sbi == nil { + sbi := d.getSBI() + if sbi == nil { return nil, fmt.Errorf("%s: %w", d.config.Name, targettypes.ErrNotConnected) } - rsp, err = d.sbi.Set(ctx, source) + rsp, err = sbi.Set(ctx, source) if err != nil { return nil, fmt.Errorf("%s: %w", d.config.Name, err) } diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index 785385ac..05a51ef3 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -160,48 +160,31 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.RootEntry) ([]string, error) { log := logger.FromContext(ctx) - // releases the cache producer if we return before consuming all intents + // Releases the cache producer when returning before all intents are consumed. ctx, cancel := context.WithCancel(ctx) defer cancel() intentNames := []string{} - IntentChan := make(chan *tree_persist.Intent) - ErrChan := make(chan error, 1) - - go d.cacheClient.IntentGetAll(ctx, []string{consts.RunningIntentName}, IntentChan, ErrChan) - - for { - selectLoop: - select { - case err, ok := <-ErrChan: - if !ok { - // ErrChan already closed which is fine, continue - ErrChan = nil - break selectLoop - } - return nil, err - case <-ctx.Done(): - return nil, fmt.Errorf("context closed while retrieving all intents") - case intent, ok := <-IntentChan: - if !ok { - // IntentChan closed due to finish - IntentChan = nil - break selectLoop - } - log.V(logger.VDebug).Info("adding intent to tree", "intent", intent.GetIntentName()) - log.V(logger.VTrace).Info("adding intent to tree", "intent", intent.GetIntentName(), "content", utils.ProtoJSON(intent)) + intentChan := make(chan *tree_persist.Intent) + errChan := make(chan error, 1) - intentNames = append(intentNames, intent.GetIntentName()) - protoLoader := treeproto.NewProtoTreeImporter(intent) - _, err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags(), d.taskPool) - if err != nil { - return nil, err - } - } - if ErrChan == nil && IntentChan == nil { - return intentNames, nil + go d.cacheClient.IntentGetAll(ctx, []string{consts.RunningIntentName}, intentChan, errChan) + + for intent := range intentChan { + log.V(logger.VDebug).Info("adding intent to tree", "intent", intent.GetIntentName()) + log.V(logger.VTrace).Info("adding intent to tree", "intent", intent.GetIntentName(), "content", utils.ProtoJSON(intent)) + + intentNames = append(intentNames, intent.GetIntentName()) + protoLoader := treeproto.NewProtoTreeImporter(intent) + _, err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags(), d.taskPool) + if err != nil { + return nil, err } } + if err, ok := <-errChan; ok { + return nil, err + } + return intentNames, nil } // lowlevelTransactionSet diff --git a/pkg/datastore/transaction_rpc_test.go b/pkg/datastore/transaction_rpc_test.go index ec441ba4..075bd230 100644 --- a/pkg/datastore/transaction_rpc_test.go +++ b/pkg/datastore/transaction_rpc_test.go @@ -3,6 +3,8 @@ package datastore import ( "context" "encoding/json" + "errors" + "fmt" "runtime" "sync" "testing" @@ -227,3 +229,71 @@ func TestTransactionSet_PreviouslyApplied(t *testing.T) { }) } } + +func TestLoadAllButRunningIntents_PropagatesStreamErrorAndCancelsEarly(t *testing.T) { + ctx := context.Background() + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatalf("InitSDCIOSchema: %v", err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + + t.Run("stream error", func(t *testing.T) { + ctrl := gomock.NewController(t) + wantErr := fmt.Errorf("cache read failure") + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + errChan <- wantErr + close(errChan) + close(intentChan) + }) + + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + root, err := tree.NewTreeRoot(ctx, tree.NewTreeContext(scb, vpf)) + if err != nil { + t.Fatal(err) + } + ds := &Datastore{cacheClient: ccb, taskPool: vpf} + if _, err := ds.LoadAllButRunningIntents(ctx, root); !errors.Is(err, wantErr) { + t.Fatalf("got err %v, want %v", err, wantErr) + } + }) + + t.Run("import error cancels producer ctx", func(t *testing.T) { + ctrl := gomock.NewController(t) + producerCancelled := make(chan struct{}) + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(streamCtx context.Context, _ []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + defer close(intentChan) + defer close(errChan) + intentChan <- &tree_persist.Intent{ + IntentName: "bad", + Root: &tree_persist.TreeElement{ + Childs: []*tree_persist.TreeElement{{Name: "nonexistent-schema-element"}}, + }, + } + <-streamCtx.Done() + close(producerCancelled) + }) + + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + root, err := tree.NewTreeRoot(ctx, tree.NewTreeContext(scb, vpf)) + if err != nil { + t.Fatal(err) + } + ds := &Datastore{cacheClient: ccb, taskPool: vpf} + if _, err := ds.LoadAllButRunningIntents(ctx, root); err == nil { + t.Fatal("expected import error, got nil") + } + select { + case <-producerCancelled: + case <-time.After(5 * time.Second): + t.Fatal("producer ctx was not cancelled on early return") + } + }) +} + diff --git a/pkg/server/datastore.go b/pkg/server/datastore.go index 9d39628c..ec3164d2 100644 --- a/pkg/server/datastore.go +++ b/pkg/server/datastore.go @@ -16,6 +16,7 @@ package server import ( "context" + "errors" "fmt" "math" "strings" @@ -93,6 +94,9 @@ func (s *Server) CreateDataStore(ctx context.Context, req *sdcpb.CreateDataStore return nil, status.Error(codes.InvalidArgument, "datastore name attribute too long") } + if err := s.datastores.WaitPending(ctx, name); err != nil { + return nil, err + } if _, err := s.datastores.GetDataStore(name); err == nil { log.Error(nil, "datastore already exists") return nil, status.Errorf(codes.InvalidArgument, "datastore %s already exists", name) @@ -214,8 +218,9 @@ func (s *Server) CreateDataStore(ctx context.Context, req *sdcpb.CreateDataStore log.Error(err, "failed creating new datastore") return nil, err } - err = s.datastores.AddDatastore(ds) + err = s.datastores.AddDatastore(ctx, ds) if err != nil { + _ = ds.Stop(ctx) log.Error(err, "failed adding new datastore") return nil, err } @@ -234,16 +239,10 @@ func (s *Server) DeleteDataStore(ctx context.Context, req *sdcpb.DeleteDataStore return nil, status.Error(codes.InvalidArgument, "missing datastore name attribute") } - ds, err := s.datastores.GetDataStore(name) - if err != nil { + err := s.datastores.DeleteDatastore(ctx, name) + if errors.Is(err, errUnknownDatastore) { return nil, err } - - err = ds.Stop(s.ctx) - if err != nil { - log.Error(err, "failed to stop datastore") - } - err = s.datastores.DeleteDatastore(ctx, name) if err != nil { log.Error(err, "failed to delete datastore") return nil, fmt.Errorf("failed to delete datastore: %w", err) diff --git a/pkg/server/datastore_test.go b/pkg/server/datastore_test.go index 784072ae..ccda1d86 100644 --- a/pkg/server/datastore_test.go +++ b/pkg/server/datastore_test.go @@ -17,9 +17,12 @@ package server import ( "bytes" "context" + "errors" + "fmt" "log/slog" "strings" "testing" + "time" "github.com/go-logr/logr" "github.com/sdcio/data-server/mocks/mockcacheclient" @@ -233,3 +236,119 @@ func TestCreateDataStore_NoopType_WithSyncConfig_Succeeds(t *testing.T) { t.Fatalf("expected no error for noop type with sync config, got: %v", err) } } + +func TestDeleteDataStore_DoesNotBlockOtherDatastoresAndSerializesSameName(t *testing.T) { + s := newTestServer(t) + mockCC := s.cacheClient.(*mockcacheclient.MockClient) + ctx := context.Background() + + for _, name := range []string{"victim", "other"} { + if _, err := s.CreateDataStore(ctx, noopCreateReq(name, "noop")); err != nil { + t.Fatalf("CreateDataStore(%s): %v", name, err) + } + } + + deleting := make(chan struct{}) + release := make(chan struct{}) + mockCC.EXPECT(). + InstanceDelete(gomock.Any(), "victim"). + DoAndReturn(func(context.Context, string) error { + close(deleting) + <-release + return nil + }) + mockCC.EXPECT().InstanceDelete(gomock.Any(), "third").Return(nil) + + delErr := make(chan error, 1) + go func() { + _, err := s.DeleteDataStore(ctx, &sdcpb.DeleteDataStoreRequest{Name: "victim"}) + delErr <- err + }() + select { + case <-deleting: + case <-time.After(5 * time.Second): + t.Fatal("InstanceDelete was not called") + } + + // Unrelated datastores remain accessible while victim's delete is stuck in cache I/O. + otherDone := make(chan error, 1) + go func() { + if _, err := s.datastores.GetDataStore("other"); err != nil { + otherDone <- err + return + } + if _, err := s.CreateDataStore(ctx, noopCreateReq("third", "noop")); err != nil { + otherDone <- err + return + } + _, err := s.DeleteDataStore(ctx, &sdcpb.DeleteDataStoreRequest{Name: "third"}) + otherDone <- err + }() + select { + case err := <-otherDone: + if err != nil { + t.Fatalf("operations on unrelated datastores failed: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("unrelated datastore operations blocked on in-flight delete") + } + + // Recreating the deleting datastore waits for the delete and respects ctx cancellation. + shortCtx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + if _, err := s.CreateDataStore(shortCtx, noopCreateReq("victim", "noop")); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("CreateDataStore during delete got %v, want %v", err, context.DeadlineExceeded) + } + + recreateErr := make(chan error, 1) + go func() { + _, err := s.CreateDataStore(ctx, noopCreateReq("victim", "noop")) + recreateErr <- err + }() + close(release) + + for what, ch := range map[string]chan error{"DeleteDataStore": delErr, "CreateDataStore": recreateErr} { + select { + case err := <-ch: + if err != nil { + t.Fatalf("%s: %v", what, err) + } + case <-time.After(5 * time.Second): + t.Fatalf("%s did not finish after release", what) + } + } + if _, err := s.datastores.GetDataStore("victim"); err != nil { + t.Fatalf("victim not present after recreate: %v", err) + } +} + +func TestDeleteDataStore_RestoresEntryOnFailureSoRetrySucceeds(t *testing.T) { + s := newTestServer(t) + mockCC := s.cacheClient.(*mockcacheclient.MockClient) + ctx := context.Background() + + if _, err := s.CreateDataStore(ctx, noopCreateReq("ds-retry", "noop")); err != nil { + t.Fatalf("CreateDataStore: %v", err) + } + + wantErr := fmt.Errorf("transient delete failure") + gomock.InOrder( + mockCC.EXPECT().InstanceDelete(gomock.Any(), "ds-retry").Return(wantErr), + mockCC.EXPECT().InstanceDelete(gomock.Any(), "ds-retry").Return(nil), + ) + + if _, err := s.DeleteDataStore(ctx, &sdcpb.DeleteDataStoreRequest{Name: "ds-retry"}); !errors.Is(err, wantErr) { + t.Fatalf("first DeleteDataStore got %v, want %v", err, wantErr) + } + if _, err := s.datastores.GetDataStore("ds-retry"); err != nil { + t.Fatalf("datastore should remain registered after failed delete: %v", err) + } + + if _, err := s.DeleteDataStore(ctx, &sdcpb.DeleteDataStoreRequest{Name: "ds-retry"}); err != nil { + t.Fatalf("retry DeleteDataStore failed: %v", err) + } + if _, err := s.datastores.GetDataStore("ds-retry"); !errors.Is(err, errUnknownDatastore) { + t.Fatalf("GetDataStore after delete got %v, want %v", err, errUnknownDatastore) + } +} + diff --git a/pkg/server/server.go b/pkg/server/server.go index 76def6eb..23176591 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -16,6 +16,7 @@ package server import ( "context" + "errors" "fmt" "net" "net/http" @@ -65,15 +66,22 @@ type Server struct { gnmiOpts []grpc.DialOption } +var errUnknownDatastore = errors.New("unknown datastore") + +// DatastoreMap indexes active datastores by name. +// md guards short map lookups and mutations; pending serializes add and delete +// lifecycle operations per name so slow Stop/Delete I/O runs without holding md. type DatastoreMap struct { md *sync.RWMutex - datastores map[string]*datastore.Datastore // datastore group with sbi + datastores map[string]*datastore.Datastore + pending map[string]chan struct{} } func NewDatastoreMap() *DatastoreMap { return &DatastoreMap{ md: &sync.RWMutex{}, datastores: map[string]*datastore.Datastore{}, + pending: map[string]chan struct{}{}, } } func (d *DatastoreMap) StopAll(ctx context.Context) { @@ -82,30 +90,114 @@ func (d *DatastoreMap) StopAll(ctx context.Context) { } } -func (d *DatastoreMap) AddDatastore(ds *datastore.Datastore) error { +// withPending runs fn with exclusive lifecycle ownership of name, waiting for +// any in-flight add or delete on name to finish first (or ctx to expire). +func (d *DatastoreMap) withPending(ctx context.Context, name string, fn func() error) error { + for { + done, ok := d.tryAcquire(name) + if ok { + defer d.release(name) + return fn() + } + select { + case <-done: + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// tryAcquire marks name as in flight and returns (nil, true) when idle; +// otherwise it returns the active completion channel and false so callers can wait without holding md. +func (d *DatastoreMap) tryAcquire(name string) (<-chan struct{}, bool) { d.md.Lock() defer d.md.Unlock() - if existingDs, _ := d.getDataStore(ds.Name()); existingDs != nil { - return fmt.Errorf("datastore %s already exists", ds.Name()) + if done, busy := d.pending[name]; busy { + return done, false } + d.pending[name] = make(chan struct{}) + return nil, true +} - d.datastores[ds.Name()] = ds - return nil +// release closes name's completion channel to wake waiting callers and removes its pending entry. +func (d *DatastoreMap) release(name string) { + d.md.Lock() + defer d.md.Unlock() + close(d.pending[name]) + delete(d.pending, name) +} + +// pendingDone returns the completion channel for an in-flight operation on name, or nil when idle. +func (d *DatastoreMap) pendingDone(name string) <-chan struct{} { + d.md.RLock() + defer d.md.RUnlock() + return d.pending[name] +} + +// WaitPending waits until no add or delete is in flight for name, or ctx expires. +func (d *DatastoreMap) WaitPending(ctx context.Context, name string) error { + for { + done := d.pendingDone(name) + if done == nil { + return nil + } + select { + case <-done: + case <-ctx.Done(): + return ctx.Err() + } + } } +// AddDatastore registers ds under its name after waiting for any in-flight lifecycle operation on that name. +func (d *DatastoreMap) AddDatastore(ctx context.Context, ds *datastore.Datastore) error { + return d.withPending(ctx, ds.Name(), func() error { + d.md.Lock() + defer d.md.Unlock() + if existingDs, _ := d.getDataStore(ds.Name()); existingDs != nil { + return fmt.Errorf("datastore %s already exists", ds.Name()) + } + d.datastores[ds.Name()] = ds + return nil + }) +} + +// DeleteDatastore unmaps name and stops and deletes its datastore without holding the map lock. +// If cache deletion fails, the entry is restored so a retry can finish cleanup. func (d *DatastoreMap) DeleteDatastore(ctx context.Context, name string) error { + return d.withPending(ctx, name, func() error { + ds, err := d.takeDataStore(name) + if err != nil { + return err + } + if err := ds.Stop(ctx); err != nil { + logf.FromContext(ctx).Error(err, "failed to stop datastore") + } + if err := ds.Delete(ctx); err != nil { + d.putDataStore(name, ds) + return err + } + return nil + }) +} + +// takeDataStore removes and returns name from the map so teardown can run outside md. +func (d *DatastoreMap) takeDataStore(name string) (*datastore.Datastore, error) { d.md.Lock() defer d.md.Unlock() ds, err := d.getDataStore(name) if err != nil { - return err - } - // keep the entry on failure so a retried delete can clean up the cache instance - if err := ds.Delete(ctx); err != nil { - return err + return nil, err } delete(d.datastores, name) - return nil + return ds, nil +} + +// putDataStore restores a datastore entry after a failed delete. +func (d *DatastoreMap) putDataStore(name string, ds *datastore.Datastore) { + d.md.Lock() + defer d.md.Unlock() + d.datastores[name] = ds } func (d *DatastoreMap) GetDataStore(name string) (*datastore.Datastore, error) { @@ -128,7 +220,7 @@ func (d *DatastoreMap) GetDatastoreAll() []*datastore.Datastore { func (d *DatastoreMap) getDataStore(name string) (*datastore.Datastore, error) { ds, exists := d.datastores[name] if !exists { - return nil, fmt.Errorf("unknown datastore %s", name) + return nil, fmt.Errorf("%w %s", errUnknownDatastore, name) } return ds, nil } @@ -296,8 +388,9 @@ func (s *Server) createInitialDatastores(ctx context.Context) { log.Error(err, "failed to create datastore") return } - err = s.datastores.AddDatastore(ds) + err = s.datastores.AddDatastore(ctx, ds) if err != nil { + _ = ds.Stop(ctx) log.Error(err, "failed to create datastore") } }(dsCfg)