diff --git a/go.mod b/go.mod index 6e727fde..16a197b1 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/openconfig/ygot v0.36.0 github.com/prometheus/client_golang v1.25.0 github.com/scrapli/scrapligo v1.4.4 - github.com/sdcio/cache v0.0.38 + github.com/sdcio/cache v0.0.39 github.com/sdcio/logger v0.0.3 github.com/sdcio/schema-server v0.0.34 github.com/sdcio/sdc-protos v0.0.56 @@ -46,8 +46,8 @@ require ( github.com/clipperhouse/uax29/v2 v2.6.0 // indirect github.com/creack/pty v1.1.24 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/dgraph-io/badger/v4 v4.9.0 // indirect - github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect + github.com/dgraph-io/badger/v4 v4.9.2 // indirect + github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/fatih/color v1.18.0 // indirect diff --git a/go.sum b/go.sum index b088473d..7c3c354c 100644 --- a/go.sum +++ b/go.sum @@ -27,10 +27,10 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v4 v4.9.0 h1:tpqWb0NewSrCYqTvywbcXOhQdWcqephkVkbBmaaqHzc= -github.com/dgraph-io/badger/v4 v4.9.0/go.mod h1:5/MEx97uzdPUHR4KtkNt8asfI2T4JiEiQlV7kWUo8c0= -github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= -github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= +github.com/dgraph-io/badger/v4 v4.9.2 h1:Wb5qw8gElqwV1a8msHTeQKova9b1V10heFKMIiPd80E= +github.com/dgraph-io/badger/v4 v4.9.2/go.mod h1:nJjaJTUOSsQEBhsq209FmwCvMJzEA3e74RjZw6V2pQI= +github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU= +github.com/dgraph-io/ristretto/v2 v2.4.0/go.mod h1:0KsrXtXvnv0EqnzyowllbVJB8yBonswa2lTCK2gGo9E= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -183,8 +183,8 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/scrapli/scrapligo v1.4.4 h1:F3BiLL1tmFvEsKraq8iOqFuRRIjPlKC2SXxnCaZlzXY= github.com/scrapli/scrapligo v1.4.4/go.mod h1:pOWxVyPsQRrWTrkoSSDg05tjOqtWfLffAZtAsCc0w3M= -github.com/sdcio/cache v0.0.38 h1:8meykZC/DAgzKI+R3GtZvTs0/OXWRCeNN5mxVKG0Ba8= -github.com/sdcio/cache v0.0.38/go.mod h1:PHJd4pTmfFZBLdPohtXJAVBNmAKhhKdZHTi3MejGrk0= +github.com/sdcio/cache v0.0.39 h1:8Vk5pJueBrp4fK58aC+c9pkOtUcI2GNDp/9iuJA2Ko8= +github.com/sdcio/cache v0.0.39/go.mod h1:sOhPItnWwrdr3b8QwPTrXxEk3PtcnHkhL1KQ+ey3wzs= github.com/sdcio/goyang v1.6.2-2 h1:qfeUKBmoKpiKAruuEc3+V8wgHKP/n1jRDEnTy23knV8= github.com/sdcio/goyang v1.6.2-2/go.mod h1:5WolITjek1NF8yrNERyVZ7jqjOClJTpO8p/+OwmETM4= github.com/sdcio/logger v0.0.3 h1:IFUbObObGry+S8lHGwOQKKRxJSuOphgRU/hxVhOdMOM= diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 8000da53..2a455a19 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -31,5 +31,8 @@ type Client interface { InstanceIntentModify(ctx context.Context, cacheName string, intent *tree_persist.Intent) error InstanceIntentDelete(ctx context.Context, cacheName string, intentName string, IgnoreNonExisting bool) error InstanceIntentExists(ctx context.Context, cacheName string, intentName string) (bool, error) + // InstanceIntentGetAll streams all intents not in excludeIntentNames to intentChan, sends at most + // one error on errChan, and closes errChan before intentChan. A buffered errChan always receives + // the error; an unbuffered errChan receives it unless ctx is cancelled with no reader waiting. InstanceIntentGetAll(ctx context.Context, cacheName string, excludeIntentNames []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) } diff --git a/pkg/cache/local.go b/pkg/cache/local.go index 5235dd92..999b66bc 100644 --- a/pkg/cache/local.go +++ b/pkg/cache/local.go @@ -61,41 +61,52 @@ func (l *LocalCache) InstanceIntentGet(ctx context.Context, cacheName string, in } func (l *LocalCache) InstanceIntentGetAll(ctx context.Context, cacheName string, excludeIntentNames []string, intentChanOrig chan<- *tree_persist.Intent, errChanOrig chan<- error) { - // create new channels + var streamErr error + defer func() { + finishIntentStream(ctx, streamErr, intentChanOrig, errChanOrig) + }() + + // Cancel before finishIntentStream so the store stream releases before any errChanOrig wait. + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + intentChan := make(chan *types.Intent, 5) errChan := make(chan error, 1) - go l.Cache.InstanceIntentGetAll(ctx, cacheName, excludeIntentNames, intentChan, errChan) - - defer close(intentChanOrig) - defer close(errChanOrig) + go l.Cache.InstanceIntentGetAll(streamCtx, cacheName, excludeIntentNames, intentChan, errChan) - for { - select { - case <-ctx.Done(): // Or stop if context is canceled + for intent := range intentChan { + tpIntent := &tree_persist.Intent{} + if err := proto.Unmarshal(intent.Data(), tpIntent); err != nil { + streamErr = err return - case intent, ok := <-intentChan: // retieve intent - if !ok { - return - } - // unmarshall it into a tree_persit.Intent - tpIntent := &tree_persist.Intent{} - err := proto.Unmarshal(intent.Data(), tpIntent) - if err != nil { - errChanOrig <- err - return - } - // forward to caller - intentChanOrig <- tpIntent - case err, ok := <-errChan: // Handle errors after intents - if !ok { - errChan = nil // Mark errChan as nil so select ignores it - continue - } - errChanOrig <- err + } + select { + case intentChanOrig <- tpIntent: + case <-ctx.Done(): + streamErr = ctx.Err() return } } + streamErr = <-errChan +} + +func finishIntentStream(ctx context.Context, err error, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + defer close(intentChan) + defer close(errChan) + if err == nil { + return + } + // Buffered or waiting errChan receives err even when ctx is already done. + select { + case errChan <- err: + return + default: + } + select { + case errChan <- err: + case <-ctx.Done(): + } } func (l *LocalCache) InstanceIntentModify(ctx context.Context, cacheName string, intent *tree_persist.Intent) error { diff --git a/pkg/cache/local_test.go b/pkg/cache/local_test.go new file mode 100644 index 00000000..96249493 --- /dev/null +++ b/pkg/cache/local_test.go @@ -0,0 +1,142 @@ +package cache + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/sdcio/cache/pkg/config" + "github.com/sdcio/sdc-protos/tree_persist" +) + +// An abandoned InstanceIntentGetAll must exit cleanly and release the store for both buffered and unbuffered errChan. +func TestLocalCache_InstanceIntentGetAll_AbandonedReleasesStore(t *testing.T) { + for _, errCap := range []int{0, 1} { + t.Run(fmt.Sprintf("errChanCap=%d", errCap), func(t *testing.T) { + ctx := context.Background() + c, err := NewLocalCache(&config.CacheConfig{Dir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + const name = "ds" + if err := c.InstanceCreate(ctx, name); err != nil { + t.Fatal(err) + } + for i := range 20 { + if err := c.InstanceIntentModify(ctx, name, &tree_persist.Intent{IntentName: fmt.Sprintf("intent-%d", i)}); err != nil { + t.Fatal(err) + } + } + + getCtx, cancel := context.WithCancel(ctx) + intentChan := make(chan *tree_persist.Intent) + errChan := make(chan error, errCap) + streamDone := make(chan struct{}) + go func() { + defer close(streamDone) + c.InstanceIntentGetAll(getCtx, name, nil, intentChan, errChan) + }() + + <-intentChan + time.Sleep(50 * time.Millisecond) + cancel() + + select { + case <-streamDone: + case <-time.After(5 * time.Second): + t.Fatal("InstanceIntentGetAll blocked after ctx cancellation") + } + if errCap > 0 { + if err := <-errChan; !errors.Is(err, context.Canceled) { + t.Fatalf("got err %v, want %v", err, context.Canceled) + } + } + + done := make(chan error, 1) + go func() { done <- c.InstanceDelete(ctx, name) }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("InstanceDelete blocked: store lock still held by abandoned InstanceIntentGetAll") + } + }) + } +} + +func TestLocalCache_InstanceIntentGetAll_UnmarshalErrorReleasesStore(t *testing.T) { + ctx := context.Background() + raw, err := NewLocalCache(&config.CacheConfig{Dir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + lc := raw.(*LocalCache) + + const name = "ds" + if err := lc.InstanceCreate(ctx, name); err != nil { + t.Fatal(err) + } + if err := lc.Cache.InstanceIntentModify(ctx, name, "corrupt", []byte{0xff, 0xff, 0xff}); err != nil { + t.Fatal(err) + } + for i := range 20 { + if err := lc.InstanceIntentModify(ctx, name, &tree_persist.Intent{IntentName: fmt.Sprintf("intent-%d", i)}); err != nil { + t.Fatal(err) + } + } + + // Unbuffered errChan with no reader yet: the store stream must still be released immediately. + getCtx, cancel := context.WithCancel(ctx) + defer cancel() + intentChan := make(chan *tree_persist.Intent) + errChan := make(chan error) + streamDone := make(chan struct{}) + go func() { + defer close(streamDone) + lc.InstanceIntentGetAll(getCtx, name, nil, intentChan, errChan) + }() + + // Drain any valid intents emitted before the corrupt one, then stop reading errChan briefly. + readLoop: + for { + select { + case _, ok := <-intentChan: + if !ok { + break readLoop + } + case <-time.After(50 * time.Millisecond): + break readLoop + } + } + + // Even before errChan is read or getCtx is cancelled, InstanceDelete must not deadlock on the store. + deleted := make(chan error, 1) + go func() { deleted <- lc.InstanceDelete(ctx, name) }() + select { + case err := <-deleted: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("InstanceDelete blocked on unmarshal error with unbuffered errChan") + } + + // Reading errChan receives the unmarshal error and lets InstanceIntentGetAll finish. + select { + case err := <-errChan: + if err == nil { + t.Fatal("expected unmarshal error, got nil") + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for unmarshal error") + } + select { + case <-streamDone: + case <-time.After(5 * time.Second): + t.Fatal("InstanceIntentGetAll did not return after delivering unmarshal error") + } +} diff --git a/pkg/datastore/datastore_rpc.go b/pkg/datastore/datastore_rpc.go index 15efbce4..ca8b4c8e 100644 --- a/pkg/datastore/datastore_rpc.go +++ b/pkg/datastore/datastore_rpc.go @@ -21,6 +21,7 @@ import ( "sync" "time" + sdccache "github.com/sdcio/cache/pkg/cache" logf "github.com/sdcio/logger" sdcpb "github.com/sdcio/sdc-protos/sdcpb" "google.golang.org/grpc" @@ -44,7 +45,8 @@ type Datastore struct { cacheClient cache.CacheClientBound // SBI target of this datastore - sbi target.Target + sbiMutex sync.RWMutex + sbi target.Target // schema server client // schemaClient sdcpb.SchemaServerClient @@ -122,7 +124,10 @@ func New(ctx context.Context, c *config.DatastoreConfig, sc schema.Client, cc ca // create cache instance if needed // this is a blocking call - ds.initCache(ctx) + if err := ds.initCache(ctx); err != nil { + cancel() + return nil, err + } go func() { // init sbi, this is a blocking call @@ -146,31 +151,61 @@ func (d *Datastore) IntentsList(ctx context.Context) ([]string, error) { return d.cacheClient.IntentsList(ctx) } -func (d *Datastore) initCache(ctx context.Context) { +func (d *Datastore) initCache(ctx context.Context) error { log := logf.FromContext(ctx) exists := d.cacheClient.InstanceExists(ctx) if exists { log.V(logf.VDebug).Info("cache already exists") - return + return nil } log.Info("creating cache instance") -CREATE: - err := d.cacheClient.InstanceCreate(ctx) - if err != nil { + for { + err := d.cacheClient.InstanceCreate(ctx) + if err == nil || errors.Is(err, sdccache.ErrorCacheInstanceAlreadyExists) { + return nil + } log.Error(err, "failed to create cache") - time.Sleep(time.Second) - goto CREATE + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } } } +// installSBI publishes sbi onto the datastore unless ctx was cancelled while connecting, +// in which case sbi is closed immediately so Stop cannot miss it. +func (d *Datastore) installSBI(ctx context.Context, sbi target.Target) error { + if !d.setSBI(ctx, sbi) { + _ = sbi.Close(ctx) + return ctx.Err() + } + return nil +} + +func (d *Datastore) setSBI(ctx context.Context, sbi target.Target) bool { + d.sbiMutex.Lock() + defer d.sbiMutex.Unlock() + if ctx.Err() != nil { + return false + } + d.sbi = sbi + return true +} + +func (d *Datastore) getSBI() target.Target { + d.sbiMutex.RLock() + defer d.sbiMutex.RUnlock() + return d.sbi +} + func (d *Datastore) connectSBI(ctx context.Context, opts ...grpc.DialOption) error { log := logf.FromContext(ctx) - var err error - d.sbi, err = target.New(ctx, d.config.Name, d.config.SBI, d.schemaClient, d, d.config.Sync.Config, d.taskPool, opts...) + sbi, err := target.New(ctx, d.config.Name, d.config.SBI, d.schemaClient, d, d.config.Sync.Config, d.taskPool, opts...) if err == nil { - return nil + return d.installSBI(ctx, sbi) } log.Error(err, "failed to create DS target") @@ -182,12 +217,12 @@ func (d *Datastore) connectSBI(ctx context.Context, opts ...grpc.DialOption) err case <-ctx.Done(): return ctx.Err() case <-ticker.C: - d.sbi, err = target.New(ctx, d.config.Name, d.config.SBI, d.schemaClient, d, d.config.Sync.Config, d.taskPool, opts...) + sbi, err = target.New(ctx, d.config.Name, d.config.SBI, d.schemaClient, d, d.config.Sync.Config, d.taskPool, opts...) if err != nil { log.Error(err, "failed to create DS target") continue } - return nil + return d.installSBI(ctx, sbi) } } } @@ -205,14 +240,19 @@ func (d *Datastore) Config() *config.DatastoreConfig { } func (d *Datastore) Delete(ctx context.Context) error { - return d.cacheClient.InstanceDelete(ctx) + err := d.cacheClient.InstanceDelete(ctx) + if errors.Is(err, sdccache.ErrorCacheInstanceNotFound) { + return nil + } + return err } func (d *Datastore) ConnectionState() *targettypes.TargetStatus { - if d.sbi == nil { + sbi := d.getSBI() + if sbi == nil { return targettypes.NewTargetStatus(sdcpb.TargetStatus_NOT_CONNECTED) } - return d.sbi.Status() + return sbi.Status() } func (d *Datastore) Stop(ctx context.Context) error { @@ -220,10 +260,11 @@ func (d *Datastore) Stop(ctx context.Context) error { return nil } d.cfn() - if d.sbi == nil { + sbi := d.getSBI() + if sbi == nil { return nil } - err := d.sbi.Close(ctx) + err := sbi.Close(ctx) if err != nil { logf.DefaultLogger.Error(err, "datastore failed to close the target connection", "datastore-name", d.Name()) } diff --git a/pkg/datastore/intent_rpc.go b/pkg/datastore/intent_rpc.go index 62e34bb3..e4efc8c6 100644 --- a/pkg/datastore/intent_rpc.go +++ b/pkg/datastore/intent_rpc.go @@ -45,11 +45,12 @@ func (d *Datastore) applyIntent(ctx context.Context, source targettypes.TargetSo return &sdcpb.SetDataResponse{}, nil } - if d.sbi == nil { + sbi := d.getSBI() + if sbi == nil { return nil, fmt.Errorf("%s: %w", d.config.Name, targettypes.ErrNotConnected) } - rsp, err = d.sbi.Set(ctx, source) + rsp, err = sbi.Set(ctx, source) if err != nil { return nil, fmt.Errorf("%s: %w", d.config.Name, err) } diff --git a/pkg/datastore/target/netconf/driver/scrapligo/scrapligo.go b/pkg/datastore/target/netconf/driver/scrapligo/scrapligo.go index 6fbe2398..f3194e0b 100644 --- a/pkg/datastore/target/netconf/driver/scrapligo/scrapligo.go +++ b/pkg/datastore/target/netconf/driver/scrapligo/scrapligo.go @@ -19,6 +19,7 @@ import ( "fmt" "io" "os" + "sync" "time" "github.com/beevik/etree" @@ -60,6 +61,9 @@ const closeTimeout = 3 * time.Second type ScrapligoNetconfTarget struct { driver *scraplinetconf.Driver + // scrapligo's Driver.Close blocks forever when called a second time + closeOnce sync.Once + closeErr error } // NewScrapligoNetconfTarget inits a new ScrapligoNetconfTarget which is already connected to the target node @@ -111,18 +115,21 @@ func (snt *ScrapligoNetconfTarget) Close() error { if snt.driver == nil { return nil } - done := make(chan error, 1) - go func() { done <- snt.driver.Close() }() - - select { - case err := <-done: - return err - case <-time.After(closeTimeout): - if snt.driver.Transport != nil { - _ = snt.driver.Transport.Close(true) + snt.closeOnce.Do(func() { + done := make(chan error, 1) + go func() { done <- snt.driver.Close() }() + + select { + case err := <-done: + snt.closeErr = err + case <-time.After(closeTimeout): + if snt.driver.Transport != nil { + _ = snt.driver.Transport.Close(true) + } + snt.closeErr = fmt.Errorf("closing netconf session: %w", os.ErrDeadlineExceeded) } - return fmt.Errorf("closing netconf session: %w", os.ErrDeadlineExceeded) - } + }) + return snt.closeErr } func (snt *ScrapligoNetconfTarget) IsAlive() bool { diff --git a/pkg/datastore/transaction_rpc.go b/pkg/datastore/transaction_rpc.go index d165a7a4..1b3a61f4 100644 --- a/pkg/datastore/transaction_rpc.go +++ b/pkg/datastore/transaction_rpc.go @@ -164,45 +164,31 @@ func (d *Datastore) replaceIntent(ctx context.Context, transaction *types.Transa func (d *Datastore) LoadAllButRunningIntents(ctx context.Context, root *tree.RootEntry) ([]string, error) { log := logger.FromContext(ctx) + // Releases the cache producer when returning before all intents are consumed. + ctx, cancel := context.WithCancel(ctx) + defer cancel() intentNames := []string{} - IntentChan := make(chan *tree_persist.Intent) - ErrChan := make(chan error, 1) - - go d.cacheClient.IntentGetAll(ctx, []string{consts.RunningIntentName}, IntentChan, ErrChan) - - for { - selectLoop: - select { - case err, ok := <-ErrChan: - if !ok { - // ErrChan already closed which is fine, continue - ErrChan = nil - break selectLoop - } - return nil, err - case <-ctx.Done(): - return nil, fmt.Errorf("context closed while retrieving all intents") - case intent, ok := <-IntentChan: - if !ok { - // IntentChan closed due to finish - IntentChan = nil - break selectLoop - } - log.V(logger.VDebug).Info("adding intent to tree", "intent", intent.GetIntentName()) - log.V(logger.VTrace).Info("adding intent to tree", "intent", intent.GetIntentName(), "content", utils.ProtoJSON(intent)) + intentChan := make(chan *tree_persist.Intent) + errChan := make(chan error, 1) - intentNames = append(intentNames, intent.GetIntentName()) - protoLoader := treeproto.NewProtoTreeImporter(intent) - _, err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags(), d.taskPool) - if err != nil { - return nil, err - } - } - if ErrChan == nil && IntentChan == nil { - return intentNames, nil + go d.cacheClient.IntentGetAll(ctx, []string{consts.RunningIntentName}, intentChan, errChan) + + for intent := range intentChan { + log.V(logger.VDebug).Info("adding intent to tree", "intent", intent.GetIntentName()) + log.V(logger.VTrace).Info("adding intent to tree", "intent", intent.GetIntentName(), "content", utils.ProtoJSON(intent)) + + intentNames = append(intentNames, intent.GetIntentName()) + protoLoader := treeproto.NewProtoTreeImporter(intent) + _, err := root.ImportConfig(ctx, nil, protoLoader, treetypes.NewUpdateInsertFlags(), d.taskPool) + if err != nil { + return nil, err } } + if err, ok := <-errChan; ok { + return nil, err + } + return intentNames, nil } // lowlevelTransactionSet diff --git a/pkg/datastore/transaction_rpc_test.go b/pkg/datastore/transaction_rpc_test.go index 00231c57..86f31491 100644 --- a/pkg/datastore/transaction_rpc_test.go +++ b/pkg/datastore/transaction_rpc_test.go @@ -3,6 +3,8 @@ package datastore import ( "context" "encoding/json" + "errors" + "fmt" "runtime" "sync" "testing" @@ -227,3 +229,71 @@ func TestTransactionSet_PreviouslyApplied(t *testing.T) { }) } } + +func TestLoadAllButRunningIntents_PropagatesStreamErrorAndCancelsEarly(t *testing.T) { + ctx := context.Background() + sc, schema, err := testhelper.InitSDCIOSchema() + if err != nil { + t.Fatalf("InitSDCIOSchema: %v", err) + } + scb := schemaClient.NewSchemaClientBound(schema, sc) + + t.Run("stream error", func(t *testing.T) { + ctrl := gomock.NewController(t) + wantErr := fmt.Errorf("cache read failure") + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, _ []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + errChan <- wantErr + close(errChan) + close(intentChan) + }) + + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + root, err := tree.NewTreeRoot(ctx, tree.NewTreeContext(scb, vpf)) + if err != nil { + t.Fatal(err) + } + ds := &Datastore{cacheClient: ccb, taskPool: vpf} + if _, err := ds.LoadAllButRunningIntents(ctx, root); !errors.Is(err, wantErr) { + t.Fatalf("got err %v, want %v", err, wantErr) + } + }) + + t.Run("import error cancels producer ctx", func(t *testing.T) { + ctrl := gomock.NewController(t) + producerCancelled := make(chan struct{}) + ccb := mockcacheclient.NewMockCacheClientBound(ctrl) + ccb.EXPECT(). + IntentGetAll(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(streamCtx context.Context, _ []string, intentChan chan<- *tree_persist.Intent, errChan chan<- error) { + defer close(intentChan) + defer close(errChan) + intentChan <- &tree_persist.Intent{ + IntentName: "bad", + Root: &tree_persist.TreeElement{ + Childs: []*tree_persist.TreeElement{{Name: "nonexistent-schema-element"}}, + }, + } + <-streamCtx.Done() + close(producerCancelled) + }) + + vpf := pool.NewSharedTaskPool(ctx, runtime.GOMAXPROCS(0)) + root, err := tree.NewTreeRoot(ctx, tree.NewTreeContext(scb, vpf)) + if err != nil { + t.Fatal(err) + } + ds := &Datastore{cacheClient: ccb, taskPool: vpf} + if _, err := ds.LoadAllButRunningIntents(ctx, root); err == nil { + t.Fatal("expected import error, got nil") + } + select { + case <-producerCancelled: + case <-time.After(5 * time.Second): + t.Fatal("producer ctx was not cancelled on early return") + } + }) +} + diff --git a/pkg/server/datastore.go b/pkg/server/datastore.go index 889f6fb7..5c931dc0 100644 --- a/pkg/server/datastore.go +++ b/pkg/server/datastore.go @@ -16,6 +16,7 @@ package server import ( "context" + "errors" "fmt" "math" "strings" @@ -93,6 +94,9 @@ func (s *Server) CreateDataStore(ctx context.Context, req *sdcpb.CreateDataStore return nil, status.Error(codes.InvalidArgument, "datastore name attribute too long") } + if err := s.datastores.WaitPending(ctx, name); err != nil { + return nil, err + } if _, err := s.datastores.GetDataStore(name); err == nil { log.Error(nil, "datastore already exists") return nil, status.Errorf(codes.InvalidArgument, "datastore %s already exists", name) @@ -214,8 +218,9 @@ func (s *Server) CreateDataStore(ctx context.Context, req *sdcpb.CreateDataStore log.Error(err, "failed creating new datastore") return nil, err } - err = s.datastores.AddDatastore(ds) + err = s.datastores.AddDatastore(ctx, ds) if err != nil { + _ = ds.Stop(ctx) log.Error(err, "failed adding new datastore") return nil, err } @@ -234,16 +239,10 @@ func (s *Server) DeleteDataStore(ctx context.Context, req *sdcpb.DeleteDataStore return nil, status.Error(codes.InvalidArgument, "missing datastore name attribute") } - ds, err := s.datastores.GetDataStore(name) - if err != nil { + err := s.datastores.DeleteDatastore(ctx, name) + if errors.Is(err, errUnknownDatastore) { return nil, err } - - err = ds.Stop(s.ctx) - if err != nil { - log.Error(err, "failed to stop datastore") - } - err = s.datastores.DeleteDatastore(ctx, name) if err != nil { log.Error(err, "failed to delete datastore") return nil, fmt.Errorf("failed to delete datastore: %w", err) diff --git a/pkg/server/datastore_test.go b/pkg/server/datastore_test.go index 784072ae..ccda1d86 100644 --- a/pkg/server/datastore_test.go +++ b/pkg/server/datastore_test.go @@ -17,9 +17,12 @@ package server import ( "bytes" "context" + "errors" + "fmt" "log/slog" "strings" "testing" + "time" "github.com/go-logr/logr" "github.com/sdcio/data-server/mocks/mockcacheclient" @@ -233,3 +236,119 @@ func TestCreateDataStore_NoopType_WithSyncConfig_Succeeds(t *testing.T) { t.Fatalf("expected no error for noop type with sync config, got: %v", err) } } + +func TestDeleteDataStore_DoesNotBlockOtherDatastoresAndSerializesSameName(t *testing.T) { + s := newTestServer(t) + mockCC := s.cacheClient.(*mockcacheclient.MockClient) + ctx := context.Background() + + for _, name := range []string{"victim", "other"} { + if _, err := s.CreateDataStore(ctx, noopCreateReq(name, "noop")); err != nil { + t.Fatalf("CreateDataStore(%s): %v", name, err) + } + } + + deleting := make(chan struct{}) + release := make(chan struct{}) + mockCC.EXPECT(). + InstanceDelete(gomock.Any(), "victim"). + DoAndReturn(func(context.Context, string) error { + close(deleting) + <-release + return nil + }) + mockCC.EXPECT().InstanceDelete(gomock.Any(), "third").Return(nil) + + delErr := make(chan error, 1) + go func() { + _, err := s.DeleteDataStore(ctx, &sdcpb.DeleteDataStoreRequest{Name: "victim"}) + delErr <- err + }() + select { + case <-deleting: + case <-time.After(5 * time.Second): + t.Fatal("InstanceDelete was not called") + } + + // Unrelated datastores remain accessible while victim's delete is stuck in cache I/O. + otherDone := make(chan error, 1) + go func() { + if _, err := s.datastores.GetDataStore("other"); err != nil { + otherDone <- err + return + } + if _, err := s.CreateDataStore(ctx, noopCreateReq("third", "noop")); err != nil { + otherDone <- err + return + } + _, err := s.DeleteDataStore(ctx, &sdcpb.DeleteDataStoreRequest{Name: "third"}) + otherDone <- err + }() + select { + case err := <-otherDone: + if err != nil { + t.Fatalf("operations on unrelated datastores failed: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("unrelated datastore operations blocked on in-flight delete") + } + + // Recreating the deleting datastore waits for the delete and respects ctx cancellation. + shortCtx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + if _, err := s.CreateDataStore(shortCtx, noopCreateReq("victim", "noop")); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("CreateDataStore during delete got %v, want %v", err, context.DeadlineExceeded) + } + + recreateErr := make(chan error, 1) + go func() { + _, err := s.CreateDataStore(ctx, noopCreateReq("victim", "noop")) + recreateErr <- err + }() + close(release) + + for what, ch := range map[string]chan error{"DeleteDataStore": delErr, "CreateDataStore": recreateErr} { + select { + case err := <-ch: + if err != nil { + t.Fatalf("%s: %v", what, err) + } + case <-time.After(5 * time.Second): + t.Fatalf("%s did not finish after release", what) + } + } + if _, err := s.datastores.GetDataStore("victim"); err != nil { + t.Fatalf("victim not present after recreate: %v", err) + } +} + +func TestDeleteDataStore_RestoresEntryOnFailureSoRetrySucceeds(t *testing.T) { + s := newTestServer(t) + mockCC := s.cacheClient.(*mockcacheclient.MockClient) + ctx := context.Background() + + if _, err := s.CreateDataStore(ctx, noopCreateReq("ds-retry", "noop")); err != nil { + t.Fatalf("CreateDataStore: %v", err) + } + + wantErr := fmt.Errorf("transient delete failure") + gomock.InOrder( + mockCC.EXPECT().InstanceDelete(gomock.Any(), "ds-retry").Return(wantErr), + mockCC.EXPECT().InstanceDelete(gomock.Any(), "ds-retry").Return(nil), + ) + + if _, err := s.DeleteDataStore(ctx, &sdcpb.DeleteDataStoreRequest{Name: "ds-retry"}); !errors.Is(err, wantErr) { + t.Fatalf("first DeleteDataStore got %v, want %v", err, wantErr) + } + if _, err := s.datastores.GetDataStore("ds-retry"); err != nil { + t.Fatalf("datastore should remain registered after failed delete: %v", err) + } + + if _, err := s.DeleteDataStore(ctx, &sdcpb.DeleteDataStoreRequest{Name: "ds-retry"}); err != nil { + t.Fatalf("retry DeleteDataStore failed: %v", err) + } + if _, err := s.datastores.GetDataStore("ds-retry"); !errors.Is(err, errUnknownDatastore) { + t.Fatalf("GetDataStore after delete got %v, want %v", err, errUnknownDatastore) + } +} + diff --git a/pkg/server/intent.go b/pkg/server/intent.go index 336234e7..b419abe1 100644 --- a/pkg/server/intent.go +++ b/pkg/server/intent.go @@ -26,7 +26,7 @@ func (s *Server) ListIntent(ctx context.Context, req *sdcpb.ListIntentRequest) ( return nil, status.Error(codes.InvalidArgument, "missing datastore name") } - ds, err := s.datastores.getDataStore(req.GetDatastoreName()) + ds, err := s.datastores.GetDataStore(req.GetDatastoreName()) if err != nil { return nil, status.Error(codes.NotFound, err.Error()) } @@ -62,7 +62,7 @@ func (s *Server) GetIntent(ctx context.Context, req *sdcpb.GetIntentRequest) (*s return nil, status.Error(codes.InvalidArgument, "missing intent name") } // retrieve the referenced datastore - ds, err := s.datastores.getDataStore(req.GetDatastoreName()) + ds, err := s.datastores.GetDataStore(req.GetDatastoreName()) if err != nil { return nil, status.Error(codes.NotFound, err.Error()) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 471b7173..23176591 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -16,6 +16,7 @@ package server import ( "context" + "errors" "fmt" "net" "net/http" @@ -65,44 +66,138 @@ type Server struct { gnmiOpts []grpc.DialOption } +var errUnknownDatastore = errors.New("unknown datastore") + +// DatastoreMap indexes active datastores by name. +// md guards short map lookups and mutations; pending serializes add and delete +// lifecycle operations per name so slow Stop/Delete I/O runs without holding md. type DatastoreMap struct { md *sync.RWMutex - datastores map[string]*datastore.Datastore // datastore group with sbi + datastores map[string]*datastore.Datastore + pending map[string]chan struct{} } func NewDatastoreMap() *DatastoreMap { return &DatastoreMap{ md: &sync.RWMutex{}, datastores: map[string]*datastore.Datastore{}, + pending: map[string]chan struct{}{}, } } func (d *DatastoreMap) StopAll(ctx context.Context) { - for _, ds := range d.datastores { + for _, ds := range d.GetDatastoreAll() { _ = ds.Stop(ctx) } } -func (d *DatastoreMap) AddDatastore(ds *datastore.Datastore) error { +// withPending runs fn with exclusive lifecycle ownership of name, waiting for +// any in-flight add or delete on name to finish first (or ctx to expire). +func (d *DatastoreMap) withPending(ctx context.Context, name string, fn func() error) error { + for { + done, ok := d.tryAcquire(name) + if ok { + defer d.release(name) + return fn() + } + select { + case <-done: + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// tryAcquire marks name as in flight and returns (nil, true) when idle; +// otherwise it returns the active completion channel and false so callers can wait without holding md. +func (d *DatastoreMap) tryAcquire(name string) (<-chan struct{}, bool) { d.md.Lock() defer d.md.Unlock() - if existingDs, _ := d.getDataStore(ds.Name()); existingDs != nil { - return fmt.Errorf("datastore %s already exists", ds.Name()) + if done, busy := d.pending[name]; busy { + return done, false } + d.pending[name] = make(chan struct{}) + return nil, true +} - d.datastores[ds.Name()] = ds - return nil +// release closes name's completion channel to wake waiting callers and removes its pending entry. +func (d *DatastoreMap) release(name string) { + d.md.Lock() + defer d.md.Unlock() + close(d.pending[name]) + delete(d.pending, name) +} + +// pendingDone returns the completion channel for an in-flight operation on name, or nil when idle. +func (d *DatastoreMap) pendingDone(name string) <-chan struct{} { + d.md.RLock() + defer d.md.RUnlock() + return d.pending[name] } +// WaitPending waits until no add or delete is in flight for name, or ctx expires. +func (d *DatastoreMap) WaitPending(ctx context.Context, name string) error { + for { + done := d.pendingDone(name) + if done == nil { + return nil + } + select { + case <-done: + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// AddDatastore registers ds under its name after waiting for any in-flight lifecycle operation on that name. +func (d *DatastoreMap) AddDatastore(ctx context.Context, ds *datastore.Datastore) error { + return d.withPending(ctx, ds.Name(), func() error { + d.md.Lock() + defer d.md.Unlock() + if existingDs, _ := d.getDataStore(ds.Name()); existingDs != nil { + return fmt.Errorf("datastore %s already exists", ds.Name()) + } + d.datastores[ds.Name()] = ds + return nil + }) +} + +// DeleteDatastore unmaps name and stops and deletes its datastore without holding the map lock. +// If cache deletion fails, the entry is restored so a retry can finish cleanup. func (d *DatastoreMap) DeleteDatastore(ctx context.Context, name string) error { + return d.withPending(ctx, name, func() error { + ds, err := d.takeDataStore(name) + if err != nil { + return err + } + if err := ds.Stop(ctx); err != nil { + logf.FromContext(ctx).Error(err, "failed to stop datastore") + } + if err := ds.Delete(ctx); err != nil { + d.putDataStore(name, ds) + return err + } + return nil + }) +} + +// takeDataStore removes and returns name from the map so teardown can run outside md. +func (d *DatastoreMap) takeDataStore(name string) (*datastore.Datastore, error) { d.md.Lock() defer d.md.Unlock() ds, err := d.getDataStore(name) if err != nil { - return err + return nil, err } - _ = ds.Delete(ctx) delete(d.datastores, name) - return nil + return ds, nil +} + +// putDataStore restores a datastore entry after a failed delete. +func (d *DatastoreMap) putDataStore(name string, ds *datastore.Datastore) { + d.md.Lock() + defer d.md.Unlock() + d.datastores[name] = ds } func (d *DatastoreMap) GetDataStore(name string) (*datastore.Datastore, error) { @@ -125,7 +220,7 @@ func (d *DatastoreMap) GetDatastoreAll() []*datastore.Datastore { func (d *DatastoreMap) getDataStore(name string) (*datastore.Datastore, error) { ds, exists := d.datastores[name] if !exists { - return nil, fmt.Errorf("unknown datastore %s", name) + return nil, fmt.Errorf("%w %s", errUnknownDatastore, name) } return ds, nil } @@ -291,9 +386,11 @@ func (s *Server) createInitialDatastores(ctx context.Context) { ds, err := datastore.New(ctx, dsCfg, s.schemaClient, s.cacheClient, s.gnmiOpts...) if err != nil { log.Error(err, "failed to create datastore") + return } - err = s.datastores.AddDatastore(ds) + err = s.datastores.AddDatastore(ctx, ds) if err != nil { + _ = ds.Stop(ctx) log.Error(err, "failed to create datastore") } }(dsCfg) diff --git a/pkg/server/transaction.go b/pkg/server/transaction.go index bbc91b85..89f1a56c 100644 --- a/pkg/server/transaction.go +++ b/pkg/server/transaction.go @@ -52,7 +52,7 @@ func (s *Server) TransactionSet(ctx context.Context, req *sdcpb.TransactionSetRe } // retrieve the referenced datastore - ds, err := s.datastores.getDataStore(req.DatastoreName) + ds, err := s.datastores.GetDataStore(req.DatastoreName) if err != nil { return nil, status.Error(codes.NotFound, err.Error()) }