diff --git a/bs3/bs3test/bs3test.go b/bs3/bs3test/bs3test.go new file mode 100644 index 00000000000..57b514680a3 --- /dev/null +++ b/bs3/bs3test/bs3test.go @@ -0,0 +1,74 @@ +// Package bs3test provides an in-memory fake of the S3 client subset +// provided by the bs3 package, for use in tests. +package bs3test + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "sync" + + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/service/s3" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +// StoredObject is an object held by a FakeS3. +type StoredObject struct { + Data []byte + ContentEncoding *string +} + +// FakeS3 implements the PutObject/GetObject/Bucket subset of the S3 API +// used by the tiles package with an in-memory map. +// +// Calling PutObject twice for the same key result in an error. +type FakeS3 struct { + mu sync.Mutex + + Objects map[string]StoredObject +} + +func New() *FakeS3 { + return &FakeS3{Objects: make(map[string]StoredObject)} +} + +func (f *FakeS3) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + _, ok := f.Objects[*params.Key] + if ok { + return nil, &awshttp.ResponseError{ + ResponseError: &smithyhttp.ResponseError{ + Response: &smithyhttp.Response{Response: &http.Response{StatusCode: http.StatusPreconditionFailed}}, + Err: errors.New("PreconditionFailed"), + }, + } + } + body, err := io.ReadAll(params.Body) + if err != nil { + return nil, err + } + f.Objects[*params.Key] = StoredObject{Data: body, ContentEncoding: params.ContentEncoding} + return nil, nil +} + +func (f *FakeS3) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + f.mu.Lock() + defer f.mu.Unlock() + o, ok := f.Objects[*params.Key] + if ok { + return &s3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader(o.Data)), + ContentEncoding: o.ContentEncoding, + }, nil + } + return nil, fmt.Errorf("not found") +} + +func (f *FakeS3) Bucket() string { + return "fakebucket" +} diff --git a/cmd/boulder-mtca/main.go b/cmd/boulder-mtca/main.go index 5d5087ec97f..a3e88997146 100644 --- a/cmd/boulder-mtca/main.go +++ b/cmd/boulder-mtca/main.go @@ -129,7 +129,7 @@ func main() { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) defer cancel() err = mtcaImpl.InitLog(ctx) - cmd.FailOnError(err, "Initializing log") + cmd.FailOnError(err, "Initializing MTC issuance log") return } if *initLogForTest { @@ -137,10 +137,13 @@ func main() { defer cancel() err = mtcaImpl.InitLog(ctx) if err != nil && !errors.Is(err, mtca.ErrIssuanceLogAlreadyInitialized) { - cmd.FailOnError(err, "Initializing MTC log DB for test") + cmd.FailOnError(err, "Initializing MTC issuance log for test") } } + err = mtcaImpl.Preflight(context.Background()) + cmd.FailOnError(err, "Loading log state") + srv := bgrpc.NewServer(c.MTCA.GRPCMTCA, logger).Add( &mtcapb.MTCA_ServiceDesc, mtcaImpl) diff --git a/mtca/mtca.go b/mtca/mtca.go index 7cf2c29a7ce..7a828c858c8 100644 --- a/mtca/mtca.go +++ b/mtca/mtca.go @@ -3,13 +3,15 @@ package mtca import ( + "bytes" "context" "crypto" - "crypto/rand" "crypto/sha256" "crypto/x509" "database/sql" "encoding/asn1" + "encoding/base64" + "encoding/hex" "errors" "fmt" "sync" @@ -26,6 +28,7 @@ import ( mtcapb "github.com/letsencrypt/boulder/mtca/proto" "github.com/letsencrypt/boulder/trees/cosigned" "github.com/letsencrypt/boulder/trees/entry" + "github.com/letsencrypt/boulder/trees/tiles" ) var ErrIssuanceLogAlreadyInitialized = errors.New("issuance log already initialized") @@ -79,6 +82,11 @@ type mtca struct { logNumber uint16 pool *pool + // frontier contains all the tiles on the right edge of the tree. + // It will be used to accumulate entries for writing to storage. + // Not safe for concurrent reading and writing. + frontier *tiles.Frontier + sequencingPeriod time.Duration // TODO: factor our sa.InitWrappedDb() so we get metrics and other goodies. @@ -122,7 +130,17 @@ func initDB(dbMap *borp.DbMap) *db.WrappedMap { // InitLog creates the database metadata for a new, empty log: one checkpoint and the row // in `latestCheckpoint` that refers to it. Should only be run once in a log's lifetime. func (m *mtca) InitLog(ctx context.Context) error { - _, err := db.WithTransaction(ctx, m.db, func(tx db.Executor) (any, error) { + candidate := &tiles.Frontier{} + + nullEntry := &entry.MTCLogEntry{} + err := candidate.AppendEntry(nullEntry) + if err != nil { + return err + } + + rootHash := candidate.RootHash() + + _, err = db.WithTransaction(ctx, m.db, func(tx db.Executor) (any, error) { var numLatestCheckpoints int64 err := tx.SelectOne(ctx, &numLatestCheckpoints, "SELECT COUNT(*) FROM latestCheckpoint WHERE mtcLogID = ?", m.mtcLogID()) @@ -146,19 +164,9 @@ func (m *mtca) InitLog(ctx context.Context) error { m.mtcLogID(), numCheckpoints, numLatestCheckpoints) } - // null_entry has empty extensions and a MTCLogEntryType of 0. Since extensions can be up to 2^16 long - // there's two bytes of length prefix. Since MTCLogEntryType can have up to 2^16 values, it's also two bytes. - // All the bytes are zero: empty extensions, null_entry type is enum value zero. - // https://ietf-plants-wg.github.io/merkle-tree-certs/draft-ietf-plants-merkle-tree-certs.html#name-log-entries - // To calculate the Merkle Tree Hash of a single-entry list, we prepend 0x00 (as compared with 0x01 when hashing - // two nodes). So five zeroes total. - // https://www.rfc-editor.org/info/rfc9162/#name-definition-of-the-merkle-tr - nullEntry := []byte{0, 0, 0, 0, 0} - rootHash := sha256.Sum256(nullEntry) - firstCheckpoint := checkpoint{ MTCLogID: m.mtcLogID(), - TreeSize: 1, + TreeSize: candidate.TreeSize(), RootHash: rootHash[:], } @@ -188,9 +196,24 @@ func (m *mtca) InitLog(ctx context.Context) error { return nil, nil }) if err != nil { + if errors.Is(err, ErrIssuanceLogAlreadyInitialized) { + // The DB thinks the log is initialized; make sure the tiles are there. + err2 := m.Preflight(ctx) + if err2 != nil { + return fmt.Errorf("DB is initialized but Preflight returns: %s", err2) + } + return err + } return err } + err = candidate.Publish(ctx, m.s3c, m.tileStoragePrefix()) + if err != nil { + return err + } + + m.frontier = candidate + _, err = m.latestCheckpoint(ctx) if err != nil { return fmt.Errorf("fetching first checkpoint: %s", err) @@ -199,13 +222,37 @@ func (m *mtca) InitLog(ctx context.Context) error { return err } +// Preflight gets the latest checkpoint from the database and reads the corresponding +// frontier tiles from storage. It must be called on startup, before Loop(). +func (m *mtca) Preflight(ctx context.Context) error { + latest, err := m.latestCheckpoint(ctx) + if err != nil { + return err + } + frontier, err := tiles.LoadFrontier(ctx, m.s3c, latest.TreeSize, m.tileStoragePrefix()) + if err != nil { + return err + } + + tileBasedHash := frontier.RootHash() + if !bytes.Equal(latest.RootHash, tileBasedHash[:]) { + return fmt.Errorf("state mismatch: at tree size %d, DB contains RootHash %s, but frontier tiles calculate %s", + latest.TreeSize, + base64.StdEncoding.EncodeToString(latest.RootHash[:]), + tileBasedHash) + } + + m.frontier = frontier + return nil +} + type pool struct { sync.RWMutex entries []pendingEntry maxSize int } -// pendingEntry represents an pending entry in the pool, along with a channel to notify a pending RPC. +// pendingEntry represents a pending entry in the pool, along with a channel to notify a pending RPC. type pendingEntry struct { mtcle *entry.MTCLogEntry ch chan<- int64 @@ -242,8 +289,29 @@ func (m *mtca) mtcLogID() string { return fmt.Sprintf("%s.0.%d", m.mtcaID, m.logNumber) } +// tileStoragePrefix returns the path within a bucket where we will store tiles. +// +// https://github.com/C2SP/C2SP/blob/main/mtc-tlog.md#serving-issuance-logs +// +// "Each log's prefix URL is the concatenation of the CA prefix URL and the log number, encoded as an +// ASCII decimal integer with no additional leading zeros: +// +// /" +// +// We assume we will serve directly from tile storage (likely sync'ed somewhere), so we +// want to store tiles compatible with that pattern, with log number as the last component +// of the path before "tile/". +// +// As a matter of local convention we will put the MTCA ID as the path component right before +// log number. +func (m *mtca) tileStoragePrefix() string { + return fmt.Sprintf("%s/%d", m.mtcaID, m.logNumber) +} + // Issue requests a TBSCertificateLogEntry be issued and returns after it's been sequenced into the log // and a new checkpoint signed by the CA. It does not wait for a mirror cosignature. +// +// Safe for concurrent calls. Implements a gRPC method. func (m *mtca) Issue(ctx context.Context, req *mtcapb.IssueRequest) (*mtcapb.IssueResponse, error) { key, err := x509.ParsePKIXPublicKey(req.Pubkey) if err != nil { @@ -279,7 +347,7 @@ func (m *mtca) Issue(ctx context.Context, req *mtcapb.IssueRequest) (*mtcapb.Iss mtcle, err := entry.FromX509(lintCertBytes, crypto.SHA256) if err != nil { - return nil, fmt.Errorf("generating MTCLogEntry: %s '%x'", err, lintCertBytes) + return nil, fmt.Errorf("generating MTCLogEntry: %s", err) } // We'll get notification of sequencing on this channel. Buffer it so `sequence()` doesn't @@ -309,6 +377,8 @@ func (m *mtca) Issue(ctx context.Context, req *mtcapb.IssueRequest) (*mtcapb.Iss // Loop periodically sequences all entries in the pool and sends notifications to the waiting RPCs. // +// Must be called after Preflight() returns success. +// // At process shutdown, this context should be canceled _after_ GracefulStop returns. That ensures // there are no inflight RPCs from clients, which in turn ensures that we have sequenced everything // had in the pool. @@ -385,7 +455,13 @@ func (m *mtca) fakePublisher(ctx context.Context) { // // Each entry in the pool will get a notification on its channel: either the index at which // it was sequenced, or -1 if there was an error during sequencing. +// +// Must only be called after Preflight() returns success. func (m *mtca) sequence(ctx context.Context) error { + if m.frontier == nil { + return fmt.Errorf("call mtca.Preflight() before sequencing") + } + if m.pool.len() == 0 { return nil } @@ -414,18 +490,36 @@ func (m *mtca) sequence(ctx context.Context) error { } }() - // Simulate writing to tile storage - latestTreeSize := latest.TreeSize - var entryIndexes []int64 + candidate := m.frontier.Clone() + + // Add leaves to the candidate. for _, e := range entries { - m.log.AuditInfo("Preparing to issue: %x", e) - entryIndexes = append(entryIndexes, latestTreeSize) - latestTreeSize++ + err = candidate.AppendEntry(e.mtcle) + if err != nil { + return err + } } - // TODO: calculate new root hash for real - var newRootHash [sha256.Size]byte - rand.Read(newRootHash[:]) + newRootHash := candidate.RootHash() + + // Log each leaf along with the root hash it will be included in. + for i, e := range entries { + m.log.AuditInfo("issuing", map[string]any{ + "TBSCertificateLogEntry": hex.EncodeToString(e.mtcle.TBS()), + "entryIndex": latest.TreeSize + int64(i), + "mtcLogID": m.mtcLogID(), + "newRootHash": newRootHash.String(), + }) + } + + // First stage to a pending area. After signing and storing to the DB + // (but before publishing a new checkpoint signed note), we will flush + // to the live location. This ensures we've persisted the tiles before + // committing to a tree hash by signing it. + err = candidate.Stage(ctx, m.s3c, m.tileStoragePrefix()) + if err != nil { + return fmt.Errorf("staging candidate tiles: %s", err) + } newCheckpoint := checkpoint{ ID: 0, @@ -433,7 +527,7 @@ func (m *mtca) sequence(ctx context.Context) error { MTCASignature: nil, MirrorID: "", MirrorSignature: nil, - TreeSize: latestTreeSize, + TreeSize: candidate.TreeSize(), RootHash: newRootHash[:], } @@ -510,9 +604,24 @@ func (m *mtca) sequence(ctx context.Context) error { return err } + m.frontier = candidate + + // Write the tiles to a live serving location. + // + // TODO(#8902): This should include indefinite retries on error. We've committed to the + // tree hash by signing it, so nothing can make progress until we've published the tiles. + // + // Once we add publishing of checkpoints as signed notes, publication of the signed note + // should come after this flush succeeds, so monitors don't try to fetch tiles that aren't + // yet available. + err = m.frontier.Publish(ctx, m.s3c, m.tileStoragePrefix()) + if err != nil { + return fmt.Errorf("publishing tiles: %s", err) + } + // Notify waiting RPCs. for i, e := range entries { - e.ch <- entryIndexes[i] + e.ch <- latest.TreeSize + int64(i) } // Empty out the entries list so the deferred error path doesn't try to notify them. entries = nil diff --git a/mtca/mtca_test.go b/mtca/mtca_test.go index 64dac42d2ee..dcf70623351 100644 --- a/mtca/mtca_test.go +++ b/mtca/mtca_test.go @@ -4,16 +4,20 @@ package mtca import ( "bytes" + "compress/gzip" "context" "crypto/ecdsa" "crypto/elliptic" "crypto/mldsa" + "crypto/rand" + "crypto/sha256" "crypto/x509" "database/sql" "encoding/base64" "encoding/hex" "errors" "fmt" + "io" "strings" "sync" "testing" @@ -23,6 +27,7 @@ import ( "github.com/jmhodges/clock" "github.com/letsencrypt/borp" + "github.com/letsencrypt/boulder/bs3/bs3test" "github.com/letsencrypt/boulder/config" corepb "github.com/letsencrypt/boulder/core/proto" "github.com/letsencrypt/boulder/issuance" @@ -30,10 +35,13 @@ import ( "github.com/letsencrypt/boulder/mtca/proto" "github.com/letsencrypt/boulder/test/vars" "github.com/letsencrypt/boulder/trees/cosigned" + "github.com/letsencrypt/boulder/trees/entry" + "github.com/letsencrypt/boulder/trees/tiles" ) -// setup returns a working mtca plus a cleanup function, or an error. -func setup() (*mtca, func(), error) { +// setup returns a working mtca, its fake tile storage, and a cleanup +// function, or an error. +func setup() (*mtca, *bs3test.FakeS3, func(), error) { issuer, err := issuance.LoadIssuer(issuance.IssuerConfig{ Profiles: []string{"some profile"}, IssuerURL: "http://ignored.letsencrypt.org", @@ -45,12 +53,12 @@ func setup() (*mtca, func(), error) { }, }, clock.NewFake()) if err != nil { - return nil, nil, err + return nil, nil, nil, err } db, err := sql.Open("mysql", vars.DBConnMTCMeta_44947_4_1_0_44FullPerms) if err != nil { - return nil, nil, err + return nil, nil, nil, err } dbMap := &borp.DbMap{Db: db, Dialect: borp.MySQLDialect{}} truncateTables(db) @@ -64,7 +72,7 @@ func setup() (*mtca, func(), error) { OmitClientAuth: true, OmitSKID: true, MTC: true, - MaxValidityPeriod: config.Duration{time.Hour}, + MaxValidityPeriod: config.Duration{Duration: time.Hour}, LintConfig: "", IgnoredLints: []string{ "w_ext_subject_key_identifier_missing_sub_cert", @@ -73,24 +81,32 @@ func setup() (*mtca, func(), error) { }, }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } + fs3 := bs3test.New() mtca, err := New( issuer, - profile, + map[string]*issuance.Profile{"mtcExample": profile}, 100*time.Millisecond, dbMap, - &fakeS3{}, + fs3, logger, clk) - mtca.InitLog(context.Background()) + if err != nil { + return nil, nil, nil, err + } + + err = mtca.InitLog(context.Background()) + if err != nil { + return nil, nil, nil, fmt.Errorf("initializing log: %w", err) + } cleanup := func() { truncateTables(db) } - return mtca, cleanup, nil + return mtca, fs3, cleanup, nil } func TestPool(t *testing.T) { @@ -169,20 +185,19 @@ func truncateTables(db *sql.DB) { db.Exec("TRUNCATE TABLE latestCheckpoint") } -func TestSequence(t *testing.T) { - mtca, cleanup, err := setup() - if err != nil { - t.Fatalf("setting up mtca: %s", err) - } - t.Cleanup(cleanup) - // An empty pool is a no-op regardless of checkpoint state. - err = mtca.sequence(t.Context()) - if err != nil { - t.Fatalf("sequencing with empty pool: %s", err) - } - // Fill the pool with five concurrent requests. - mtca.pool.maxSize = 5 +// issueResult is the outcome of one async Issue call, along with the values +// we expect to find in the entry sequenced for it. +type issueResult struct { + *proto.IssueResponse + err error + expectedSPKIHash [sha256.Size]byte + expectedDNSName string +} +// makeIssueRequest returns an IssueRequest with a freshly generated key and +// a random DNS name under example.com. +func makeIssueRequest(t *testing.T) *proto.IssueRequest { + t.Helper() key, err := ecdsa.GenerateKey(elliptic.P256(), nil) if err != nil { t.Fatalf("generating key: %s", err) @@ -192,32 +207,208 @@ func TestSequence(t *testing.T) { t.Fatalf("marshaling pubkey: %s", err) } - req := &proto.IssueRequest{ + var buf [4]byte + _, err = rand.Read(buf[:]) + if err != nil { + t.Fatalf("generating random DNS name: %s", err) + } + dnsName := fmt.Sprintf("%x.example.com", buf) + + return &proto.IssueRequest{ Pubkey: pubkeyBytes, Identifiers: []*corepb.Identifier{ - {Type: "dns", Value: "example.com"}, + {Type: "dns", Value: dnsName}, }, Profile: "mtcExample", } +} - type result struct { - *proto.IssueResponse - error - } - ch := make(chan result) - for i := 0; i < 5; i++ { +// issueMany calls Issue() `n` times concurrently and waits for all of the requests +// to be included in the pool. Returns a channel that will supply results once `Issue()` +// returns. +// +// Does not call `sequence()`. The caller is responsible for that. +func issueMany(t *testing.T, m *mtca, n int) <-chan issueResult { + t.Helper() + ch := make(chan issueResult, n) + for i := 0; i < n; i++ { + req := makeIssueRequest(t) go func() { - resp, err := mtca.Issue(t.Context(), req) - ch <- result{IssueResponse: resp, error: err} + resp, err := m.Issue(t.Context(), req) + ch <- issueResult{ + IssueResponse: resp, + err: err, + expectedSPKIHash: sha256.Sum256(req.Pubkey), + expectedDNSName: req.Identifiers[0].Value, + } }() } - // Issue blocks until sequencing, so wait for all five to be pooled. - for mtca.pool.len() < 5 { + // Waiting for inclusion in the pool lets us check "pool full" conditions. + for m.pool.len() < n { time.Sleep(time.Millisecond) } + return ch +} + +// errorS3 wraps a bs3test.FakeS3, failing every PutObject with `err` while +// it is non-nil and passing through to the wrapped fake otherwise. +type errorS3 struct { + *bs3test.FakeS3 + err error +} + +func (e *errorS3) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + if e.err != nil { + return nil, e.err + } + return e.FakeS3.PutObject(ctx, params, optFns...) +} + +// fakeMirror simulates mirror cosigning of the latest checkpoint so +// sequencing can proceed. +func fakeMirror(t *testing.T, m *mtca) { + t.Helper() + latest, err := m.latestCheckpoint(t.Context()) + if err != nil { + t.Fatalf("getting latest: %s", err) + } + latest.MirrorID = "fake" + latest.MirrorSignature = []byte("fake") + _, err = m.db.Update(t.Context(), latest) + if err != nil { + t.Fatalf("updating checkpoint with fake mirror signature: %s", err) + } +} + +// verifyStores checks at a given point in time that tree size and root hash +// are the same between: +// +// - m.frontier +// - m.latestCheckpoint() +// - fake tile storage +func verifyStores(t *testing.T, m *mtca, fs3 *bs3test.FakeS3) *checkpoint { + t.Helper() + latest, err := m.latestCheckpoint(t.Context()) + if err != nil { + t.Fatalf("getting latest: %s", err) + } + + memTreeSize := m.frontier.TreeSize() + if memTreeSize != latest.TreeSize { + t.Errorf("in-memory frontier TreeSize %d != DB TreeSize %d", memTreeSize, latest.TreeSize) + } + memHash := m.frontier.RootHash() + if !bytes.Equal(latest.RootHash, memHash[:]) { + t.Errorf("in-memory frontier RootHash %s != DB RootHash %s", + memHash, base64.StdEncoding.EncodeToString(latest.RootHash)) + } + + loaded, err := tiles.LoadFrontier(t.Context(), fs3, latest.TreeSize, m.tileStoragePrefix()) + if err != nil { + t.Fatalf("loading frontier from tile storage: %s", err) + } + tileHash := loaded.RootHash() + if !bytes.Equal(latest.RootHash, tileHash[:]) { + t.Errorf("tile storage RootHash %s != DB RootHash %s", + tileHash, base64.StdEncoding.EncodeToString(latest.RootHash)) + } + return latest +} + +// validateStoredEntries reads and parses the rightmost entries tile from +// storage, then verifies that the entry at each index in `got` contains the +// SPKI hash and DNS name of the request that was assigned that index. +// +// Only valid for tree sizes below 256. +func validateStoredEntries(t *testing.T, fs3 *bs3test.FakeS3, prefix string, treeSize int64, got map[int64]issueResult) { + t.Helper() + obj, ok := fs3.Objects[fmt.Sprintf("%s/tile/entries/000.p/%d", prefix, treeSize)] + if !ok { + t.Fatalf("no entries tile in storage for tree size %d", treeSize) + } + data := obj.Data + if obj.ContentEncoding != nil && *obj.ContentEncoding == "gzip" { + r, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + t.Fatalf("decompressing entries tile: %s", err) + } + data, err = io.ReadAll(r) + if err != nil { + t.Fatalf("decompressing entries tile: %s", err) + } + } + br := entry.NewBundleReader(data) + var entries []*entry.MTCLogEntry + for { + mtcle, _, err := br.ReadEntry() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("parsing entries tile: %s", err) + } + entries = append(entries, mtcle) + } + if int64(len(entries)) != treeSize { + t.Fatalf("entries tile at tree size %d: got %d entries", treeSize, len(entries)) + } + + for idx, res := range got { + tbs := entries[idx].TBS() + if !bytes.Contains(tbs, res.expectedSPKIHash[:]) { + t.Errorf("entry at index %d does not contain the SPKI hash of the request that was assigned that index", idx) + } + if !bytes.Contains(tbs, []byte(res.expectedDNSName)) { + t.Errorf("entry at index %d does not contain the DNS name %q of the request that was assigned that index", idx, res.expectedDNSName) + } + } +} + +// collectResults reads n results. The entryIndexes must exactly cover [firstIndex, firstIndex+n). +// +// Returns a map from entry index to the result of the request that received that index. +func collectResults(t *testing.T, results <-chan issueResult, firstIndex int64, n int) map[int64]issueResult { + t.Helper() + got := map[int64]issueResult{} + for i := 0; i < n; i++ { + res := <-results + if res.err != nil { + t.Errorf("Issue: %s", res.err) + continue + } + _, ok := got[res.MtcEntryIndex] + if ok { + t.Errorf("entryIndex %d returned twice", res.MtcEntryIndex) + } + got[res.MtcEntryIndex] = res + } + for i := firstIndex; i < firstIndex+int64(n); i++ { + _, ok := got[i] + if !ok { + t.Errorf("no Issue call got entryIndex %d", i) + } + } + return got +} + +func TestSequence(t *testing.T) { + mtca, fs3, cleanup, err := setup() + if err != nil { + t.Fatalf("setting up mtca: %s", err) + } + t.Cleanup(cleanup) + // An empty pool is a no-op regardless of checkpoint state. + err = mtca.sequence(t.Context()) + if err != nil { + t.Fatalf("sequencing with empty pool: %s", err) + } + // Fill the pool with five concurrent requests. + mtca.pool.maxSize = 5 + results := issueMany(t, mtca, 5) // With the pool full, a sixth request should fail. + req := makeIssueRequest(t) _, err = mtca.Issue(t.Context(), req) if err == nil { t.Fatal("Issue with a full pool: got nil error, want error") @@ -236,46 +427,82 @@ func TestSequence(t *testing.T) { if mtca.pool.len() != 5 { t.Errorf("pool after refused sequencing: got len %d, want 5", mtca.pool.len()) } - // Fake publication - latest, err := mtca.latestCheckpoint(t.Context()) + + fakeMirror(t, mtca) + + // Now sequencing should succeed, assigning indexes 1 through 5 (the + // genesis null entry occupies index 0). + err = mtca.sequence(t.Context()) if err != nil { - t.Fatalf("getting latest: %s", err) + t.Fatalf("sequencing with waiting entries: %s", err) } - latest.MirrorID = "fake" - latest.MirrorSignature = []byte("fake") - _, err = mtca.db.Update(t.Context(), latest) + got := collectResults(t, results, 1, 5) + + // The in-memory frontier, DB checkpoint, and stored tiles must agree, + // and the resulting checkpoint signature must be valid. + latest := verifyStores(t, mtca, fs3) + verifyCheckpoint(t, mtca, latest) + + // Each client's returned index must point at its own entry in the + // published entries tile. + validateStoredEntries(t, fs3, mtca.tileStoragePrefix(), latest.TreeSize, got) +} + +// TestSequenceStorageFailure checks that a failed sequencing pass leaves the +// in-memory frontier consistent with the database, and that sequencing +// recovers cleanly once storage is healthy again. +func TestSequenceStorageFailure(t *testing.T) { + mtca, fs3, cleanup, err := setup() if err != nil { - t.Fatalf("updating checkpoint with fake mirror signature: %s", err) + t.Fatalf("setting up mtca: %s", err) } - // Now sequencing should succeed. + t.Cleanup(cleanup) + fakeMirror(t, mtca) + + mtca.pool.maxSize = 2 + results := issueMany(t, mtca, 2) + + // Fail writing the tiles. + es3 := &errorS3{FakeS3: fs3, err: errors.New("the tiles will not tesselate")} + mtca.s3c = es3 err = mtca.sequence(t.Context()) - if err != nil { - t.Fatalf("sequencing with waiting entries: %s", err) + if err == nil { + t.Fatal("sequencing with failing storage: got nil error, want error") } - // The five requests should now be unblocked and return results. - seenIDs := map[int64]bool{} - for i := 0; i < 5; i++ { - res := <-ch - if res.error != nil { - t.Errorf("Issue: %s", res.error) - continue - } - if res.MtcEntryIndex < 1 || res.MtcEntryIndex > 5 || seenIDs[res.MtcEntryIndex] { - t.Errorf("entryIndex %d out of range or seen twice", res.MtcEntryIndex) + if !strings.Contains(err.Error(), "staging") { + t.Errorf("sequencing with failing storage: got %q, want a staging error", err) + } + + // The waiting RPCs should get their responses. + for i := 0; i < 2; i++ { + res := <-results + if res.err == nil { + t.Errorf("Issue with failing storage: got entryIndex %d, want error", res.MtcEntryIndex) } - seenIDs[res.MtcEntryIndex] = true } - // Lastly, verify the resulting checkpoint is valid. - latest, err = mtca.latestCheckpoint(t.Context()) + // The in-memory frontier must be unchanged, still matching the DB. + verifyStores(t, mtca, fs3) + + // Storage is working again! + es3.err = nil + results = issueMany(t, mtca, 2) + err = mtca.sequence(t.Context()) if err != nil { - t.Fatalf("getting latest: %s", err) + t.Fatalf("sequencing after storage recovered: %s", err) } - verify(t, mtca, latest) + // After storage recovered, we should pick up where we left off + // and sequence 2 entries starting at entryIndex 1. + got := collectResults(t, results, 1, 2) + + latest := verifyStores(t, mtca, fs3) + verifyCheckpoint(t, mtca, latest) + + validateStoredEntries(t, fs3, mtca.tileStoragePrefix(), latest.TreeSize, got) } func TestInitLog(t *testing.T) { - mtca, cleanup, err := setup() + mtca, _, cleanup, err := setup() if err != nil { t.Fatalf("setting up mtca: %s", err) } @@ -300,10 +527,10 @@ func TestInitLog(t *testing.T) { t.Errorf("just-initialized log: got RootHash %x, want %x", latest.RootHash, expected) } - verify(t, mtca, latest) + verifyCheckpoint(t, mtca, latest) } -func verify(t *testing.T, mtca *mtca, checkpoint *checkpoint) { +func verifyCheckpoint(t *testing.T, mtca *mtca, checkpoint *checkpoint) { t.Helper() message := cosigned.Message{ CosignerName: fmt.Sprintf("oid/1.3.6.1.4.1.%s", mtca.mtcaID), @@ -358,18 +585,3 @@ DgQIBAaC3xMBAgEwCgYIKoZIzj0EAwIDQQAwPgIdAMebuq7759hyFC3hjrVUEaXk t.Errorf("getMTCAID(): got %s, want %s", mtcaID, expected) } } - -type fakeS3 struct { -} - -func (f *fakeS3) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { - return nil, fmt.Errorf("unimplemented") -} - -func (f *fakeS3) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { - return nil, fmt.Errorf("unimplemented") -} - -func (f *fakeS3) Bucket() string { - return "fake bucket" -} diff --git a/test/config-next/mtca.json b/test/config-next/mtca.json index fc64cb983d1..8abad92e7b6 100644 --- a/test/config-next/mtca.json +++ b/test/config-next/mtca.json @@ -68,7 +68,7 @@ } }, "syslog": { - "stdoutlevel": 4, + "stdoutlevel": 6, "sysloglevel": -1 }, "openTelemetry": { diff --git a/test/mtca/README.md b/test/mtca/README.md index b3eb48a5a4f..cf3eda0f768 100644 --- a/test/mtca/README.md +++ b/test/mtca/README.md @@ -1,5 +1,23 @@ This directory contains a handful of utility scripts for manually interacting -with boulder-mtca. +with boulder-mtca within a running boulder container. - clear.sh: clear the demo log SQL DB (only works for MariaDB). - start.sh: run boulder-mtca as a single component. + +To list all tiles (while bminio is running): + + alias mc="cd ~/boulder && docker compose exec bminio mc" + mc ls -r local/boulder-mtc-tiles/ + +To see a specific tile: + + mc cat local/boulder-mtc-tiles/tile/entries/011.p/9 > 9.gz + +To clear tile storage (note: also run clear.sh to clear the DB): + + mc rm -r --force local/boulder-mtc-tiles/ + +Alternatively, to reset all storage (DB and tiles), you can always run, +from outside the containers: + + docker compose down --volumes diff --git a/trees/tiles/tiles.go b/trees/tiles/tiles.go index cf8cce857b3..67b5f6735a6 100644 --- a/trees/tiles/tiles.go +++ b/trees/tiles/tiles.go @@ -19,6 +19,7 @@ import ( "bytes" "compress/gzip" "context" + "encoding/hex" "errors" "fmt" "io" @@ -88,6 +89,38 @@ type Frontier struct { treeSize int64 } +// Clone creates a copy of Frontier that doesn't share any memory with the original. +// +// During sequencing in the MTCA we want to use a temporary copy of the frontier, so +// that if we error out the original in-memory Frontier stays untouched. +func (f *Frontier) Clone() *Frontier { + var hashesTiles []*hashesTile + for _, ht := range f.hashesTiles { + hashesTiles = append(hashesTiles, ht.clone()) + } + + var fullHashesTiles []*hashesTile + for _, ht := range f.fullHashesTiles { + fullHashesTiles = append(fullHashesTiles, ht.clone()) + } + + entries := f.entryTile.clone() + + var fullEntryTiles []*entryTile + for _, et := range f.fullEntryTiles { + fullEntryTiles = append(fullEntryTiles, et.clone()) + } + + return &Frontier{ + hashesTiles: hashesTiles, + entryTile: entries, + dirtyLevel: f.dirtyLevel, + fullHashesTiles: fullHashesTiles, + fullEntryTiles: fullEntryTiles, + treeSize: f.treeSize, + } +} + // hashesTile represents a tile containing hashes. // It may be empty, partial, or full. If it's full it can only // be part of fullHashesTiles. @@ -99,6 +132,13 @@ type hashesTile struct { data []tlog.Hash } +func (h *hashesTile) clone() *hashesTile { + return &hashesTile{ + coords: h.coords, + data: slices.Clone(h.data), + } +} + func (h *hashesTile) append(val tlog.Hash) { h.coords.W++ h.data = append(h.data, val) @@ -113,6 +153,16 @@ type entryTile struct { data []byte } +func (e *entryTile) clone() *entryTile { + if e == nil { + return nil + } + return &entryTile{ + coords: e.coords, + data: bytes.Clone(e.data), + } +} + func (e *entryTile) append(val []byte) { e.coords.W++ e.data = append(e.data, val...) @@ -345,13 +395,39 @@ func (f *Frontier) appendHash(val tlog.Hash, level int) { f.dirtyLevel = max(f.dirtyLevel, level) } -// Flush writes all dirty tiles to storage and clears their dirty status. +// Stage writes all dirty tiles to storage but does not clear their dirty status. // -// If it errors partway, dirty status is not reset but subsequent flushes -// will likely fail (duplicate writes). +// Tiles are written to a prefix determined by the tree size and root hash, so as +// not to conflict with live-published tiles. // // Errors if the tree is empty. -func (f *Frontier) Flush(ctx context.Context, s3c simpleS3, prefix string) error { +func (f *Frontier) Stage(ctx context.Context, s3c simpleS3, prefix string) error { + rootHash := f.RootHash() + return f.store(ctx, s3c, + fmt.Sprintf("pending/%s/%d-%s", prefix, f.TreeSize(), + hex.EncodeToString(rootHash[:]))) +} + +// Publish writes all dirty tiles to storage and clears their dirty status. +// +// Errors if the tree is empty. +// +// TODO(#8902): This should use CopyObject, and allow overwriting. +func (f *Frontier) Publish(ctx context.Context, s3c simpleS3, prefix string) error { + err := f.store(ctx, s3c, prefix) + if err != nil { + return err + } + + f.fullHashesTiles = nil + f.fullEntryTiles = nil + f.dirtyLevel = -1 + + return nil +} + +// store stores all tiles to the given prefix. +func (f *Frontier) store(ctx context.Context, s3c simpleS3, prefix string) error { if f.treeSize == 0 { return fmt.Errorf("an empty tree has nothing to write") } @@ -386,9 +462,6 @@ func (f *Frontier) Flush(ctx context.Context, s3c simpleS3, prefix string) error } } - f.fullHashesTiles = nil - f.fullEntryTiles = nil - f.dirtyLevel = -1 return nil } @@ -523,7 +596,7 @@ func getTile(ctx context.Context, s3c simpleS3, coords tlog.Tile, prefix string) Key: &key, }) if err != nil { - return nil, err + return nil, fmt.Errorf("fetching s3://%s/%s: %w", bucket, key, err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) diff --git a/trees/tiles/tiles_test.go b/trees/tiles/tiles_test.go index 85cd19c45c4..7c531e0acf5 100644 --- a/trees/tiles/tiles_test.go +++ b/trees/tiles/tiles_test.go @@ -3,22 +3,17 @@ package tiles import ( "bytes" "compress/gzip" - "context" "encoding/base64" "errors" "fmt" - "io" - "net/http" "reflect" "slices" "testing" - awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" - "github.com/aws/aws-sdk-go-v2/service/s3" - smithyhttp "github.com/aws/smithy-go/transport/http" "golang.org/x/crypto/cryptobyte" "golang.org/x/mod/sumdb/tlog" + "github.com/letsencrypt/boulder/bs3/bs3test" "github.com/letsencrypt/boulder/trees/entry" "github.com/letsencrypt/boulder/trees/subtree" ) @@ -57,54 +52,6 @@ func TestPath(t *testing.T) { } } -type fakeS3Object struct { - data []byte - contentEncoding *string -} - -type fakeS3 struct { - objects map[string]fakeS3Object - puts int -} - -func newFakeS3() *fakeS3 { - return &fakeS3{objects: make(map[string]fakeS3Object)} -} - -func (f *fakeS3) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { - _, ok := f.objects[*params.Key] - if ok { - return nil, &awshttp.ResponseError{ - ResponseError: &smithyhttp.ResponseError{ - Response: &smithyhttp.Response{Response: &http.Response{StatusCode: http.StatusPreconditionFailed}}, - Err: errors.New("PreconditionFailed"), - }, - } - } - body, err := io.ReadAll(params.Body) - if err != nil { - return nil, err - } - f.objects[*params.Key] = fakeS3Object{data: body, contentEncoding: params.ContentEncoding} - f.puts++ - return nil, nil -} - -func (f *fakeS3) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { - o, ok := f.objects[*params.Key] - if ok { - return &s3.GetObjectOutput{ - Body: io.NopCloser(bytes.NewReader(o.data)), - ContentEncoding: o.contentEncoding, - }, nil - } - return nil, fmt.Errorf("not found") -} - -func (f *fakeS3) Bucket() string { - return "fakebucket" -} - // testEntryBody returns the marshaled MTCLogEntry bytes for index i: empty // extensions, type tbs_cert_entry, and a unique value with length varying by // index (not a real TBSCertificateLogEntry). @@ -132,7 +79,7 @@ func testEntry(i int) *entry.MTCLogEntry { // appendEntries appends test entries [start, end) to `f`, flushing every // `flushEvery“ appends and once at the end. -func appendEntries(t *testing.T, f *Frontier, fs3 *fakeS3, start, end int, prefix string, flushEvery int) { +func appendEntries(t *testing.T, f *Frontier, fs3 *bs3test.FakeS3, start, end int, prefix string, flushEvery int) { t.Helper() for i := start; i < end; i++ { err := f.AppendEntry(testEntry(i)) @@ -140,13 +87,13 @@ func appendEntries(t *testing.T, f *Frontier, fs3 *fakeS3, start, end int, prefi t.Fatalf("AppendEntry(%d): %s", i, err) } if (i+1)%flushEvery == 0 { - err := f.Flush(t.Context(), fs3, prefix) + err := f.Publish(t.Context(), fs3, prefix) if err != nil { t.Fatalf("Flush after %d entries: %s", i+1, err) } } } - err := f.Flush(t.Context(), fs3, prefix) + err := f.Publish(t.Context(), fs3, prefix) if err != nil { t.Fatalf("final Flush: %s", err) } @@ -154,7 +101,7 @@ func appendEntries(t *testing.T, f *Frontier, fs3 *fakeS3, start, end int, prefi // buildFrontier appends `n` test entries to a new Frontier, flushing every // `flushEvery` appends and once at the end. -func buildFrontier(t *testing.T, fs3 *fakeS3, n int, prefix string, flushEvery int) *Frontier { +func buildFrontier(t *testing.T, fs3 *bs3test.FakeS3, n int, prefix string, flushEvery int) *Frontier { t.Helper() f := &Frontier{} appendEntries(t, f, fs3, 0, n, prefix, flushEvery) @@ -177,7 +124,7 @@ func parseEntries(t *testing.T, body []byte) [][]byte { } func TestGetTile(t *testing.T) { - fs3 := newFakeS3() + fs3 := bs3test.New() gzipStr := "gzip" gz := func(data []byte) []byte { @@ -204,7 +151,7 @@ func TestGetTile(t *testing.T) { }) t.Run("no content encoding", func(t *testing.T) { - fs3.objects["tile/0/000.p/1"] = fakeS3Object{data: raw} + fs3.Objects["tile/0/000.p/1"] = bs3test.StoredObject{Data: raw} got, err := getTile(t.Context(), fs3, tlog.Tile{L: 0, N: 0, W: 1}, "") if err != nil { t.Fatalf("getTile: %s", err) @@ -215,7 +162,7 @@ func TestGetTile(t *testing.T) { }) t.Run("gzip", func(t *testing.T) { - fs3.objects["tile/entries/000.p/1"] = fakeS3Object{data: gz([]byte("compressed tile bytes")), contentEncoding: &gzipStr} + fs3.Objects["tile/entries/000.p/1"] = bs3test.StoredObject{Data: gz([]byte("compressed tile bytes")), ContentEncoding: &gzipStr} got, err := getTile(t.Context(), fs3, tlog.Tile{L: -1, N: 0, W: 1}, "") if err != nil { t.Fatalf("getTile: %s", err) @@ -226,7 +173,7 @@ func TestGetTile(t *testing.T) { }) t.Run("corrupt gzip", func(t *testing.T) { - fs3.objects["tile/entries/000.p/2"] = fakeS3Object{data: []byte("not gzip"), contentEncoding: &gzipStr} + fs3.Objects["tile/entries/000.p/2"] = bs3test.StoredObject{Data: []byte("not gzip"), ContentEncoding: &gzipStr} _, err := getTile(t.Context(), fs3, tlog.Tile{L: -1, N: 0, W: 2}, "") if err == nil { t.Errorf("getTile of corrupt gzip body: got nil, want error") @@ -234,7 +181,7 @@ func TestGetTile(t *testing.T) { }) t.Run("prefix", func(t *testing.T) { - fs3.objects["pre/tile/entries/000.p/3"] = fakeS3Object{data: gz([]byte("prefixed")), contentEncoding: &gzipStr} + fs3.Objects["pre/tile/entries/000.p/3"] = bs3test.StoredObject{Data: gz([]byte("prefixed")), ContentEncoding: &gzipStr} got, err := getTile(t.Context(), fs3, tlog.Tile{L: -1, N: 0, W: 3}, "pre/") if err != nil { t.Fatalf("getTile with prefix: %s", err) @@ -254,7 +201,7 @@ func TestRoundTrip(t *testing.T) { sizes := []int{1, 2, 100, 255, 256, 257, 511, 512, 513, 767, 768, 1000} for _, n := range sizes { t.Run(fmt.Sprintf("size %d", n), func(t *testing.T) { - fs3 := newFakeS3() + fs3 := bs3test.New() prefix := "p/" f := buildFrontier(t, fs3, n, prefix, 100) @@ -293,7 +240,7 @@ func TestStorageCorrectness(t *testing.T) { leafHashes[i] = tlog.RecordHash(bodies[i]) } - fs3 := newFakeS3() + fs3 := bs3test.New() buildFrontier(t, fs3, reloadAt, prefix, 999) f, err := LoadFrontier(t.Context(), fs3, reloadAt, prefix) @@ -376,13 +323,13 @@ func TestStorageCorrectness(t *testing.T) { } func TestLoadBadHashTile(t *testing.T) { - fs3 := newFakeS3() + fs3 := bs3test.New() buildFrontier(t, fs3, 1, "", 100) // Corrupt the level-0 hash tile so its length no longer matches its width. - o := fs3.objects["tile/0/000.p/1"] - o.data = o.data[:len(o.data)-1] - fs3.objects["tile/0/000.p/1"] = o + o := fs3.Objects["tile/0/000.p/1"] + o.Data = o.Data[:len(o.Data)-1] + fs3.Objects["tile/0/000.p/1"] = o _, err := LoadFrontier(t.Context(), fs3, 1, "") if err == nil { @@ -409,12 +356,12 @@ func TestLoadBadEntryTile(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fs3 := newFakeS3() + fs3 := bs3test.New() buildFrontier(t, fs3, 2, "", 100) // Replace the entry tile with the bad body (uncompressed, so no // ContentEncoding). - fs3.objects["tile/entries/000.p/2"] = fakeS3Object{data: tc.body} + fs3.Objects["tile/entries/000.p/2"] = bs3test.StoredObject{Data: tc.body} _, err := LoadFrontier(t.Context(), fs3, 2, "") if err == nil { @@ -425,7 +372,7 @@ func TestLoadBadEntryTile(t *testing.T) { } func TestLoadEmptyTree(t *testing.T) { - _, err := LoadFrontier(t.Context(), newFakeS3(), 0, "") + _, err := LoadFrontier(t.Context(), bs3test.New(), 0, "") if err == nil { t.Errorf("Load(0): got nil, want error") } @@ -444,7 +391,7 @@ func TestAppendMaxSizeEntry(t *testing.T) { if err != nil { t.Fatalf("AppendEntry(65535-byte entry): %s", err) } - err = f.Flush(t.Context(), newFakeS3(), "") + err = f.Publish(t.Context(), bs3test.New(), "") if err != nil { t.Fatalf("Flush: %s", err) } @@ -455,7 +402,7 @@ func TestAppendMaxSizeEntry(t *testing.T) { func TestFlushEmptyTree(t *testing.T) { f := &Frontier{} - err := f.Flush(t.Context(), newFakeS3(), "") + err := f.Publish(t.Context(), bs3test.New(), "") if err == nil { t.Errorf("Flush of empty tree: got nil, want error") } @@ -464,16 +411,16 @@ func TestFlushEmptyTree(t *testing.T) { // TestFlushClean checks the dirtyLevel behavior. After a Flush, // a re-Flush should write nothing. func TestFlushClean(t *testing.T) { - fs3 := newFakeS3() + fs3 := bs3test.New() f := buildFrontier(t, fs3, 10, "", 100) - putsBefore := fs3.puts - err := f.Flush(t.Context(), fs3, "") + objectsBefore := len(fs3.Objects) + err := f.Publish(t.Context(), fs3, "") if err != nil { t.Fatalf("second Flush: %s", err) } - if fs3.puts != putsBefore { - t.Errorf("second Flush wrote %d objects, want 0", fs3.puts-putsBefore) + if len(fs3.Objects) != objectsBefore { + t.Errorf("second Flush wrote %d objects, want 0", len(fs3.Objects)-objectsBefore) } } @@ -506,7 +453,7 @@ func TestRootHash(t *testing.T) { } func TestWriteTileThatAlreadyExists(t *testing.T) { - fs3 := newFakeS3() + fs3 := bs3test.New() coords := tlog.Tile{L: 0, N: 0, W: 1} body := bytes.Repeat([]byte{'h'}, tlog.HashSize) @@ -520,3 +467,72 @@ func TestWriteTileThatAlreadyExists(t *testing.T) { t.Errorf("second writeTile: got %s, want ErrTileExists", err) } } + +// TestClone checks that a cloned Frontier shares no mutable state with the +// original: growing the clone across a full-tile boundary must leave the +// original unchanged, and appending the same entries to the original must +// produce an identical tree that publishes identical tiles. +func TestClone(t *testing.T) { + // Append more than one full tile's worth of entries, to exercise + // fullHashesTiles and fullEntryTiles. + frontier := &Frontier{} + for i := 0; i < 260; i++ { + err := frontier.AppendEntry(testEntry(i)) + if err != nil { + t.Fatalf("AppendEntry(%d): %s", i, err) + } + } + + clonedAt := frontier.TreeSize() + + candidate := frontier.Clone() + if !reflect.DeepEqual(frontier, candidate) { + t.Fatalf("clone differs from original:\ngot %#v\nwant %#v", candidate, frontier) + } + + rootBefore := frontier.RootHash() + + // Add another full tile's worth of entries. + const growTo = 520 + for i := clonedAt; i < growTo; i++ { + err := candidate.AppendEntry(testEntry(int(i))) + if err != nil { + t.Fatalf("AppendEntry(%d) to clone: %s", i, err) + } + } + + gotTreeSize := frontier.TreeSize() + if gotTreeSize != clonedAt { + t.Errorf("original TreeSize after appending to clone: got %d, want %d", gotTreeSize, clonedAt) + } + + gotRootHash := frontier.RootHash() + if gotRootHash != rootBefore { + t.Errorf("original RootHash after appending to clone: got %s, want %s", gotRootHash, rootBefore) + } + + // Append the same entries to the original and check for equality. + for i := clonedAt; i < growTo; i++ { + err := frontier.AppendEntry(testEntry(int(i))) + if err != nil { + t.Fatalf("AppendEntry(%d) to original: %s", i, err) + } + } + if !reflect.DeepEqual(frontier, candidate) { + t.Errorf("after identical appends, original differs from clone:\ngot %#v\nwant %#v", frontier, candidate) + } + + // Published tiles must be the same, too. + fs3f, fs3c := bs3test.New(), bs3test.New() + err := frontier.Publish(t.Context(), fs3f, "") + if err != nil { + t.Fatalf("publishing original: %s", err) + } + err = candidate.Publish(t.Context(), fs3c, "") + if err != nil { + t.Fatalf("publishing clone: %s", err) + } + if !reflect.DeepEqual(fs3f.Objects, fs3c.Objects) { + t.Errorf("original and clone published different tiles") + } +}