diff --git a/.gitignore b/.gitignore index e15aea4c0f76..8e4e6abc3444 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,6 @@ CLAUDE.md docs .codegraph .mcp.json + +# Scratch plans (not part of the codebase) +plans/2*.md diff --git a/Jenkinsfile b/Jenkinsfile index a5911f72f0ba..57b375581145 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -257,6 +257,17 @@ helpers.rootLinuxNode(env, { "NODE_PATH=${env.HOME}/.node/lib/node_modules:${env.NODE_PATH}", "NODE_OPTIONS=--max-old-space-size=4096", ]) { + // Native C++ framing tests. Needs only a C++ compiler and the + // vendored msgpack headers (the script fetches them if yarn's + // darwin-only postinstall didn't), no JSI/RN, so run it before + // the JS suite rather than behind it -- it takes seconds and + // failFast would otherwise hide it whenever JS is red. + stage("Native framing tests") { + sh "./rnmodules/react-native-kb/scripts/test-framing.sh" + // Same deal, and not even msgpack: pure arithmetic extracted + // from the iOS reader loop's engine-reset emit backoff. + sh "./rnmodules/react-native-kb/scripts/test-engine-reset-backoff.sh" + } dir("shared") { stage("JS Tests") { sh "git config --global user.name 'Keybase Jenkins'" @@ -526,11 +537,30 @@ def testGo(prefix, packagesToTest, hasKBFSChanges) { test_go_test_suite: { testGoTestSuite(prefix, packagesToTest) }, + test_go_bind: { + testGoBind(prefix) + }, failFast: true ) }} } +// go/bind is deliberately absent from the generic sweep (getTestDirsNix, +// getTestDirsWindows, getPackagesToTest): it is the gomobile entry point and +// used to fail to link a test binary at all because of duplicate quarantineFile +// C symbols (fixed in fb1420066c / a4281e4026). It links now, so run its suite +// explicitly rather than reintroducing it to the sweep -- the sweep runs under +// citogo with per-package flags, and this package wants nothing but -race. +// Linux/darwin only: the Windows path has never built this package. +def testGoBind(prefix) { + if (prefix != "test_linux_go_") { + return + } + timeout(activity: true, time: 10, unit: 'MINUTES') { + sh "go test -race -count=1 ./bind/..." + } +} + def testGoBuilds(prefix, packagesToTest, hasKBFSChanges) { if (prefix == "test_linux_go_") { dir("keybase") { diff --git a/go/bind/keybase.go b/go/bind/keybase.go index 84983ccf4a73..8a0de20c60a7 100644 --- a/go/bind/keybase.go +++ b/go/bind/keybase.go @@ -58,9 +58,22 @@ var ( var ( jsReadyOnce sync.Once jsReadyCh = make(chan struct{}) - connMutex sync.Mutex // Protects conn operations + connMutex sync.Mutex // Protects conn and connEpoch ) +// connEpoch increments every time ensureConnection successfully (re)dials. +// Callers that observed a failure on a particular conn capture the epoch +// alongside it (while holding connMutex) and pass it to ResetIfCurrent, +// which only tears down the connection if nothing has re-dialed since — +// otherwise some other caller already recovered and this call is a stale, +// no-op complaint. Protected by connMutex. +var connEpoch int64 + +// lastReadEpoch is the epoch of the connection the most recent ReadArr call +// read (or is about to read) from, captured atomically with the connection +// reference itself. See LastReadEpoch. Protected by connMutex. +var lastReadEpoch int64 + func describeConn(c net.Conn) string { if c == nil { return "" @@ -571,6 +584,7 @@ func WriteArr(b []byte) (err error) { } } currentConn := conn + currentEpoch := connEpoch connMutex.Unlock() if currentConn == nil { @@ -586,7 +600,17 @@ func WriteArr(b []byte) (err error) { if n != len(bytes) { log("Go: WriteArr short write conn=%s wrote=%d expected=%d appState=%s", describeConn(currentConn), n, len(bytes), appStateForLog()) - return errors.New("Did not write all the data") + // Not reachable through LoopbackConn today, whose Write is + // all-or-nothing. If a future transport can short-write, the peer's + // framer is left holding a partial frame and every later write would + // be consumed as its remainder, so drop the connection rather than + // corrupt the stream indefinitely. ResetIfCurrent (rather than Reset) + // so this doesn't tear down a connection some concurrent caller has + // already redialed since currentConn was captured above. + if ierr := ResetIfCurrent(currentEpoch); ierr != nil { + log("failed to Reset after short write: %v", ierr) + } + return fmt.Errorf("Did not write all the data: wrote %d of %d", n, len(bytes)) } return nil } @@ -595,7 +619,12 @@ func WriteArr(b []byte) (err error) { var buffer []byte // ReadArr is a blocking read for msgpack rpc data. -// It is called serially by the mobile run loops. +// It must still be called serially by the mobile run loops: conn.Read +// ordering depends on it, and the msgpack::unpacker behind onDataFromGo +// on the C++ side is not thread-safe. The returned slice is this call's own +// copy, which only protects the caller of ReadArr — a second concurrent +// caller would still interleave its Read into the shared package-level +// buffer above, garbling frame content, not just ordering. func ReadArr() (data []byte, err error) { defer func() { err = flattenError(err) }() @@ -611,6 +640,8 @@ func ReadArr() (data []byte, err error) { } } currentConn := conn + currentEpoch := connEpoch + lastReadEpoch = currentEpoch connMutex.Unlock() if currentConn == nil { @@ -633,15 +664,21 @@ func ReadArr() (data []byte, err error) { } // Deliver data even if err != nil (allowed by the net.Conn // contract); a persistent failure is returned by a later call. - // Returning a view of the shared buffer is safe because gomobile - // copies the bytes across the boundary and ReadArr is called - // serially. - return buffer[0:n], nil + // + // Copy out of the shared buffer rather than returning a view of it: + // gomobile copies again at the JNI/ObjC boundary regardless, so this + // costs one extra memcpy of a few KB on a call already crossing a + // language boundary, and it removes buffer aliasing as a hazard. + out := make([]byte, n) + copy(out, buffer[:n]) + return out, nil } if err != nil { - // Attempt to fix the connection - if ierr := Reset(); ierr != nil { + // Attempt to fix the connection. ResetIfCurrent so a stale failure + // on an already-superseded conn doesn't tear down one a concurrent + // WriteArr just redialed. + if ierr := ResetIfCurrent(currentEpoch); ierr != nil { log("failed to Reset: %v", ierr) } return nil, fmt.Errorf("Read error: %s", err) @@ -677,21 +714,98 @@ func ensureConnection() error { log("ensureConnection: Dial failed after restart: %v", err) return fmt.Errorf("failed to dial after loopback restart: %s", err) } - log("ensureConnection: loopback server restarted successfully conn=%s appState=%s", - describeConn(conn), appStateForLog()) + connEpoch++ + log("ensureConnection: loopback server restarted successfully conn=%s epoch=%d appState=%s", + describeConn(conn), connEpoch, appStateForLog()) return nil } - log("Go: Established loopback connection conn=%s appState=%s", - describeConn(conn), appStateForLog()) + connEpoch++ + log("Go: Established loopback connection conn=%s epoch=%d appState=%s", + describeConn(conn), connEpoch, appStateForLog()) return nil } -// Reset resets the socket connection +// Reset unconditionally resets the socket connection. Use this only when the +// caller genuinely means "tear down whatever connection is current" (e.g. +// iOS invalidate, Android destroy/engineReset) — it will happily close a +// connection some concurrent failure-driven caller never saw fail. Callers +// reacting to a failure on a specific connection should use ResetIfCurrent +// instead so a stale complaint can't clobber a connection that has already +// been recovered. func Reset() error { connMutex.Lock() defer connMutex.Unlock() + return resetLocked() +} + +// LastReadEpoch returns the epoch of the connection the most recent ReadArr +// call read its bytes from. A caller that is about to hand off a batch of +// bytes it just got back from ReadArr (e.g. a platform reader loop, right +// before passing the data into the native bridge for parsing) should call +// this immediately after ReadArr returns and capture the result there, not +// later once some asynchronous handler for that batch actually runs — +// ensureConnection may have re-dialed by then, and passing the wrong +// (newer) epoch to ResetIfCurrent would tear down a good connection +// instead of leaving it alone. +// +// This is exact only because there is exactly one permanent ReadArr caller +// for the life of the process (enforced on both platforms: a single serial +// reader loop). With one reader, nothing can start a second ReadArr call +// between this one's internal epoch capture and this accessor being read +// afterward, so the value can never be stale by the time the caller reads +// it. A design with concurrent readers would need to thread the epoch +// through ReadArr's return value instead. +func LastReadEpoch() int64 { + connMutex.Lock() + defer connMutex.Unlock() + return lastReadEpoch +} + +// ResetIfCurrent closes the connection only if epoch (obtained implicitly by +// whoever last dialed) still matches the live connection's epoch. If +// ensureConnection has re-dialed since, the +// epoch has moved on and this call is a stale no-op: some other caller +// already recovered, and closing the newer connection would silently +// discard whatever was just written to it. +func ResetIfCurrent(epoch int64) error { + _, err := resetIfCurrent(epoch) + return err +} + +// ResetIfCurrentDidReset behaves exactly like ResetIfCurrent, additionally +// reporting whether the reset actually ran (epoch matched connEpoch) as +// opposed to being a stale no-op (epoch was already superseded). +// +// Platform readers need this distinction: they also reset their local parser +// state (resetRecv/nativeResetRecv) alongside the Go connection, and doing so +// unconditionally after a stale no-op drops bytes already in flight on a +// connection nothing here actually touched. See Kb.mm and KbModule.kt. +func ResetIfCurrentDidReset(epoch int64) bool { + didReset, err := resetIfCurrent(epoch) + if err != nil { + log("Go: ResetIfCurrentDidReset: reset error: %v", err) + } + return didReset +} + +// resetIfCurrent is the shared implementation behind ResetIfCurrent and +// ResetIfCurrentDidReset. didReset reports whether epoch matched connEpoch +// (i.e. whether resetLocked actually ran), independent of whether resetLocked +// itself returned an error. +func resetIfCurrent(epoch int64) (didReset bool, err error) { + connMutex.Lock() + defer connMutex.Unlock() + if epoch != connEpoch { + log("Go: ResetIfCurrent skipped, stale epoch=%d current=%d appState=%s", + epoch, connEpoch, appStateForLog()) + return false, nil + } + return true, resetLocked() +} - log("Go: Reset start conn=%s appState=%s", describeConn(conn), appStateForLog()) +// resetLocked does the actual teardown. Must be called with connMutex held. +func resetLocked() error { + log("Go: Reset start conn=%s epoch=%d appState=%s", describeConn(conn), connEpoch, appStateForLog()) if conn != nil { conn.Close() conn = nil diff --git a/go/bind/keybase_test.go b/go/bind/keybase_test.go new file mode 100644 index 000000000000..f9d7193d61d8 --- /dev/null +++ b/go/bind/keybase_test.go @@ -0,0 +1,1030 @@ +// Copyright 2015 Keybase, Inc. All rights reserved. Use of +// this source code is governed by the included BSD license. + +package keybase + +import ( + "bytes" + "errors" + "net" + "os" + "os/exec" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/keybase/client/go/libkb" + "github.com/keybase/client/go/logger" +) + +// fakeLogContext satisfies libkb.LogContext so tests can build a real +// libkb.LoopbackListener without standing up a full libkb.GlobalContext. +type fakeLogContext struct{} + +func (fakeLogContext) GetLog() logger.Logger { return logger.NewNull() } + +// trackingConn is a minimal net.Conn whose only job is recording whether +// Close was called on it, and how many times. The embedded nil net.Conn +// satisfies the interface; none of these tests exercise Read/Write on it +// (that would panic), only Close, which is all resetLocked ever calls on the +// connection it tears down. +type trackingConn struct { + net.Conn + closed atomic.Bool + closes atomic.Int32 +} + +func (c *trackingConn) Close() error { + c.closes.Add(1) + c.closed.Store(true) + return nil +} + +// resetConnStateForTest snapshots every package-level var the epoch/reset +// mechanism touches, clears it to a known-empty state for the test, and +// restores the original values on cleanup. These are the real production +// globals (guarded by connMutex / initMutex), not a test-only copy — tests +// seed them directly instead of routing through ensureConnection's dial +// machinery, then exercise the real Reset/ResetIfCurrent/ReadArr/WriteArr/ +// LastReadEpoch functions against that seeded state. +// +// One piece of state this cannot restore: any test that calls NotifyJSReady +// closes the package-level jsReadyCh via a sync.Once, and that close is +// irreversible for the life of the test binary. Later tests that call +// ReadArr never block on jsReadyCh regardless of what this helper resets. +// A test that needs ReadArr to actually block on JS readiness must therefore +// run in a fresh process and guard itself with requireJSNotYetReady, which +// hard-fails rather than silently passing for the wrong reason. See +// TestReadArr_BlocksUntilJSReady. +func resetConnStateForTest(t *testing.T) { + t.Helper() + + connMutex.Lock() + savedConn := conn + savedEpoch := connEpoch + savedLastRead := lastReadEpoch + savedKbCtx := kbCtx + conn = nil + connEpoch = 0 + lastReadEpoch = 0 + connMutex.Unlock() + + initMutex.Lock() + savedInitComplete := initComplete + initMutex.Unlock() + + t.Cleanup(func() { + connMutex.Lock() + conn = savedConn + connEpoch = savedEpoch + lastReadEpoch = savedLastRead + kbCtx = savedKbCtx + connMutex.Unlock() + + initMutex.Lock() + initComplete = savedInitComplete + initMutex.Unlock() + }) +} + +// setConn seeds the package-level conn/connEpoch under connMutex, exactly +// the state ensureConnection would leave behind after a successful dial. +func setConn(c net.Conn, epoch int64) { + connMutex.Lock() + conn = c + connEpoch = epoch + connMutex.Unlock() +} + +func getConnState() (net.Conn, int64) { + connMutex.Lock() + defer connMutex.Unlock() + return conn, connEpoch +} + +// jsReadyFired reports whether NotifyJSReady has already run in this process. +// sync.Once has no query API, but the observable effect — jsReadyCh being +// closed — does: a receive on a closed channel never blocks. +func jsReadyFired() bool { + select { + case <-jsReadyCh: + return true + default: + return false + } +} + +// requireJSNotYetReady hard-fails a test that depends on ReadArr blocking +// before JS readiness when NotifyJSReady has already fired in this process. +// The close is irreversible (sync.Once + close of a package-level channel), +// so once any earlier test has called NotifyJSReady, ReadArr sails straight +// past `<-jsReadyCh` and a "did ReadArr block?" assertion would pass +// vacuously. Failing loudly here is the point: a future test must not +// silently prove nothing. +func requireJSNotYetReady(t *testing.T) { + t.Helper() + if jsReadyFired() { + t.Fatal("NotifyJSReady has already fired in this process; jsReadyCh is " + + "closed irreversibly, so a pre-ready blocking assertion cannot be " + + "trusted here. Run this test in a fresh process (see " + + "TestReadArr_BlocksUntilJSReady).") + } +} + +// setupReadArrBuffer installs a test-sized read buffer for ReadArr (normally +// allocated by Init, which these tests never call) and restores the original. +func setupReadArrBuffer(t *testing.T, size int) { + t.Helper() + savedBuffer := buffer + buffer = make([]byte, size) + t.Cleanup(func() { buffer = savedBuffer }) +} + +// Test 1: ResetIfCurrent closes the connection when the epoch matches the +// live one, and is a no-op that leaves the connection intact when the epoch +// is stale. +func TestResetIfCurrent_MatchingVsStaleEpoch(t *testing.T) { + resetConnStateForTest(t) + + t.Run("matching epoch closes", func(t *testing.T) { + c := &trackingConn{} + setConn(c, 5) + + if err := ResetIfCurrent(5); err != nil { + t.Fatalf("ResetIfCurrent returned error: %v", err) + } + if !c.closed.Load() { + t.Error("expected connection to be closed when epoch matches, but it was not") + } + if gotConn, _ := getConnState(); gotConn != nil { + t.Error("expected conn to be nil after a matching-epoch reset") + } + }) + + t.Run("stale epoch is a no-op", func(t *testing.T) { + c := &trackingConn{} + setConn(c, 6) + + if err := ResetIfCurrent(5); err != nil { + t.Fatalf("ResetIfCurrent returned error: %v", err) + } + if c.closed.Load() { + t.Error("expected connection to be left open on a stale epoch, but it was closed") + } + gotConn, gotEpoch := getConnState() + if gotConn != c { + t.Error("expected conn to be left untouched on a stale epoch") + } + if gotEpoch != 6 { + t.Errorf("expected epoch to remain 6, got %d", gotEpoch) + } + }) +} + +// Test 2: the exact interleaving the mechanism exists to prevent. Caller A +// captures epoch N against connection A. Before A calls ResetIfCurrent, a +// redial happens (simulating some other caller recovering first), advancing +// to a new connection at epoch N+1. A's stale ResetIfCurrent(N) must not +// close the new connection. +func TestResetIfCurrent_StaleEpochDoesNotClobberRedial(t *testing.T) { + resetConnStateForTest(t) + + connA := &trackingConn{} + setConn(connA, 1) + + // Caller A observes a failure on connA and captures its epoch, exactly + // as ReadArr/WriteArr do under connMutex before releasing it. + capturedEpoch := int64(1) + + // A redial happens concurrently, before A gets to call ResetIfCurrent: + // this is what ensureConnection leaves behind after a successful dial. + connB := &trackingConn{} + setConn(connB, 2) + + // A's stale complaint must be a no-op. + if err := ResetIfCurrent(capturedEpoch); err != nil { + t.Fatalf("ResetIfCurrent returned error: %v", err) + } + + if connB.closed.Load() { + t.Fatal("stale ResetIfCurrent closed the redialed connection") + } + if connA.closed.Load() { + t.Error("connA should not be reachable/closed either; it was already superseded") + } + gotConn, gotEpoch := getConnState() + if gotConn != connB { + t.Error("expected the live connection to still be connB") + } + if gotEpoch != 2 { + t.Errorf("expected epoch to remain 2, got %d", gotEpoch) + } +} + +// Test 3: two callers both holding epoch N. The first reset closes the +// connection; the second, redundant call against the same (now-stale-by- +// virtue-of-nil-conn but epoch-matching) state must be harmless and must +// not panic. +// +// The assertions here are deliberately ones a no-op resetLocked would fail: +// the connection is closed exactly once (not zero times, not twice), and +// connEpoch is left alone by the reset itself. That last point is not +// incidental — only ensureConnection advances the epoch, which is precisely +// why the second caller's captured epoch still matches and this double-reset +// path is reachable at all. If resetLocked ever started bumping connEpoch, +// the second call would take the stale branch instead and this test would +// catch the change in meaning. +func TestResetIfCurrent_DoubleResetSameEpochIsHarmless(t *testing.T) { + resetConnStateForTest(t) + + c := &trackingConn{} + setConn(c, 3) + + if didReset := ResetIfCurrentDidReset(3); !didReset { + t.Fatal("expected the first ResetIfCurrent to report that it acted") + } + if got := c.closes.Load(); got != 1 { + t.Fatalf("expected the first reset to close the connection exactly once, got %d closes", got) + } + gotConn, gotEpoch := getConnState() + if gotConn != nil { + t.Fatal("expected conn to be nil after the first reset") + } + if gotEpoch != 3 { + t.Fatalf("expected reset to leave connEpoch at 3 (only ensureConnection advances it), got %d", gotEpoch) + } + + // The second caller captured the same epoch, so it still matches and + // re-enters resetLocked — this time with conn == nil. + if didReset := ResetIfCurrentDidReset(3); !didReset { + t.Fatal("expected the second ResetIfCurrent to still match the (unadvanced) epoch") + } + if got := c.closes.Load(); got != 1 { + t.Errorf("expected the redundant reset to be observably a no-op (still 1 close), got %d closes", got) + } + gotConn, gotEpoch = getConnState() + if gotConn != nil { + t.Error("expected conn to remain nil after the redundant reset") + } + if gotEpoch != 3 { + t.Errorf("expected connEpoch to remain 3 after the redundant reset, got %d", gotEpoch) + } +} + +// Test 4: Reset is the unconditional escape hatch used by invalidate/ +// destroy/engineReset. It must close whatever connection is current +// regardless of any epoch bookkeeping. +func TestReset_UnconditionallyClosesCurrentConnection(t *testing.T) { + resetConnStateForTest(t) + + c := &trackingConn{} + setConn(c, 42) + + if err := Reset(); err != nil { + t.Fatalf("Reset returned error: %v", err) + } + if !c.closed.Load() { + t.Error("expected Reset to unconditionally close the current connection") + } + if gotConn, _ := getConnState(); gotConn != nil { + t.Error("expected conn to be nil after Reset") + } +} + +// Test 5: connEpoch is monotonically increasing and never reused across +// redials. This drives the real ensureConnection dial path (not a +// reimplementation of it) against a real libkb.LoopbackListener, so the +// increment under test is the actual production statement. +func TestEnsureConnection_EpochMonotonicAcrossRedials(t *testing.T) { + resetConnStateForTest(t) + + ll := libkb.NewLoopbackListener(fakeLogContext{}) + t.Cleanup(func() { _ = ll.Close() }) + + // Drain the Accept() side in the background so Dial() (called inside + // ensureConnection) doesn't block forever; ensureConnection only needs + // the dial to succeed, not a live peer. + go func() { + for { + if _, err := ll.Accept(); err != nil { + return + } + } + }() + + connMutex.Lock() + kbCtx = &libkb.GlobalContext{LoopbackListener: ll} + connMutex.Unlock() + setInited() + + seen := map[int64]bool{} + var prev int64 = -1 + for i := 0; i < 20; i++ { + connMutex.Lock() + conn = nil // force ensureConnection to dial again + err := ensureConnection() + gotEpoch := connEpoch + connMutex.Unlock() + + if err != nil { + t.Fatalf("ensureConnection failed on iteration %d: %v", i, err) + } + if gotEpoch <= prev { + t.Fatalf("epoch did not strictly increase: prev=%d got=%d", prev, gotEpoch) + } + if seen[gotEpoch] { + t.Fatalf("epoch %d reused across redials", gotEpoch) + } + seen[gotEpoch] = true + prev = gotEpoch + } +} + +// Test 6: LastReadEpoch reflects the epoch of the connection the most +// recent ReadArr call actually used. +func TestReadArr_LastReadEpochMatchesConnectionUsed(t *testing.T) { + resetConnStateForTest(t) + + setupReadArrBuffer(t, 4096) + + NotifyJSReady() // sync.Once; safe to call unconditionally + + local, remote := net.Pipe() + t.Cleanup(func() { _ = local.Close(); _ = remote.Close() }) + + setConn(remote, 9) + + go func() { + _, _ = local.Write([]byte("hello")) + }() + + data, err := ReadArr() + if err != nil { + t.Fatalf("ReadArr returned error: %v", err) + } + if string(data) != "hello" { + t.Fatalf("unexpected data: %q", data) + } + + if got := LastReadEpoch(); got != 9 { + t.Errorf("expected LastReadEpoch to be 9, got %d", got) + } + + // Now redial to a new connection/epoch and read again; LastReadEpoch + // must track the new one, not the old. + local2, remote2 := net.Pipe() + t.Cleanup(func() { _ = local2.Close(); _ = remote2.Close() }) + setConn(remote2, 10) + + go func() { + _, _ = local2.Write([]byte("world")) + }() + + if _, err := ReadArr(); err != nil { + t.Fatalf("second ReadArr returned error: %v", err) + } + if got := LastReadEpoch(); got != 10 { + t.Errorf("expected LastReadEpoch to be 10 after redial, got %d", got) + } +} + +// Test 7: concurrency through the real entry points. Earlier revisions of +// this test took connMutex in its own goroutines and called ensureConnection +// directly, which is not how anything in production reaches it — that +// demonstrated the test's own lock discipline, not the callers'. This drives +// ReadArr / WriteArr / Reset / ResetIfCurrent instead, so the lazy-init, +// epoch-capture and failure-reset code under test is the code that actually +// ships. +// +// The goroutine mix mirrors production exactly: ONE permanent ReadArr caller +// (a single serial reader loop on both platforms), many concurrent WriteArr +// callers, and concurrent resetters. It is deliberately not N concurrent +// readers: ReadArr reads into the shared package-level `buffer`, so a second +// concurrent reader is a genuine data race on that buffer (verified: -race +// flags keybase.go's `currentConn.Read(buffer)` against its own +// `copy(out, buffer[:n])`). The per-call copy ReadArr returns bounds the +// blast radius of such a regression — a caller's slice can never be +// retroactively rewritten by a later read, see +// TestReadArr_ReturnsPrivateCopy — but it does not make concurrent ReadArr +// safe, and asserting otherwise here would be asserting a property the code +// does not have. +func TestConcurrentReadWriteAndResetsThroughRealEntryPoints(t *testing.T) { + resetConnStateForTest(t) + setupReadArrBuffer(t, 4096) + NotifyJSReady() + + // Peers send only 'R' bytes and the app only ever sends 'W' bytes, so any + // byte the reader sees that isn't 'R' means a frame got crossed or the + // shared buffer got garbled. + peerPayload := bytes.Repeat([]byte{'R'}, 128) + appPayload := bytes.Repeat([]byte{'W'}, 96) + + ll := libkb.NewLoopbackListener(fakeLogContext{}) + t.Cleanup(func() { _ = ll.Close() }) + + // Every accepted peer is recorded so teardown can poke it (see below). + var peerMu sync.Mutex + var peers []net.Conn + pokeAllPeers := func() { + peerMu.Lock() + snapshot := append([]net.Conn(nil), peers...) + peerMu.Unlock() + // Fire-and-forget: LoopbackConn.Write parks on an unbuffered channel + // until its partner reads, so a poke at a connection nobody is reading + // must not block the poker. + for _, pc := range snapshot { + go func(pc net.Conn) { _, _ = pc.Write(peerPayload) }(pc) + } + } + + // Accept every dial ensureConnection makes. Each peer gets one proactive + // payload (so the reader loop has something to return) and then drains + // whatever WriteArr sends, so LoopbackConn.Write — which blocks on an + // unbuffered channel until its partner reads — can always make progress. + // A peer whose app-side conn is reset while its proactive Write is still + // parked stays parked; that is bounded at one goroutine per redial and + // dies with the test process. + go func() { + for { + peer, err := ll.Accept() + if err != nil { + return + } + peerMu.Lock() + peers = append(peers, peer) + peerMu.Unlock() + go func(pc net.Conn) { _, _ = pc.Write(peerPayload) }(peer) + go func(pc net.Conn) { + sink := make([]byte, 4096) + for { + if _, err := pc.Read(sink); err != nil { + return + } + } + }(peer) + } + }() + + connMutex.Lock() + kbCtx = &libkb.GlobalContext{LoopbackListener: ll} + connMutex.Unlock() + setInited() + + const writers = 8 + const resetters = 4 + const iterations = 60 + + stop := make(chan struct{}) + var reads, writes atomic.Int64 + + // The single permanent reader. + readerDone := make(chan struct{}) + go func() { + defer close(readerDone) + for { + select { + case <-stop: + return + default: + } + data, err := ReadArr() + if err != nil { + continue + } + if len(data) == 0 { + continue + } + reads.Add(1) + if i := bytes.IndexFunc(data, func(r rune) bool { return r != 'R' }); i >= 0 { + t.Errorf("ReadArr returned a garbled slice: byte %d is %q, want all 'R'", i, data[i]) + return + } + } + }() + + var wg sync.WaitGroup + + for g := 0; g < writers; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + if err := WriteArr(appPayload); err == nil { + writes.Add(1) + } + } + }() + } + + // Failure-driven resetters: capture an epoch the way ReadArr/WriteArr do, + // then race to reset it. + for g := 0; g < resetters; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + connMutex.Lock() + epoch := connEpoch + connMutex.Unlock() + _ = ResetIfCurrent(epoch) + } + }() + } + + // Unconditional resetters: e.g. concurrent invalidate/engineReset. + for g := 0; g < resetters; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < iterations; i++ { + _ = Reset() + } + }() + } + + workersDone := make(chan struct{}) + go func() { + wg.Wait() + close(workersDone) + }() + + select { + case <-workersDone: + case <-time.After(60 * time.Second): + t.Fatal("concurrent read/write/reset test deadlocked") + } + + // Unblock the reader, which is very likely parked in conn.Read. Closing + // the app-side conn does not help: LoopbackConn.Read blocks on the + // *partner's* channel, so only the peer writing (or closing) wakes it. + // Poke every peer until the reader observes stop and returns. + close(stop) + deadline := time.After(30 * time.Second) + for { + select { + case <-readerDone: + goto readerStopped + case <-deadline: + t.Fatal("reader loop did not stop") + case <-time.After(20 * time.Millisecond): + pokeAllPeers() + } + } +readerStopped: + + if reads.Load() == 0 { + t.Error("no ReadArr call ever returned data; the test never exercised the read path") + } + if writes.Load() == 0 { + t.Error("no WriteArr call ever succeeded; the test never exercised the write path") + } + + // No torn state: the mechanism must still be usable afterward. A fresh + // write must succeed and drive a redial to a strictly higher epoch. + _, preEpoch := getConnState() + if err := Reset(); err != nil { + t.Fatalf("post-concurrency Reset failed: %v", err) + } + if err := WriteArr(appPayload); err != nil { + t.Fatalf("post-concurrency WriteArr failed: %v", err) + } + postConn, postEpoch := getConnState() + if postConn == nil { + t.Fatal("post-concurrency conn is nil after a successful WriteArr redial") + } + if postEpoch <= preEpoch { + t.Fatalf("post-concurrency epoch did not advance: pre=%d post=%d", preEpoch, postEpoch) + } +} + +// readSignalConn wraps a net.Conn and closes entered the first time Read is +// called, letting a test observe "the reader has entered conn.Read" without +// racing on it. +type readSignalConn struct { + net.Conn + entered chan struct{} + once sync.Once +} + +func (c *readSignalConn) Read(p []byte) (int, error) { + c.once.Do(func() { close(c.entered) }) + return c.Conn.Read(p) +} + +// Test 8: the regression this whole mechanism guards against. ReadArr must +// capture lastReadEpoch from the connection/epoch pair it is actually about +// to read from, before that read can be raced by a redial — not after +// currentConn.Read returns, and not from whatever connEpoch happens to be +// when the async caller gets around to reading it back. +// +// This interleaves a redial with an in-flight ReadArr: the reader parks in +// Read() on connA/epoch A, a redial swaps in connB/epoch B while the read is +// still outstanding, and only then does connA's peer supply the bytes that +// unblock it. LastReadEpoch() must report A (the connection actually read +// from), not B (the connection live at the time the caller happens to check). +// +// Falsified: moving `lastReadEpoch = currentEpoch` in ReadArr to after +// currentConn.Read returns (the exact regression this guards against) makes +// this test fail, reporting epoch B instead of A. +func TestReadArr_LastReadEpochCapturedBeforeRaceableRead(t *testing.T) { + resetConnStateForTest(t) + + setupReadArrBuffer(t, 4096) + + NotifyJSReady() // sync.Once; safe to call unconditionally; see the note + // on resetConnStateForTest above about why this can't be undone. + + localA, remoteA := net.Pipe() + t.Cleanup(func() { _ = localA.Close(); _ = remoteA.Close() }) + + entered := make(chan struct{}) + setConn(&readSignalConn{Conn: remoteA, entered: entered}, 100) + + type readResult struct { + data []byte + err error + } + resultCh := make(chan readResult, 1) + go func() { + data, err := ReadArr() + resultCh <- readResult{data, err} + }() + + // Wait until the reader has actually entered conn.Read on connA/epoch + // 100 (and, in correct code, has already captured lastReadEpoch=100 + // under connMutex before releasing it and blocking here). + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("ReadArr never entered conn.Read") + } + + // A redial races in while the read above is still outstanding, exactly + // as ensureConnection would do for a concurrent WriteArr/ReadArr caller + // recovering from a failure on a different connection. + localB, remoteB := net.Pipe() + t.Cleanup(func() { _ = localB.Close(); _ = remoteB.Close() }) + setConn(remoteB, 101) + + // Only now does connA's peer supply the bytes that unblock the + // already-in-flight Read on connA. + go func() { _, _ = localA.Write([]byte("hello")) }() + + var res readResult + select { + case res = <-resultCh: + case <-time.After(5 * time.Second): + t.Fatal("ReadArr did not return") + } + if res.err != nil { + t.Fatalf("ReadArr returned error: %v", res.err) + } + if string(res.data) != "hello" { + t.Fatalf("unexpected data: %q", res.data) + } + + if got := LastReadEpoch(); got != 100 { + t.Errorf("expected LastReadEpoch to report the epoch actually read from (100), got %d", got) + } +} + +// erroringReadConn's Read always fails, driving ReadArr's error path. +type erroringReadConn struct { + trackingConn +} + +func (c *erroringReadConn) Read(p []byte) (int, error) { + return 0, errors.New("simulated read error") +} + +// Test 9: ReadArr's error path (a real production caller of ResetIfCurrent, +// keybase.go ~:681) must reset the connection it just failed to read from. +func TestReadArr_ErrorPathResetsConnection(t *testing.T) { + resetConnStateForTest(t) + + setupReadArrBuffer(t, 4096) + + NotifyJSReady() + + c := &erroringReadConn{} + setConn(c, 7) + + if _, err := ReadArr(); err == nil { + t.Fatal("expected ReadArr to return an error") + } + + if !c.closed.Load() { + t.Error("expected ReadArr's error path to reset (close) the connection it read from") + } + if gotConn, _ := getConnState(); gotConn != nil { + t.Error("expected conn to be nil after ReadArr's error-path reset") + } +} + +// shortWriteConn's Write always reports writing one byte fewer than given, +// with no error, driving WriteArr's short-write path. +type shortWriteConn struct { + trackingConn +} + +func (c *shortWriteConn) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + return len(p) - 1, nil +} + +// Test 10: WriteArr's short-write path (a real production caller of +// ResetIfCurrent, keybase.go ~:610) must reset the connection rather than +// leave the peer's framer holding a partial frame. +func TestWriteArr_ShortWriteResetsConnection(t *testing.T) { + resetConnStateForTest(t) + + c := &shortWriteConn{} + setConn(c, 11) + + if err := WriteArr([]byte("hello world")); err == nil { + t.Fatal("expected WriteArr to return an error on a short write") + } + + if !c.closed.Load() { + t.Error("expected WriteArr's short-write path to reset (close) the connection") + } + if gotConn, _ := getConnState(); gotConn != nil { + t.Error("expected conn to be nil after WriteArr's short-write reset") + } +} + +// Test 11: ResetIfCurrentDidReset reports whether it actually reset the +// connection (epoch matched) as opposed to being a stale no-op, which is +// exactly what platform readers (Kb.mm, KbModule.kt) now gate their local +// parser reset on. +func TestResetIfCurrentDidReset_ReportsWhetherItActed(t *testing.T) { + resetConnStateForTest(t) + + t.Run("matching epoch resets and reports true", func(t *testing.T) { + c := &trackingConn{} + setConn(c, 20) + + if didReset := ResetIfCurrentDidReset(20); !didReset { + t.Error("expected ResetIfCurrentDidReset to report true for a matching epoch") + } + if !c.closed.Load() { + t.Error("expected the connection to be closed when epoch matches") + } + }) + + t.Run("stale epoch is a no-op and reports false", func(t *testing.T) { + c := &trackingConn{} + setConn(c, 21) + + if didReset := ResetIfCurrentDidReset(20); didReset { + t.Error("expected ResetIfCurrentDidReset to report false for a stale epoch") + } + if c.closed.Load() { + t.Error("expected the connection to be left open on a stale epoch") + } + }) +} + +// Test 12: ReadArr returns the caller its own copy, not a view of the shared +// package-level `buffer` (commit 731f1074f9). The distinguishing property is +// that a slice already handed out cannot be retroactively rewritten by a +// later read into that same buffer — which is what a returned `buffer[:n]` +// view would do, silently, to bytes the caller has not finished with. +// +// This is deliberately serial. The single-permanent-reader invariant makes a +// second concurrent ReadArr a data race on `buffer` itself (see the comment +// on TestConcurrentReadWriteAndResetsThroughRealEntryPoints), so the copy is +// not what makes concurrency safe; it is what keeps a regression in that +// invariant from corrupting data the caller already owns. +// +// Falsified: replacing ReadArr's `out := make([]byte, n); copy(out, +// buffer[:n]); return out, nil` with `return buffer[:n], nil` makes both +// assertions below fail. +func TestReadArr_ReturnsPrivateCopy(t *testing.T) { + resetConnStateForTest(t) + setupReadArrBuffer(t, 4096) + NotifyJSReady() + + local, remote := net.Pipe() + t.Cleanup(func() { _ = local.Close(); _ = remote.Close() }) + setConn(remote, 1) + + first := bytes.Repeat([]byte{'a'}, 64) + go func() { _, _ = local.Write(first) }() + + got, err := ReadArr() + if err != nil { + t.Fatalf("ReadArr returned error: %v", err) + } + if !bytes.Equal(got, first) { + t.Fatalf("unexpected first read: %q", got) + } + + // A later read into the shared buffer must not touch the slice already + // handed out. Same length, different content, so a buffer view would be + // rewritten in place with no length change to hint at it. + second := bytes.Repeat([]byte{'b'}, 64) + go func() { _, _ = local.Write(second) }() + if _, err := ReadArr(); err != nil { + t.Fatalf("second ReadArr returned error: %v", err) + } + if !bytes.Equal(got, first) { + t.Errorf("a later ReadArr rewrote a slice already returned to the caller: %q", got) + } + + // Nor may anything else scribbling on the shared buffer reach it. + for i := range buffer { + buffer[i] = 'z' + } + if !bytes.Equal(got, first) { + t.Errorf("returned slice aliases the shared buffer: %q", got) + } +} + +// blockingShortWriteConn parks in Write until release is closed, signalling +// entry on entered, and then reports a short write (one byte fewer than +// asked, no error) to drive WriteArr's short-write reset path. +type blockingShortWriteConn struct { + trackingConn + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (c *blockingShortWriteConn) Write(p []byte) (int, error) { + c.once.Do(func() { close(c.entered) }) + <-c.release + if len(p) == 0 { + return 0, nil + } + return len(p) - 1, nil +} + +// Test 13: the mirror image of Test 8, on the write side. WriteArr captures +// connEpoch under connMutex alongside the connection it is about to write to, +// and its short-write path hands that captured epoch to ResetIfCurrent. If it +// instead called Reset(), or re-read connEpoch at reset time, a write that +// fails on an already-superseded connection would tear down the healthy one a +// concurrent reader just dialed — which is the original bug (f57a55cf69) in +// the opposite direction. +// +// The interleaving: a write parks in conn.Write on connA/epoch 200; a redial +// swaps in connB/epoch 201 while it is still outstanding; only then does the +// write complete, short. connB must survive untouched. +// +// Falsified two ways, each making this test fail by closing connB and nilling +// conn: (1) replacing `ResetIfCurrent(currentEpoch)` in WriteArr's short-write +// branch with `Reset()`, and (2) re-reading the live connEpoch at reset time +// instead of using the epoch captured before the write. +func TestWriteArr_ShortWriteEpochCapturedBeforeRaceableWrite(t *testing.T) { + resetConnStateForTest(t) + + connA := &blockingShortWriteConn{ + entered: make(chan struct{}), + release: make(chan struct{}), + } + setConn(connA, 200) + + errCh := make(chan error, 1) + go func() { errCh <- WriteArr([]byte("hello world")) }() + + // Wait until the writer is actually inside conn.Write on connA/epoch 200 + // (and, in correct code, has already captured currentEpoch=200 under + // connMutex before releasing it). + select { + case <-connA.entered: + case <-time.After(5 * time.Second): + t.Fatal("WriteArr never entered conn.Write") + } + + // A redial races in while the write above is still outstanding, exactly + // as ensureConnection would do for a concurrent ReadArr caller recovering + // from a failure on a different connection. + connB := &trackingConn{} + setConn(connB, 201) + + // Only now does the write complete — short, so WriteArr takes its reset + // path with the stale epoch it captured before the redial. + close(connA.release) + + select { + case err := <-errCh: + if err == nil { + t.Fatal("expected WriteArr to return an error on a short write") + } + case <-time.After(5 * time.Second): + t.Fatal("WriteArr did not return") + } + + if connB.closed.Load() { + t.Fatal("a stale short write tore down the connection a concurrent caller had redialed") + } + gotConn, gotEpoch := getConnState() + if gotConn != connB { + t.Errorf("expected the live connection to still be connB, got %v", gotConn) + } + if gotEpoch != 201 { + t.Errorf("expected epoch to remain 201, got %d", gotEpoch) + } + if connA.closes.Load() != 0 { + t.Errorf("expected the stale ResetIfCurrent to be a complete no-op, but connA was closed %d times", + connA.closes.Load()) + } +} + +// jsReadyChildEnv, when set, tells the re-executed test binary to run the +// child half of the JS-readiness test rather than skipping it. +const jsReadyChildEnv = "KEYBASE_BIND_JSREADY_CHILD" + +// Test 14 (parent): ReadArr blocks on jsReadyCh until NotifyJSReady fires. +// This gates the entire startup handshake — until JS says it is ready, the Go +// side must not pull bytes off the loopback that nothing is there to receive. +// +// It cannot run in-process: NotifyJSReady is a sync.Once that closes a +// package-level channel, so the first test in this binary to call it makes +// `<-jsReadyCh` a no-op forever afterward and any "did it block?" assertion +// would pass vacuously. So re-exec this same test binary and run the child +// half in a fresh process where the once has not fired. requireJSNotYetReady +// in the child is the backstop: if that assumption is ever broken, the child +// hard-fails instead of quietly proving nothing. +// +// Falsified: deleting `<-jsReadyCh` from ReadArr makes the child fail with +// "ReadArr returned before JS signalled ready", failing this parent too. +func TestReadArr_BlocksUntilJSReady(t *testing.T) { + if os.Getenv(jsReadyChildEnv) != "" { + t.Skip("this is the parent half; the child runs as TestReadArrJSReadyChild") + } + if _, err := os.Stat(os.Args[0]); err != nil { + t.Skipf("cannot re-exec the test binary (%v)", err) + } + + cmd := exec.Command(os.Args[0], "-test.run=^TestReadArrJSReadyChild$", "-test.v") + cmd.Env = append(os.Environ(), jsReadyChildEnv+"=1") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("child process failed: %v\n--- child output ---\n%s", err, out) + } + if !bytes.Contains(out, []byte("PASS")) { + t.Fatalf("child process did not report PASS\n--- child output ---\n%s", out) + } +} + +// Test 14 (child): runs only in the re-executed process spawned by +// TestReadArr_BlocksUntilJSReady, where NotifyJSReady has not yet fired. +func TestReadArrJSReadyChild(t *testing.T) { + if os.Getenv(jsReadyChildEnv) == "" { + t.Skip("child half; spawned by TestReadArr_BlocksUntilJSReady") + } + // The whole point of the subprocess. If this ever trips, some earlier + // test in the child's -test.run set called NotifyJSReady and the + // assertions below would be meaningless. + requireJSNotYetReady(t) + + resetConnStateForTest(t) + setupReadArrBuffer(t, 4096) + + local, remote := net.Pipe() + t.Cleanup(func() { _ = local.Close(); _ = remote.Close() }) + setConn(remote, 1) + + type readResult struct { + data []byte + err error + } + resultCh := make(chan readResult, 1) + go func() { + data, err := ReadArr() + resultCh <- readResult{data, err} + }() + + // Bytes are available on the connection the whole time, so the only thing + // that can keep ReadArr from returning is the jsReadyCh gate. + go func() { _, _ = local.Write([]byte("hello")) }() + + select { + case res := <-resultCh: + t.Fatalf("ReadArr returned before JS signalled ready: data=%q err=%v", res.data, res.err) + case <-time.After(500 * time.Millisecond): + } + + NotifyJSReady() + + select { + case res := <-resultCh: + if res.err != nil { + t.Fatalf("ReadArr returned error after NotifyJSReady: %v", res.err) + } + if string(res.data) != "hello" { + t.Fatalf("unexpected data after NotifyJSReady: %q", res.data) + } + case <-time.After(5 * time.Second): + t.Fatal("ReadArr did not return after NotifyJSReady") + } + + if !jsReadyFired() { + t.Error("expected jsReadyFired to report true after NotifyJSReady") + } +} diff --git a/go/chat/attachments/quarantine_darwin.go b/go/chat/attachments/quarantine_darwin.go index 68dff5388c15..9deec2929729 100644 --- a/go/chat/attachments/quarantine_darwin.go +++ b/go/chat/attachments/quarantine_darwin.go @@ -5,7 +5,7 @@ package attachments /* #cgo CFLAGS: -x objective-c -fobjc-arc #include -void quarantineFile(const char* inFilename) { +void chatAttachmentsQuarantineFile(const char* inFilename) { NSError* error = NULL; NSString* filename = [NSString stringWithUTF8String:inFilename]; NSURL* url = [NSURL fileURLWithPath:filename]; @@ -26,6 +26,6 @@ import ( func Quarantine(ctx context.Context, path string) error { cpath := C.CString(path) defer C.free(unsafe.Pointer(cpath)) - C.quarantineFile(cpath) + C.chatAttachmentsQuarantineFile(cpath) return nil } diff --git a/go/kbfs/simplefs/platform_darwin.go b/go/kbfs/simplefs/platform_darwin.go index 8f370cfac887..64a65d396cb6 100644 --- a/go/kbfs/simplefs/platform_darwin.go +++ b/go/kbfs/simplefs/platform_darwin.go @@ -8,7 +8,7 @@ package simplefs #cgo LDFLAGS: -framework Foundation -framework CoreServices -lobjc #include -void quarantineFile(const char* inFilename) { +void simpleFSQuarantineFile(const char* inFilename) { NSError* error = NULL; NSString* filename = [NSString stringWithUTF8String:inFilename]; NSURL* url = [NSURL fileURLWithPath:filename]; @@ -30,7 +30,7 @@ import ( func Quarantine(ctx context.Context, path string) error { cpath := C.CString(path) defer C.free(unsafe.Pointer(cpath)) - C.quarantineFile(cpath) + C.simpleFSQuarantineFile(cpath) return nil } diff --git a/rnmodules/react-native-kb/android/CMakeLists.txt b/rnmodules/react-native-kb/android/CMakeLists.txt index 30091fd94228..03182308880d 100644 --- a/rnmodules/react-native-kb/android/CMakeLists.txt +++ b/rnmodules/react-native-kb/android/CMakeLists.txt @@ -8,6 +8,7 @@ set (NODE_MODULES_DIR "${CMAKE_SOURCE_DIR}/../..") add_library(cpp SHARED ../cpp/react-native-kb.cpp + ../cpp/frame-parser.cpp cpp-adapter.cpp ) diff --git a/rnmodules/react-native-kb/android/build.gradle b/rnmodules/react-native-kb/android/build.gradle index 02df114634b7..c0b155825fdb 100644 --- a/rnmodules/react-native-kb/android/build.gradle +++ b/rnmodules/react-native-kb/android/build.gradle @@ -132,6 +132,10 @@ dependencies { implementation "androidx.media3:media3-transformer:1.4.1" implementation "androidx.media3:media3-effect:1.4.1" implementation "androidx.media3:media3-common:1.4.1" + // Plain JVM unit tests for the dependency-free reader-loop throttles + // (src/test/java/com/reactnativekb). Run with + // `./gradlew :react-native-kb:testDebugUnitTest` from shared/android. + testImplementation "junit:junit:4.13.2" } react { diff --git a/rnmodules/react-native-kb/android/cpp-adapter.cpp b/rnmodules/react-native-kb/android/cpp-adapter.cpp index b11e52b4e737..1bd1f907fce2 100644 --- a/rnmodules/react-native-kb/android/cpp-adapter.cpp +++ b/rnmodules/react-native-kb/android/cpp-adapter.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include using namespace facebook; using namespace facebook::jsi; @@ -16,58 +18,245 @@ struct JKbModule : jni::JavaClass { class KbNativeAdapter { public: jni::global_ref jModule_; - std::shared_ptr bridge_; explicit KbNativeAdapter(jni::alias_ref jModule) : jModule_(jni::make_global(jModule)) {} - void writeToGo(void *ptr, size_t size) { + bool writeToGo(void *ptr, size_t size) { jni::ThreadScope scope; auto env = jni::Environment::current(); auto jba = env->NewByteArray(size); + if (jba == nullptr) { + // NewByteArray leaves OutOfMemoryError pending. Returning with it still + // set means the JS thread makes its next JNI call with a live exception + // -- UB, and an abort under CheckJNI. Describe it to logcat before + // clearing, and route a line through onLog so an OOM here also reaches + // the uploadable log, not just logcat. + env->ExceptionDescribe(); + env->ExceptionClear(); + // onLog itself allocates a jstring (jni::make_jstring); if that + // allocation throws -- plausible right after an OOM -- an fbjni + // JniException must not escape writeToGo in place of the documented + // `return false`, or the caller's exception-handling contract breaks. + try { + onLog("writeToGo: NewByteArray failed (out of memory), dropping message"); + } catch (...) { + env->ExceptionClear(); + } + return false; + } + // Adopt into a local_ref so the ref is released even if the call below + // throws a JniException. + auto arr = jni::adopt_local(static_cast(jba)); env->SetByteArrayRegion(jba, 0, size, (jbyte *)ptr); static auto method = JKbModule::javaClassStatic() - ->getMethod)>("rpcOnGo"); - method(jModule_, jni::wrap_alias(jba)); - env->DeleteLocalRef(jba); + ->getMethod)>("rpcOnGo"); + auto ok = method(jModule_, arr); + return ok != JNI_FALSE; + } + + void onFatal(int64_t epoch) { + jni::ThreadScope scope; + static auto method = + JKbModule::javaClassStatic()->getMethod( + "onRpcStreamFatal"); + method(jModule_, static_cast(epoch)); + } + + // Called from JNI. Routes native bridge errors into the uploadable log -- + // __android_log_print only reaches logcat, which a field log send does not + // include. + void onLog(const std::string &message) { + jni::ThreadScope scope; + static auto method = + JKbModule::javaClassStatic() + ->getMethod)>("onNativeLog"); + method(jModule_, jni::make_jstring(message)); } }; +// The bridge is created on the JS thread and consumed by the native reader +// thread, so both the adapter and the bridge live behind this lock. +static std::mutex g_mutex; static std::shared_ptr g_adapter; +static std::shared_ptr g_bridge; + +static std::shared_ptr getBridge() { + std::lock_guard lock(g_mutex); + return g_bridge; +} + +static bool isCurrentAdapter(const std::shared_ptr &adapter) { + std::lock_guard lock(g_mutex); + return g_adapter == adapter; +} + +// Any thread. Drops this module's C++-side state: flips the bridge's atomic +// teardown flag so the permanent reader stops delivering into a runtime that +// is going away, and releases the globals that would otherwise pin a dead +// KbModule (and its ReactContext/Activity) and a destroyed runtime's +// CallInvoker for the life of the process. +// +// Only atomics and shared_ptr slots are touched — the old bridge's jsi +// handles belong to its own runtime and are released by its kbTeardown host +// object on the JS thread. +// A stale invalidate can run after the next module already installed its own +// bridge -- RN's module teardown is not ordered against the next module's +// installJSIBindings/getBindingsInstaller, so without an identity check an +// unconditional clear would tear down the LIVE bridge with no desync and no +// recovery. Mirrors iOS's kbClearBridgeIfCurrent: only clear if the module +// invoking invalidate is still the one that installed the current +// adapter/bridge. +// +// Returns true only when it actually matched `thiz` and cleared, so the Kotlin +// caller can gate Keybase.reset() on it exactly like iOS gates KeybaseReset on +// kbClearBridgeIfCurrent. The gate is the whole point: a stale invalidate that +// reset the Go connection would tear down the loopback the NEXT module already +// owns, killing its in-flight RPCs for no reason. +static jboolean nativeInvalidate(jni::alias_ref thiz) { + std::shared_ptr oldBridge; + std::shared_ptr oldAdapter; + { + std::lock_guard lock(g_mutex); + if (!g_adapter || !(g_adapter->jModule_ == thiz)) { + return JNI_FALSE; + } + oldBridge = std::move(g_bridge); + oldAdapter = std::move(g_adapter); + g_bridge = nullptr; + g_adapter = nullptr; + } + if (oldBridge) { + oldBridge->markTornDown(); + } + return JNI_TRUE; +} + +static void nativeResetRecv(jni::alias_ref) { + if (auto bridge = getBridge()) { + bridge->resetRecv(); + } +} static jni::local_ref getBindingsInstaller(jni::alias_ref thiz) { - g_adapter = std::make_shared(thiz); + auto adapter = std::make_shared(thiz); + // INVARIANT: g_adapter and g_bridge are only ever published as a PAIR, and + // g_bridge is only ever published while its own adapter is still current. + // Publishing the adapter here without also dropping the previous module's + // bridge would leave that bridge live with no owner -- a later + // nativeInvalidate from the previous module no longer matches g_adapter, so + // nothing would ever tear it down. A new module installing means the + // previous one is definitively dead, so take its bridge over now. + // + // Known window, accepted: from this displace until the bindings-installer + // lambda below publishes the new bridge, the permanent reader finds + // g_bridge null and drops whatever it reads -- and nothing resets the Go + // connection here, so those bytes are consumed from a live stream. If the + // drop lands mid-frame, the new bridge's first feed desyncs and the + // fatal/reset path re-syncs it: one needless reset cycle on some reloads, + // in exchange for never touching a connection that might not need it. + std::shared_ptr displaced; + { + std::lock_guard lock(g_mutex); + displaced = std::move(g_bridge); + g_bridge = nullptr; + g_adapter = adapter; + } + // Outside the lock: markTornDown only flips an atomic, but nothing that can + // reenter this file may run while g_mutex is held. + if (displaced) { + displaced->markTornDown(); + } + // Captured strongly: the adapter must outlive the bridge that calls it. + // Unpublishing a bridge only flips its teardown flag -- another thread can + // still be inside one of these callbacks -- so the adapter cannot be kept + // alive by the g_adapter slot alone. No cycle: the adapter holds a + // global_ref to the Java module and never the bridge. return BindingsInstallerHolder::newObjectCxxArgs( - [adapter = g_adapter](jsi::Runtime &runtime, - const std::shared_ptr &callInvoker) { - if (adapter->bridge_) { - adapter->bridge_->teardown(); - } - adapter->bridge_ = std::make_shared(); - adapter->bridge_->install( + [adapter]( + jsi::Runtime &runtime, + const std::shared_ptr &callInvoker) { + auto bridge = std::make_shared(); + bridge->install( runtime, callInvoker, - [weak = std::weak_ptr(adapter)](void *ptr, size_t size) { - if (auto a = weak.lock()) - a->writeToGo(ptr, size); + // false means the RPC never reached Go, so the caller fails that + // invocation instead of waiting forever for a reply. + [adapter](void *ptr, size_t size) -> bool { + return adapter->writeToGo(ptr, size); }, - [](const std::string &err) { + [adapter](const std::string &err) { __android_log_print(ANDROID_LOG_ERROR, "KBBridge", "JSI error: %s", err.c_str()); + adapter->onLog("jsi error: " + err); + }, + // The incoming stream desynced; reset the Go connection (if + // `epoch` -- the connection the desynced bytes actually came + // from -- is still current) and tell JS so it fails outstanding + // RPCs rather than hanging forever. + [adapter](int64_t epoch) { + // Identity gate mirroring iOS's Kb.mm: only act if the module + // that faulted still owns the installed adapter/bridge. A batch + // queued by a dying runtime can hit its conversion-failure + // fatal after the next module has already published its pair, + // and the epoch check can't catch that (nothing re-dialed, so + // `epoch` is still current) -- acting here would tear down the + // connection the new runtime is already using, then clear the + // new bridge's parser mid-frame, forcing a needless second + // fatal/reset cycle. + if (!isCurrentAdapter(adapter)) { + __android_log_print( + ANDROID_LOG_WARN, "KBBridge", + "rpc stream desync from superseded bridge, ignoring"); + return; + } + __android_log_print(ANDROID_LOG_ERROR, "KBBridge", + "rpc stream desync, resetting connection"); + adapter->onLog("rpc stream desync, resetting connection"); + adapter->onFatal(epoch); }); + + // Identity-gated publish: this bridge belongs to `adapter`, so it may + // only be published while `adapter` is still the current one. Without + // the gate, a nativeInvalidate that lands between the g_adapter store + // above and this lambda (RN runs them at different times, on + // different threads) clears both slots and then this store resurrects + // g_bridge -- targeting a runtime that is going away, with a null + // g_adapter. A bridge installed into a runtime whose module was + // already invalidated must be inert, not published. + std::shared_ptr old; + bool published = false; + { + std::lock_guard lock(g_mutex); + if (g_adapter == adapter) { + old = std::move(g_bridge); + g_bridge = bridge; + published = true; + } + } + // Only flips an atomic. The old bridge's jsi handles belong to its + // own runtime and are released by its kbTeardown host object. Kept + // outside the lock -- nothing may reenter this file under g_mutex. + if (old) { + old->markTornDown(); + } + if (!published) { + bridge->markTornDown(); + } }); } static void nativeOnDataFromGo(jni::alias_ref thiz, - jni::alias_ref data) { - auto adapter = g_adapter; - if (!adapter || !adapter->bridge_ || !data) + jni::alias_ref data, + jlong epoch) { + auto bridge = getBridge(); + if (!bridge || !data) return; auto pinned = data->pin(); - adapter->bridge_->onDataFromGo(reinterpret_cast(pinned.get()), - pinned.size()); + bridge->onDataFromGo(reinterpret_cast(pinned.get()), + pinned.size(), static_cast(epoch)); } JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) { @@ -76,6 +265,8 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) { ->registerNatives({ makeNativeMethod("getBindingsInstaller", getBindingsInstaller), makeNativeMethod("nativeOnDataFromGo", nativeOnDataFromGo), + makeNativeMethod("nativeInvalidate", nativeInvalidate), + makeNativeMethod("nativeResetRecv", nativeResetRecv), }); }); } diff --git a/rnmodules/react-native-kb/android/src/main/java/com/reactnativekb/KbModule.kt b/rnmodules/react-native-kb/android/src/main/java/com/reactnativekb/KbModule.kt index a21166f549cb..56226389262c 100644 --- a/rnmodules/react-native-kb/android/src/main/java/com/reactnativekb/KbModule.kt +++ b/rnmodules/react-native-kb/android/src/main/java/com/reactnativekb/KbModule.kt @@ -33,9 +33,7 @@ import java.io.FileNotFoundException import java.io.FileReader import java.io.IOException import java.lang.reflect.Method -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean import keybase.Keybase import keybase.Keybase.readArr import keybase.Keybase.version @@ -47,9 +45,12 @@ class KbModule(reactContext: ReactApplicationContext?) : KbSpec(reactContext), T @DoNotStrip external override fun getBindingsInstaller(): BindingsInstallerHolder - private external fun nativeOnDataFromGo(data: ByteArray) + private external fun nativeOnDataFromGo(data: ByteArray, epoch: Long) + // Returns true only if this module was still the one owning the installed + // adapter/bridge (see cpp-adapter's nativeInvalidate). + private external fun nativeInvalidate(): Boolean + private external fun nativeResetRecv() - private var executor: ExecutorService? = null private var lifecycleListenerRegistered = false override fun getName(): String { @@ -299,9 +300,11 @@ class KbModule(reactContext: ReactApplicationContext?) : KbSpec(reactContext), T init { this.reactContext = reactContext!! - instance = this misTestDevice = isTestDevice(reactContext) Thread { cachedConstants }.start() + // Published last: the reader thread reads `instance` and must never + // see a partially constructed module. + instance = this } private fun normalizePath(path: String): String { @@ -458,10 +461,17 @@ class KbModule(reactContext: ReactApplicationContext?) : KbSpec(reactContext), T } } + // Synchronous, best-effort mirror of relayReset()'s own check, callable + // from the reader thread: lets a caller decide whether an emit has any + // chance of being delivered before committing to it. + internal fun canDeliverReset(): Boolean = reactContext.hasActiveReactInstance() && canEmit() + + // No current caller (kept for future use). @ReactMethod override fun engineReset() { try { Keybase.reset() + nativeResetRecv() relayReset() } catch (e: Exception) { NativeLogger.error("Exception in engineReset", e) @@ -472,17 +482,18 @@ class KbModule(reactContext: ReactApplicationContext?) : KbSpec(reactContext), T override fun notifyJSReady() { NativeLogger.info("JS signaled ready, starting ReadFromKBLib loop") try { - // Signal to Go that JS is ready + // Signal to Go that JS is ready. This is a sync.Once on the Go + // side, so calling it again after a reload is free. Keybase.notifyJSReady() startReadLoop() - // Register once; restart the read loop on resume, tear down on destroy. + // Register once; tear down the Go connection on destroy. The read + // loop itself is never restarted — see startReadLoop. if (!lifecycleListenerRegistered) { lifecycleListenerRegistered = true reactContext.addLifecycleEventListener(object : LifecycleEventListener { override fun onHostResume() { - startReadLoop() } override fun onHostPause() { @@ -498,72 +509,229 @@ class KbModule(reactContext: ReactApplicationContext?) : KbSpec(reactContext), T } } + // Exactly one reader exists for the life of the process. Go's readArr + // hands back a view of a single shared buffer and is documented as + // "called serially by the mobile run loops": a second concurrent reader + // corrupts both deliveries and the (not thread safe) msgpack unpacker + // behind nativeOnDataFromGo. Stopping it isn't possible either — a thread + // parked in the JNI readArr call ignores interrupts, so shutdownNow() + // returns while the old reader is still live and about to swallow one + // more message. So the loop outlives any module instance and forwards to + // whichever bridge is currently installed. private fun startReadLoop() { - if (executor == null) { - val ex = Executors.newSingleThreadExecutor() - executor = ex - ex.execute(ReadFromKBLib(reactContext)) + if (readLoopStarted.compareAndSet(false, true)) { + Thread(ReadFromKBLib(), "ReadFromKBLib").apply { + isDaemon = true + start() + } } } - // JSI - private inner class ReadFromKBLib(private val reactContext: ReactApplicationContext) : Runnable { + // JSI. Deliberately not an inner class: the thread outlives every module + // instance, so capturing one would pin its ReactContext (and Activity) for + // the life of the process. It forwards to whichever module is current. + private class ReadFromKBLib : Runnable { + private var loggedEmptyRead = false + // See ReadLoopThrottles.kt for the arithmetic and its unit tests. + private val readErrorLog = ReadErrorLogThrottle() + private val emitThrottle = EngineResetEmitThrottle { android.os.SystemClock.elapsedRealtime() } + override fun run() { - do { + while (true) { try { - Thread.currentThread().name = "ReadFromKBLib" - val data: ByteArray = readArr() - if (!reactContext.hasActiveReactInstance()) { - NativeLogger.info("$NAME: JS Bridge is dead, dropping engine message") + val data: ByteArray? = readArr() + // Read immediately after readArr returns, not before it: + // Go's ReadArr records the epoch of the connection it + // actually read from under connMutex as part of the call, + // and with exactly one permanent reader for the life of + // the process (this loop) nothing can have started a + // second readArr in between, so this is exact for the + // bytes just returned -- unlike capturing the epoch + // before the blocking call, which could race a redial + // that happens while this read is in flight. + val epoch = Keybase.lastReadEpoch() + if (data == null || data.isEmpty()) { + // Not the idle path: readArr blocks until there is + // data, so an empty non-error result is degenerate -- + // reachable if Init never ran and the shared buffer is + // zero-length, which would otherwise spin silently. + if (!loggedEmptyRead) { + NativeLogger.warn("$NAME: read returned no data; is Keybase initialized?") + loggedEmptyRead = true + } + Thread.sleep(10) continue } - nativeOnDataFromGo(data) + readErrorLog.reset() + emitThrottle.reset() + instance?.nativeOnDataFromGo(data, epoch) + } catch (e: InterruptedException) { + Thread.currentThread().interrupt() + return } catch (e: Exception) { - if (e.message != null && e.message.equals("Read error: EOF")) { - NativeLogger.info("Got EOF from read. Likely because of reset.") - } else { - NativeLogger.error("Exception in ReadFromKBLib.run", e) + // readArr already called Reset() on the Go side, so the + // connection JS thinks it has is gone and every in-flight + // RPC is dead. EOF is the ordinary shape of this, not a + // surprise -- but JS still has to be told, or its callers + // hang forever. + val isEof = e.message != null && e.message.equals("Read error: EOF") + // EOF is the ordinary shape of a persistent-read outage, + // not a surprise, but at this loop's ~100ms retry it is + // still a ~10Hz flood into the uploadable log if left + // unthrottled. Non-EOF gets its own counter -- see + // ReadErrorLogThrottle. + val logCount = readErrorLog.next(isEof) + if (logCount != null) { + if (isEof) { + NativeLogger.info("Got EOF from read, connection reset (count=$logCount).") + } else { + NativeLogger.error("Exception in ReadFromKBLib.run (count=$logCount)", e) + } + } + val inst = instance + if (inst != null) { + inst.onRpcConnectionReset(emitThrottle.shouldEmit(inst.canDeliverReset())) } - // Back off on error to avoid spinning at full CPU speed when Go is - // unavailable (e.g. during init or loopback restart). - try { Thread.sleep(100) } catch (ie: InterruptedException) { Thread.currentThread().interrupt() } + try { Thread.sleep(100) } catch (ie: InterruptedException) { Thread.currentThread().interrupt(); return } } - } while (!Thread.currentThread().isInterrupted && reactContext.hasActiveReactInstance()) + } } } fun destroy() { + // `instance` is deliberately NOT cleared here. onHostDestroy fires when + // the last Activity goes away, but the ReactInstance and this module + // survive, so init{} never re-runs and nothing would ever restore it — + // every later inbound message would be dropped for the life of the + // process (back button to home, then reopen). It is only a gate: the + // JNI callee ignores the receiver and routes through g_bridge, so a + // stale instance delivers to the correct current bridge regardless. + // Bridge teardown belongs to invalidate() below, which fires on real + // ReactInstance teardown, not Activity death. try { Keybase.reset() relayReset() } catch (e: Exception) { NativeLogger.error("Exception in KeybaseEngine.destroy", e) } - try { - executor?.shutdownNow() + } - // We often hit this timeout during app resume, e.g. hit the back - // button to go to home screen and then tap Keybase app icon again. - if (executor?.awaitTermination(3, TimeUnit.SECONDS)== false) { - NativeLogger.warn("$NAME: Executor pool didn't shut down cleanly") + // Fires on real ReactInstance/TurboModule teardown (reload, instance + // recreation) — unlike onHostDestroy, which fires when the last Activity + // dies while the ReactInstance and this module survive. This is the only + // place that should ever clear the native bridge. + override fun invalidate() { + val wasCurrent = nativeInvalidate() + // Mirrors iOS invalidate's identity-gated KeybaseReset. Replacing the + // runtime replaces the JS transport: it starts with fresh seqids and + // an empty parser, while the still-live Go loopback may hold + // outstanding replies or a partial frame -- which would mis-settle a + // new invocation with an old response, or desync on the first read. + // Resetting the connection along with the bridge keeps the two ends in + // step. + // + // Gated on wasCurrent because a stale invalidate can run after the next + // module already installed its own bridge; resetting then would tear + // down a connection that module already owns and kill its in-flight + // RPCs for nothing. + if (wasCurrent) { + try { + Keybase.reset() + } catch (e: Exception) { + NativeLogger.error("Exception resetting connection in invalidate", e) } - executor = null - } catch (e: Exception) { - NativeLogger.error("Exception in JSI.destroy", e) } + super.invalidate() } // Called from JNI (cpp-adapter writeToGo), not from JS. DoNotStrip keeps it // from being removed/renamed by ProGuard since the only caller is reflective. + // Returns false when the payload never reached Go, so the caller can fail + // that RPC instead of leaving it outstanding forever. @DoNotStrip - fun rpcOnGo(arr: ByteArray) { - try { + fun rpcOnGo(arr: ByteArray): Boolean { + return try { writeArr(arr) + true } catch (e: Exception) { NativeLogger.error("Exception in GoJSIBridge.rpcOnGo", e) + false } } + // Called from JNI when the incoming byte stream desyncs -- also true + // since this can escalate for a msgpack->JSI conversion failure or a + // missing rpcOnJs, arriving on the JS thread rather than the reader + // thread. Either way recv_ still holds bytes from the now-dead + // connection; nativeResetRecv() drops them so the next connection + // doesn't desync on its very first frame. Resetting the Go connection + // and relaying the meta event lets JS fail its outstanding RPCs instead + // of waiting on a channel that can no longer deliver. + @DoNotStrip + fun onRpcStreamFatal(epoch: Long) { + NativeLogger.warn("$NAME: rpc stream desync, resetting connection") + // Reset the Go connection before the parser: the reader thread is + // still live on this path, so clearing the parser first would leave a + // window where bytes from the OLD connection land in the + // freshly-reset unpacker mid-frame, causing a second desync. + // + // resetIfCurrentDidReset(epoch), not reset(): `epoch` is the epoch of + // the connection the desynced bytes actually came from, captured by + // the reader loop at read time. Three outcomes, and only one of them + // keeps the parser's bytes: + // - true: the desynced connection was still current and Go reset it, + // so the buffered partial frame belongs to a dead connection -> + // nativeResetRecv(). + // - threw: we have no idea what state Go is in, and we already know + // this stream desynced -> nativeResetRecv(); keeping the bytes + // would feed old frame fragments into whatever connection exists + // and re-desync immediately. + // - false (stale epoch): Go already re-dialed (e.g. a concurrent + // writeArr recovered first), so this is a no-op instead of tearing + // down a connection that already worked -- and the buffered partial + // frame belongs to that healthy current connection, so dropping it + // would force a second, needless fatal/reset cycle. This is the + // ONLY case that skips nativeResetRecv(). + var didReset = false + var resetThrew = false + try { + didReset = Keybase.resetIfCurrentDidReset(epoch) + } catch (e: Exception) { + resetThrew = true + NativeLogger.error("Exception resetting after rpc desync", e) + } + if (didReset || resetThrew) { + nativeResetRecv() + } + reactContext.runOnUiQueueThread { relayReset() } + } + + // Called from the reader thread when readArr failed and Go reset the + // connection underneath us. Unlike onRpcStreamFatal we must NOT call + // Keybase.reset() -- ReadArr already did, and a second reset would close a + // connection a concurrent writeArr may have just dialed. + // + // nativeResetRecv() runs every time regardless of `emit`: it is cheap and + // must happen on every failed read. `emit` only throttles the + // kb-engine-reset meta event, which the caller rate-limits with a + // backoff -- a connection that cannot be re-dialed retries this path at + // ~10Hz, and disconnectCallback/connectCallback on the JS side are not + // free to re-run at that rate. + fun onRpcConnectionReset(emit: Boolean) { + nativeResetRecv() + if (emit) { + reactContext.runOnUiQueueThread { relayReset() } + } + } + + // Called from JNI. Routes native bridge errors into the uploadable log -- + // __android_log_print only reaches logcat, which a field log send does not + // include. + @DoNotStrip + fun onNativeLog(message: String) { + NativeLogger.error("$NAME: $message") + } + @ReactMethod override fun iosGetHasShownPushPrompt(promise: Promise) { promise.reject(Exception("wrong platform")) @@ -582,9 +750,16 @@ class KbModule(reactContext: ReactApplicationContext?) : KbSpec(reactContext), T const val NAME: String = "Kb" private const val RPC_META_EVENT_ENGINE_RESET: String = "kb-engine-reset" + + // Process-wide, not per-instance: a reload creates a new KbModule but + // must not create a second reader for the one shared Go connection. + private val readLoopStarted = AtomicBoolean(false) private const val MAX_TEXT_FILE_SIZE = 100 * 1024 // 100 kiB private val LINE_SEPARATOR: String? = System.getProperty("line.separator") + // Written on init/destroy, read on the permanent reader thread; needs a + // visibility guarantee so the reader never sees a stale instance. + @Volatile var instance: KbModule? = null @JvmStatic internal var initialNotificationBundle: Bundle? = null diff --git a/rnmodules/react-native-kb/android/src/main/java/com/reactnativekb/ReadLoopThrottles.kt b/rnmodules/react-native-kb/android/src/main/java/com/reactnativekb/ReadLoopThrottles.kt new file mode 100644 index 000000000000..0a7238c027bd --- /dev/null +++ b/rnmodules/react-native-kb/android/src/main/java/com/reactnativekb/ReadLoopThrottles.kt @@ -0,0 +1,105 @@ +package com.reactnativekb + +// Rate-limiting state for the permanent RPC reader loop (see +// KbModule.ReadFromKBLib). Kept here, free of any Android/Keybase dependency +// and with an injectable clock, so the arithmetic is exercisable by a plain +// JVM unit test -- the reader loop itself is a blocking, process-lifetime +// thread that cannot be driven from a test. + +/** + * Reports the first [FIRST_ALWAYS] occurrences of a repeating condition, then + * every [EVERY_NTH]th. + * + * A read error retries every ~100ms; if the connection can't be + * re-established that is a ~10Hz flood into the uploadable log. Unlike a + * one-shot degenerate case, a recurring read error is exactly what an + * operator needs to see recur, so this logs the first few occurrences, then + * backs off to every Nth rather than going silent. + */ +class LogThrottle { + private var count = 0 + + /** The occurrence number to put in the log line, or null if throttled. */ + fun next(): Int? { + count++ + return if (count <= FIRST_ALWAYS || count % EVERY_NTH == 0) count else null + } + + fun reset() { + count = 0 + } + + companion object { + const val FIRST_ALWAYS = 5 + const val EVERY_NTH = 50 + } +} + +/** + * Log throttling for reader-loop read errors, split by kind. + * + * The split is the whole point: the counter only resets on a successful read, + * so a sustained EOF outage drives it into the hundreds before a genuine + * non-EOF failure arrives. Sharing one counter would let the every-Nth + * throttle swallow the very log line (the one with the exception and stack + * trace) an operator needs at exactly that transition. A dedicated counter + * per kind guarantees the first few non-EOF exceptions always log regardless + * of how long the preceding EOF flood ran. + */ +class ReadErrorLogThrottle { + private val eof = LogThrottle() + private val nonEof = LogThrottle() + + /** The occurrence number to put in the log line, or null if throttled. */ + fun next(isEof: Boolean): Int? = if (isEof) eof.next() else nonEof.next() + + fun reset() { + eof.reset() + nonEof.reset() + } +} + +/** + * Throttles the kb-engine-reset EMIT, separately from the log throttles above + * -- they have different cadences and must not share a counter. JS's + * disconnectCallback does a full session-cancel sweep (with a log of its own) + * and connectCallback re-dispatches the bootstrap path, so a connection that + * cannot be re-dialed must not re-trigger those at ~10Hz. The first failure + * emits immediately so JS learns promptly, then backs off exponentially to a + * ceiling. [reset] on the next successful read, so a later, unrelated episode + * again emits promptly. + * + * [nowMs] must be a monotonic clock (SystemClock.elapsedRealtime, not + * System.currentTimeMillis): a backward wall-clock correction during an + * outage must not push the next allowed emit into the future. + */ +class EngineResetEmitThrottle(private val nowMs: () -> Long) { + private var backoffMs = 0L + private var notBeforeMs = 0L + + /** + * [deliverable] gates the backoff advance itself, not just the emit: an + * emit that has nowhere to go (no active react instance / can't emit yet) + * must not cost a full backoff window, or a dropped notification during + * e.g. a reload delays the next one that could actually be delivered. + */ + fun shouldEmit(deliverable: Boolean): Boolean { + val now = nowMs() + if (now < notBeforeMs || !deliverable) { + return false + } + backoffMs = if (backoffMs == 0L) INITIAL_MS else minOf(backoffMs * 2, CEILING_MS) + notBeforeMs = now + backoffMs + return true + } + + fun reset() { + backoffMs = 0L + notBeforeMs = 0L + } + + companion object { + const val INITIAL_MS = 500L + const val CEILING_MS = 5000L + } +} diff --git a/rnmodules/react-native-kb/android/src/test/java/com/reactnativekb/ReadLoopThrottlesTest.kt b/rnmodules/react-native-kb/android/src/test/java/com/reactnativekb/ReadLoopThrottlesTest.kt new file mode 100644 index 000000000000..87f72fd73732 --- /dev/null +++ b/rnmodules/react-native-kb/android/src/test/java/com/reactnativekb/ReadLoopThrottlesTest.kt @@ -0,0 +1,207 @@ +package com.reactnativekb + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class EngineResetEmitThrottleTest { + // Stand-in for SystemClock.elapsedRealtime so the backoff windows can be + // stepped exactly instead of slept through. + private class FakeClock(var nowMs: Long = 0L) { + fun advance(ms: Long) { + nowMs += ms + } + } + + private fun throttle(clock: FakeClock) = EngineResetEmitThrottle { clock.nowMs } + + @Test + fun firstFailureEmitsImmediately() { + val clock = FakeClock() + val t = throttle(clock) + assertTrue(t.shouldEmit(true)) + } + + @Test + fun secondEmitIsSuppressedUntilTheWindowElapses() { + val clock = FakeClock() + val t = throttle(clock) + assertTrue(t.shouldEmit(true)) + // ~10Hz retry loop: nine more failures inside the first 500ms window. + for (i in 1..9) { + clock.advance(49) + assertFalse("emit at ${clock.nowMs}ms should be suppressed", t.shouldEmit(true)) + } + clock.advance(EngineResetEmitThrottle.INITIAL_MS - clock.nowMs) + assertTrue(t.shouldEmit(true)) + } + + @Test + fun intervalDoublesPerEmit() { + val clock = FakeClock() + val t = throttle(clock) + val expected = listOf(500L, 1000L, 2000L, 4000L) + var last = 0L + assertTrue(t.shouldEmit(true)) + for (want in expected) { + // One tick short of the window: still suppressed. + clock.nowMs = last + want - 1 + assertFalse("emit at +${want - 1}ms should be suppressed", t.shouldEmit(true)) + clock.nowMs = last + want + assertTrue("emit at +${want}ms should be allowed", t.shouldEmit(true)) + last = clock.nowMs + } + } + + @Test + fun clampsAtCeiling() { + val clock = FakeClock() + val t = throttle(clock) + assertTrue(t.shouldEmit(true)) + // Drive well past the point where doubling exceeds the ceiling. + var last = 0L + var window = EngineResetEmitThrottle.INITIAL_MS + repeat(20) { + clock.nowMs = last + window + assertTrue(t.shouldEmit(true)) + last = clock.nowMs + window = minOf(window * 2, EngineResetEmitThrottle.CEILING_MS) + } + assertEquals(EngineResetEmitThrottle.CEILING_MS, window) + // Now confirm the live window really is the ceiling and not larger: + // one tick short is suppressed, exactly the ceiling is allowed. + clock.nowMs = last + EngineResetEmitThrottle.CEILING_MS - 1 + assertFalse(t.shouldEmit(true)) + clock.nowMs = last + EngineResetEmitThrottle.CEILING_MS + assertTrue(t.shouldEmit(true)) + } + + @Test + fun resetOnSuccessfulReadEmitsPromptlyAgain() { + val clock = FakeClock() + val t = throttle(clock) + assertTrue(t.shouldEmit(true)) + clock.advance(500) + assertTrue(t.shouldEmit(true)) // backoff now 1000ms + // A successful read lands. + t.reset() + // A later, unrelated episode must notify JS immediately, and start + // over at the initial interval rather than resuming at 2000ms. + clock.advance(1) + assertTrue(t.shouldEmit(true)) + clock.nowMs += EngineResetEmitThrottle.INITIAL_MS - 1 + assertFalse(t.shouldEmit(true)) + clock.advance(1) + assertTrue(t.shouldEmit(true)) + } + + @Test + fun undeliverableDoesNotAdvanceTheBackoff() { + val clock = FakeClock() + val t = throttle(clock) + // A whole episode's worth of failures while JS can't receive them + // (reload in flight): none of them may cost a backoff window. + repeat(100) { + clock.advance(100) + assertFalse(t.shouldEmit(false)) + } + // The moment JS can receive again, the very next failure emits + // immediately -- not after a multi-second window. + assertTrue(t.shouldEmit(true)) + // ...and it is the FIRST emit, so the window that follows is the + // initial one, not a ceiling-sized one grown by the undeliverable run. + val at = clock.nowMs + clock.nowMs = at + EngineResetEmitThrottle.INITIAL_MS - 1 + assertFalse(t.shouldEmit(true)) + clock.advance(1) + assertTrue(t.shouldEmit(true)) + } + + @Test + fun undeliverableDuringAnOpenWindowStillDoesNotAdvance() { + val clock = FakeClock() + val t = throttle(clock) + assertTrue(t.shouldEmit(true)) // window 500ms + clock.advance(500) // window is open again + assertFalse(t.shouldEmit(false)) // ...but nothing can be delivered + // The open window must survive: the next deliverable failure emits. + assertTrue(t.shouldEmit(true)) + } +} + +class LogThrottleTest { + @Test + fun logsFirstFiveThenEveryFiftiethWithTheOccurrenceNumber() { + val t = LogThrottle() + for (i in 1..5) { + assertEquals("occurrence $i should log", i, t.next()) + } + for (i in 6..49) { + assertNull("occurrence $i should be throttled", t.next()) + } + assertEquals(50, t.next()) + for (i in 51..99) { + assertNull("occurrence $i should be throttled", t.next()) + } + assertEquals(100, t.next()) + } + + @Test + fun resetRestartsTheWindow() { + val t = LogThrottle() + repeat(300) { t.next() } + t.reset() + assertEquals(1, t.next()) + } +} + +class ReadErrorLogThrottleTest { + // The entire point of the split: an EOF flood must not be able to + // throttle away the FIRST non-EOF exception -- that is the one log line + // carrying the exception and stack trace, and it arrives at exactly the + // transition an operator needs. The counters only reset on a successful + // read, so during an outage there is no upper bound on how far the EOF + // count has run. + @Test + fun eofFloodDoesNotSwallowTheFirstNonEofError() { + val t = ReadErrorLogThrottle() + repeat(300) { t.next(isEof = true) } + assertEquals("first non-EOF error must always log", 1, t.next(isEof = false)) + // ...and so must the next few. + for (i in 2..5) { + assertEquals("non-EOF error $i must log", i, t.next(isEof = false)) + } + assertNull(t.next(isEof = false)) + } + + // A single shared counter is what this guards against: after a 300-deep + // EOF flood the first non-EOF error lands on occurrence 301, which the + // every-50th throttle drops. Pinned here so the mutation is visible. + @Test + fun aSharedCounterWouldHaveDroppedIt() { + val shared = LogThrottle() + repeat(300) { shared.next() } + assertNull(shared.next()) + } + + @Test + fun theTwoKindsCountIndependently() { + val t = ReadErrorLogThrottle() + assertEquals(1, t.next(isEof = true)) + assertEquals(1, t.next(isEof = false)) + assertEquals(2, t.next(isEof = true)) + assertEquals(2, t.next(isEof = false)) + } + + @Test + fun resetClearsBothKinds() { + val t = ReadErrorLogThrottle() + repeat(300) { t.next(isEof = true) } + repeat(300) { t.next(isEof = false) } + t.reset() + assertEquals(1, t.next(isEof = true)) + assertEquals(1, t.next(isEof = false)) + } +} diff --git a/rnmodules/react-native-kb/cpp/engine-reset-backoff.h b/rnmodules/react-native-kb/cpp/engine-reset-backoff.h new file mode 100644 index 000000000000..c2cd02193ced --- /dev/null +++ b/rnmodules/react-native-kb/cpp/engine-reset-backoff.h @@ -0,0 +1,59 @@ +#pragma once + +// Emit backoff for the permanent RPC reader loop (ios/Kb.mm's ReadArr loop). +// +// Header-only, with the clock passed IN rather than read here, so the +// arithmetic is exercisable by a plain unit test (cpp/tests/ +// engine-reset-backoff-test.cpp, built by scripts/test-engine-reset-backoff.sh) +// -- the reader loop itself is a blocking, process-lifetime dispatch queue +// that cannot be driven from a test. The Android twin is +// EngineResetEmitThrottle in android/src/main/java/com/reactnativekb/ +// ReadLoopThrottles.kt; keep the two in step. + +namespace kb { + +// Throttles the kb-engine-reset EMIT, separately from the read-error log line +// -- they have different cadences and must not share a counter. JS's +// disconnectCallback does a full session-cancel sweep (with a log of its own) +// and connectCallback re-dispatches the bootstrap path, so a connection that +// cannot be re-dialed must not re-trigger those at the ~10Hz this loop retries +// at. The first failure emits immediately so JS learns promptly; each later +// failure in the same episode backs off exponentially to a ceiling. reset() on +// the next successful read, so a later, unrelated episode again emits +// promptly. +class EngineResetEmitBackoff { +public: + static constexpr double kInitialSeconds = 0.5; + static constexpr double kCeilingSeconds = 5.0; + + // `now` must come from a monotonic clock (CACurrentMediaTime, not NSDate): + // a backward wall-clock correction during a read-error episode (plausible at + // cold boot) must not push the next allowed emit into the future and + // suppress the notification entirely. + // + // `deliverable` gates the backoff advance itself, not just the emit: an emit + // that has nowhere to go (no shared instance / can't emit yet) must not cost + // a full backoff window, or a notification dropped during e.g. a reload + // delays the next one that could actually be delivered. + bool shouldEmit(double now, bool deliverable) { + if (now < notBefore_ || !deliverable) { + return false; + } + backoff_ = backoff_ == 0 ? kInitialSeconds + : (backoff_ * 2 < kCeilingSeconds ? backoff_ * 2 + : kCeilingSeconds); + notBefore_ = now + backoff_; + return true; + } + + void reset() { + backoff_ = 0; + notBefore_ = 0; + } + +private: + double backoff_ = 0; + double notBefore_ = 0; +}; + +} // namespace kb diff --git a/rnmodules/react-native-kb/cpp/frame-parser.cpp b/rnmodules/react-native-kb/cpp/frame-parser.cpp new file mode 100644 index 000000000000..b5f3e71ce5fc --- /dev/null +++ b/rnmodules/react-native-kb/cpp/frame-parser.cpp @@ -0,0 +1,73 @@ +#include "frame-parser.h" +#include +#include + +namespace kb { + +void FrameParser::feed(const uint8_t *data, size_t size, + std::vector &out) { + unpacker_->reserve_buffer(size); + std::memcpy(unpacker_->buffer(), data, size); + unpacker_->buffer_consumed(size); + totalFed_ += size; + + while (true) { + msgpack::object_handle result; + if (!unpacker_->next(result)) { + break; + } + if (state_ == ReadState::needSize) { + // The framing prefix must be a msgpack uint. Anything else means the + // stream desynced; without this check the parity flips and every + // later frame is silently swallowed as a "size". + const auto &o = result.get(); + if (o.type != msgpack::type::POSITIVE_INTEGER || + o.as() > kMaxFrameSize) { + throw std::runtime_error("bad rpc frame header"); + } + declaredSize_ = static_cast(o.as()); + peakFrameSize_ = std::max(peakFrameSize_, declaredSize_); + consumedAtHeader_ = totalFed_ - unpacker_->nonparsed_size(); + state_ = ReadState::needContent; + } else { + // The header is only a plausibility check on its own: a fixint sitting + // inside a string payload parses as a valid header after a resync. + // Requiring the content object to consume exactly the declared byte + // count makes the framing self-checking, so a truncated or overlong + // frame is caught here instead of being handed to JS as a bogus + // [type, seqid, ...]. + // + // parsed_size() is NOT usable here: unpacker::next() resets it to 0 on + // every successful parse, so it can't measure a span across two next() + // calls. totalFed - nonparsed_size() is a genuine monotonic count of + // bytes consumed from the stream. + const size_t consumed = + (totalFed_ - unpacker_->nonparsed_size()) - consumedAtHeader_; + if (consumed != declaredSize_) { + throw std::runtime_error("rpc frame length mismatch"); + } + out.push_back(std::move(result)); + state_ = ReadState::needSize; + } + } + + if (unpacker_->nonparsed_size() > kMaxFrameSize + kMaxFrameSlack) { + throw std::runtime_error("rpc frame exceeds size limit"); + } +} + +void FrameParser::reset() { + unpacker_ = std::make_unique(); + state_ = ReadState::needSize; + declaredSize_ = 0; + consumedAtHeader_ = 0; + totalFed_ = 0; + peakFrameSize_ = 0; +} + +bool FrameParser::atSafeShrinkPoint() const { + return state_ == ReadState::needSize && unpacker_->nonparsed_size() == 0 && + peakFrameSize_ > kRecvBufKeepCapacity; +} + +} // namespace kb diff --git a/rnmodules/react-native-kb/cpp/frame-parser.h b/rnmodules/react-native-kb/cpp/frame-parser.h new file mode 100644 index 000000000000..29ce7c98f9be --- /dev/null +++ b/rnmodules/react-native-kb/cpp/frame-parser.h @@ -0,0 +1,98 @@ +// Pure msgpack framing state machine, deliberately free of jsi/React/platform +// dependencies so it can be unit tested without a JS runtime. See +// react-native-kb.cpp's onDataFromGo for how the JSI-facing half (delivery to +// JS, epoch handling, error reporting) wraps this. +#pragma once + +#include "msgpack-safe.hpp" +#include +#include +#include +#include +#include +#include + +namespace kb { + +// Decodes a stream of `` frames, matching the +// JS-side packetizer. Not thread safe; the caller (KBBridge::onDataFromGo) +// serializes access under recvMutex_. +class FrameParser { +public: + // A desynced length prefix can otherwise ask us to buffer gigabytes. + // Matches the JS-side packetizer limit. + static constexpr uint64_t kMaxFrameSize = 64ull * 1024 * 1024; + // The end-of-feed() check `nonparsed_size() > kMaxFrameSize + kMaxFrameSlack` + // guards against a garbage/corrupt length prefix (e.g. a malicious or + // desynced header claiming a multi-gigabyte size) that would otherwise make + // the unpacker buffer unboundedly. It is not needed to protect legal + // traffic: nonparsed_size() at that point can only be an incomplete + // (not-yet-fully-parsed) frame, and every declared size is already checked + // against kMaxFrameSize as soon as its header is read, so a legal frame's + // nonparsed_size() can never exceed kMaxFrameSize even with zero slack. + // This margin exists purely to keep the check from being a razor's-edge + // `> kMaxFrameSize` on the garbage-detection path; it does not need to be + // large, just nonzero. + static constexpr size_t kMaxFrameSlack = 1024 * 1024; + // msgpack::unpacker rewinds its buffer in place but never shrinks the + // realloc'd allocation, so this bounds how long a single large frame's peak + // stays resident. + static constexpr size_t kRecvBufKeepCapacity = 4u * 1024 * 1024; + + FrameParser() = default; + FrameParser(const FrameParser &) = delete; + FrameParser &operator=(const FrameParser &) = delete; + + // Feeds `size` bytes of newly received data into the unpacker and decodes + // as many complete frames as are now available, appending them to `out` in + // wire order. Throws std::runtime_error (with a descriptive message) on any + // framing violation: bad header type/value, length mismatch, or exceeding + // the size limit. On throw, `out` may already contain frames decoded + // earlier in this same call -- the caller must reset() before continuing, + // since the stream can no longer be trusted from that point on. + void feed(const uint8_t *data, size_t size, + std::vector &out); + + // Drops any partially parsed frame and all buffered state, starting over + // from a fresh unpacker. Must be called after feed() throws, or whenever + // the underlying byte stream is known to have been replaced (e.g. the Go + // connection was redialed), since resuming mid-frame on a new stream would + // fail the next header check on otherwise-valid data. + void reset(); + + // True if state == needSize, nonparsed_size() == 0 (no partial frame and no + // unparsed bytes buffered), and peakFrameSize since the last reset exceeds + // kRecvBufKeepCapacity. This is a provably safe resync point at which the + // caller may choose to reset() purely to release a large realloc'd buffer, + // independent of any framing error. + bool atSafeShrinkPoint() const; + +private: + enum class ReadState { needSize, needContent }; + + // Held by pointer so reset() can replace it with a freshly constructed one. + // msgpack::unpacker must never be move-assigned: its parser base holds a + // *reference* to the finalizer living inside the unpacker object, and the + // move constructor copies that reference rather than rebinding it, so a + // moved-into unpacker keeps pointing at the (destroyed) source. The next + // buffer expansion then writes through that dangling reference. + std::unique_ptr unpacker_ = + std::make_unique(); + ReadState state_ = ReadState::needSize; + // Persist across calls: a frame's header and its content routinely arrive + // in separate reads. + size_t declaredSize_ = 0; + size_t consumedAtHeader_ = 0; + // Every byte ever handed to the unpacker. parsed_size() cannot serve this + // role -- next() zeroes it on each success -- but totalFed minus + // nonparsed_size() is an accurate running count of what has been consumed. + size_t totalFed_ = 0; + // High-water mark of declaredSize since the last reset. declaredSize + // itself gets overwritten by the next header before we necessarily reach a + // safe (nonparsed_size() == 0) point to act on it, so this survives that + // overwrite and lets the shrink check fire for the frame that actually + // grew the buffer. + size_t peakFrameSize_ = 0; +}; + +} // namespace kb diff --git a/rnmodules/react-native-kb/cpp/react-native-kb.cpp b/rnmodules/react-native-kb/cpp/react-native-kb.cpp index 21ae0b552600..bf08f2693da3 100644 --- a/rnmodules/react-native-kb/cpp/react-native-kb.cpp +++ b/rnmodules/react-native-kb/cpp/react-native-kb.cpp @@ -2,44 +2,101 @@ #include #include #include +#include "frame-parser.h" #include "msgpack-safe.hpp" +#include #include +#include +#include using namespace facebook; using namespace facebook::jsi; namespace kb { -struct KBBridge::MsgpackState { - msgpack::unpacker unpacker; +namespace { +// A desynced length prefix can otherwise ask us to buffer gigabytes. Matches +// the JS-side packetizer limit. +constexpr uint64_t kMaxFrameSize = FrameParser::kMaxFrameSize; +// Conversion is iterative, so nesting costs heap rather than native stack. +// This is a sanity bound on pathological payloads, not a stack guard. +constexpr size_t kMaxDepth = 1024; +// Don't let one huge attachment frame permanently retain its peak size. +constexpr size_t kSendBufKeepCapacity = 4u * 1024 * 1024; +constexpr size_t kMaxCachedPropNames = 4096; +} // namespace + +struct KBBridge::RecvState { + // Owns the unpacker and all pure framing arithmetic; see frame-parser.h. + // This is only a meaningful shrink trigger because Go's ReadArr + // (go/bind/keybase.go) reads into a fixed 300KB buffer per call: a stream + // of many small frames delivered across many small reads can't durably + // grow the unpacker, since msgpack::unpacker rewinds its buffer in place + // once fully drained. If that Go-side buffer ever grows, peakFrameSize + // stops tracking "did one big frame arrive" and starts tracking "did + // several small frames arrive in the same read" instead. + FrameParser parser; +}; + +struct KBBridge::SendState { msgpack::sbuffer sendBuf; }; KBBridge::KBBridge() = default; KBBridge::~KBBridge() = default; +void KBBridge::markTornDown() { isTornDown_.store(true); } + void KBBridge::teardown() { isTornDown_.store(true); - // Clear cached JSI objects while the runtime is still alive. - // This prevents stale jsi::Function destructors from crashing - // if the bridge outlives the runtime (due to shared_ptr captures). + releaseJSIState(); +} + +void KBBridge::tearup() { isTornDown_.store(false); } + +void KBBridge::resetRecv() { + std::lock_guard lock(recvMutex_); + if (recv_) { + resetRecvLocked(); + } +} + +// Clears cached JSI objects. Must run on the runtime's thread while the +// runtime is still alive — destroying jsi handles elsewhere is UB. +void KBBridge::releaseJSIState() { cachedUint8ArrayCtor_.reset(); - cachedRpcOnJs_.reset(); + cachedIsView_.reset(); + cachedRpcOnJsName_.reset(); cachedPropNames_.clear(); cachedRuntime_ = nullptr; } -void KBBridge::tearup() { isTornDown_.store(false); } - void KBBridge::resetCaches(Runtime &runtime) { if (cachedRuntime_ != &runtime) { - cachedUint8ArrayCtor_.reset(); - cachedRpcOnJs_.reset(); - cachedPropNames_.clear(); + releaseJSIState(); cachedRuntime_ = &runtime; } } +void KBBridge::reportError(const std::string &msg) { + if (onError_) { + onError_(msg); + } +} + +// Resets framing state in place when a RecvState already exists (the common +// case: a fresh connection or post-error recovery), constructing one only the +// first time (recv_ starts null). RecvState currently owns nothing besides +// the parser, so recv_->parser.reset() is behavior-equivalent to discarding +// and rebuilding the whole RecvState, without the reallocation. +void KBBridge::resetRecvLocked() { + if (recv_) { + recv_->parser.reset(); + } else { + recv_ = std::make_unique(); + } +} + Function &KBBridge::uint8ArrayCtor(Runtime &runtime) { resetCaches(runtime); if (!cachedUint8ArrayCtor_) { @@ -49,6 +106,21 @@ Function &KBBridge::uint8ArrayCtor(Runtime &runtime) { return *cachedUint8ArrayCtor_; } +Function &KBBridge::arrayBufferIsView(Runtime &runtime) { + resetCaches(runtime); + if (!cachedIsView_) { + auto ab = runtime.global().getPropertyAsObject(runtime, "ArrayBuffer"); + auto fn = ab.getPropertyAsFunction(runtime, "isView"); + cachedIsView_ = std::make_unique(std::move(fn)); + } + return *cachedIsView_; +} + +bool KBBridge::isArrayBufferView(Runtime &runtime, const Object &obj) { + auto res = arrayBufferIsView(runtime).call(runtime, Value(runtime, obj)); + return res.isBool() && res.getBool(); +} + namespace { // Single-pass copy into an uninitialized buffer. Constructing a // Uint8Array of the target size would zero-fill it before we memcpy @@ -68,37 +140,8 @@ class CopiedBuffer : public MutableBuffer { std::unique_ptr buf_; size_t size_; }; -} // namespace -Value KBBridge::binaryFromBytes(Runtime &runtime, const char *ptr, - size_t size) { - ArrayBuffer buffer(runtime, std::make_shared(ptr, size)); - return uint8ArrayCtor(runtime).callAsConstructor(runtime, - std::move(buffer)); -} - -const PropNameID &KBBridge::mapKeyPropName(Runtime &runtime, const char *ptr, - size_t size) { - // RPC maps reuse a small set of string keys; caching the PropNameIDs - // avoids re-interning the same symbols on every message. - propNameScratch_.assign(ptr, size); - auto it = cachedPropNames_.find(propNameScratch_); - if (it == cachedPropNames_.end()) { - if (cachedPropNames_.size() >= 4096) { - // Some maps are keyed by dynamic IDs; cap growth and restart. - cachedPropNames_.clear(); - } - it = cachedPropNames_ - .emplace(propNameScratch_, - PropNameID::forUtf8( - runtime, reinterpret_cast(ptr), - size)) - .first; - } - return it->second; -} - -static std::string mpToString(msgpack::object &o) { +std::string mpToString(const msgpack::object &o) { switch (o.type) { case msgpack::type::STR: return o.as(); @@ -107,7 +150,6 @@ static std::string mpToString(msgpack::object &o) { case msgpack::type::NEGATIVE_INTEGER: return std::to_string(o.as()); case msgpack::type::FLOAT32: - return std::to_string(o.as()); case msgpack::type::FLOAT64: return std::to_string(o.as()); default: @@ -115,245 +157,447 @@ static std::string mpToString(msgpack::object &o) { } } +void packNumber(msgpack::packer &pk, double d) { + // Encode exact integers as msgpack int/uint so Go's decoder sees integer + // types, not float64. @msgpack/msgpack does the same only for safe + // integers (|d| <= 2^53, Number.isSafeInteger); this goes further and + // packs any integer-valued double in [INT64_MIN, UINT64_MAX] as an + // integer, which typed Go decoding coerces identically -- only an + // interface{}/reflect consumer could see uint64 where the old JS encoder + // produced float64, and only for values above 2^53. Integer-valued + // doubles outside [INT64_MIN, UINT64_MAX] would be UB to cast, so those + // stay float64. 18446744073709551616.0 == 2^64 and + // -9223372036854775808.0 == -2^63 are both exact doubles. + // + // The range check comes first so the cast is always defined, and the + // round-trip through the integer is what proves d was integral -- doing it + // that way instead of floor()/isfinite() keeps this off libm, which is + // worth ~20% of total pack time on real traffic since every number on the + // wire goes through here. NaN and the infinities fail both range checks and + // fall through to pack(d), as they did before. + if (d >= 0 && d < 18446744073709551616.0) { + const uint64_t u = static_cast(d); + if (static_cast(u) == d) { + pk.pack(u); + return; + } + } else if (d < 0 && d >= -9223372036854775808.0) { + const int64_t i = static_cast(d); + if (static_cast(i) == d) { + pk.pack(i); + return; + } + } + pk.pack(d); +} + +void packBytes(msgpack::packer &pk, const uint8_t *data, + size_t length) { + if (length > std::numeric_limits::max()) { + throw std::runtime_error("binary payload too large"); + } + pk.pack_bin(static_cast(length)); + pk.pack_bin_body(reinterpret_cast(data), + static_cast(length)); +} + +// Frame for the iterative msgpack -> JSI walk. Exactly one of arr/obj is set. +struct BuildFrame { + const msgpack::object *node = nullptr; + uint32_t i = 0; + bool isArray = false; + std::optional arr; + std::optional obj; +}; + +// Frame for the iterative JSI -> msgpack walk. `names` is the property-name +// snapshot for plain objects; `len` is captured up front because the msgpack +// map/array header is written before any of the children. +struct PackFrame { + size_t i = 0; + size_t len = 0; + bool isArray = false; + std::optional arr; + std::optional obj; + std::optional names; +}; +} // namespace + +Value KBBridge::binaryFromBytes(Runtime &runtime, const char *ptr, + size_t size) { + ArrayBuffer buffer(runtime, std::make_shared(ptr, size)); + return uint8ArrayCtor(runtime).callAsConstructor(runtime, + std::move(buffer)); +} + +void KBBridge::setObjectKey(Runtime &runtime, Object &obj, const char *ptr, + size_t size, const Value &value) { + // RPC maps reuse a small set of string keys; caching the PropNameIDs + // avoids re-interning the same symbols on every message. The cache is + // never cleared once full — maps keyed by dynamic IDs would otherwise + // invalidate entries other frames still reference — so overflow keys + // just build a throwaway PropNameID. + propNameScratch_.assign(ptr, size); + auto it = cachedPropNames_.find(propNameScratch_); + if (it != cachedPropNames_.end()) { + obj.setProperty(runtime, it->second, value); + return; + } + auto name = PropNameID::forUtf8( + runtime, reinterpret_cast(ptr), size); + if (cachedPropNames_.size() < kMaxCachedPropNames) { + it = cachedPropNames_.emplace(propNameScratch_, std::move(name)).first; + obj.setProperty(runtime, it->second, value); + return; + } + obj.setProperty(runtime, name, value); +} + Value KBBridge::convertMPToJSI(Runtime &runtime, void *mpObj) { - auto &o = *static_cast(mpObj); - switch (o.type) { - case msgpack::type::STR: - return jsi::String::createFromUtf8(runtime, - reinterpret_cast(o.via.str.ptr), o.via.str.size); - case msgpack::type::POSITIVE_INTEGER: - return jsi::Value(o.as()); - case msgpack::type::NEGATIVE_INTEGER: - return jsi::Value(o.as()); - case msgpack::type::FLOAT32: - return jsi::Value(o.as()); - case msgpack::type::FLOAT64: - return jsi::Value(o.as()); - case msgpack::type::BOOLEAN: - return jsi::Value(o.as()); - case msgpack::type::NIL: - return jsi::Value::null(); - case msgpack::type::EXT: - return jsi::Value::undefined(); - case msgpack::type::MAP: { - jsi::Object obj = jsi::Object(runtime); - auto *p = o.via.map.ptr; - auto *const pend = o.via.map.ptr + o.via.map.size; - for (; p < pend; ++p) { - auto val = convertMPToJSI(runtime, &p->val); - auto &k = p->key; + // Iterative rather than recursive: nesting depth is driven by the wire + // payload, and a recursive walk would overflow the native stack. + auto scalar = [&](const msgpack::object &o, Value &out) -> bool { + switch (o.type) { + case msgpack::type::STR: + out = String::createFromUtf8( + runtime, reinterpret_cast(o.via.str.ptr), + o.via.str.size); + return true; + case msgpack::type::POSITIVE_INTEGER: + case msgpack::type::NEGATIVE_INTEGER: + case msgpack::type::FLOAT32: + case msgpack::type::FLOAT64: + out = Value(o.as()); + return true; + case msgpack::type::BOOLEAN: + out = Value(o.as()); + return true; + case msgpack::type::NIL: + out = Value::null(); + return true; + case msgpack::type::BIN: + out = binaryFromBytes(runtime, o.via.bin.ptr, o.via.bin.size); + return true; + case msgpack::type::MAP: + case msgpack::type::ARRAY: + return false; + default: + // EXT and anything unknown. + out = Value::undefined(); + return true; + } + }; + + auto makeFrame = [&](const msgpack::object &o) { + BuildFrame f; + f.node = &o; + f.isArray = (o.type == msgpack::type::ARRAY); + if (f.isArray) { + f.arr.emplace(runtime, o.via.array.size); + } else { + f.obj.emplace(runtime); + } + return f; + }; + + // Attaches a finished child into its parent at the parent's cursor, then + // advances the cursor. + auto attach = [&](BuildFrame &f, const Value &value) { + if (f.isArray) { + f.arr->setValueAtIndex(runtime, f.i, value); + } else { + const auto &k = f.node->via.map.ptr[f.i].key; if (k.type == msgpack::type::STR) { - obj.setProperty( - runtime, mapKeyPropName(runtime, k.via.str.ptr, k.via.str.size), - val); + setObjectKey(runtime, *f.obj, k.via.str.ptr, k.via.str.size, value); } else { auto keyStr = mpToString(k); - obj.setProperty(runtime, - mapKeyPropName(runtime, keyStr.data(), keyStr.size()), - val); + setObjectKey(runtime, *f.obj, keyStr.data(), keyStr.size(), value); } } - return obj; - } - case msgpack::type::BIN: { - auto ptr = o.via.bin.ptr; - auto size = o.via.bin.size; - return binaryFromBytes(runtime, ptr, size); - } - case msgpack::type::ARRAY: { - auto size = o.via.array.size; - jsi::Array arr(runtime, size); - for (uint32_t i = 0; i < size; ++i) { - arr.setValueAtIndex(runtime, i, - convertMPToJSI(runtime, &o.via.array.ptr[i])); - } - return arr; + ++f.i; + }; + + const auto &root = *static_cast(mpObj); + Value out; + if (scalar(root, out)) { + return out; } - default: - return jsi::Value::undefined(); + + std::vector stack; + stack.reserve(16); + stack.push_back(makeFrame(root)); + + while (true) { + BuildFrame &f = stack.back(); + const uint32_t size = + f.isArray ? f.node->via.array.size : f.node->via.map.size; + if (f.i >= size) { + Value done = + f.isArray ? Value(std::move(*f.arr)) : Value(std::move(*f.obj)); + stack.pop_back(); + if (stack.empty()) { + return done; + } + attach(stack.back(), done); + continue; + } + + const msgpack::object &child = + f.isArray ? f.node->via.array.ptr[f.i] : f.node->via.map.ptr[f.i].val; + Value value; + if (scalar(child, value)) { + attach(f, value); + continue; + } + if (stack.size() >= kMaxDepth) { + throw std::runtime_error("msgpack nesting too deep"); + } + // `f` is invalidated by the push; nothing below touches it. + stack.push_back(makeFrame(child)); } } void KBBridge::convertJSIToMP(Runtime &runtime, const Value &value, void *packer) { auto &pk = *static_cast *>(packer); - if (value.isNull() || value.isUndefined()) { - pk.pack_nil(); - } else if (value.isBool()) { - pk.pack(value.getBool()); - } else if (value.isNumber()) { - double d = value.getNumber(); - // Doubles can exactly represent integers up to 2^53. Encode exact - // integers as msgpack int/uint (matching @msgpack/msgpack JS behavior) - // so Go's decoder sees integer types, not float64. Integer-valued - // doubles outside [INT64_MIN, UINT64_MAX] would be UB to cast, so - // those stay float64. 18446744073709551616.0 == 2^64 and - // -9223372036854775808.0 == -2^63 are both exact doubles. - if (d == std::floor(d) && std::isfinite(d)) { - if (d >= 0 && d < 18446744073709551616.0) { - pk.pack(static_cast(d)); - } else if (d < 0 && d >= -9223372036854775808.0) { - pk.pack(static_cast(d)); - } else { - pk.pack(d); - } - } else { - pk.pack(d); - } - } else if (value.isString()) { - auto str = value.getString(runtime).utf8(runtime); - pk.pack(str); - } else if (value.isObject()) { - auto obj = value.getObject(runtime); - auto packArrayBufferBytesUnchecked = [&](ArrayBuffer &arrayBuf, - size_t offset, size_t length) { - pk.pack_bin(static_cast(length)); - pk.pack_bin_body( - reinterpret_cast(arrayBuf.data(runtime)) + offset, - static_cast(length)); - }; - auto packArrayBufferBytes = [&](ArrayBuffer &arrayBuf, size_t offset, - size_t length) { - auto bufferSize = arrayBuf.size(runtime); - if (offset > bufferSize || length > bufferSize - offset || - length > std::numeric_limits::max()) { - throw std::runtime_error("ArrayBuffer view is out of range"); - } - pk.pack_bin(static_cast(length)); - pk.pack_bin_body( - reinterpret_cast(arrayBuf.data(runtime)) + offset, - static_cast(length)); - }; + std::vector stack; + stack.reserve(16); + + auto packArrayBufferRange = [&](ArrayBuffer &arrayBuf, double offsetNum, + double lengthNum) { + auto bufferSize = arrayBuf.size(runtime); + if (!std::isfinite(offsetNum) || !std::isfinite(lengthNum) || + offsetNum < 0 || lengthNum < 0 || + offsetNum != std::floor(offsetNum) || + lengthNum != std::floor(lengthNum) || + offsetNum > static_cast(bufferSize) || + lengthNum > static_cast(bufferSize)) { + throw std::runtime_error("ArrayBuffer view is out of range"); + } + auto offset = static_cast(offsetNum); + auto length = static_cast(lengthNum); + if (offset > bufferSize || length > bufferSize - offset) { + throw std::runtime_error("ArrayBuffer view is out of range"); + } + packBytes(pk, arrayBuf.data(runtime) + offset, length); + }; + + // TypedArray/DataView detection. The cheap byteLength probe lets plain RPC + // objects bail after one property get; ArrayBuffer.isView then gives a + // definitive answer, so an object that merely happens to carry + // byteLength/buffer fields is never mistaken for binary. The range is + // always bounds-checked against the backing buffer. + auto tryPackTypedArray = [&](Object &obj) -> bool { + auto byteLengthProp = obj.getProperty(runtime, "byteLength"); + if (!byteLengthProp.isNumber()) { + return false; + } + if (!isArrayBufferView(runtime, obj)) { + return false; + } + auto bufferProp = obj.getProperty(runtime, "buffer"); + if (!bufferProp.isObject()) { + return false; + } + auto bufferObj = bufferProp.asObject(runtime); + if (!bufferObj.isArrayBuffer(runtime)) { + return false; + } + auto arrayBuf = bufferObj.getArrayBuffer(runtime); + auto byteOffsetProp = obj.getProperty(runtime, "byteOffset"); + packArrayBufferRange( + arrayBuf, byteOffsetProp.isNumber() ? byteOffsetProp.getNumber() : 0, + byteLengthProp.getNumber()); + return true; + }; + + // Packs `v`. Containers write only their header here and push a frame; the + // driver loop below walks their children. + auto packOne = [&](const Value &v) { + if (v.isNull() || v.isUndefined()) { + pk.pack_nil(); + return; + } + if (v.isBool()) { + pk.pack(v.getBool()); + return; + } + if (v.isNumber()) { + packNumber(pk, v.getNumber()); + return; + } + if (v.isString()) { + pk.pack(v.getString(runtime).utf8(runtime)); + return; + } + if (!v.isObject()) { + // Symbol/BigInt. Packing nothing would desync the frame: the enclosing + // map header already promised a value for this key. + pk.pack_nil(); + return; + } + + auto obj = v.getObject(runtime); if (obj.isArrayBuffer(runtime)) { auto buf = obj.getArrayBuffer(runtime); - packArrayBufferBytes(buf, 0, buf.size(runtime)); - } else if (obj.isArray(runtime)) { + packBytes(pk, buf.data(runtime), buf.size(runtime)); + return; + } + if (obj.isFunction(runtime)) { + pk.pack_nil(); + return; + } + if (stack.size() >= kMaxDepth) { + throw std::runtime_error("object nesting too deep"); + } + if (obj.isArray(runtime)) { auto arr = obj.getArray(runtime); auto len = arr.size(runtime); - pk.pack_array(static_cast(len)); - for (size_t i = 0; i < len; ++i) { - convertJSIToMP(runtime, arr.getValueAtIndex(runtime, i), &pk); + if (len > std::numeric_limits::max()) { + throw std::runtime_error("array too large"); } - } else { - auto tryPackTypedArray = [&](bool requireUint8Array, - bool validateView) -> bool { - // Check byteLength before instanceOf: plain objects bail after a - // single property get instead of a prototype-chain walk. - auto byteLengthProp = obj.getProperty(runtime, "byteLength"); - if (!byteLengthProp.isNumber()) return false; - if (requireUint8Array && - !obj.instanceOf(runtime, uint8ArrayCtor(runtime))) { - return false; - } - auto bufferProp = obj.getProperty(runtime, "buffer"); - if (!bufferProp.isObject()) return false; - auto bufferObj = bufferProp.asObject(runtime); - if (!bufferObj.isArrayBuffer(runtime)) return false; - auto arrayBuf = bufferObj.getArrayBuffer(runtime); - auto byteOffset = obj.getProperty(runtime, "byteOffset"); - auto byteLengthNum = byteLengthProp.getNumber(); - auto byteOffsetNum = - byteOffset.isNumber() ? byteOffset.getNumber() : 0; - if (validateView) { - if (!std::isfinite(byteOffsetNum) || - !std::isfinite(byteLengthNum) || byteOffsetNum < 0 || - byteLengthNum < 0 || - byteOffsetNum != std::floor(byteOffsetNum) || - byteLengthNum != std::floor(byteLengthNum)) { - return false; - } - auto offset = static_cast(byteOffsetNum); - auto length = static_cast(byteLengthNum); - packArrayBufferBytes(arrayBuf, offset, length); - } else { - auto offset = static_cast(byteOffsetNum); - auto length = static_cast(byteLengthNum); - packArrayBufferBytesUnchecked(arrayBuf, offset, length); - } - return true; - }; - - // Uint8Array is common for RPC binary payloads; check it before - // enumerating indexed properties. - if (tryPackTypedArray(true, true)) { - return; - } - - auto names = obj.getPropertyNames(runtime); - auto len = names.size(runtime); - - // Empty object: could be {} or empty TypedArray — must probe + pk.pack_array(static_cast(len)); if (len == 0) { - if (tryPackTypedArray(false, false)) return; - pk.pack_map(0); return; } + PackFrame f; + f.isArray = true; + f.len = len; + f.arr.emplace(std::move(arr)); + stack.push_back(std::move(f)); + return; + } + if (tryPackTypedArray(obj)) { + return; + } - // Get first property name (needed for MAP encoding anyway). - // TypedArrays have numeric index keys ("0", "1", ...); regular - // RPC objects have string keys. Only probe for TypedArray when - // the first key looks numeric — saves a JSI call per regular object. - auto firstName = names.getValueAtIndex(runtime, 0).getString(runtime); - auto firstStr = firstName.utf8(runtime); - if (!firstStr.empty() && firstStr[0] >= '0' && firstStr[0] <= '9') { - if (tryPackTypedArray(false, false)) return; - } + auto names = obj.getPropertyNames(runtime); + auto len = names.size(runtime); + if (len > std::numeric_limits::max()) { + throw std::runtime_error("object too large"); + } + pk.pack_map(static_cast(len)); + if (len == 0) { + return; + } + PackFrame f; + f.isArray = false; + f.len = len; + f.obj.emplace(std::move(obj)); + f.names.emplace(std::move(names)); + stack.push_back(std::move(f)); + }; + + packOne(value); - // Regular object — encode as MAP - pk.pack_map(static_cast(len)); - pk.pack(firstStr); - convertJSIToMP(runtime, obj.getProperty(runtime, firstName), &pk); - for (size_t i = 1; i < len; ++i) { - auto name = names.getValueAtIndex(runtime, i).getString(runtime); - auto nameStr = name.utf8(runtime); - pk.pack(nameStr); - convertJSIToMP(runtime, obj.getProperty(runtime, name), &pk); + while (!stack.empty()) { + const size_t top = stack.size() - 1; + if (stack[top].i >= stack[top].len) { + stack.pop_back(); + continue; + } + const size_t idx = stack[top].i++; + // Lengths are snapshots, so a getter that mutates the container can't + // desync the frame: missing entries read back as undefined -> nil. + Value child; + if (stack[top].isArray) { + child = stack[top].arr->getValueAtIndex(runtime, idx); + } else { + auto name = stack[top].names->getValueAtIndex(runtime, idx); + if (!name.isString()) { + // getPropertyNames doesn't enumerate symbol keys, but stay in sync + // with the map header regardless. + pk.pack(std::string()); + pk.pack_nil(); + continue; } + auto nameStr = name.getString(runtime); + pk.pack(nameStr.utf8(runtime)); + child = stack[top].obj->getProperty(runtime, nameStr); } + // May push and invalidate references into `stack`; only indices are used + // above, and nothing below touches the old frame. + packOne(child); } } -void KBBridge::packAndSend(Runtime &runtime, const Value &value) { +bool KBBridge::packAndSend(Runtime &runtime, const Value &value) { + // convertJSIToMP runs JS getters, which could re-enter rpcOnGo and clobber + // the shared scratch buffer mid-frame. + if (packing_) { + throw std::runtime_error("rpcOnGo re-entered from a property getter"); + } + packing_ = true; + struct Guard { + bool &flag; + ~Guard() { flag = false; } + } guard{packing_}; + // Reserve space for the frame header (msgpack uint32 length prefix: // 0xce followed by 4 big-endian bytes) up front, then patch it in // place after packing — avoids copying the payload into a second // buffer just to prepend the header. constexpr size_t headerLen = 5; - mp_->sendBuf.clear(); + auto &sendBuf = send_->sendBuf; + sendBuf.clear(); const char placeholder[headerLen] = {0}; - mp_->sendBuf.write(placeholder, headerLen); - msgpack::packer pk(&mp_->sendBuf); + sendBuf.write(placeholder, headerLen); + msgpack::packer pk(&sendBuf); convertJSIToMP(runtime, value, &pk); - auto contentSize = static_cast(mp_->sendBuf.size() - headerLen); - auto *header = reinterpret_cast(mp_->sendBuf.data()); + auto contentBytes = sendBuf.size() - headerLen; + if (contentBytes > kMaxFrameSize) { + throw std::runtime_error("outgoing rpc frame too large"); + } + auto contentSize = static_cast(contentBytes); + auto *header = reinterpret_cast(sendBuf.data()); header[0] = 0xce; header[1] = static_cast(contentSize >> 24); header[2] = static_cast(contentSize >> 16); header[3] = static_cast(contentSize >> 8); header[4] = static_cast(contentSize); - writeToGo_(mp_->sendBuf.data(), mp_->sendBuf.size()); + const bool ok = writeToGo_ ? writeToGo_(sendBuf.data(), sendBuf.size()) + : false; + + if (sendBuf.size() > kSendBufKeepCapacity) { + // Drop the whole buffer rather than keep one attachment's peak + // allocation alive for the rest of the session. `sendBuf` dangles past + // this point. + send_ = std::make_unique(); + } + return ok; } void KBBridge::install( Runtime &runtime, std::shared_ptr callInvoker, - std::function writeToGo, - std::function onError) { + std::function writeToGo, + std::function onError, + std::function onFatal) { callInvoker_ = std::move(callInvoker); onError_ = std::move(onError); + onFatal_ = std::move(onFatal); writeToGo_ = std::move(writeToGo); - mp_ = std::make_unique(); + send_ = std::make_unique(); + { + std::lock_guard lock(recvMutex_); + resetRecvLocked(); + } auto rpcOnGo = Function::createFromHostFunction( runtime, PropNameID::forAscii(runtime, "rpcOnGo"), 1, [self = shared_from_this()](Runtime &runtime, const Value &thisValue, const Value *arguments, size_t count) -> Value { + if (count < 1) { + throw std::runtime_error("rpcOnGo requires one argument"); + } + if (self->isTornDown_.load() || !self->send_) { + return Value(false); + } try { - self->packAndSend(runtime, arguments[0]); - return Value(true); + return Value(self->packAndSend(runtime, arguments[0])); } catch (const std::exception &e) { throw std::runtime_error("Error in rpcOnGo: " + std::string(e.what())); @@ -364,7 +608,9 @@ void KBBridge::install( runtime.global().setProperty(runtime, "rpcOnGo", std::move(rpcOnGo)); - // HostObject that calls teardown when the JS runtime is destroyed + // HostObject that tears down when the JS runtime is destroyed. Its + // destructor runs on the runtime's thread, which is the only place the + // cached jsi handles may be released. class KBTearDownSimple : public jsi::HostObject { public: KBTearDownSimple(std::weak_ptr bridge) : bridge_(bridge) { @@ -394,98 +640,172 @@ void KBBridge::install( std::make_shared(shared_from_this()))); } -void KBBridge::onDataFromGo(uint8_t *data, int size) { - if (isTornDown_.load() || size <= 0) { +void KBBridge::onDataFromGo(uint8_t *data, int size, int64_t epoch) { + if (isTornDown_.load() || size <= 0 || data == nullptr) { return; } - try { - auto values = std::make_shared>(); - mp_->unpacker.reserve_buffer(size); - std::copy(data, data + size, mp_->unpacker.buffer()); - mp_->unpacker.buffer_consumed(size); - while (true) { - msgpack::object_handle result; - if (mp_->unpacker.next(result)) { - if (readState_ == ReadState::needSize) { - readState_ = ReadState::needContent; - } else { - values->push_back(std::move(result)); - readState_ = ReadState::needSize; - } - } else { - break; + auto values = std::make_shared>(); + bool fatal = false; + std::string fatalMsg; + + { + // Serialized against a stray second reader thread: msgpack::unpacker is + // not thread safe, and a duplicate reader would corrupt the heap. + std::lock_guard lock(recvMutex_); + if (!recv_) { + return; + } + try { + recv_->parser.feed(data, static_cast(size), *values); + + // A provably safe resync point: no partial frame and no unparsed + // bytes to lose. resetRecvLocked() resets the parser's unpacker, + // totalFed, and consumedAtHeader together, so that invariant restarts + // from a consistent fresh baseline (and the realloc'd unpacker buffer + // is released) rather than desyncing. + if (recv_->parser.atSafeShrinkPoint()) { + resetRecvLocked(); } + } catch (const std::exception &e) { + fatal = true; + fatalMsg = std::string("Error in onDataFromGo: ") + e.what(); + resetRecvLocked(); + } catch (...) { + fatal = true; + fatalMsg = "Unknown error in onDataFromGo"; + resetRecvLocked(); } - if (values->empty()) { - return; + } + + if (fatal) { + reportError(fatalMsg); + // The stream can no longer be trusted, so anything decoded in this batch + // is dropped. The platform layer resets the Go connection (if `epoch` + // is still current) and signals JS so outstanding RPCs fail instead of + // hanging forever. + if (onFatal_) { + onFatal_(epoch); } + return; + } + + if (values->empty()) { + return; + } - auto self = shared_from_this(); - callInvoker_->invokeAsync([values, self](jsi::Runtime &runtime) { - try { + auto self = shared_from_this(); + // `epoch` is captured by value here, fixed at the moment this batch was + // parsed from the reader thread -- not re-queried once this lambda + // actually runs on the JS thread, which could be arbitrarily later and + // after Go has re-dialed. + callInvoker_->invokeAsync([values, self, epoch](jsi::Runtime &runtime) { + try { + if (self->isTornDown_.load()) { + return; + } + + self->resetCaches(runtime); + if (!self->cachedRpcOnJsName_) { + self->cachedRpcOnJsName_ = std::make_unique( + PropNameID::forAscii(runtime, "rpcOnJs")); + } + // Deliberately re-read the global each batch instead of caching the + // function: JS installs a fresh rpcOnJs whenever the engine client is + // recreated, and a cached handle would keep feeding the dead one. + auto onJsValue = + runtime.global().getProperty(runtime, *self->cachedRpcOnJsName_); + if (!onJsValue.isObject() || + !onJsValue.getObject(runtime).isFunction(runtime)) { + // These messages are already consumed from the unpacker and cannot be + // replayed, so dropping them silently would strand their callers. + self->reportError("rpcOnJs is not installed"); + if (self->onFatal_) { + self->onFatal_(epoch); + } + return; + } + auto onJs = onJsValue.getObject(runtime).getFunction(runtime); + + if (values->size() == 1) { + // Single message: pass directly (no array wrapper) + msgpack::object obj((*values)[0].get()); + Value value = self->convertMPToJSI(runtime, &obj); if (self->isTornDown_.load()) { return; } - - self->resetCaches(runtime); - if (!self->cachedRpcOnJs_) { + onJs.call(runtime, std::move(value), jsi::Value(1)); + } else { + // Convert per message: a single bad message (non-scalar map key, + // nesting over the limit) must not take the whole batch with it and + // strand every other reply's caller. JS iterates the delivered + // array with a plain for-of (index.platform.tsx's global.rpcOnJs + // only uses `count` to decide array-vs-single, not to bound the + // loop), so the array must be sized to what actually converted -- + // a hole left at `values->size()` would hand JS an undefined + // message to dispatch instead of just skipping it. + std::vector converted; + converted.reserve(values->size()); + for (size_t i = 0; i < values->size(); ++i) { try { - auto func = - runtime.global().getPropertyAsFunction(runtime, "rpcOnJs"); - self->cachedRpcOnJs_ = - std::make_unique(std::move(func)); + msgpack::object obj((*values)[i].get()); + converted.push_back(self->convertMPToJSI(runtime, &obj)); + } catch (const std::exception &e) { + self->reportError(std::string("dropping undecodable message: ") + + e.what()); } catch (...) { - if (self->onError_) { - self->onError_("Failed to get rpcOnJs function"); - } - return; + self->reportError("dropping undecodable message: unknown error"); } } - - if (values->size() == 1) { - // Single message: pass directly (no array wrapper) - msgpack::object obj((*values)[0].get()); - Value value = self->convertMPToJSI(runtime, &obj); - if (self->isTornDown_.load()) { - return; - } - self->cachedRpcOnJs_->call(runtime, std::move(value), - jsi::Value(1)); - } else { - // Multiple messages: batch into array, pass count - jsi::Array arr(runtime, values->size()); - for (size_t i = 0; i < values->size(); ++i) { - msgpack::object obj((*values)[i].get()); - arr.setValueAtIndex(runtime, i, - self->convertMPToJSI(runtime, &obj)); - } - if (self->isTornDown_.load()) { - return; + if (converted.empty()) { + // Every message in the batch was already irrevocably consumed off + // the wire during parsing, so whatever seqids they carried will + // never get a reply -- their JS-side callers would hang forever. + // `values` is never empty here (onDataFromGo returns before + // scheduling this lambda otherwise), so this only fires when + // something was genuinely dropped. + self->reportError("dropped entire batch: all " + + std::to_string(values->size()) + + " message(s) failed to convert"); + if (self->onFatal_) { + self->onFatal_(epoch); } - self->cachedRpcOnJs_->call( - runtime, std::move(arr), - jsi::Value(static_cast(values->size()))); + return; } - } catch (const std::exception &e) { - if (self->onError_) { - self->onError_(e.what()); + if (self->isTornDown_.load()) { + return; } - } catch (...) { - if (self->onError_) { - self->onError_("unknown error in onDataFromGo JS callback"); + // The wire shape must match what actually survived conversion, not + // the original batch size: JS's rpcOnJs only unwraps the array when + // count > 1, so a single surviving message must go through as a bare + // value like the values->size() == 1 path above, or JS hands the + // wrapper array itself to isRPCMessage and silently drops it. + if (converted.size() == 1) { + onJs.call(runtime, std::move(converted[0]), jsi::Value(1)); + } else { + jsi::Array arr(runtime, converted.size()); + for (size_t i = 0; i < converted.size(); ++i) { + arr.setValueAtIndex(runtime, i, std::move(converted[i])); + } + onJs.call(runtime, std::move(arr), + jsi::Value(static_cast(converted.size()))); } } - }); - } catch (const std::exception &e) { - if (onError_) { - onError_(std::string("Error in onDataFromGo: ") + e.what()); - } - } catch (...) { - if (onError_) { - onError_("Unknown error in onDataFromGo"); + } catch (const std::exception &e) { + // Nothing decoded reached JS, so every reply in this batch is lost and + // its caller would wait forever. Treat it like a desync: reset the + // connection so JS fails its outstanding RPCs. + self->reportError(e.what()); + if (self->onFatal_) { + self->onFatal_(epoch); + } + } catch (...) { + self->reportError("unknown error in onDataFromGo JS callback"); + if (self->onFatal_) { + self->onFatal_(epoch); + } } - } + }); } } // namespace kb diff --git a/rnmodules/react-native-kb/cpp/react-native-kb.h b/rnmodules/react-native-kb/cpp/react-native-kb.h index 2a8e0debdbaf..1a3d937c9cc8 100644 --- a/rnmodules/react-native-kb/cpp/react-native-kb.h +++ b/rnmodules/react-native-kb/cpp/react-native-kb.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -24,45 +25,96 @@ class KBBridge : public std::enable_shared_from_this { KBBridge &operator=(const KBBridge &) = delete; KBBridge(KBBridge &&) = delete; KBBridge &operator=(KBBridge &&) = delete; + + // `writeToGo` returns false if the native write failed, so the caller's + // RPC can be failed instead of hanging forever waiting for a reply. + // `onFatal` is invoked when the incoming byte stream can no longer be + // trusted (desynced framing, oversized frame), when a decoded message + // fails msgpack->JSI conversion (single message, or an entire batch), or + // when rpcOnJs is not installed. The first case runs synchronously on the + // native reader thread inside onDataFromGo; the latter three run + // asynchronously on the JS thread, from the callInvoker lambda scheduled + // by onDataFromGo. Its int64_t argument is the epoch (go/bind/keybase.go's + // connEpoch) of the Go connection the parsed bytes were actually read + // from — the same value onDataFromGo was called with, carried through to + // wherever onFatal ends up running, sync or async. The platform layer is + // expected to call Go's ResetIfCurrent(epoch) (not an unconditional + // reset) so a fatal blamed on a since-superseded connection can't tear + // down a connection that has already recovered, drop any buffered recv + // state, and emit the engine-reset meta event so JS fails its outstanding + // RPCs. void install(facebook::jsi::Runtime &runtime, std::shared_ptr callInvoker, - std::function writeToGo, - std::function onError); + std::function writeToGo, + std::function onError, + std::function onFatal); + + // Any thread. `epoch` is the epoch of the Go connection `data` was read + // from, captured by the caller at read time (see the platform reader + // loops) — not whatever Go's connection epoch happens to be when this + // function runs. + void onDataFromGo(uint8_t *data, int size, int64_t epoch); + + // Any thread. Drops any partially parsed frame. Must be called whenever the + // Go connection is replaced, or the unpacker resumes mid-frame on a fresh + // stream and the next header check fails on valid data. + void resetRecv(); - void onDataFromGo(uint8_t *data, int size); + // Any thread. Stops all further work. Does NOT touch JSI state, so it is + // safe to call from the main thread / module invalidation. + void markTornDown(); + + // JS thread ONLY — destroys cached jsi handles, which is undefined + // behavior off the runtime's thread. void teardown(); void tearup(); private: std::shared_ptr callInvoker_; std::function onError_; + std::function onFatal_; + std::function writeToGo_; std::atomic isTornDown_{false}; - enum class ReadState { needSize, needContent }; - ReadState readState_ = ReadState::needSize; + // Incoming stream state. Touched only from the native reader thread, and + // under recvMutex_ so a stray second reader can't corrupt the unpacker. + struct RecvState; + std::mutex recvMutex_; + std::unique_ptr recv_; - struct MsgpackState; - std::unique_ptr mp_; + // Outgoing scratch buffer. JS thread only. + struct SendState; + std::unique_ptr send_; + bool packing_ = false; std::unique_ptr cachedUint8ArrayCtor_; - std::unique_ptr cachedRpcOnJs_; + std::unique_ptr cachedIsView_; + std::unique_ptr cachedRpcOnJsName_; std::unordered_map cachedPropNames_; std::string propNameScratch_; facebook::jsi::Runtime *cachedRuntime_ = nullptr; - std::function writeToGo_; void resetCaches(facebook::jsi::Runtime &runtime); - const facebook::jsi::PropNameID & - mapKeyPropName(facebook::jsi::Runtime &runtime, const char *ptr, - size_t size); + void releaseJSIState(); + // Requires recvMutex_. + void resetRecvLocked(); + void reportError(const std::string &msg); + + // Sets obj[key] = value, reusing an interned PropNameID when possible. + void setObjectKey(facebook::jsi::Runtime &runtime, + facebook::jsi::Object &obj, const char *ptr, size_t size, + const facebook::jsi::Value &value); facebook::jsi::Function &uint8ArrayCtor(facebook::jsi::Runtime &runtime); + facebook::jsi::Function &arrayBufferIsView(facebook::jsi::Runtime &runtime); + bool isArrayBufferView(facebook::jsi::Runtime &runtime, + const facebook::jsi::Object &obj); facebook::jsi::Value binaryFromBytes(facebook::jsi::Runtime &runtime, const char *ptr, size_t size); facebook::jsi::Value convertMPToJSI(facebook::jsi::Runtime &runtime, void *mpObj); void convertJSIToMP(facebook::jsi::Runtime &runtime, const facebook::jsi::Value &value, void *packer); - void packAndSend(facebook::jsi::Runtime &runtime, + bool packAndSend(facebook::jsi::Runtime &runtime, const facebook::jsi::Value &value); }; diff --git a/rnmodules/react-native-kb/cpp/tests/engine-reset-backoff-test.cpp b/rnmodules/react-native-kb/cpp/tests/engine-reset-backoff-test.cpp new file mode 100644 index 000000000000..de54d8555214 --- /dev/null +++ b/rnmodules/react-native-kb/cpp/tests/engine-reset-backoff-test.cpp @@ -0,0 +1,116 @@ +// Unit tests for kb::EngineResetEmitBackoff (cpp/engine-reset-backoff.h), the +// kb-engine-reset emit throttle used by ios/Kb.mm's ReadArr loop. The clock is +// passed in, so every window is stepped exactly rather than slept through. +// +// Mirror of android/src/test/java/com/reactnativekb/ReadLoopThrottlesTest.kt -- +// the two platforms must agree. + +#include "../engine-reset-backoff.h" +#include "test-harness.h" + +using kb::EngineResetEmitBackoff; + +namespace { + +constexpr double kInitial = EngineResetEmitBackoff::kInitialSeconds; +constexpr double kCeiling = EngineResetEmitBackoff::kCeilingSeconds; +// One reader-loop retry tick (the loop sleeps 0.1s between failed reads). +constexpr double kTick = 0.1; + +void firstFailureEmitsImmediately() { + EngineResetEmitBackoff b; + CHECK(b.shouldEmit(0, true)); +} + +void secondEmitIsSuppressedUntilTheWindowElapses() { + EngineResetEmitBackoff b; + CHECK(b.shouldEmit(0, true)); + // ~10Hz retry: four more failures inside the first 0.5s window. + for (int i = 1; i <= 4; ++i) { + CHECK(!b.shouldEmit(i * kTick, true)); + } + CHECK(b.shouldEmit(kInitial, true)); +} + +void intervalDoublesPerEmit() { + EngineResetEmitBackoff b; + CHECK(b.shouldEmit(0, true)); + double last = 0; + for (double want : {0.5, 1.0, 2.0, 4.0}) { + CHECK_MSG(!b.shouldEmit(last + want * 0.999, true), + "emit just short of the window should be suppressed"); + CHECK_MSG(b.shouldEmit(last + want, true), + "emit at the window boundary should be allowed"); + last += want; + } +} + +void clampsAtCeiling() { + EngineResetEmitBackoff b; + CHECK(b.shouldEmit(0, true)); + double last = 0; + double window = kInitial; + for (int i = 0; i < 20; ++i) { + last += window; + CHECK(b.shouldEmit(last, true)); + window = window * 2 < kCeiling ? window * 2 : kCeiling; + } + CHECK_EQ(window, kCeiling); + // The live window really is the ceiling, not something larger. + CHECK(!b.shouldEmit(last + kCeiling * 0.999, true)); + CHECK(b.shouldEmit(last + kCeiling, true)); +} + +void resetOnSuccessfulReadEmitsPromptlyAgain() { + EngineResetEmitBackoff b; + CHECK(b.shouldEmit(0, true)); + CHECK(b.shouldEmit(0.5, true)); // backoff now 1.0s + b.reset(); // a successful read landed + // A later, unrelated episode notifies JS immediately and starts over at the + // initial interval rather than resuming at 2.0s. + CHECK(b.shouldEmit(0.6, true)); + CHECK(!b.shouldEmit(0.6 + kInitial * 0.999, true)); + CHECK(b.shouldEmit(0.6 + kInitial, true)); +} + +void undeliverableDoesNotAdvanceTheBackoff() { + EngineResetEmitBackoff b; + // A whole episode's worth of failures while JS can't receive them (reload in + // flight): none may cost a backoff window. + double now = 0; + for (int i = 0; i < 100; ++i) { + now += kTick; + CHECK(!b.shouldEmit(now, false)); + } + // The moment JS can receive again the very next failure emits immediately, + // not after a multi-second window... + CHECK(b.shouldEmit(now, true)); + // ...and it is the FIRST emit, so the window that follows is the initial + // one, not a ceiling-sized one grown by the undeliverable run. + CHECK(!b.shouldEmit(now + kInitial * 0.999, true)); + CHECK(b.shouldEmit(now + kInitial, true)); +} + +void undeliverableDuringAnOpenWindowStillDoesNotAdvance() { + EngineResetEmitBackoff b; + CHECK(b.shouldEmit(0, true)); // window 0.5s + CHECK(!b.shouldEmit(0.5, false)); // open again, but nothing to deliver to + // The open window must survive: the next deliverable failure emits. + CHECK(b.shouldEmit(0.5, true)); +} + +} // namespace + +int main() { + kbtest::Runner r; +#define ADD(fn) r.add(#fn, fn) + ADD(firstFailureEmitsImmediately); + ADD(secondEmitIsSuppressedUntilTheWindowElapses); + ADD(intervalDoublesPerEmit); + ADD(clampsAtCeiling); + ADD(resetOnSuccessfulReadEmitsPromptlyAgain); + ADD(undeliverableDoesNotAdvanceTheBackoff); + ADD(undeliverableDuringAnOpenWindowStillDoesNotAdvance); +#undef ADD + return r.run(); +} diff --git a/rnmodules/react-native-kb/cpp/tests/frame-builder.h b/rnmodules/react-native-kb/cpp/tests/frame-builder.h new file mode 100644 index 000000000000..3ac865b405c4 --- /dev/null +++ b/rnmodules/react-native-kb/cpp/tests/frame-builder.h @@ -0,0 +1,152 @@ +// Test-only helpers for constructing raw `` +// frames byte-for-byte, mirroring the JS-side packetizer that FrameParser +// decodes. +#pragma once + +#include "../msgpack-safe.hpp" +#include +#include +#include +#include + +namespace kbtest { + +using Bytes = std::vector; + +inline Bytes sbufferToBytes(const msgpack::sbuffer &buf) { + const auto *p = reinterpret_cast(buf.data()); + return Bytes(p, p + buf.size()); +} + +// Packs `fn(packer)` into a standalone msgpack object and returns its wire +// bytes. +inline Bytes packContent( + const std::function &)> &fn) { + msgpack::sbuffer buf; + msgpack::packer pk(&buf); + fn(pk); + return sbufferToBytes(buf); +} + +// Packs a bare msgpack unsigned integer header (the smallest encoding msgpack +// chooses for `value`). Useful for exercising the parser against header +// encodings other than the one production actually emits (e.g. small values +// that fit in a 1-byte fixint). +inline Bytes packHeader(uint64_t value) { + msgpack::sbuffer buf; + msgpack::packer pk(&buf); + pk.pack_uint64(value); + return sbufferToBytes(buf); +} + +// Packs the frame header exactly as production writes it +// (react-native-kb.cpp's packAndSend): always 5 bytes -- a 0xce ("uint 32") +// tag byte followed by a big-endian uint32 length -- regardless of how small +// `value` is. Unlike packHeader() above, this never picks a smaller encoding, +// so it's the encoding that must be used to faithfully exercise the header +// split-boundary arithmetic. +inline Bytes packHeaderUint32(uint32_t value) { + Bytes out(5); + out[0] = 0xce; + out[1] = static_cast(value >> 24); + out[2] = static_cast(value >> 16); + out[3] = static_cast(value >> 8); + out[4] = static_cast(value); + return out; +} + +// A well-formed frame: header declares content.size(), content follows. +inline Bytes buildFrame(const Bytes &content) { + Bytes out = packHeader(content.size()); + out.insert(out.end(), content.begin(), content.end()); + return out; +} + +// Same as buildFrame(), but with the header encoded exactly as production +// writes it (see packHeaderUint32()) rather than msgpack's minimal encoding. +inline Bytes buildFrameProdHeader(const Bytes &content) { + Bytes out = packHeaderUint32(static_cast(content.size())); + out.insert(out.end(), content.begin(), content.end()); + return out; +} + +// A frame whose declared size does not match content.size() -- for testing +// the length-mismatch detection. +inline Bytes buildFrameWithDeclaredSize(uint64_t declaredSize, + const Bytes &content) { + Bytes out = packHeader(declaredSize); + out.insert(out.end(), content.begin(), content.end()); + return out; +} + +// Convenience content generators. + +inline Bytes contentSmallMap() { + return packContent([](auto &pk) { + pk.pack_map(2); + pk.pack(std::string("type")); + pk.pack(1); + pk.pack(std::string("seqid")); + pk.pack(42); + }); +} + +inline Bytes contentEmptyMap() { + return packContent([](auto &pk) { pk.pack_map(0); }); +} + +inline Bytes contentEmptyArray() { + return packContent([](auto &pk) { pk.pack_array(0); }); +} + +// Binary payload of exactly `totalWireSize` bytes on the wire (header + +// body), using bin32 encoding (5-byte header) which applies once the body +// exceeds 65535 bytes. Lets a caller hit an exact target frame size, such as +// exactly kMaxFrameSize. +inline Bytes contentBinOfWireSize(size_t totalWireSize) { + constexpr size_t kBin32HeaderLen = 5; + size_t bodyLen = totalWireSize - kBin32HeaderLen; + msgpack::sbuffer buf; + msgpack::packer pk(&buf); + pk.pack_bin(static_cast(bodyLen)); + std::string body(bodyLen, 'x'); + pk.pack_bin_body(body.data(), static_cast(bodyLen)); + return sbufferToBytes(buf); +} + +// A bare msgpack bin32 *tag* (0xc6 + big-endian uint32 length) with no body. +// Feeding this plus fewer than `bodyLen` body bytes leaves the unpacker with a +// msgpack object that can never complete, which is how a corrupt/desynced +// length prefix makes it buffer without bound. Used to drive the +// nonparsed_size() > kMaxFrameSize + kMaxFrameSlack guard. +inline Bytes bin32TagOnly(uint32_t bodyLen) { + Bytes out(5); + out[0] = 0xc6; + out[1] = static_cast(bodyLen >> 24); + out[2] = static_cast(bodyLen >> 16); + out[3] = static_cast(bodyLen >> 8); + out[4] = static_cast(bodyLen); + return out; +} + +inline void packNestedArray(msgpack::packer &pk, + int depth) { + if (depth == 0) { + pk.pack(42); + return; + } + pk.pack_array(1); + packNestedArray(pk, depth - 1); +} + +inline Bytes contentNested(int depth) { + return packContent( + [depth](auto &pk) { packNestedArray(pk, depth); }); +} + +// Non-integer header content (a string), for the "bad header type" case. +inline Bytes headerAsString() { + return packContent([](auto &pk) { pk.pack(std::string("not a size")); }); +} + +} // namespace kbtest diff --git a/rnmodules/react-native-kb/cpp/tests/frame-parser-test.cpp b/rnmodules/react-native-kb/cpp/tests/frame-parser-test.cpp new file mode 100644 index 000000000000..6b91d0695763 --- /dev/null +++ b/rnmodules/react-native-kb/cpp/tests/frame-parser-test.cpp @@ -0,0 +1,694 @@ +// Unit tests for FrameParser, the pure msgpack framing state machine +// extracted from KBBridge::onDataFromGo (see ../frame-parser.h). No jsi/React +// dependency: these exercise the exact production framing arithmetic without +// a JS runtime. +#include "../frame-parser.h" +#include "frame-builder.h" +#include "test-harness.h" +#include +#include + +using kb::FrameParser; +using namespace kbtest; + +namespace { + +// Feeds `frame` to `parser` in chunks of `chunkSize` bytes (last chunk may be +// smaller), appending decoded objects to `out`. +void feedInChunks(FrameParser &parser, const Bytes &frame, size_t chunkSize, + std::vector &out) { + size_t pos = 0; + while (pos < frame.size()) { + size_t n = std::min(chunkSize, frame.size() - pos); + parser.feed(frame.data() + pos, n, out); + pos += n; + } +} + +void testSingleFrameDecodesOne() { + FrameParser parser; + std::vector out; + Bytes frame = buildFrame(contentSmallMap()); + parser.feed(frame.data(), frame.size(), out); + CHECK_EQ(out.size(), 1u); + CHECK(out[0].get().type == msgpack::type::MAP); +} + +void testMultipleFramesCoalescedInOrder() { + FrameParser parser; + std::vector out; + Bytes all; + for (int i = 0; i < 5; ++i) { + Bytes content = packContent( + [i](auto &pk) { pk.pack_array(1); pk.pack(i); }); + Bytes frame = buildFrame(content); + all.insert(all.end(), frame.begin(), frame.end()); + } + parser.feed(all.data(), all.size(), out); + CHECK_EQ(out.size(), 5u); + for (int i = 0; i < 5; ++i) { + const auto &o = out[static_cast(i)].get(); + CHECK(o.type == msgpack::type::ARRAY); + CHECK_EQ(o.via.array.ptr[0].as(), i); + } +} + +void testFrameSplitAcrossTwoFeeds() { + FrameParser parser; + std::vector out; + Bytes frame = buildFrame(contentSmallMap()); + // Header in one feed, content in the next. + Bytes header = packHeader(contentSmallMap().size()); + parser.feed(frame.data(), header.size(), out); + CHECK_EQ(out.size(), 0u); + parser.feed(frame.data() + header.size(), frame.size() - header.size(), + out); + CHECK_EQ(out.size(), 1u); +} + +// `padBytes` inflates the "b" value so callers can steer which msgpack +// encoding packHeader() picks for the frame header (fixint / uint16 / uint32) +// while keeping the object shape identical. +Bytes contentSplitBoundarySample(size_t padBytes = 0) { + return packContent([padBytes](auto &pk) { + pk.pack_map(3); + pk.pack(std::string("a")); + pk.pack(1); + pk.pack(std::string("b")); + pk.pack(std::string("some string value") + std::string(padBytes, 'p')); + pk.pack(std::string("c")); + pk.pack_array(3); + pk.pack(1); + pk.pack(2); + pk.pack(3); + }); +} + +// Feeds `frame` in two pieces at every possible split point (including the +// no-op splits at 0 and frame.size()) and checks that exactly one object of +// `wantType` is always decoded. Exercises totalFed - nonparsed_size() +// arithmetic, and in particular the header-partial-read bookkeeping +// (consumedAtHeader_ in frame-parser.h), at every possible split point across +// a single frame. +void sweepSplitBoundaries(const Bytes &frame, msgpack::type::object_type wantType) { + for (size_t split = 0; split <= frame.size(); ++split) { + FrameParser parser; + std::vector out; + if (split > 0) { + parser.feed(frame.data(), split, out); + } + if (split < frame.size()) { + parser.feed(frame.data() + split, frame.size() - split, out); + } + CHECK_MSG(out.size() == 1, + "split at " + std::to_string(split) + " produced " + + std::to_string(out.size()) + " messages, want 1"); + CHECK(out[0].get().type == wantType); + } +} + +void testFrameSplitAtEveryByteBoundaryUint16Header() { + // The third header encoding the parser can meet: padded past 255 bytes so + // packHeader() picks msgpack's uint16 form (0xcd + 2 bytes) rather than the + // 1-byte fixint a small payload collapses to, or the uint32 form + // ..._prod_header sweeps. A 3-byte header is the only length for which + // consumedAtHeader_ can be latched after a split that lands at header byte + // 1 or 2, so this sweep is load-bearing rather than a subset of the + // production-header sweep. + Bytes content = contentSplitBoundarySample(300); + CHECK(content.size() > 255 && content.size() < 65536); + Bytes frame = buildFrame(content); + CHECK_EQ(frame.size(), content.size() + 3u); + sweepSplitBoundaries(frame, msgpack::type::MAP); +} + +void testFrameSplitAtEveryByteBoundaryProdHeader() { + // Same sweep and same content as testFrameSplitAtEveryByteBoundary, but + // with the header encoded exactly as production writes it (0xce tag + a + // fixed 4-byte big-endian length, see packHeaderUint32() and + // react-native-kb.cpp's packAndSend). This is what real traffic looks like + // for small frames, and is the only encoding that actually exercises a + // split landing inside bytes 1-4 of a real header. + Bytes frame = buildFrameProdHeader(contentSplitBoundarySample()); + sweepSplitBoundaries(frame, msgpack::type::MAP); +} + +void testMaxSizeFrameHeaderSplitBoundaries() { + // A maximal (kMaxFrameSize) frame's header already requires msgpack's + // uint32 format under minimal encoding, so this is production-accurate + // without even needing packHeaderUint32 -- but the existing max-size + // coverage (testOversizedFrameRejectedAndLegalMaxNotFalselyRejected) always + // buffers the whole 5-byte header in a single chunk (its fixed + // 65537-byte chunking never happens to split within the first 5 bytes). + // This sweeps every split point within and just past the header so the + // "header split across two feeds" case is covered for the largest frame + // the parser accepts, not just for small frames. (A full byte-by-byte + // sweep across the entire 64MB body would be O(n^2) and impractically + // slow, so this only sweeps the header region.) + Bytes content = contentBinOfWireSize(FrameParser::kMaxFrameSize); + CHECK_EQ(content.size(), FrameParser::kMaxFrameSize); + Bytes frame = buildFrameProdHeader(content); + constexpr size_t kHeaderLen = 5; + for (size_t split = 0; split <= kHeaderLen + 2; ++split) { + FrameParser parser; + std::vector out; + if (split > 0) { + parser.feed(frame.data(), split, out); + } + parser.feed(frame.data() + split, frame.size() - split, out); + CHECK_MSG(out.size() == 1, + "split at " + std::to_string(split) + " produced " + + std::to_string(out.size()) + " messages, want 1"); + CHECK(out[0].get().type == msgpack::type::BIN); + } +} + +void testConsumedEqualsDeclaredAcrossSizes() { + // Includes a payload well past msgpack::unpacker's default initial buffer, + // forcing at least one internal reserve/grow. + const std::vector sizes = {0, 1, 100, 4096, 1u << 20}; + for (size_t n : sizes) { + FrameParser parser; + std::vector out; + Bytes content = packContent([n](auto &pk) { + std::string body(n, 'z'); + pk.pack(body); + }); + Bytes frame = buildFrame(content); + parser.feed(frame.data(), frame.size(), out); + CHECK_MSG(out.size() == 1, "size " + std::to_string(n)); + CHECK(out[0].get().type == msgpack::type::STR); + CHECK_EQ(out[0].get().via.str.size, static_cast(n)); + } +} + +void testDeclaredLengthMismatchDetected() { + FrameParser parser; + std::vector out; + Bytes content = contentSmallMap(); + Bytes frame = buildFrameWithDeclaredSize(content.size() + 3, content); + bool threw = false; + try { + parser.feed(frame.data(), frame.size(), out); + } catch (const std::runtime_error &e) { + threw = true; + CHECK_MSG(std::string(e.what()).find("length mismatch") != + std::string::npos, + std::string("unexpected message: ") + e.what()); + } + CHECK_MSG(threw, "expected a length-mismatch throw"); +} + +void testNonIntegerHeaderDetected() { + FrameParser parser; + std::vector out; + Bytes badHeader = headerAsString(); + bool threw = false; + try { + parser.feed(badHeader.data(), badHeader.size(), out); + } catch (const std::runtime_error &e) { + threw = true; + CHECK_MSG(std::string(e.what()).find("bad rpc frame header") != + std::string::npos, + std::string("unexpected message: ") + e.what()); + } + CHECK_MSG(threw, "expected a bad-header throw"); +} + +void testDeclaredSizeZeroRejected() { + FrameParser parser; + std::vector out; + // No msgpack object is zero bytes, so a declared size of 0 can never be + // satisfied by real content -- it must surface as a length mismatch. + Bytes content = contentEmptyMap(); + Bytes frame = buildFrameWithDeclaredSize(0, content); + bool threw = false; + try { + parser.feed(frame.data(), frame.size(), out); + } catch (const std::runtime_error &e) { + threw = true; + // Must be the content-vs-declared check, not the slack guard and not a + // throw escaping msgpack's own parser. + CHECK_MSG(std::string(e.what()).find("length mismatch") != + std::string::npos, + std::string("unexpected message: ") + e.what()); + } + CHECK_MSG(threw, "expected declared size 0 to be rejected"); + CHECK_EQ(out.size(), 0u); +} + +void testOversizedFrameRejectedAndLegalMaxNotFalselyRejected() { + // A header declaring more than kMaxFrameSize is rejected immediately. + { + FrameParser parser; + std::vector out; + Bytes header = packHeader(FrameParser::kMaxFrameSize + 1); + bool threw = false; + try { + parser.feed(header.data(), header.size(), out); + } catch (const std::runtime_error &e) { + threw = true; + // Specifically the header's `> kMaxFrameSize` check. The slack guard + // ("exceeds size limit") cannot be what fires here: only 9 bytes have + // been fed, so this pins the rejection to the header, not to buffering. + CHECK_MSG(std::string(e.what()).find("bad rpc frame header") != + std::string::npos, + std::string("unexpected message: ") + e.what()); + } + CHECK_MSG(threw, "expected a >max-size header to be rejected"); + } + + // A LEGAL maximal frame (declaredSize == kMaxFrameSize exactly) arriving in + // small chunks must NOT be falsely rejected. next() consumes the msgpack + // object as soon as it's fully buffered, so by the time the + // nonparsed_size() > kMaxFrameSize + kMaxFrameSlack check runs at the end + // of feed(), nonparsed_size() is already back to 0 for this scenario -- + // this pins down that the header check's `> kMaxFrameSize` (not `>=`) + // comparison accepts a declared size of exactly kMaxFrameSize, not the + // kMaxFrameSlack margin. + { + FrameParser parser; + std::vector out; + Bytes content = contentBinOfWireSize(FrameParser::kMaxFrameSize); + CHECK_EQ(content.size(), FrameParser::kMaxFrameSize); + Bytes frame = buildFrame(content); + constexpr size_t kChunkSize = 65537; // deliberately not a divisor + feedInChunks(parser, frame, kChunkSize, out); + CHECK_EQ(out.size(), 1u); + CHECK(out[0].get().type == msgpack::type::BIN); + } +} + +void testRecoveryAfterErrorAndReset() { + FrameParser parser; + std::vector out; + Bytes bad = headerAsString(); + bool threw = false; + try { + parser.feed(bad.data(), bad.size(), out); + } catch (const std::runtime_error &) { + threw = true; + } + CHECK_MSG(threw, "setup: expected the bad header to throw"); + parser.reset(); + + for (int i = 0; i < 3; ++i) { + Bytes content = packContent([i](auto &pk) { pk.pack(i); }); + Bytes frame = buildFrame(content); + parser.feed(frame.data(), frame.size(), out); + } + CHECK_EQ(out.size(), 3u); + for (int i = 0; i < 3; ++i) { + CHECK_EQ(out[static_cast(i)].get().as(), i); + } +} + +// reset() is called from its own frame, which is then deliberately overwritten +// before the parser is used again. If reset() left the unpacker holding a +// reference into that frame (which move-assignment does), the reference now +// points at the fill pattern rather than at plausible-looking leftovers, so the +// test fails on every run instead of only when the stack happens to be reused. +[[gnu::noinline]] void resetInOwnFrame(FrameParser &parser) { parser.reset(); } + +[[gnu::noinline]] void clobberStack() { + volatile char scratch[8192]; + for (size_t i = 0; i < sizeof(scratch); ++i) { + scratch[i] = static_cast(0xab); + } +} + +void testResetThenBufferGrowth() { + // reset() must leave a parser that is safe to grow, not just safe to reuse. + // msgpack::unpacker cannot be move-assigned: its parser base keeps a + // reference to the finalizer object inside the unpacker, and the move + // constructor copies that reference instead of rebinding it, so a + // move-assigned unpacker points at the destroyed source. Nothing goes wrong + // until the buffer has to expand -- which is why every existing reset test + // passed while the real reader crashed on the first sizeable frame after a + // reset. + FrameParser parser; + std::vector out; + + Bytes first = buildFrame(packContent([](auto &pk) { pk.pack(1); })); + parser.feed(first.data(), first.size(), out); + CHECK_EQ(out.size(), 1u); + + for (int round = 0; round < 3; ++round) { + resetInOwnFrame(parser); + clobberStack(); + out.clear(); + + // The finalizer is only consulted when the buffer has to grow while it is + // still referenced, i.e. when a frame that has already had a string parsed + // out of it is still incomplete and the buffer must be reallocated to hold + // the rest. A multi-field frame arriving across several reads -- an + // everyday RPC reply -- does exactly that. + const size_t big = 2u * 1024 * 1024; + Bytes frame = buildFrame(packContent([big](auto &pk) { + pk.pack_array(2); + pk.pack(std::string(1024, 'r')); + pk.pack(std::string(big, 'z')); + })); + feedInChunks(parser, frame, 64 * 1024, out); + CHECK_EQ(out.size(), 1u); + CHECK_EQ(out[0].get().via.array.ptr[1].via.str.size, big); + } +} + +void testBufferShrinkFiresAndArithmeticHoldsAfterShrink() { + FrameParser parser; + std::vector out; + + // A frame comfortably larger than kRecvBufKeepCapacity. + const size_t bigSize = FrameParser::kRecvBufKeepCapacity + (1u << 20); + Bytes bigContent = packContent( + [bigSize](auto &pk) { pk.pack(std::string(bigSize, 'q')); }); + Bytes bigFrame = buildFrame(bigContent); + parser.feed(bigFrame.data(), bigFrame.size(), out); + CHECK_EQ(out.size(), 1u); + + // A small frame right after it. peakFrameSize must remember the earlier + // large frame, not get overwritten by this small one's declaredSize -- this + // is the regression the peakFrameSize fix addressed. + Bytes smallContent = packContent([](auto &pk) { pk.pack(7); }); + Bytes smallFrame = buildFrame(smallContent); + parser.feed(smallFrame.data(), smallFrame.size(), out); + CHECK_EQ(out.size(), 2u); + + CHECK_MSG(parser.atSafeShrinkPoint(), + "expected a safe shrink point after a big frame followed by a " + "small one"); + + // Mirror what onDataFromGo does at a safe shrink point: drop and rebuild. + parser.reset(); + + // Framing arithmetic must still hold for frames decoded after the shrink. + for (int i = 0; i < 4; ++i) { + Bytes content = packContent([i](auto &pk) { pk.pack(100 + i); }); + Bytes frame = buildFrame(content); + parser.feed(frame.data(), frame.size(), out); + } + CHECK_EQ(out.size(), 6u); + for (int i = 0; i < 4; ++i) { + CHECK_EQ(out[2 + static_cast(i)].get().as(), 100 + i); + } +} + +void testUnboundedBufferingRejectedBySlackGuard() { + // Pins the end-of-feed() guard in frame-parser.cpp: + // if (unpacker_->nonparsed_size() > kMaxFrameSize + kMaxFrameSlack) throw + // + // Every other size test is rejected at the *header* check instead, because + // they feed a complete msgpack object whose declared size next() can read. + // The case that guard actually exists for is a corrupt length prefix that + // never completes an object at all: next() keeps returning false, the header + // check is never reached, and the unpacker buffers every byte forever. Here + // a bin32 tag claims a 4GB body, so no amount of data completes it and + // nonparsed_size() grows monotonically. Without the guard the only outcome + // is an OOM kill. + FrameParser parser; + std::vector out; + + Bytes tag = bin32TagOnly(0xffffffffu); + parser.feed(tag.data(), tag.size(), out); + CHECK_EQ(out.size(), 0u); + + const size_t kLimit = FrameParser::kMaxFrameSize + FrameParser::kMaxFrameSlack; + const size_t kChunk = 4u * 1024 * 1024; + // Enough body bytes to push nonparsed_size() past the limit, plus a chunk of + // margin so the crossing definitely happens inside the loop. + const size_t kTotal = kLimit + kChunk; + Bytes chunk(kChunk, 'x'); + + bool threw = false; + size_t fed = tag.size(); + while (fed < kTotal && !threw) { + try { + parser.feed(chunk.data(), chunk.size(), out); + fed += chunk.size(); + // While still under the limit the parser must stay quiet, not deliver + // anything, and not reject a merely-large-but-legal buffer. + CHECK_MSG(fed <= kLimit, + "buffered " + std::to_string(fed) + + " bytes past the limit without throwing"); + } catch (const std::runtime_error &e) { + threw = true; + CHECK_MSG(std::string(e.what()).find("exceeds size limit") != + std::string::npos, + std::string("unexpected message: ") + e.what()); + // The crossing must not happen early: a legal 64MB frame's bytes are + // still in flight below the limit. + CHECK_MSG(fed + kChunk > kLimit, + "rejected at " + std::to_string(fed) + + " bytes, before the limit"); + } + } + CHECK_MSG(threw, "expected unbounded buffering to be rejected"); + CHECK_EQ(out.size(), 0u); +} + +void testSafeShrinkPointFalseWhenUnsafe() { + // atSafeShrinkPoint() has three conjuncts and the existing coverage only + // asserts the all-true case, so `return true;` passes the whole suite. Each + // block below leaves exactly one conjunct false. Acting on a false positive + // means the caller reset()s -- zeroing totalFed_/consumedAtHeader_ -- while + // a partial frame is still buffered, which guarantees a length mismatch on + // the very next frame and a fatal/reset cycle on healthy traffic. + + const size_t bigSize = FrameParser::kRecvBufKeepCapacity + (1u << 20); + Bytes bigContent = + packContent([bigSize](auto &pk) { pk.pack(std::string(bigSize, 'q')); }); + Bytes bigFrame = buildFrame(bigContent); + Bytes bigHeader = packHeader(bigContent.size()); + + // (1) peakFrameSize_ <= kRecvBufKeepCapacity: nothing large has been seen, + // so there is no oversized allocation to release. state_/nonparsed_size() + // are both at their "safe" values here, isolating the peak conjunct. + { + FrameParser parser; + std::vector out; + CHECK_MSG(!parser.atSafeShrinkPoint(), + "fresh parser must not report a safe shrink point"); + for (int i = 0; i < 3; ++i) { + Bytes frame = buildFrame(packContent([i](auto &pk) { pk.pack(i); })); + parser.feed(frame.data(), frame.size(), out); + } + CHECK_EQ(out.size(), 3u); + CHECK_MSG(!parser.atSafeShrinkPoint(), + "small frames only must not report a safe shrink point"); + } + + // (2) state_ == needContent: the header of a large frame has been consumed + // and its content is still arriving. peakFrameSize_ is already big and + // nonparsed_size() is 0, so only the state conjunct holds this false. + // Shrinking here would discard declaredSize_/consumedAtHeader_ mid-frame. + { + FrameParser parser; + std::vector out; + parser.feed(bigHeader.data(), bigHeader.size(), out); + CHECK_EQ(out.size(), 0u); + CHECK_MSG(!parser.atSafeShrinkPoint(), + "mid-frame (header read, content pending) must not be a safe " + "shrink point"); + // Partway into the content is equally unsafe. + parser.feed(bigContent.data(), 1024, out); + CHECK_MSG(!parser.atSafeShrinkPoint(), + "mid-frame (content partially buffered) must not be a safe " + "shrink point"); + // Finishing the frame gets us back to a genuinely safe point, proving the + // negatives above are not just an always-false stub. + parser.feed(bigContent.data() + 1024, bigContent.size() - 1024, out); + CHECK_EQ(out.size(), 1u); + CHECK_MSG(parser.atSafeShrinkPoint(), + "a completed big frame must be a safe shrink point"); + } + + // (3) nonparsed_size() != 0: a complete big frame followed by the first two + // bytes of the next frame's production 5-byte header. state_ is needSize and + // peakFrameSize_ is big, so only the nonparsed_size conjunct holds this + // false. reset()ing here would drop those two buffered bytes and desync. + { + FrameParser parser; + std::vector out; + parser.feed(bigFrame.data(), bigFrame.size(), out); + CHECK_EQ(out.size(), 1u); + CHECK_MSG(parser.atSafeShrinkPoint(), "setup: expected a safe shrink point"); + + Bytes nextFrame = buildFrameProdHeader(contentSmallMap()); + parser.feed(nextFrame.data(), 2, out); + CHECK_EQ(out.size(), 1u); + CHECK_MSG(!parser.atSafeShrinkPoint(), + "a partially buffered next header must not be a safe shrink " + "point"); + + // And the withheld bytes really were load-bearing: the frame completes. + parser.feed(nextFrame.data() + 2, nextFrame.size() - 2, out); + CHECK_EQ(out.size(), 2u); + } +} + +void testRecoveryFromPartialStaleFrameAfterReset() { + // The existing recovery tests feed complete, well-formed frames after + // reset(). Real recovery is messier: the tail of the frame that desynced us + // is still draining out of the dead connection, so the first bytes after + // reset() are mid-frame garbage rather than a header. This is exactly the + // case the content-length check (`consumed != declaredSize_`) exists for -- + // a byte inside a payload can parse as a perfectly plausible fixint header, + // and only the length check stops whatever follows it from being handed to + // JS as a real [type, seqid, ...] message. + FrameParser parser; + std::vector out; + + auto strContent = [](const std::string &s) { + return packContent([&s](auto &pk) { pk.pack(s); }); + }; + const std::string kGoodOne = "good-one"; + const std::string kGoodTwo = "good-two"; + const std::string kGarbage = "GARBAGE"; + + // A good frame lands first, so we can prove nothing already delivered gets + // clobbered by the recovery path either. + Bytes g1 = buildFrame(strContent(kGoodOne)); + parser.feed(g1.data(), g1.size(), out); + CHECK_EQ(out.size(), 1u); + + // Now the stream desyncs. The trigger is deliberately the *header-type* + // check rather than the length check, so that this test's real assertion + // (below: nothing bogus is ever delivered) is what fails if the length check + // is removed -- not this setup step. + Bytes bad = headerAsString(); + bool threw = false; + try { + parser.feed(bad.data(), bad.size(), out); + } catch (const std::runtime_error &) { + threw = true; + } + CHECK_MSG(threw, "setup: expected the corrupt header to throw"); + CHECK_EQ(out.size(), 1u); + + parser.reset(); + + // The tail of that dead frame is still arriving. Its leading byte is a bare + // fixint 3 -- a syntactically valid header declaring 3 bytes -- followed by + // an 8-byte string. If the length check were dropped, "GARBAGE" would be + // delivered as an RPC message. + Bytes garbage = strContent(kGarbage); + CHECK_EQ(garbage.size(), 8u); + Bytes staleTail; + staleTail.push_back(0x03); + staleTail.insert(staleTail.end(), garbage.begin(), garbage.end()); + + Bytes g2 = buildFrame(strContent(kGoodTwo)); + Bytes tailThenGood = staleTail; + tailThenGood.insert(tailThenGood.end(), g2.begin(), g2.end()); + + bool rejected = false; + try { + parser.feed(tailThenGood.data(), tailThenGood.size(), out); + } catch (const std::runtime_error &e) { + rejected = true; + CHECK_MSG(std::string(e.what()).find("rpc frame") != std::string::npos, + std::string("unexpected message: ") + e.what()); + } + + // The parser may reject cleanly or resync -- both are acceptable. What is + // never acceptable is delivering the stale tail as a message. + for (size_t i = 0; i < out.size(); ++i) { + const auto &o = out[i].get(); + CHECK_MSG(o.type == msgpack::type::STR, + "delivered a non-string object at index " + std::to_string(i)); + std::string s(o.via.str.ptr, o.via.str.size); + CHECK_MSG(s == kGoodOne || s == kGoodTwo, + "delivered garbage as an rpc message: " + s); + } + + // After the caller does what production does on a framing error -- reset and + // keep going -- the stream must be usable again. + if (rejected) { + parser.reset(); + const size_t before = out.size(); + parser.feed(g2.data(), g2.size(), out); + CHECK_EQ(out.size(), before + 1u); + const auto &o = out.back().get(); + CHECK(o.type == msgpack::type::STR); + CHECK(std::string(o.via.str.ptr, o.via.str.size) == kGoodTwo); + } +} + +void testNestedAndEmptyContainersRoundtrip() { + { + FrameParser parser; + std::vector out; + Bytes frame = buildFrame(contentEmptyMap()); + parser.feed(frame.data(), frame.size(), out); + CHECK_EQ(out.size(), 1u); + CHECK(out[0].get().type == msgpack::type::MAP); + CHECK_EQ(out[0].get().via.map.size, 0u); + } + { + FrameParser parser; + std::vector out; + Bytes frame = buildFrame(contentEmptyArray()); + parser.feed(frame.data(), frame.size(), out); + CHECK_EQ(out.size(), 1u); + CHECK(out[0].get().type == msgpack::type::ARRAY); + CHECK_EQ(out[0].get().via.array.size, 0u); + } + { + // Near but under a typical application-level depth limit (1024 in + // react-native-kb.cpp); FrameParser itself has no depth limit of its + // own, so this proves the raw decode handles realistic nesting. + FrameParser parser; + std::vector out; + const int depth = 900; + Bytes frame = buildFrame(contentNested(depth)); + parser.feed(frame.data(), frame.size(), out); + CHECK_EQ(out.size(), 1u); + const msgpack::object *cur = &out[0].get(); + int seen = 0; + while (cur->type == msgpack::type::ARRAY) { + CHECK_EQ(cur->via.array.size, 1u); + cur = &cur->via.array.ptr[0]; + ++seen; + } + CHECK_EQ(seen, depth); + CHECK_EQ(cur->as(), 42); + } +} + +} // namespace + +int main() { + Runner runner; + runner.add("single_frame_decodes_one_message", testSingleFrameDecodesOne); + runner.add("multiple_frames_coalesced_decode_in_order", + testMultipleFramesCoalescedInOrder); + runner.add("frame_split_across_two_feeds", testFrameSplitAcrossTwoFeeds); + runner.add("frame_split_at_every_byte_boundary_uint16_header", + testFrameSplitAtEveryByteBoundaryUint16Header); + runner.add("frame_split_at_every_byte_boundary_prod_header", + testFrameSplitAtEveryByteBoundaryProdHeader); + runner.add("max_size_frame_header_split_boundaries", + testMaxSizeFrameHeaderSplitBoundaries); + runner.add("consumed_equals_declared_across_sizes", + testConsumedEqualsDeclaredAcrossSizes); + runner.add("declared_length_mismatch_detected", + testDeclaredLengthMismatchDetected); + runner.add("non_integer_header_detected", testNonIntegerHeaderDetected); + runner.add("declared_size_zero_rejected", testDeclaredSizeZeroRejected); + runner.add("oversized_frame_rejected_and_legal_max_not_falsely_rejected", + testOversizedFrameRejectedAndLegalMaxNotFalselyRejected); + runner.add("unbounded_buffering_rejected_by_slack_guard", + testUnboundedBufferingRejectedBySlackGuard); + runner.add("safe_shrink_point_false_when_unsafe", + testSafeShrinkPointFalseWhenUnsafe); + runner.add("recovery_after_error_and_reset", testRecoveryAfterErrorAndReset); + runner.add("recovery_from_partial_stale_frame_after_reset", + testRecoveryFromPartialStaleFrameAfterReset); + runner.add("reset_then_buffer_growth", testResetThenBufferGrowth); + runner.add("buffer_shrink_fires_and_arithmetic_holds_after_shrink", + testBufferShrinkFiresAndArithmeticHoldsAfterShrink); + runner.add("nested_and_empty_containers_roundtrip", + testNestedAndEmptyContainersRoundtrip); + return runner.run(); +} diff --git a/rnmodules/react-native-kb/cpp/tests/jsi-convert-bench.cpp b/rnmodules/react-native-kb/cpp/tests/jsi-convert-bench.cpp new file mode 100644 index 000000000000..12c12c6af813 --- /dev/null +++ b/rnmodules/react-native-kb/cpp/tests/jsi-convert-bench.cpp @@ -0,0 +1,198 @@ +// Benchmark for the msgpack <-> JSI conversion in KBBridge, run on a real +// Hermes runtime against a corpus of real RPC payloads. +// +// The corpus comes from an actual mobile session's Metro log; see +// scripts/make-bench-corpus.mjs. Build and run both this and the master +// baseline with scripts/bench-jsi-convert.sh. +// +// Decode direction (msgpack -> JSI): frames are handed to onDataFromGo exactly +// as the platform reader would, and the installed rpcOnJs receives the +// converted values. +// +// Encode direction (JSI -> msgpack): the values rpcOnJs received are handed +// back to the global rpcOnGo the bridge installs, which packs them and calls +// writeToGo. +// +// Both directions therefore run through the public API, not through internals +// copied out of the implementation, so the two builds are measuring the same +// thing the app does. + +#include "react-native-kb.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace facebook; + +namespace { + +// The bridge only ever schedules work meant to run on the JS thread, and the +// benchmark is single threaded, so running inline is faithful. +class InlineCallInvoker : public react::CallInvoker { +public: + explicit InlineCallInvoker(jsi::Runtime &rt) : rt_(rt) {} + void invokeAsync(react::CallFunc &&func) noexcept override { func(rt_); } + void invokeSync(react::CallFunc &&func) override { func(rt_); } + +private: + jsi::Runtime &rt_; +}; + +std::vector readFile(const char *path) { + std::ifstream in(path, std::ios::binary); + if (!in) { + fprintf(stderr, "cannot open corpus %s\n", path); + exit(1); + } + return std::vector((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); +} + +double msSince(std::chrono::steady_clock::time_point start) { + return std::chrono::duration( + std::chrono::steady_clock::now() - start) + .count(); +} + +} // namespace + +int main(int argc, char **argv) { + const char *corpusPath = + argc > 1 ? argv[1] : "/tmp/kb-bench-corpus.bin"; + const int iterations = argc > 2 ? atoi(argv[2]) : 5; + + auto corpus = readFile(corpusPath); + auto runtime = facebook::hermes::makeHermesRuntime(); + auto &rt = *runtime; + auto invoker = std::make_shared(rt); + auto bridge = std::make_shared(); + + size_t bytesWritten = 0; + size_t errors = 0; + auto onError = [&](const std::string &msg) { + if (errors++ < 5) { + fprintf(stderr, "bridge error: %s\n", msg.c_str()); + } + }; + +#ifdef KB_BASELINE + bridge->install( + rt, invoker, + [&](void *, size_t size) { bytesWritten += size; }, + onError); +#else + bridge->install( + rt, invoker, + [&](void *, size_t size) { + bytesWritten += size; + return true; + }, + onError, [&](int64_t) { fprintf(stderr, "fatal\n"); }); +#endif + + // Collects everything the decode pass produced so the encode pass has real + // JSI values to pack. Held in JS so the values stay exactly what rpcOnJs got. + rt.global().setProperty(rt, "received", jsi::Array(rt, 0)); + auto rpcOnJs = jsi::Function::createFromHostFunction( + rt, jsi::PropNameID::forAscii(rt, "rpcOnJs"), 2, + [](jsi::Runtime &rt, const jsi::Value &, const jsi::Value *args, + size_t argCount) -> jsi::Value { + if (argCount < 2) { + return jsi::Value::undefined(); + } + auto arr = rt.global() + .getPropertyAsObject(rt, "received") + .getArray(rt); + auto push = arr.getPropertyAsFunction(rt, "push"); + // Same unwrapping rule as JS's rpcOnJs: count > 1 means the argument + // is a batch array, anything else is a single message. + const int count = int(args[1].asNumber()); + if (count > 1) { + auto batch = args[0].getObject(rt).getArray(rt); + for (size_t i = 0, n = batch.size(rt); i < n; ++i) { + push.callWithThis(rt, arr, batch.getValueAtIndex(rt, i)); + } + } else { + push.callWithThis(rt, arr, args[0]); + } + return jsi::Value::undefined(); + }); + rt.global().setProperty(rt, "rpcOnJs", std::move(rpcOnJs)); + + size_t frames = 0; + for (size_t off = 0; off + 5 <= corpus.size();) { + uint32_t len = (uint32_t(corpus[off + 1]) << 24) | + (uint32_t(corpus[off + 2]) << 16) | + (uint32_t(corpus[off + 3]) << 8) | uint32_t(corpus[off + 4]); + off += 5 + len; + frames++; + } + + printf("corpus %s: %zu frames, %.2f MB\n", corpusPath, frames, + double(corpus.size()) / (1024 * 1024)); +#ifdef KB_BASELINE + printf("build: baseline (origin/master)\n"); +#else + printf("build: branch\n"); +#endif + + double decodeBest = 1e18; + double encodeBest = 1e18; + for (int iter = 0; iter < iterations; ++iter) { + // Decode: hand the corpus over in reads no larger than Go's ReadArr + // buffer (go/bind/keybase.go), so batching and partial frames happen the + // way they do on device rather than as one impossible 10MB read. + constexpr size_t kReadSize = 300 * 1024; + rt.global().setProperty(rt, "received", jsi::Array(rt, 0)); +#ifndef KB_BASELINE + bridge->resetRecv(); +#endif + auto t0 = std::chrono::steady_clock::now(); + for (size_t off = 0; off < corpus.size(); off += kReadSize) { + const size_t n = std::min(kReadSize, corpus.size() - off); +#ifdef KB_BASELINE + bridge->onDataFromGo(corpus.data() + off, int(n)); +#else + bridge->onDataFromGo(corpus.data() + off, int(n), 1); +#endif + } + double decodeMs = msSince(t0); + + auto received = + rt.global().getPropertyAsObject(rt, "received").getArray(rt); + size_t n = received.size(rt); + auto rpcOnGo = rt.global().getPropertyAsFunction(rt, "rpcOnGo"); + bytesWritten = 0; + auto t1 = std::chrono::steady_clock::now(); + for (size_t i = 0; i < n; ++i) { + rpcOnGo.call(rt, received.getValueAtIndex(rt, i)); + } + double encodeMs = msSince(t1); + + if (iter == 0) { + printf("delivered %zu messages, repacked %zu bytes\n", n, bytesWritten); + } + decodeBest = std::min(decodeBest, decodeMs); + encodeBest = std::min(encodeBest, encodeMs); + printf(" iter %d: decode %8.2f ms encode %8.2f ms\n", iter, decodeMs, + encodeMs); + } + + printf("best: decode %.2f ms encode %.2f ms (%d iterations)\n", + decodeBest, encodeBest, iterations); + // Any conversion error makes the timings meaningless (a build where every + // message fails is the fastest one), so exit nonzero -- this doubles as a + // smoke test of the bridge rather than reporting green on total failure. + if (errors) { + printf("errors reported: %zu\n", errors); + return 1; + } + return 0; +} diff --git a/rnmodules/react-native-kb/cpp/tests/jsi-convert-test.cpp b/rnmodules/react-native-kb/cpp/tests/jsi-convert-test.cpp new file mode 100644 index 000000000000..d9ad5b3cbecc --- /dev/null +++ b/rnmodules/react-native-kb/cpp/tests/jsi-convert-test.cpp @@ -0,0 +1,1237 @@ +// Correctness tests for the msgpack <-> JSI bridge in react-native-kb.cpp: +// packNumber's integer-encoding range checks, convertMPToJSI / +// convertJSIToMP, the batch delivery shape onDataFromGo hands to rpcOnJs, and +// packAndSend's framing. +// +// Everything runs against a real Hermes runtime through the public API -- +// frames go in via onDataFromGo and values come back out through the rpcOnGo +// the bridge installs -- so what is exercised is what the app actually does. +// Build and run with scripts/test-jsi-convert.sh. +// +// react-native-kb.cpp is #included rather than linked: packNumber lives in an +// anonymous namespace, and pulling the translation unit in is the only way to +// reach it without changing production code purely for the tests' benefit. +// Nothing else in here depends on that; the rest goes through the header's +// public surface. +#include "../react-native-kb.cpp" // IWYU pragma: keep + +#include "../frame-parser.h" +#include "frame-builder.h" +#include "test-harness.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +using kbtest::Bytes; + +namespace { + +// --------------------------------------------------------------------------- +// packNumber +// --------------------------------------------------------------------------- + +// The formulation packNumber replaced (commit 4b3d2120e2), kept verbatim as +// the oracle for the random sweep: obviously correct, but goes through libm. +void packNumberReference(msgpack::packer &pk, double d) { + if (d == std::floor(d) && std::isfinite(d)) { + if (d >= 0 && d < 18446744073709551616.0) { + pk.pack(static_cast(d)); + } else if (d < 0 && d >= -9223372036854775808.0) { + pk.pack(static_cast(d)); + } else { + pk.pack(d); + } + } else { + pk.pack(d); + } +} + +Bytes packWith(void (*fn)(msgpack::packer &, double), + double d) { + msgpack::sbuffer buf; + msgpack::packer pk(&buf); + fn(pk, d); + return kbtest::sbufferToBytes(buf); +} + +Bytes packNum(double d) { return packWith(&kb::packNumber, d); } + +msgpack::object_handle unpackBytes(const Bytes &b) { + return msgpack::unpack(reinterpret_cast(b.data()), b.size()); +} + +std::string hex(const Bytes &b) { + static const char *digits = "0123456789abcdef"; + std::string out; + for (uint8_t c : b) { + out += digits[c >> 4]; + out += digits[c & 0xf]; + } + return out; +} + +// Human-readable double, exact enough to identify which value failed. +std::string dstr(double d) { + char buf[64]; + snprintf(buf, sizeof(buf), "%.17g", d); + return buf; +} + +enum class Want { Uint, Int, Float }; + +const char *wantName(Want w) { + switch (w) { + case Want::Uint: + return "POSITIVE_INTEGER"; + case Want::Int: + return "NEGATIVE_INTEGER"; + default: + return "FLOAT"; + } +} + +// Asserts the msgpack type packNumber chose, and that the packed value still +// means what the double meant: exact integer equality for the integer +// encodings, bit equality for the float fallback (so NaN payloads and -0.0 are +// checked, not glossed over by ==). +void expectPacked(double d, Want want, int64_t asInt = 0) { + auto bytes = packNum(d); + auto oh = unpackBytes(bytes); + const auto &o = oh.get(); + const std::string where = "packNumber(" + dstr(d) + ") -> " + hex(bytes); + + switch (want) { + case Want::Uint: + CHECK_MSG(o.type == msgpack::type::POSITIVE_INTEGER, + where + ": expected POSITIVE_INTEGER, got type " + + std::to_string(int(o.type))); + CHECK_MSG(o.as() == static_cast(asInt), + where + ": value " + std::to_string(o.as()) + + " != " + std::to_string(static_cast(asInt))); + break; + case Want::Int: + CHECK_MSG(o.type == msgpack::type::NEGATIVE_INTEGER, + where + ": expected NEGATIVE_INTEGER, got type " + + std::to_string(int(o.type))); + CHECK_MSG(o.as() == asInt, + where + ": value " + std::to_string(o.as()) + + " != " + std::to_string(asInt)); + break; + case Want::Float: { + CHECK_MSG(o.type == msgpack::type::FLOAT64 || + o.type == msgpack::type::FLOAT32, + where + ": expected FLOAT, got type " + + std::to_string(int(o.type))); + double got = o.as(); + CHECK_MSG(std::memcmp(&got, &d, sizeof(double)) == 0, + where + ": float payload " + dstr(got) + " != " + dstr(d)); + break; + } + } +} + +// Also assert the encoding matches the old formulation byte for byte -- the +// table is the interesting half of that comparison, so run both. +void expectMatchesReference(double d) { + auto a = packNum(d); + auto b = packWith(&packNumberReference, d); + CHECK_MSG(a == b, "packNumber(" + dstr(d) + ") = " + hex(a) + + " but reference = " + hex(b)); +} + +constexpr double kInf = std::numeric_limits::infinity(); +constexpr double k2p53 = 9007199254740992.0; // 2^53 +constexpr double k2p63 = 9223372036854775808.0; // 2^63 +constexpr double k2p64 = 18446744073709551616.0; // 2^64 + +void testPackNumberTable() { + struct Row { + double d; + Want want; + int64_t asInt; + const char *name; + }; + + // 2^53+1 is not representable; the nearest double above 2^53 is 2^53+2. + // 2^63-1 is not representable either; nextafter(2^63, 0) is 2^63 - 1024. + const double justUnder2p63 = std::nextafter(k2p63, 0.0); + const double justUnder2p64 = std::nextafter(k2p64, 0.0); + const double justOver2p64 = std::nextafter(k2p64, kInf); + const double justBelowNeg2p63 = std::nextafter(-k2p63, -kInf); + const double justAboveNeg2p63 = std::nextafter(-k2p63, 0.0); + + const Row rows[] = { + {0.0, Want::Uint, 0, "0"}, + // -0.0 >= 0 is true, so it takes the unsigned path and lands on uint 0. + {-0.0, Want::Uint, 0, "-0"}, + {1.0, Want::Uint, 1, "1"}, + {-1.0, Want::Int, -1, "-1"}, + {k2p53 - 1, Want::Uint, 9007199254740991LL, "2^53-1"}, + {k2p53, Want::Uint, 9007199254740992LL, "2^53"}, + {k2p53 + 2, Want::Uint, 9007199254740994LL, "2^53+2 (2^53+1 unrepr.)"}, + {-k2p53, Want::Int, -9007199254740992LL, "-2^53"}, + {justUnder2p63, Want::Uint, 9223372036854774784LL, "nextafter(2^63,0)"}, + // 2^63 itself is < 2^64 so it packs as an unsigned, not as a negative. + {k2p63, Want::Uint, static_cast(9223372036854775808ULL), "2^63"}, + {justAboveNeg2p63, Want::Int, -9223372036854774784LL, + "nextafter(-2^63,0)"}, + {-k2p63, Want::Int, INT64_MIN, "-2^63"}, + // The whole point of the `<` in `d < 2^64`: one ulp lower is the largest + // double that still fits a uint64, 2^64 itself must not be cast. + {justUnder2p64, Want::Uint, + static_cast(18446744073709549568ULL), "nextafter(2^64,0)"}, + {k2p64, Want::Float, 0, "2^64"}, + {justOver2p64, Want::Float, 0, "nextafter(2^64,inf)"}, + {justBelowNeg2p63, Want::Float, 0, "nextafter(-2^63,-inf)"}, + {std::numeric_limits::infinity(), Want::Float, 0, "+inf"}, + {-std::numeric_limits::infinity(), Want::Float, 0, "-inf"}, + {std::numeric_limits::quiet_NaN(), Want::Float, 0, "NaN"}, + {0.5, Want::Float, 0, "0.5"}, + {-0.5, Want::Float, 0, "-0.5"}, + {1.5, Want::Float, 0, "1.5"}, + {-1.5, Want::Float, 0, "-1.5"}, + {1e300, Want::Float, 0, "1e300"}, + {-1e300, Want::Float, 0, "-1e300"}, + {std::numeric_limits::denorm_min(), Want::Float, 0, "denorm_min"}, + {-std::numeric_limits::denorm_min(), Want::Float, 0, + "-denorm_min"}, + {std::numeric_limits::max(), Want::Float, 0, "DBL_MAX"}, + {std::numeric_limits::lowest(), Want::Float, 0, "-DBL_MAX"}, + }; + + for (const auto &r : rows) { + try { + expectPacked(r.d, r.want, r.asInt); + expectMatchesReference(r.d); + } catch (const kbtest::CheckFailed &e) { + throw kbtest::CheckFailed(std::string("[") + r.name + " expected " + + wantName(r.want) + "] " + e.message); + } + } +} + +// A dense ulp-level walk around every constant the range checks compare +// against -- this is where a `<` vs `<=` slip lives. +void testPackNumberBoundaryNeighborhoods() { + const double anchors[] = {0.0, -0.0, 1.0, -1.0, k2p53, + -k2p53, k2p63, -k2p63, k2p64, -k2p64}; + for (double anchor : anchors) { + double up = anchor; + double down = anchor; + for (int i = 0; i < 8; ++i) { + expectMatchesReference(up); + expectMatchesReference(down); + up = std::nextafter(up, kInf); + down = std::nextafter(down, -kInf); + } + } +} + +// Fixed seed, stated here so a failure is reproducible: 0xC0FFEE12345678ULL. +constexpr uint64_t kSweepSeed = 0xC0FFEE12345678ULL; +constexpr int kSweepCount = 1000000; + +void testPackNumberRandomSweep() { + std::mt19937_64 rng(kSweepSeed); + size_t mismatches = 0; + std::string firstMismatch; + + for (int i = 0; i < kSweepCount; ++i) { + uint64_t bits = rng(); + double d; + switch (i % 5) { + case 0: { + // Arbitrary bit patterns: hits NaNs, infinities, denormals, huge + // magnitudes, non-integral values. + std::memcpy(&d, &bits, sizeof(d)); + break; + } + case 1: + // Exactly representable integers across the whole uint64 range. + d = static_cast(bits); + break; + case 2: + d = static_cast(static_cast(bits)); + break; + case 3: + // Straddles 2^53, where doubles stop representing every integer. + d = static_cast(static_cast(bits % (1ULL << 55))) - + (1LL << 54); + break; + default: + // Small non-integral values. + d = static_cast(static_cast(bits % 100000)) / + static_cast(1 + (bits >> 40)); + break; + } + + auto got = packNum(d); + auto want = packWith(&packNumberReference, d); + if (got != want) { + if (mismatches++ == 0) { + firstMismatch = "i=" + std::to_string(i) + " d=" + dstr(d) + " got " + + hex(got) + " want " + hex(want); + } + } + } + + CHECK_MSG(mismatches == 0, + "random sweep (seed 0xC0FFEE12345678, " + + std::to_string(kSweepCount) + " doubles): " + + std::to_string(mismatches) + " mismatches, first: " + + firstMismatch); +} + +// --------------------------------------------------------------------------- +// Hermes harness +// --------------------------------------------------------------------------- + +// The bridge only schedules work meant for the JS thread and these tests are +// single threaded, so running inline is faithful and keeps assertions +// synchronous with the onDataFromGo call that produced them. +class InlineCallInvoker : public facebook::react::CallInvoker { +public: + explicit InlineCallInvoker(jsi::Runtime &rt) : rt_(rt) {} + void invokeAsync(facebook::react::CallFunc &&func) noexcept override { + func(rt_); + } + void invokeSync(facebook::react::CallFunc &&func) override { func(rt_); } + +private: + jsi::Runtime &rt_; +}; + +struct Delivery { + jsi::Value value; + int count = 0; +}; + +struct Harness { + // Declaration order is destruction order in reverse: the jsi::Values in + // `deliveries` must die before the runtime does. + std::unique_ptr runtime; + std::shared_ptr bridge; + std::vector writes; + std::vector errors; + std::vector fatals; + std::vector deliveries; + bool writeResult = true; + + Harness() { + runtime = facebook::hermes::makeHermesRuntime(); + auto &rt = *runtime; + bridge = std::make_shared(); + bridge->install( + rt, std::make_shared(rt), + [this](void *ptr, size_t size) { + const auto *p = static_cast(ptr); + writes.emplace_back(p, p + size); + return writeResult; + }, + [this](const std::string &msg) { errors.push_back(msg); }, + [this](int64_t epoch) { fatals.push_back(epoch); }); + installRpcOnJs(); + } + + jsi::Runtime &rt() { return *runtime; } + + void installRpcOnJs() { + auto fn = jsi::Function::createFromHostFunction( + rt(), jsi::PropNameID::forAscii(rt(), "rpcOnJs"), 2, + [this](jsi::Runtime &r, const jsi::Value &, const jsi::Value *args, + size_t argCount) -> jsi::Value { + Delivery d; + d.value = argCount > 0 ? jsi::Value(r, args[0]) + : jsi::Value::undefined(); + d.count = (argCount > 1 && args[1].isNumber()) + ? int(args[1].asNumber()) + : -1; + deliveries.push_back(std::move(d)); + return jsi::Value::undefined(); + }); + rt().global().setProperty(rt(), "rpcOnJs", std::move(fn)); + } + + void feed(const Bytes &bytes, int64_t epoch = 1) { + Bytes copy = bytes; // onDataFromGo takes a mutable pointer. + bridge->onDataFromGo(copy.data(), int(copy.size()), epoch); + } + + // jsi::JSError owns a shared_ptr, so one escaping a test would + // be destroyed after this Harness (and its runtime) is gone -- a crash, not + // a failure. Every entry point that can throw one converts it here, while + // the runtime is still alive. + jsi::Value eval(const std::string &src) { + try { + return rt().evaluateJavaScript( + std::make_shared("(" + src + ")"), "test.js"); + } catch (const jsi::JSError &e) { + throw kbtest::CheckFailed("JS threw while evaluating: " + e.getMessage()); + } + } + + // Runs `src`, hands the resulting value to the installed rpcOnGo. + jsi::Value send(const std::string &src) { + auto value = eval(src); + try { + return rt().global().getPropertyAsFunction(rt(), "rpcOnGo").call(rt(), + value); + } catch (const jsi::JSError &e) { + throw kbtest::CheckFailed("rpcOnGo threw: " + e.getMessage()); + } + } + + // Returns the exception message if the send threw, empty otherwise. + std::string sendExpectingThrow(const std::string &src) { + try { + send(src); + } catch (const jsi::JSError &e) { + return e.getMessage(); + } catch (const std::exception &e) { + return e.what(); + } + return {}; + } + + // The msgpack payload of the last frame written, header stripped, with the + // header itself checked against the encoding frame-builder.h describes. + msgpack::object_handle lastPayload() { + CHECK(!writes.empty()); + const auto &w = writes.back(); + CHECK(w.size() >= 5); + auto expectedHeader = + kbtest::packHeaderUint32(static_cast(w.size() - 5)); + Bytes actualHeader(w.begin(), w.begin() + 5); + CHECK_MSG(actualHeader == expectedHeader, + "frame header " + hex(actualHeader) + " != packHeaderUint32(" + + std::to_string(w.size() - 5) + ") = " + hex(expectedHeader)); + return msgpack::unpack(reinterpret_cast(w.data()) + 5, + w.size() - 5); + } +}; + +Bytes frameOf(const std::function &)> + &fn) { + return kbtest::buildFrameProdHeader(kbtest::packContent(fn)); +} + +Bytes concatFrames(const std::vector &frames) { + Bytes out; + for (const auto &f : frames) { + out.insert(out.end(), f.begin(), f.end()); + } + return out; +} + +// A frame that decodes fine as msgpack but fails convertMPToJSI: an array is +// not a valid map key, so mpToString throws. +Bytes badKeyFrame() { + return frameOf([](auto &pk) { + pk.pack_map(1); + pk.pack_array(0); + pk.pack(1); + }); +} + +Bytes markerFrame(int id) { + return frameOf([id](auto &pk) { + pk.pack_map(1); + pk.pack(std::string("id")); + pk.pack(id); + }); +} + +int markerId(jsi::Runtime &rt, const jsi::Value &v) { + CHECK(v.isObject()); + auto prop = v.getObject(rt).getProperty(rt, "id"); + CHECK(prop.isNumber()); + return int(prop.asNumber()); +} + +bool isTypedArray(jsi::Runtime &rt, const jsi::Value &v) { + if (!v.isObject()) { + return false; + } + auto isView = rt.global() + .getPropertyAsObject(rt, "ArrayBuffer") + .getPropertyAsFunction(rt, "isView"); + auto res = isView.call(rt, jsi::Value(rt, v.getObject(rt))); + return res.isBool() && res.getBool(); +} + +std::string ctorName(jsi::Runtime &rt, const jsi::Value &v) { + return v.getObject(rt) + .getPropertyAsObject(rt, "constructor") + .getProperty(rt, "name") + .getString(rt) + .utf8(rt); +} + +Bytes typedArrayBytes(jsi::Runtime &rt, const jsi::Value &v) { + auto obj = v.getObject(rt); + size_t len = size_t(obj.getProperty(rt, "length").asNumber()); + Bytes out; + out.reserve(len); + for (size_t i = 0; i < len; ++i) { + out.push_back( + uint8_t(obj.getProperty(rt, std::to_string(i).c_str()).asNumber())); + } + return out; +} + +std::string mpBinToString(const msgpack::object &o) { + CHECK(o.type == msgpack::type::BIN); + return std::string(o.via.bin.ptr, o.via.bin.size); +} + +// --------------------------------------------------------------------------- +// msgpack -> JSI +// --------------------------------------------------------------------------- + +void testDecodeBinBecomesUint8Array() { + Harness h; + const std::string payload("\x00\x01\xfe\xff", 4); + h.feed(frameOf([&](auto &pk) { + pk.pack_map(1); + pk.pack(std::string("b")); + pk.pack_bin(uint32_t(payload.size())); + pk.pack_bin_body(payload.data(), uint32_t(payload.size())); + })); + + CHECK_EQ(h.deliveries.size(), size_t(1)); + auto bin = h.deliveries[0].value.getObject(h.rt()).getProperty(h.rt(), "b"); + CHECK_MSG(isTypedArray(h.rt(), bin), "BIN did not decode to an ArrayBuffer view"); + CHECK_MSG(ctorName(h.rt(), bin) == "Uint8Array", + "BIN decoded to " + ctorName(h.rt(), bin) + ", want Uint8Array"); + Bytes want{0x00, 0x01, 0xfe, 0xff}; + CHECK_MSG(typedArrayBytes(h.rt(), bin) == want, "Uint8Array contents differ"); + CHECK_EQ(h.errors.size(), size_t(0)); +} + +void testDecodeStrIsNotBin() { + Harness h; + h.feed(frameOf([](auto &pk) { + pk.pack_map(2); + pk.pack(std::string("s")); + pk.pack(std::string("hi")); + pk.pack(std::string("b")); + pk.pack_bin(2); + pk.pack_bin_body("hi", 2); + })); + + CHECK_EQ(h.deliveries.size(), size_t(1)); + auto obj = h.deliveries[0].value.getObject(h.rt()); + auto s = obj.getProperty(h.rt(), "s"); + auto b = obj.getProperty(h.rt(), "b"); + CHECK_MSG(s.isString(), "STR did not decode to a JS string"); + CHECK_MSG(s.getString(h.rt()).utf8(h.rt()) == "hi", "STR contents differ"); + CHECK_MSG(isTypedArray(h.rt(), b), "BIN with identical bytes decoded as a string"); +} + +void testDecodeUtf8Strings() { + Harness h; + // Multibyte (2/3/4 byte sequences) plus an embedded NUL, which msgpack + // carries fine but a NUL-terminated path would truncate. + const std::string multi = "caf\xc3\xa9 \xe2\x9c\x93 \xf0\x9f\x94\x91"; + const std::string withNul = std::string("a\0b", 3); + h.feed(frameOf([&](auto &pk) { + pk.pack_map(3); + pk.pack(std::string("m")); + pk.pack(multi); + pk.pack(std::string("n")); + pk.pack(withNul); + pk.pack(std::string("e")); + pk.pack(std::string()); + })); + + CHECK_EQ(h.deliveries.size(), size_t(1)); + auto obj = h.deliveries[0].value.getObject(h.rt()); + auto got = obj.getProperty(h.rt(), "m").getString(h.rt()).utf8(h.rt()); + CHECK_MSG(got == multi, "multibyte UTF-8 round trip differs: " + hex(Bytes(got.begin(), got.end()))); + auto nul = obj.getProperty(h.rt(), "n").getString(h.rt()); + CHECK_EQ(size_t(nul.length(h.rt())), size_t(3)); + auto nulUtf8 = nul.utf8(h.rt()); + CHECK_MSG(nulUtf8 == withNul, "embedded NUL string round trip differs"); + auto empty = obj.getProperty(h.rt(), "e"); + CHECK_MSG(empty.isString() && empty.getString(h.rt()).utf8(h.rt()).empty(), + "empty string did not round trip"); +} + +void testDecodeEmptyContainers() { + Harness h; + h.feed(frameOf([](auto &pk) { + pk.pack_map(4); + pk.pack(std::string("arr")); + pk.pack_array(0); + pk.pack(std::string("map")); + pk.pack_map(0); + pk.pack(std::string("str")); + pk.pack(std::string()); + pk.pack(std::string("bin")); + pk.pack_bin(0); + pk.pack_bin_body("", 0); + })); + + CHECK_EQ(h.deliveries.size(), size_t(1)); + auto obj = h.deliveries[0].value.getObject(h.rt()); + auto arr = obj.getProperty(h.rt(), "arr"); + CHECK_MSG(arr.isObject() && arr.getObject(h.rt()).isArray(h.rt()), + "empty array did not decode to an Array"); + CHECK_EQ(arr.getObject(h.rt()).getArray(h.rt()).size(h.rt()), size_t(0)); + auto map = obj.getProperty(h.rt(), "map"); + CHECK_MSG(map.isObject() && !map.getObject(h.rt()).isArray(h.rt()), + "empty map did not decode to an Object"); + CHECK_EQ(map.getObject(h.rt()).getPropertyNames(h.rt()).size(h.rt()), + size_t(0)); + auto bin = obj.getProperty(h.rt(), "bin"); + CHECK_MSG(isTypedArray(h.rt(), bin), "empty BIN did not decode to a Uint8Array"); + CHECK_EQ(size_t(bin.getObject(h.rt()).getProperty(h.rt(), "length").asNumber()), + size_t(0)); + CHECK_EQ(h.errors.size(), size_t(0)); +} + +void testDecodeExtBecomesUndefined() { + Harness h; + h.feed(frameOf([](auto &pk) { + pk.pack_map(1); + pk.pack(std::string("x")); + pk.pack_ext(4, 7); + pk.pack_ext_body("abcd", 4); + })); + + CHECK_EQ(h.deliveries.size(), size_t(1)); + CHECK_EQ(h.errors.size(), size_t(0)); + auto obj = h.deliveries[0].value.getObject(h.rt()); + CHECK_MSG(obj.getProperty(h.rt(), "x").isUndefined(), + "EXT did not decode to undefined"); + // The key must still exist -- an EXT value must not silently drop its key. + CHECK_EQ(obj.getPropertyNames(h.rt()).size(h.rt()), size_t(1)); +} + +void testDecodeNonScalarMapKeyThrows() { + Harness h; + h.feed(badKeyFrame(), 42); + + CHECK_EQ(h.deliveries.size(), size_t(0)); + CHECK_EQ(h.errors.size(), size_t(1)); + CHECK_MSG(h.errors[0].find("Invalid map key") != std::string::npos, + "unexpected error: " + h.errors[0]); + CHECK_EQ(h.fatals.size(), size_t(1)); + CHECK_EQ(h.fatals[0], int64_t(42)); +} + +void testDecodeScalarMapKeysCoerce() { + Harness h; + h.feed(frameOf([](auto &pk) { + pk.pack_map(3); + pk.pack(7); + pk.pack(std::string("pos")); + pk.pack(-7); + pk.pack(std::string("neg")); + pk.pack(true); + pk.pack(std::string("bool")); + })); + + // BOOLEAN is not one of mpToString's accepted key types, so this frame is + // expected to be rejected exactly like the array-key case. + CHECK_EQ(h.deliveries.size(), size_t(0)); + CHECK_EQ(h.errors.size(), size_t(1)); + + Harness h2; + h2.feed(frameOf([](auto &pk) { + pk.pack_map(2); + pk.pack(7); + pk.pack(std::string("pos")); + pk.pack(-7); + pk.pack(std::string("neg")); + })); + CHECK_EQ(h2.deliveries.size(), size_t(1)); + auto obj = h2.deliveries[0].value.getObject(h2.rt()); + CHECK_MSG(obj.getProperty(h2.rt(), "7").getString(h2.rt()).utf8(h2.rt()) == + "pos", + "positive integer key did not stringify"); + CHECK_MSG(obj.getProperty(h2.rt(), "-7").getString(h2.rt()).utf8(h2.rt()) == + "neg", + "negative integer key did not stringify"); +} + +// convertMPToJSI pushes a frame per container and throws once the stack would +// exceed kMaxDepth, so the 1024th nested container is the last one accepted. +void testDecodeDepthLimit() { + { + Harness h; + h.feed(kbtest::buildFrameProdHeader(kbtest::contentNested(1024))); + CHECK_MSG(h.errors.empty(), + "depth 1024 should decode, got: " + + (h.errors.empty() ? std::string() : h.errors[0])); + CHECK_EQ(h.deliveries.size(), size_t(1)); + } + { + Harness h; + h.feed(kbtest::buildFrameProdHeader(kbtest::contentNested(1023))); + CHECK(h.errors.empty()); + CHECK_EQ(h.deliveries.size(), size_t(1)); + } + { + Harness h; + h.feed(kbtest::buildFrameProdHeader(kbtest::contentNested(1025)), 9); + CHECK_EQ(h.deliveries.size(), size_t(0)); + CHECK_EQ(h.errors.size(), size_t(1)); + CHECK_MSG(h.errors[0].find("nesting too deep") != std::string::npos, + "unexpected error: " + h.errors[0]); + CHECK_EQ(h.fatals.size(), size_t(1)); + CHECK_EQ(h.fatals[0], int64_t(9)); + } +} + +// --------------------------------------------------------------------------- +// JSI -> msgpack +// --------------------------------------------------------------------------- + +void testEncodeSymbolAndBigIntBecomeNil() { + Harness h; + const bool hasBigInt = + h.eval("typeof BigInt === 'function'").getBool(); + const std::string src = + hasBigInt ? "{a: 1, s: Symbol('x'), g: BigInt(7), d: 4}" + : "{a: 1, s: Symbol('x'), g: null, d: 4}"; + h.send(src); + + auto oh = h.lastPayload(); + const auto &o = oh.get(); + CHECK_MSG(o.type == msgpack::type::MAP, "expected a MAP"); + // The enclosing map header promised 4 entries; the whole point of packing + // nil is that all 4 are actually present and the frame still parses. + CHECK_EQ(o.via.map.size, uint32_t(4)); + auto valueFor = [&](const char *key) -> const msgpack::object & { + for (uint32_t i = 0; i < o.via.map.size; ++i) { + const auto &k = o.via.map.ptr[i].key; + if (k.type == msgpack::type::STR && + std::string(k.via.str.ptr, k.via.str.size) == key) { + return o.via.map.ptr[i].val; + } + } + throw kbtest::CheckFailed(std::string("missing key ") + key); + }; + CHECK_EQ(valueFor("a").as(), 1); + CHECK_MSG(valueFor("s").type == msgpack::type::NIL, "Symbol did not pack nil"); + if (hasBigInt) { + CHECK_MSG(valueFor("g").type == msgpack::type::NIL, + "BigInt did not pack nil"); + } + CHECK_EQ(valueFor("d").as(), 4); +} + +void testEncodeFunctionBecomesNil() { + Harness h; + h.send("{f: function () {}, a: 1}"); + auto oh = h.lastPayload(); + const auto &o = oh.get(); + CHECK_EQ(o.via.map.size, uint32_t(2)); + CHECK_MSG(o.via.map.ptr[0].val.type == msgpack::type::NIL, + "function did not pack nil"); +} + +// Replaces a "first key is a digit" heuristic: a plain object that happens to +// carry byteLength/buffer and a "0" first key must still pack as a map. +void testEncodeIsViewNotDigitHeuristic() { + Harness h; + h.send("(function () {" + " const o = {};" + " o['0'] = 7;" + " o.byteLength = 2;" + " o.buffer = new ArrayBuffer(2);" + " return o;" + "})()"); + auto oh = h.lastPayload(); + const auto &o = oh.get(); + CHECK_MSG(o.type == msgpack::type::MAP, + "plain object with a '0' first key packed as type " + + std::to_string(int(o.type)) + ", want MAP"); + CHECK_EQ(o.via.map.size, uint32_t(3)); + + // And an object with a digit first key and nothing typed-array-ish at all. + Harness h2; + h2.send("{'0': 'a', '1': 'b'}"); + auto oh2 = h2.lastPayload(); + CHECK_MSG(oh2.get().type == msgpack::type::MAP, "digit-keyed object is not binary"); + CHECK_EQ(oh2.get().via.map.size, uint32_t(2)); +} + +void testEncodeTypedArrayViews() { + Harness h; + // A 4-byte window into a 10-byte buffer: only bytes 3..6 may be packed. + h.send("(function () {" + " const b = new ArrayBuffer(10);" + " const full = new Uint8Array(b);" + " for (let i = 0; i < 10; i++) full[i] = i;" + " return new Uint8Array(b, 3, 4);" + "})()"); + auto oh = h.lastPayload(); + CHECK_MSG(oh.get().type == msgpack::type::BIN, "typed array did not pack as BIN"); + CHECK_MSG(mpBinToString(oh.get()) == std::string("\x03\x04\x05\x06", 4), + "subrange view packed the wrong bytes"); + + // DataView takes the same path via ArrayBuffer.isView. + Harness h2; + h2.send("(function () {" + " const b = new ArrayBuffer(10);" + " const full = new Uint8Array(b);" + " for (let i = 0; i < 10; i++) full[i] = i;" + " return new DataView(b, 2, 3);" + "})()"); + auto oh2 = h2.lastPayload(); + CHECK_MSG(oh2.get().type == msgpack::type::BIN, "DataView did not pack as BIN"); + CHECK_MSG(mpBinToString(oh2.get()) == std::string("\x02\x03\x04", 3), + "DataView subrange packed the wrong bytes"); + + // A multi-byte-element view: byteLength/byteOffset are in bytes, not + // elements, so a Uint32Array over a subrange must pack byteLength bytes. + Harness h3; + h3.send("(function () {" + " const b = new ArrayBuffer(16);" + " const full = new Uint8Array(b);" + " for (let i = 0; i < 16; i++) full[i] = i;" + " return new Uint32Array(b, 4, 2);" + "})()"); + auto oh3 = h3.lastPayload(); + CHECK_MSG(oh3.get().type == msgpack::type::BIN, "Uint32Array did not pack as BIN"); + CHECK_MSG(mpBinToString(oh3.get()) == + std::string("\x04\x05\x06\x07\x08\x09\x0a\x0b", 8), + "Uint32Array subrange packed the wrong bytes"); + + // A bare ArrayBuffer (not a view) packs its whole contents. + Harness h4; + h4.send("(function () {" + " const b = new ArrayBuffer(3);" + " new Uint8Array(b).set([9, 8, 7]);" + " return b;" + "})()"); + auto oh4 = h4.lastPayload(); + CHECK_MSG(oh4.get().type == msgpack::type::BIN, "ArrayBuffer did not pack as BIN"); + CHECK_MSG(mpBinToString(oh4.get()) == std::string("\x09\x08\x07", 3), + "ArrayBuffer packed the wrong bytes"); +} + +void testEncodeEmptyContainers() { + Harness h; + h.send("{arr: [], map: {}, str: '', bin: new Uint8Array(0)}"); + auto oh = h.lastPayload(); + const auto &o = oh.get(); + CHECK_EQ(o.via.map.size, uint32_t(4)); + CHECK_MSG(o.via.map.ptr[0].val.type == msgpack::type::ARRAY, "want ARRAY"); + CHECK_EQ(o.via.map.ptr[0].val.via.array.size, uint32_t(0)); + CHECK_MSG(o.via.map.ptr[1].val.type == msgpack::type::MAP, "want MAP"); + CHECK_EQ(o.via.map.ptr[1].val.via.map.size, uint32_t(0)); + CHECK_MSG(o.via.map.ptr[2].val.type == msgpack::type::STR, "want STR"); + CHECK_EQ(o.via.map.ptr[2].val.via.str.size, uint32_t(0)); + CHECK_MSG(o.via.map.ptr[3].val.type == msgpack::type::BIN, "want BIN"); + CHECK_EQ(o.via.map.ptr[3].val.via.bin.size, uint32_t(0)); +} + +void testEncodeUtf8Strings() { + Harness h; + h.send("{m: 'caf\\u00e9 \\u2713 \\ud83d\\udd11', n: 'a\\u0000b'}"); + auto oh = h.lastPayload(); + const auto &o = oh.get(); + const auto &m = o.via.map.ptr[0].val; + CHECK_MSG(m.type == msgpack::type::STR, "want STR"); + CHECK_MSG(std::string(m.via.str.ptr, m.via.str.size) == + "caf\xc3\xa9 \xe2\x9c\x93 \xf0\x9f\x94\x91", + "multibyte string packed wrong bytes"); + const auto &n = o.via.map.ptr[1].val; + CHECK_EQ(n.via.str.size, uint32_t(3)); + CHECK_MSG(std::string(n.via.str.ptr, n.via.str.size) == + std::string("a\0b", 3), + "embedded NUL string packed wrong bytes"); +} + +void testEncodeDepthLimit() { + Harness h; + h.send("(function () { let a = 42; for (let i = 0; i < 1024; i++) a = [a]; " + "return a; })()"); + CHECK_EQ(h.writes.size(), size_t(1)); + + auto msg = h.sendExpectingThrow( + "(function () { let a = 42; for (let i = 0; i < 1025; i++) a = [a]; " + "return a; })()"); + CHECK_MSG(msg.find("nesting too deep") != std::string::npos, + "expected a nesting error, got: " + msg); +} + +// Round-trips every scalar type back out through rpcOnGo, so decode and +// encode are checked against each other rather than each against itself. +void testRoundTripThroughBothDirections() { + Harness h; + const std::string bin("\x01\x02\x03", 3); + h.feed(frameOf([&](auto &pk) { + pk.pack_map(7); + pk.pack(std::string("i")); + pk.pack(42); + pk.pack(std::string("ni")); + pk.pack(-42); + pk.pack(std::string("f")); + pk.pack(1.5); + pk.pack(std::string("b")); + pk.pack(true); + pk.pack(std::string("n")); + pk.pack_nil(); + pk.pack(std::string("s")); + pk.pack(std::string("hello")); + pk.pack(std::string("bin")); + pk.pack_bin(3); + pk.pack_bin_body(bin.data(), 3); + })); + CHECK_EQ(h.deliveries.size(), size_t(1)); + + auto rpcOnGo = h.rt().global().getPropertyAsFunction(h.rt(), "rpcOnGo"); + rpcOnGo.call(h.rt(), h.deliveries[0].value); + + auto oh = h.lastPayload(); + const auto &o = oh.get(); + CHECK_EQ(o.via.map.size, uint32_t(7)); + auto get = [&](const char *key) -> const msgpack::object & { + for (uint32_t i = 0; i < o.via.map.size; ++i) { + const auto &k = o.via.map.ptr[i].key; + if (std::string(k.via.str.ptr, k.via.str.size) == key) { + return o.via.map.ptr[i].val; + } + } + throw kbtest::CheckFailed(std::string("missing key ") + key); + }; + CHECK_EQ(get("i").as(), int64_t(42)); + CHECK_MSG(get("i").type == msgpack::type::POSITIVE_INTEGER, + "42 did not repack as an unsigned integer"); + CHECK_EQ(get("ni").as(), int64_t(-42)); + CHECK_MSG(get("ni").type == msgpack::type::NEGATIVE_INTEGER, + "-42 did not repack as a signed integer"); + CHECK_MSG(get("f").as() == 1.5, "1.5 did not survive"); + CHECK_MSG(get("f").type == msgpack::type::FLOAT64, "1.5 did not stay a float"); + CHECK_MSG(get("b").as(), "true did not survive"); + CHECK_MSG(get("n").type == msgpack::type::NIL, "nil did not survive"); + CHECK_MSG(get("s").as() == "hello", "string did not survive"); + CHECK_MSG(get("bin").type == msgpack::type::BIN, "BIN did not survive as BIN"); + CHECK_MSG(mpBinToString(get("bin")) == bin, "BIN contents differ"); +} + +// --------------------------------------------------------------------------- +// Batch delivery shape +// --------------------------------------------------------------------------- + +void testSingleFrameDeliversBareValue() { + Harness h; + h.feed(markerFrame(1)); + CHECK_EQ(h.deliveries.size(), size_t(1)); + CHECK_EQ(h.deliveries[0].count, 1); + CHECK_MSG(!h.deliveries[0].value.getObject(h.rt()).isArray(h.rt()), + "a single message was wrapped in an array"); + CHECK_EQ(markerId(h.rt(), h.deliveries[0].value), 1); +} + +void testMultiFrameDeliversArray() { + Harness h; + h.feed(concatFrames({markerFrame(1), markerFrame(2), markerFrame(3)})); + CHECK_EQ(h.deliveries.size(), size_t(1)); + CHECK_EQ(h.deliveries[0].count, 3); + auto arr = h.deliveries[0].value.getObject(h.rt()).getArray(h.rt()); + CHECK_EQ(arr.size(h.rt()), size_t(3)); + for (int i = 0; i < 3; ++i) { + CHECK_EQ(markerId(h.rt(), arr.getValueAtIndex(h.rt(), size_t(i))), i + 1); + } +} + +// JS's rpcOnJs only unwraps the array when count > 1, so when a batch is +// whittled down to one survivor it must go out bare or JS hands the wrapper +// itself to isRPCMessage and drops it. +void testBatchWithOneSurvivorDeliversBareValue() { + Harness h; + h.feed(concatFrames({markerFrame(7), badKeyFrame()}), 5); + + CHECK_EQ(h.deliveries.size(), size_t(1)); + CHECK_EQ(h.deliveries[0].count, 1); + CHECK_MSG(h.deliveries[0].value.isObject(), "survivor is not an object"); + CHECK_MSG(!h.deliveries[0].value.getObject(h.rt()).isArray(h.rt()), + "single survivor of a batch was delivered as a length-1 array"); + CHECK_EQ(markerId(h.rt(), h.deliveries[0].value), 7); + CHECK_EQ(h.errors.size(), size_t(1)); + CHECK_MSG(h.errors[0].find("dropping undecodable message") != + std::string::npos, + "unexpected error: " + h.errors[0]); + // One bad message in the batch is not stream-fatal. + CHECK_EQ(h.fatals.size(), size_t(0)); +} + +void testBatchWithAllBadEscalatesToFatal() { + Harness h; + h.feed(concatFrames({badKeyFrame(), badKeyFrame()}), 1234); + + CHECK_EQ(h.deliveries.size(), size_t(0)); + CHECK_EQ(h.errors.size(), size_t(3)); // two drops + the batch-level report + CHECK_MSG(h.errors.back().find("dropped entire batch") != std::string::npos, + "unexpected final error: " + h.errors.back()); + CHECK_MSG(h.errors.back().find("all 2 message(s)") != std::string::npos, + "batch error lost the count: " + h.errors.back()); + CHECK_EQ(h.fatals.size(), size_t(1)); + CHECK_EQ(h.fatals[0], int64_t(1234)); +} + +void testOneBadMessageDropsAlone() { + Harness h; + h.feed(concatFrames( + {markerFrame(1), badKeyFrame(), markerFrame(3), markerFrame(4)})); + + CHECK_EQ(h.deliveries.size(), size_t(1)); + CHECK_EQ(h.deliveries[0].count, 3); + auto arr = h.deliveries[0].value.getObject(h.rt()).getArray(h.rt()); + // The array must be sized to what survived: a hole at the original index + // would hand JS an undefined message to dispatch. + CHECK_EQ(arr.size(h.rt()), size_t(3)); + CHECK_EQ(markerId(h.rt(), arr.getValueAtIndex(h.rt(), 0)), 1); + CHECK_EQ(markerId(h.rt(), arr.getValueAtIndex(h.rt(), 1)), 3); + CHECK_EQ(markerId(h.rt(), arr.getValueAtIndex(h.rt(), 2)), 4); + CHECK_EQ(h.errors.size(), size_t(1)); + CHECK_EQ(h.fatals.size(), size_t(0)); +} + +void testMissingRpcOnJsIsFatal() { + Harness h; + h.rt().global().setProperty(h.rt(), "rpcOnJs", jsi::Value::undefined()); + h.feed(markerFrame(1), 99); + CHECK_EQ(h.deliveries.size(), size_t(0)); + CHECK_EQ(h.fatals.size(), size_t(1)); + CHECK_EQ(h.fatals[0], int64_t(99)); +} + +// A framing violation is fatal on the reader thread, before anything is +// scheduled to JS. +void testFramingViolationIsFatal() { + Harness h; + auto bad = kbtest::buildFrameWithDeclaredSize( + 99, kbtest::packContent([](auto &pk) { pk.pack(1); })); + h.feed(bad, 8); + CHECK_EQ(h.deliveries.size(), size_t(0)); + CHECK_EQ(h.fatals.size(), size_t(1)); + CHECK_EQ(h.fatals[0], int64_t(8)); +} + +// --------------------------------------------------------------------------- +// packAndSend framing +// --------------------------------------------------------------------------- + +void testSendFrameHeaderAndRoundTripThroughParser() { + Harness h; + h.send("{method: 'ping', seqid: 3, args: [1, 'two', null]}"); + CHECK_EQ(h.writes.size(), size_t(1)); + const auto &frame = h.writes[0]; + + // Header bytes must be exactly what frame-builder.h's packHeaderUint32 + // produces (lastPayload asserts this) ... + auto oh = h.lastPayload(); + CHECK_MSG(oh.get().type == msgpack::type::MAP, "payload is not a MAP"); + + // ... and, more importantly, the real consumer must accept the real + // producer's bytes, not a test re-implementation of them. + kb::FrameParser parser; + std::vector out; + Bytes copy = frame; + parser.feed(copy.data(), copy.size(), out); + CHECK_EQ(out.size(), size_t(1)); + CHECK_MSG(out[0].get().type == msgpack::type::MAP, + "FrameParser decoded the wrong type from packAndSend's output"); + CHECK_EQ(out[0].get().via.map.size, uint32_t(3)); + + // Byte-split every way to prove the header encoding survives partial reads. + for (size_t split = 1; split < frame.size(); ++split) { + kb::FrameParser p; + std::vector got; + Bytes a(frame.begin(), frame.begin() + long(split)); + Bytes b(frame.begin() + long(split), frame.end()); + p.feed(a.data(), a.size(), got); + p.feed(b.data(), b.size(), got); + CHECK_MSG(got.size() == 1, + "split at " + std::to_string(split) + " decoded " + + std::to_string(got.size()) + " frames"); + } +} + +void testSendReturnsWriteResult() { + Harness h; + auto ok = h.send("{a: 1}"); + CHECK_MSG(ok.isBool() && ok.getBool(), "rpcOnGo did not return true"); + h.writeResult = false; + auto bad = h.send("{a: 1}"); + CHECK_MSG(bad.isBool() && !bad.getBool(), + "rpcOnGo did not propagate a failed write"); +} + +void testSendOversizeFrameThrows() { + Harness h; + // kMaxFrameSize is 64MiB; the bin32 header pushes this just past it. + auto msg = h.sendExpectingThrow("new Uint8Array(64 * 1024 * 1024 + 16)"); + CHECK_MSG(msg.find("too large") != std::string::npos, + "expected an oversize-frame error, got: " + msg); + CHECK_EQ(h.writes.size(), size_t(0)); +} + +// The buffer release past kSendBufKeepCapacity swaps out the SendState while +// a reference to its sbuffer is still in scope, so the next send is the thing +// worth checking. +void testSendLargePayloadThenSmall() { + Harness h; + h.send("new Uint8Array(5 * 1024 * 1024)"); + CHECK_EQ(h.writes.size(), size_t(1)); + { + auto oh = h.lastPayload(); + CHECK_MSG(oh.get().type == msgpack::type::BIN, "large payload is not BIN"); + CHECK_EQ(oh.get().via.bin.size, uint32_t(5 * 1024 * 1024)); + } + + h.send("{a: 1}"); + CHECK_EQ(h.writes.size(), size_t(2)); + auto oh = h.lastPayload(); + CHECK_MSG(oh.get().type == msgpack::type::MAP, + "small frame after a buffer release is malformed"); + CHECK_EQ(oh.get().via.map.size, uint32_t(1)); +} + +// convertJSIToMP runs JS getters; one that re-enters rpcOnGo would otherwise +// clobber the shared scratch buffer mid-frame. +void testSendReentrancyGuard() { + Harness h; + auto msg = h.sendExpectingThrow( + "{a: 1, get b() { return rpcOnGo({inner: true}); }, c: 3}"); + CHECK_MSG(msg.find("re-entered") != std::string::npos, + "expected a re-entrancy error, got: " + msg); + // Neither the inner nor the outer frame may reach the wire: a half-packed + // outer frame would be a garbled write with no detection machinery. + CHECK_EQ(h.writes.size(), size_t(0)); + + // And the guard must have been cleared, so the next send is well-formed. + h.send("{after: 1}"); + CHECK_EQ(h.writes.size(), size_t(1)); + auto oh = h.lastPayload(); + CHECK_MSG(oh.get().type == msgpack::type::MAP, "post-guard frame is malformed"); + CHECK_EQ(oh.get().via.map.size, uint32_t(1)); + + kb::FrameParser parser; + std::vector out; + Bytes copy = h.writes[0]; + parser.feed(copy.data(), copy.size(), out); + CHECK_EQ(out.size(), size_t(1)); +} + +// A getter that mutates the container mid-walk must not desync the frame. +// For objects the property-name list is a snapshot, so a key deleted by an +// earlier getter still reads back as undefined and packs nil, keeping the map +// header's promised entry count honest. +void testSendMutatingObjectGetterPacksNil() { + Harness h; + h.send("(function () {" + " const o = {a: 1, b: 2, c: 3};" + " Object.defineProperty(o, 'b', {enumerable: true, configurable: " + "true, get: function () { delete o.c; return 9; }});" + " return o;" + "})()"); + auto oh = h.lastPayload(); + const auto &o = oh.get(); + CHECK_MSG(o.type == msgpack::type::MAP, "expected a MAP"); + CHECK_EQ(o.via.map.size, uint32_t(3)); + CHECK_EQ(o.via.map.ptr[1].val.as(), 9); + CHECK_MSG(o.via.map.ptr[2].val.type == msgpack::type::NIL, + "key deleted by an earlier getter did not pack nil"); +} + +// Arrays behave differently from objects here: Hermes throws on an +// out-of-bounds getValueAtIndex rather than yielding undefined, so a getter +// that shrinks the array aborts the whole frame. That still upholds the +// invariant that matters -- nothing half-packed reaches the wire -- but it is +// a throw, not a run of nils. +void testSendShrinkingArrayGetterAbortsFrame() { + Harness h; + auto msg = h.sendExpectingThrow( + "(function () {" + " const a = [1, 2, 3, 4];" + " Object.defineProperty(a, '1', {enumerable: true, configurable: true, " + "get: function () { a.length = 2; return 9; }});" + " return a;" + "})()"); + CHECK_MSG(!msg.empty(), + "expected a shrinking array to abort the frame; it packed " + "successfully instead"); + CHECK_MSG(h.writes.empty(), + "an aborted frame reached writeToGo: " + + std::to_string(h.writes.size()) + " write(s)"); + + // The scratch buffer must still be usable afterwards. + h.send("{after: 1}"); + CHECK_EQ(h.writes.size(), size_t(1)); + auto oh = h.lastPayload(); + CHECK_MSG(oh.get().type == msgpack::type::MAP, "post-abort frame is malformed"); +} + +} // namespace + +int main() { + kbtest::Runner runner; + + runner.add("packNumber: boundary table", testPackNumberTable); + runner.add("packNumber: ulp neighborhoods", + testPackNumberBoundaryNeighborhoods); + runner.add("packNumber: 1M random doubles vs floor/isfinite reference", + testPackNumberRandomSweep); + + runner.add("decode: BIN -> Uint8Array", testDecodeBinBecomesUint8Array); + runner.add("decode: STR is not BIN", testDecodeStrIsNotBin); + runner.add("decode: UTF-8, multibyte, embedded NUL", testDecodeUtf8Strings); + runner.add("decode: empty array/map/string/binary", testDecodeEmptyContainers); + runner.add("decode: EXT -> undefined", testDecodeExtBecomesUndefined); + runner.add("decode: non-scalar map key throws", + testDecodeNonScalarMapKeyThrows); + runner.add("decode: scalar map keys stringify", testDecodeScalarMapKeysCoerce); + runner.add("decode: kMaxDepth", testDecodeDepthLimit); + + runner.add("encode: Symbol/BigInt -> nil, map stays intact", + testEncodeSymbolAndBigIntBecomeNil); + runner.add("encode: function -> nil", testEncodeFunctionBecomesNil); + runner.add("encode: ArrayBuffer.isView, not a digit-key heuristic", + testEncodeIsViewNotDigitHeuristic); + runner.add("encode: typed array byteOffset/byteLength bounds", + testEncodeTypedArrayViews); + runner.add("encode: empty containers", testEncodeEmptyContainers); + runner.add("encode: UTF-8, multibyte, embedded NUL", testEncodeUtf8Strings); + runner.add("encode: kMaxDepth", testEncodeDepthLimit); + runner.add("round trip: decode then encode", testRoundTripThroughBothDirections); + + runner.add("batch: single frame delivers a bare value", + testSingleFrameDeliversBareValue); + runner.add("batch: multiple frames deliver an array", + testMultiFrameDeliversArray); + runner.add("batch: one survivor delivers a bare value", + testBatchWithOneSurvivorDeliversBareValue); + runner.add("batch: all undecodable escalates to onFatal", + testBatchWithAllBadEscalatesToFatal); + runner.add("batch: one undecodable message drops alone", + testOneBadMessageDropsAlone); + runner.add("batch: missing rpcOnJs is fatal", testMissingRpcOnJsIsFatal); + runner.add("batch: framing violation is fatal", testFramingViolationIsFatal); + + runner.add("send: header matches packHeaderUint32 and FrameParser accepts it", + testSendFrameHeaderAndRoundTripThroughParser); + runner.add("send: returns the writeToGo result", testSendReturnsWriteResult); + runner.add("send: oversize frame throws", testSendOversizeFrameThrows); + runner.add("send: large payload then small", testSendLargePayloadThenSmall); + runner.add("send: re-entrancy guard", testSendReentrancyGuard); + runner.add("send: object getter that deletes a later key packs nil", + testSendMutatingObjectGetterPacksNil); + runner.add("send: array getter that shrinks the array aborts the frame", + testSendShrinkingArrayGetterAbortsFrame); + + return runner.run(); +} diff --git a/rnmodules/react-native-kb/cpp/tests/test-harness.h b/rnmodules/react-native-kb/cpp/tests/test-harness.h new file mode 100644 index 000000000000..b1cd992cc5ee --- /dev/null +++ b/rnmodules/react-native-kb/cpp/tests/test-harness.h @@ -0,0 +1,87 @@ +// Minimal assert/report harness. No gtest/catch2 dependency so the test +// binary stays buildable with the same plain clang++ invocation already used +// for syntax-checking react-native-kb.cpp. +#pragma once + +#include +#include +#include +#include +#include + +namespace kbtest { + +struct Failure { + std::string message; +}; + +// Thrown by CHECK/CHECK_EQ on failure; caught by the runner per test case. +struct CheckFailed : std::exception { + std::string message; + explicit CheckFailed(std::string m) : message(std::move(m)) {} + const char *what() const noexcept override { return message.c_str(); } +}; + +inline void check(bool cond, const std::string &msg, const char *file, + int line) { + if (!cond) { + throw CheckFailed(std::string(file) + ":" + std::to_string(line) + ": " + + msg); + } +} + +#define CHECK(cond) \ + ::kbtest::check((cond), "CHECK failed: " #cond, __FILE__, __LINE__) +#define CHECK_MSG(cond, msg) \ + ::kbtest::check((cond), (msg), __FILE__, __LINE__) + +template +void checkEq(const A &a, const B &b, const char *aExpr, const char *bExpr, + const char *file, int line) { + if (!(a == b)) { + throw CheckFailed(std::string(file) + ":" + std::to_string(line) + + ": CHECK_EQ failed: " + aExpr + " (" + + std::to_string(a) + ") != " + bExpr + " (" + + std::to_string(b) + ")"); + } +} + +#define CHECK_EQ(a, b) \ + ::kbtest::checkEq((a), (b), #a, #b, __FILE__, __LINE__) + +class Runner { +public: + void add(std::string name, std::function fn) { + cases_.push_back({std::move(name), std::move(fn)}); + } + + // Returns 0 if every case passed, 1 otherwise. Prints PASS/FAIL per case + // and a final summary. + int run() { + int failed = 0; + for (auto &c : cases_) { + try { + c.fn(); + std::cout << "[PASS] " << c.name << "\n"; + } catch (const std::exception &e) { + ++failed; + std::cout << "[FAIL] " << c.name << ": " << e.what() << "\n"; + } catch (...) { + ++failed; + std::cout << "[FAIL] " << c.name << ": unknown exception\n"; + } + } + std::cout << (cases_.size() - failed) << "/" << cases_.size() + << " passed\n"; + return failed == 0 ? 0 : 1; + } + +private: + struct Case { + std::string name; + std::function fn; + }; + std::vector cases_; +}; + +} // namespace kbtest diff --git a/rnmodules/react-native-kb/ios/Kb.mm b/rnmodules/react-native-kb/ios/Kb.mm index 82e0ef81ed2f..dffb4ee1fead 100644 --- a/rnmodules/react-native-kb/ios/Kb.mm +++ b/rnmodules/react-native-kb/ios/Kb.mm @@ -1,13 +1,17 @@ #import "Kb.h" #import "Keybasego.h" +#import "engine-reset-backoff.h" #import #import #import #import +#import #import #import #import #import +#import +#import #import #import #import "RNKbSpec.h" @@ -36,10 +40,69 @@ + (id)sharedFsPathsHolder { static NSString *const metaEventEngineReset = @"kb-engine-reset"; static __weak Kb *kbSharedInstance = nil; +// Guards the compare-and-clear in invalidate against init's write: init runs +// on the main thread, invalidate on the TurboModule shared method queue, and +// the __weak load/store are each individually synchronized but the +// read-then-write in invalidate is not, so without this lock a reload's new +// instance can be clobbered by the old instance's invalidate. +static std::mutex kbSharedInstanceMutex; static BOOL kbPasteImageEnabled = NO; static NSString *kbStoredDeviceToken = nil; static NSDictionary *kbInitialNotification = nil; +// The bridge is created on the JS thread and consumed by the reader thread, +// so every access goes through this lock — a plain shared_ptr member would be +// a data race between installJSIBindings/invalidate and the reader. +static std::mutex kbBridgeMutex; +static std::shared_ptr kbCurrentBridge; + +static std::shared_ptr kbGetBridge(void) { + std::lock_guard lock(kbBridgeMutex); + return kbCurrentBridge; +} + +// REQUIRES kbBridgeMutex. Publishes `bridge` as the current one and hands the +// displaced bridge back to the caller, which must markTornDown() it *after* +// releasing the lock (markTornDown only flips an atomic, but nothing that can +// re-enter this file may run under kbBridgeMutex; releasing the old bridge's +// jsi handles is the JS runtime's job -- see the kbTeardown host object). +// +// Lock-requiring rather than lock-taking so the caller can publish myBridge_ +// and kbCurrentBridge in ONE critical section; see +// installJSIBindingsWithRuntime. +static std::shared_ptr +kbSetBridgeLocked(std::shared_ptr bridge) { + std::shared_ptr old = std::move(kbCurrentBridge); + kbCurrentBridge = std::move(bridge); + return old; +} + +// Clears the installed bridge only if it is still `mine`. A module's +// invalidate can run *after* the next module already installed its bridge +// (RCTInstance::invalidate hops to the old JS thread asynchronously, and +// RCTTurboModuleManager only waits 10s before proceeding), so an +// unconditional clear would tear down the live bridge and wedge the app +// with no desync and no reset to recover from. +static bool kbClearBridgeIfCurrent(const std::shared_ptr &mine) { + std::shared_ptr old; + { + std::lock_guard lock(kbBridgeMutex); + if (!mine || kbCurrentBridge != mine) { + return false; + } + old = std::move(kbCurrentBridge); + kbCurrentBridge = nullptr; + } + old->markTornDown(); + return true; +} + +static void kbLogToService(NSString *message) { + KeybaseLogToService([NSString + stringWithFormat:@"dNativeLogger: [%f,\"%@\"]", + [[NSDate date] timeIntervalSince1970] * 1000, message]); +} + // from react-native-localize static bool kbUses24HourClockForLocale(NSLocale *_Nonnull locale) { NSDateFormatter *formatter = [NSDateFormatter new]; @@ -120,13 +183,13 @@ static bool kbUses24HourClockForLocale(NSLocale *_Nonnull locale) { return constants; } -@interface Kb () -@property dispatch_queue_t readQueue; -@end - @implementation Kb { - std::shared_ptr kbBridge_; - BOOL isInvalidated_; + // Guarded by kbBridgeMutex: written on the JS thread in + // installJSIBindingsWithRuntime, read and cleared on the TurboModule shared + // method queue in invalidate. Reusing kbBridgeMutex (rather than a second + // lock) keeps this ivar and kbCurrentBridge consistent with each other + // without ever nesting the two critical sections. + std::shared_ptr myBridge_; } RCT_EXPORT_MODULE() @@ -143,8 +206,10 @@ - (BOOL)canEmit { - (instancetype)init { self = [super init]; - kbSharedInstance = self; - isInvalidated_ = NO; + { + std::lock_guard lock(kbSharedInstanceMutex); + kbSharedInstance = self; + } [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(handleHardwareKeyPressed:) name:@"hardwareKeyPressed" @@ -218,15 +283,41 @@ + (void)handlePastedImages:(NSArray *)images { - (void)invalidate { [[NSNotificationCenter defaultCenter] removeObserver:self]; - isInvalidated_ = YES; kbPasteImageEnabled = NO; - if (kbBridge_) { - kbBridge_->teardown(); - kbBridge_.reset(); + // RN never nulls _eventEmitterCallback on invalidate and the __weak ref + // above only nils at dealloc, which lags this call — so without an explicit + // clear, canEmit stays YES and a push notification, token registration or + // (worst) the reader's desync meta event emits into the dying runtime's + // invoker. Guarded because a reload may already have installed a newer + // module as the shared instance; the lock makes the compare-and-clear + // atomic with init's write so a newer instance can never be clobbered. + { + std::lock_guard lock(kbSharedInstanceMutex); + if (kbSharedInstance == self) { + kbSharedInstance = nil; + } + } + // Runs on the TurboModule shared method queue (no methodQueue getter, so + // RCTTurboModuleManager assigns _sharedModuleQueue) — any thread, never the + // JS thread. Only the atomic flag may be touched here; releasing jsi + // handles off the runtime's thread is undefined behavior. + // + // Both the teardown and the Go reset are gated on still being the current + // bridge: a reload can install the next module's bridge before this runs, + // and clearing that one would leave the app wedged with no way to notice. + std::shared_ptr mine; + { + std::lock_guard lock(kbBridgeMutex); + mine = myBridge_; + } + if (kbClearBridgeIfCurrent(mine)) { + NSError *error = nil; + KeybaseReset(&error); + } + { + std::lock_guard lock(kbBridgeMutex); + myBridge_ = nullptr; } - self.readQueue = nil; - NSError *error = nil; - KeybaseReset(&error); } RCT_EXPORT_METHOD(setEnablePasteImage:(BOOL)enabled) { @@ -241,27 +332,102 @@ - (void)invalidate { // RCTTurboModuleWithJSIBindings — called automatically by RN when the module loads - (void)installJSIBindingsWithRuntime:(jsi::Runtime &)runtime callInvoker:(const std::shared_ptr &)callInvoker { - kbBridge_ = std::make_shared(); - kbBridge_->install(runtime, callInvoker, - // writeToGo callback - [](void *ptr, size_t size) { + auto bridge = std::make_shared(); + bridge->install(runtime, callInvoker, + // writeToGo callback; false means the RPC never reached Go, so the + // caller fails that invocation instead of waiting forever. + [](void *ptr, size_t size) -> bool { NSData *data = [NSData dataWithBytesNoCopy:ptr length:size freeWhenDone:NO]; NSError *error = nil; KeybaseWriteArr(data, &error); if (error) { - NSLog(@"Error writing data: %@", error); + kbLogToService([NSString stringWithFormat:@"rpc write failed: %@", + error.localizedDescription]); + return false; } + return true; }, // error callback [](const std::string &err) { - KeybaseLogToService([NSString - stringWithFormat:@"dNativeLogger: [%f,\"jsi error: %s\"]", - [[NSDate date] timeIntervalSince1970] * 1000, - err.c_str()]); + kbLogToService([NSString stringWithFormat:@"jsi error: %s", err.c_str()]); + }, + // fatal callback: the incoming stream desynced. Reset the Go + // connection and tell JS, so it fails outstanding RPCs rather than + // leaving every caller hanging on a channel that can't recover. + // + // Also true since this can escalate for a msgpack->JSI conversion + // failure or a missing rpcOnJs, arriving on the JS thread rather than + // the reader thread -- either way recv_ still holds bytes from the + // now-dead connection, so drop them here too or the next connection + // desyncs on its very first frame. Captured weakly rather than by + // shared_ptr: this lambda is stored inside the bridge's own onFatal_ + // member, so a strong capture would be a shared_ptr cycle. + [weakBridge = std::weak_ptr(bridge)](int64_t epoch) { + // Identity gate: only act if the bridge that faulted is still the + // installed one. A batch queued by a dying runtime can hit its + // conversion-failure fatal on the old JS thread after a reload + // has already published the next module's bridge, and the epoch + // check can't catch that (nothing re-dialed, so `epoch` is still + // current) -- acting here would tear down the connection the new + // runtime is already using, then clear the new bridge's parser + // mid-frame, forcing a needless second fatal/reset cycle. + auto strongBridge = weakBridge.lock(); + if (!strongBridge || kbGetBridge() != strongBridge) { + kbLogToService(@"rpc stream desync from superseded bridge, ignoring"); + return; + } + kbLogToService(@"rpc stream desync, resetting connection"); + // Reset the Go connection before the parser: the reader thread is + // still live on this path, so resetting the parser first would + // leave a window where bytes from the OLD connection land in the + // freshly-cleared unpacker mid-frame, causing a second desync. + // + // ResetIfCurrentDidReset(epoch), not Reset(): `epoch` is the + // epoch of the connection the desynced bytes actually came from, + // captured by the reader loop below at read time. If Go has + // already re-dialed since (e.g. a concurrent WriteArr recovered + // first), epoch no longer matches and this is a stale no-op + // instead of tearing down a connection that already worked -- + // and in that case resetRecv() must also be skipped, or it drops + // the new connection's already-in-flight partial frame and + // forces a second, needless fatal/reset cycle. + BOOL didReset = KeybaseResetIfCurrentDidReset(epoch); + if (didReset) { + strongBridge->resetRecv(); + } + dispatch_async(dispatch_get_main_queue(), ^{ + Kb *instance = kbSharedInstance; + if (instance && [instance canEmit]) { + [instance emitOnMetaEvent:metaEventEngineReset]; + } + }); }); - KeybaseLogToService([NSString stringWithFormat:@"dNativeLogger: [%f,\"jsi install success (via installJSIBindings)\"]", - [[NSDate date] timeIntervalSince1970] * 1000]); + // myBridge_ and kbCurrentBridge are published in ONE critical section. + // Splitting them (set myBridge_, drop the lock, then set kbCurrentBridge) + // opened a window where invalidate could read a non-null `mine` while + // kbCurrentBridge was still the previous value: kbClearBridgeIfCurrent + // then returned false, so BOTH the teardown and the KeybaseReset were + // skipped, and this method went on to publish a bridge belonging to an + // already-invalidated module that nothing would ever clean up. + // + // With the single critical section, myBridge_ is non-null only if + // kbCurrentBridge was set to that same bridge under the same lock, so + // invalidate's read of myBridge_ followed by kbClearBridgeIfCurrent can + // only ever see the publish as all-or-nothing -- never half-done. (The + // two are separate critical sections in invalidate, which is fine: they + // only need the atomicity of the *publish*, not of their own pair.) + std::shared_ptr old; + { + std::lock_guard lock(kbBridgeMutex); + myBridge_ = bridge; + old = kbSetBridgeLocked(bridge); + } + // Outside the lock, by kbSetBridgeLocked's contract. + if (old) { + old->markTornDown(); + } + kbLogToService(@"jsi install success (via installJSIBindings)"); } RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(getTypedConstants) { @@ -276,9 +442,13 @@ - (void)installJSIBindingsWithRuntime:(jsi::Runtime &)runtime RCT_EXPORT_METHOD(shareListenersRegistered) { } +// No current caller (kept for future use). RCT_EXPORT_METHOD(engineReset) { NSError *error = nil; KeybaseReset(&error); + if (auto bridge = kbGetBridge()) { + bridge->resetRecv(); + } if ([self canEmit]) { [self emitOnMetaEvent:metaEventEngineReset]; } @@ -288,41 +458,119 @@ - (void)installJSIBindingsWithRuntime:(jsi::Runtime &)runtime } RCT_EXPORT_METHOD(notifyJSReady) { - __weak __typeof__(self) weakSelf = self; - - NSLog(@"notifyJSReady: called from JS, queuing main thread block"); - dispatch_async(dispatch_get_main_queue(), ^{ - if (self.readQueue) { - NSLog(@"notifyJSReady: read loop already running, ignoring"); - return; - } - - self.readQueue = dispatch_queue_create("go_bridge_queue_read", DISPATCH_QUEUE_SERIAL); - - // Signal to Go that JS is ready - KeybaseNotifyJSReady(); - NSLog(@"Notified Go that JS is ready, starting ReadArr loop"); - - // Start the read loop - dispatch_async(self.readQueue, ^{ + // KeybaseNotifyJSReady is a sync.Once on the Go side, so repeat calls after + // a reload are free. It must not run on the JS thread — do it on the reader + // queue, which is also where ReadArr is serviced. + // + // Exactly one reader exists for the life of the process. Go's ReadArr hands + // back a view of a single shared buffer and is documented as "called + // serially by the mobile run loops": a second concurrent reader corrupts + // both deliveries. It can't be stopped either, because a parked ReadArr + // ignores cancellation and would swallow the next message on its way out. + // So the loop outlives any individual module instance and simply forwards + // to whichever bridge is currently installed. + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + dispatch_queue_t readQueue = + dispatch_queue_create("go_bridge_queue_read", DISPATCH_QUEUE_SERIAL); + dispatch_async(readQueue, ^{ + KeybaseNotifyJSReady(); + NSLog(@"Notified Go that JS is ready, starting ReadArr loop"); + + // Consecutive read-error count, used to rate-limit the log line below. + // Reset to 0 on every genuine successful read so each new failure + // episode gets its own "first 5" logging window, rather than picking + // up mid-backoff from an earlier, unrelated episode. + static int readErrorCount = 0; + // Throttles the kb-engine-reset EMIT below, separately from + // readErrorCount above -- they have different cadences and must not + // share a counter. See cpp/engine-reset-backoff.h (and its unit test) + // for the arithmetic. Reset alongside readErrorCount on the next + // successful read. + static kb::EngineResetEmitBackoff emitBackoff; while (true) { - { - __typeof__(self) strongSelf = weakSelf; - if (!strongSelf || strongSelf->isInvalidated_) { - NSLog(@"Module invalidated, bailing from ReadArr loop"); - return; + // The block never returns, so the queue's pool never drains on its + // own — each iteration needs its own. + @autoreleasepool { + NSError *error = nil; + NSData *data = KeybaseReadArr(&error); + // Read immediately after ReadArr returns, not before it: ReadArr + // records the epoch of the connection it actually read from under + // connMutex as part of the call, and with exactly one permanent + // reader for the life of the process (this loop) nothing can have + // started a second ReadArr in between, so this is exact for the + // bytes just returned -- unlike capturing the epoch before the + // blocking call, which could race a redial that happens while this + // read is in flight. + int64_t epoch = KeybaseLastReadEpoch(); + if (error) { + // ReadArr already called Reset() on the Go side, so the connection + // JS thinks it has is gone and every in-flight RPC is dead. Tell + // JS so it fails them instead of spinning forever, and drop any + // half-parsed frame so the next connection starts clean. + // + // This retries every ~100ms below, so if the connection can't be + // re-established this is a ~10Hz flood into the uploadable log. + // Unlike the empty-read case above (a one-shot degenerate state) + // a recurring read error is exactly what an operator needs to see + // recur, so log the first few, then back off to every Nth rather + // than going silent. + readErrorCount++; + if (readErrorCount <= 5 || readErrorCount % 50 == 0) { + kbLogToService([NSString + stringWithFormat:@"rpc read error, connection reset (count=%d): %@", + readErrorCount, error.localizedDescription]); + } + if (auto bridge = kbGetBridge()) { + bridge->resetRecv(); + } + // Only advance the backoff window when the emit is actually + // deliverable now -- checked synchronously here rather than + // inside the dispatched block, so a dropped notification (no + // shared instance / not yet able to emit) costs nothing and the + // very next failure gets another chance to notify JS promptly. + Kb *instance = kbSharedInstance; + bool deliverable = instance != nil && [instance canEmit]; + // CACurrentMediaTime is monotonic and immune to wall-clock/NTP + // adjustments, unlike NSDate/[NSDate timeIntervalSinceReferenceDate]: + // a backward clock correction during a read-error episode (plausible + // at cold boot) must not suppress the kb-engine-reset emit. + if (emitBackoff.shouldEmit(CACurrentMediaTime(), deliverable)) { + // Re-check kbSharedInstance/canEmit inside the dispatched + // block rather than reusing the reader-thread snapshot above: + // an invalidate/reload can land between this dispatch and the + // block running, and `_eventEmitterCallback` is never cleared + // on invalidate, so emitting on a strongly-captured `instance` + // here could deliver into a dying runtime's invoker. + dispatch_async(dispatch_get_main_queue(), ^{ + Kb *emitInstance = kbSharedInstance; + if (emitInstance && [emitInstance canEmit]) { + [emitInstance emitOnMetaEvent:metaEventEngineReset]; + } + }); + } + [NSThread sleepForTimeInterval:0.1]; + continue; } - } - - NSError *error = nil; - NSData *data = KeybaseReadArr(&error); - if (error) { - NSLog(@"Error reading data: %@", error); - [NSThread sleepForTimeInterval:0.1]; - } else if (data) { - __typeof__(self) strongSelf = weakSelf; - if (strongSelf && strongSelf->kbBridge_) { - strongSelf->kbBridge_->onDataFromGo((uint8_t *)[data bytes], (int)[data length]); + if (data.length == 0) { + // Not the idle path: ReadArr blocks in LoopbackConn.Read until + // there is data, so an empty non-error result is degenerate (it + // needs n == 0 with no error, which a blocking read does not + // produce). It is reachable if Init never ran and the shared + // buffer is zero-length, which would otherwise spin silently. + static BOOL loggedEmptyRead = NO; + if (!loggedEmptyRead) { + kbLogToService(@"rpc read returned no data; is Keybase initialized?"); + loggedEmptyRead = YES; + } + [NSThread sleepForTimeInterval:0.01]; + continue; + } + readErrorCount = 0; + emitBackoff.reset(); + auto bridge = kbGetBridge(); + if (bridge) { + bridge->onDataFromGo((uint8_t *)[data bytes], (int)[data length], epoch); } } } diff --git a/rnmodules/react-native-kb/react-native-kb.podspec b/rnmodules/react-native-kb/react-native-kb.podspec index 1ec4a7782d1e..a0c3981f7dce 100644 --- a/rnmodules/react-native-kb/react-native-kb.podspec +++ b/rnmodules/react-native-kb/react-native-kb.podspec @@ -20,6 +20,9 @@ Pod::Spec.new do |s| "cpp/**/*.{h,cpp}", "cpp/*.{h,cpp}" ] + # cpp/tests is a standalone plain-clang++ test binary (see + # rnmodules/react-native-kb/scripts/test-framing.sh), not part of the shipped module. + s.exclude_files = "cpp/tests/**/*" s.dependency "KBCommon" diff --git a/rnmodules/react-native-kb/scripts/bench-jsi-convert.sh b/rnmodules/react-native-kb/scripts/bench-jsi-convert.sh new file mode 100755 index 000000000000..dfaf8c44263c --- /dev/null +++ b/rnmodules/react-native-kb/scripts/bench-jsi-convert.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Builds and runs cpp/tests/jsi-convert-bench.cpp twice -- once against this +# branch's bridge, once against origin/master's -- on a real Hermes runtime, +# replaying the corpus built by make-bench-corpus.mjs from a real mobile +# session's RPC traffic. +# +# Hermes comes from the already-installed iOS pod, which ships a macOS slice, so +# nothing has to be built from source. Development-only; not part of the shipped +# module. +# +# Usage: rnmodules/react-native-kb/scripts/bench-jsi-convert.sh [corpus] [iters] +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +CPP_DIR="$ROOT/rnmodules/react-native-kb/cpp" +PODS="$ROOT/shared/ios/Pods" +HERMES="$PODS/hermes-engine/destroot" +MSGPACK_INCLUDE="$ROOT/shared/node_modules/msgpack-cxx-7.0.0/include" +CORPUS="${1:-/tmp/kb-bench-corpus.bin}" +ITERS="${2:-5}" + +for p in "$MSGPACK_INCLUDE" "$HERMES/include" \ + "$PODS/Headers/Public/React-callinvoker"; do + test -d "$p" || { echo "missing $p -- run yarn and yarn ios:pod:install first" >&2; exit 1; } +done +test -f "$CORPUS" || { + echo "missing corpus $CORPUS -- run:" >&2 + echo " node $ROOT/rnmodules/react-native-kb/scripts/make-bench-corpus.mjs" >&2 + exit 1 +} + +# Deliberately not `mktemp -d`, which lands in $TMPDIR: binaries run from there +# measure consistently slower on macOS, enough to swamp the difference being +# measured. +WORK="$(mktemp -d /tmp/kb-bench.XXXXXX)" +trap 'rm -rf "$WORK"' EXIT + +# The baseline is master's bridge, checked out to its own directory so the two +# builds each see their own react-native-kb.h. +mkdir -p "$WORK/baseline" +for f in react-native-kb.cpp react-native-kb.h msgpack-safe.hpp; do + git -C "$ROOT" show "origin/master:rnmodules/react-native-kb/cpp/$f" > "$WORK/baseline/$f" +done + +FRAMEWORKS="$HERMES/Library/Frameworks/macosx" +COMMON=( + -std=c++20 -O2 -DNDEBUG -DMSGPACK_NO_BOOST + -I "$MSGPACK_INCLUDE" + -I "$HERMES/include" + -I "$PODS/Headers/Public/React-callinvoker" + -F "$FRAMEWORKS" -framework hermesvm + -Wl,-rpath,"$FRAMEWORKS" +) + +build() { # build [extra flags...] + local out="$1" dir="$2"; shift 2 + local srcs=("$dir/react-native-kb.cpp") + # frame-parser.cpp only exists on the branch. + [ -f "$dir/frame-parser.cpp" ] && srcs+=("$dir/frame-parser.cpp") + clang++ "${COMMON[@]}" "$@" -I "$dir" \ + "${srcs[@]}" "$CPP_DIR/tests/jsi-convert-bench.cpp" -o "$out" +} + +echo "building baseline (origin/master)..." +build "$WORK/bench-baseline" "$WORK/baseline" -DKB_BASELINE=1 +echo "building branch..." +build "$WORK/bench-branch" "$CPP_DIR" + +echo +"$WORK/bench-baseline" "$CORPUS" "$ITERS" +echo +"$WORK/bench-branch" "$CORPUS" "$ITERS" diff --git a/rnmodules/react-native-kb/scripts/make-bench-corpus.mjs b/rnmodules/react-native-kb/scripts/make-bench-corpus.mjs new file mode 100644 index 000000000000..67856141139a --- /dev/null +++ b/rnmodules/react-native-kb/scripts/make-bench-corpus.mjs @@ -0,0 +1,111 @@ +// Builds the msgpack corpus the JSI conversion benchmark replays. +// +// Input is a real Metro dev log from a mobile run (shared/.expo/dev/logs/start.log), +// which contains every RPC payload the app received while printRPC was on. Each +// "IN >>" line ends with the console-printed first param of one incoming RPC. +// Those payloads are the actual traffic the bridge converts, so the benchmark +// measures the shapes and sizes that really occur rather than invented ones. +// +// Metro's console serializer elides objects past its depth limit as [Object] / +// [Array]; those lines cannot be reconstructed faithfully and are dropped. The +// script reports how much of the observed volume survived. +// +// Output is a length-prefixed stream of msgpack frames, encoded with the same +// @msgpack/msgpack the JS side uses, in the same 0xce + uint32 framing the +// transport writes: +// [0xce][len:u32be][msgpack([type, seqid, method, [param]])] +// +// Usage: +// node rnmodules/react-native-kb/scripts/make-bench-corpus.mjs \ +// [shared/.expo/dev/logs/start.log] [/tmp/kb-bench-corpus.bin] + +import fs from 'fs' +import path from 'path' +import readline from 'readline' +import {fileURLToPath} from 'url' + +const here = path.dirname(fileURLToPath(import.meta.url)) +const repoRoot = path.resolve(here, '../../..') +const {encode} = await import( + path.join(repoRoot, 'shared/node_modules/@msgpack/msgpack/dist.esm/index.mjs') +) + +const logPath = process.argv[2] ?? path.join(repoRoot, 'shared/.expo/dev/logs/start.log') +const outPath = process.argv[3] ?? '/tmp/kb-bench-corpus.bin' + +// "IN >>keybase.1.foo.bar[-calling] [-calling] keybase.1.foo.bar {...}" +const lineRe = /^IN >>([A-Za-z0-9._]+)\[[^\]]*\]\s+\[[^\]]*\]\s+[A-Za-z0-9._]+(?:\s+(\{[\s\S]*\}))?$/ + +const stats = {elided: 0, noParam: 0, parsed: 0, total: 0, unparsable: 0} +const byMethod = new Map() +const frames = [] + +const rl = readline.createInterface({crlfDelay: Infinity, input: fs.createReadStream(logPath)}) +for await (const raw of rl) { + if (!raw.includes('IN >>')) continue + let entry + try { + entry = JSON.parse(raw) + } catch { + continue + } + const text = entry?.data?.[0] + if (typeof text !== 'string') continue + const m = lineRe.exec(text) + if (!m) continue + const [, method, paramText] = m + stats.total++ + const seen = byMethod.get(method) ?? {kept: 0, seen: 0} + seen.seen++ + byMethod.set(method, seen) + + if (!paramText) { + // Methods that really do take no argument still cross the bridge, but an + // empty frame measures nothing. Skip rather than pad the corpus. + stats.noParam++ + continue + } + if (paramText.includes('[Object]') || paramText.includes('[Array]')) { + stats.elided++ + continue + } + let param + try { + param = JSON.parse(paramText) + } catch { + stats.unparsable++ + continue + } + stats.parsed++ + seen.kept++ + // Wire shape of an incoming call: [type=0, seqid, method, [param]]. + frames.push(encode([0, stats.parsed, method, [param]])) +} + +const out = [] +let bytes = 0 +for (const payload of frames) { + const header = Buffer.alloc(5) + header[0] = 0xce + header.writeUInt32BE(payload.length, 1) + out.push(header, Buffer.from(payload)) + bytes += payload.length +} +fs.writeFileSync(outPath, Buffer.concat(out)) + +const keptVolume = [...byMethod.values()].reduce((n, v) => n + v.kept, 0) +console.log(`corpus: ${outPath}`) +console.log(` frames ${frames.length}`) +console.log(` payload bytes ${bytes} (mean ${Math.round(bytes / Math.max(1, frames.length))})`) +console.log( + ` from ${stats.total} incoming RPCs in ${path.relative(repoRoot, logPath)} ` + + `(${((100 * keptVolume) / Math.max(1, stats.total)).toFixed(1)}% kept)` +) +console.log( + ` dropped ${stats.elided} depth-elided, ${stats.noParam} no-param, ${stats.unparsable} unparsable` +) +console.log(' top methods kept:') +;[...byMethod.entries()] + .sort((a, b) => b[1].kept - a[1].kept) + .slice(0, 12) + .forEach(([name, v]) => console.log(` ${String(v.kept).padStart(5)} ${name}`)) diff --git a/rnmodules/react-native-kb/scripts/msgpack-include.sh b/rnmodules/react-native-kb/scripts/msgpack-include.sh new file mode 100644 index 000000000000..b29a291aa9e7 --- /dev/null +++ b/rnmodules/react-native-kb/scripts/msgpack-include.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Sourced, not executed. Sets MSGPACK_INCLUDE to the vendored msgpack-cxx +# headers, fetching them first if they aren't unpacked yet. +# +# shared/desktop/yarn-helper/index.mts's getMsgPack() only unpacks these on +# darwin (they exist for the iOS pod), so on a Linux CI box `yarn` leaves them +# absent. Anything that compiles the C++ bridge outside Xcode needs them on +# every platform, so the fetch lives here -- one copy of the version+shasum, +# shared by every script under this directory. +# +# Usage: source "$(dirname "${BASH_SOURCE[0]}")/msgpack-include.sh" +# ... then use "$MSGPACK_INCLUDE" + +# Keep in sync with getMsgPack() in shared/desktop/yarn-helper/index.mts. +MSGPACK_VER=7.0.0 +MSGPACK_SHA=37bbdbf69ef44392c7af215b9cb419891a9e1c9c + +_MSGPACK_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +_MSGPACK_DEST="$_MSGPACK_ROOT/shared/node_modules" +MSGPACK_INCLUDE="$_MSGPACK_DEST/msgpack-cxx-$MSGPACK_VER/include" + +if [ ! -d "$MSGPACK_INCLUDE" ]; then + _MSGPACK_TARBALL="msgpack-cxx-$MSGPACK_VER.tar.gz" + # macOS ships shasum, most Linux images ship only sha1sum. + _MSGPACK_SHACMD=shasum + command -v shasum >/dev/null 2>&1 || _MSGPACK_SHACMD=sha1sum + mkdir -p "$_MSGPACK_DEST/.cache" + if [ ! -f "$_MSGPACK_DEST/.cache/$_MSGPACK_TARBALL" ]; then + echo "fetching $_MSGPACK_TARBALL..." + curl -sSfL -o "$_MSGPACK_DEST/.cache/$_MSGPACK_TARBALL" \ + "https://github.com/msgpack/msgpack-c/releases/download/cpp-$MSGPACK_VER/$_MSGPACK_TARBALL" + fi + ( + cd "$_MSGPACK_DEST" && + echo "$MSGPACK_SHA *.cache/$_MSGPACK_TARBALL" | "$_MSGPACK_SHACMD" -c && + tar -xf ".cache/$_MSGPACK_TARBALL" + ) + test -d "$MSGPACK_INCLUDE" || { + echo "missing $MSGPACK_INCLUDE -- msgpack-cxx fetch/untar failed" >&2 + exit 1 + } +fi diff --git a/rnmodules/react-native-kb/scripts/sync-native-kb.sh b/rnmodules/react-native-kb/scripts/sync-native-kb.sh new file mode 100755 index 000000000000..34ca17129627 --- /dev/null +++ b/rnmodules/react-native-kb/scripts/sync-native-kb.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# shared/node_modules/react-native-kb is a `file:` COPY, not a symlink, and +# expo autolinking points gradle/pods at it. Native edits under rnmodules/ +# are invisible to a build until they are copied across. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +SRC="$ROOT/rnmodules/react-native-kb" +DST="$ROOT/shared/node_modules/react-native-kb" +test -d "$DST" || { echo "missing $DST — run yarn in shared/ first" >&2; exit 1; } +for d in cpp ios android; do + rsync -a --delete "$SRC/$d/" "$DST/$d/" +done +echo "synced rnmodules/react-native-kb -> shared/node_modules/react-native-kb" diff --git a/rnmodules/react-native-kb/scripts/test-engine-reset-backoff.sh b/rnmodules/react-native-kb/scripts/test-engine-reset-backoff.sh new file mode 100755 index 000000000000..5b255d60ef58 --- /dev/null +++ b/rnmodules/react-native-kb/scripts/test-engine-reset-backoff.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Builds and runs the kb-engine-reset emit-backoff unit tests +# (rnmodules/react-native-kb/cpp/tests/engine-reset-backoff-test.cpp). +# +# engine-reset-backoff.h is header-only and depends on nothing at all -- no +# jsi/React, not even msgpack -- so unlike test-framing.sh this needs only a +# C++ compiler and no vendored headers. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +CPP_DIR="$ROOT/rnmodules/react-native-kb/cpp" + +# clang++ locally / on the mac builders, g++ on the Linux CI image. +CXX="${CXX:-}" +if [ -z "$CXX" ]; then + if command -v clang++ >/dev/null 2>&1; then CXX=clang++; else CXX=g++; fi +fi + +BIN="$(mktemp -d)/engine-reset-backoff-test" +trap 'rm -rf "$(dirname "$BIN")"' EXIT + +"$CXX" -std=c++20 -O1 -g -Wall -Wextra \ + "$CPP_DIR/tests/engine-reset-backoff-test.cpp" \ + -o "$BIN" + +"$BIN" diff --git a/rnmodules/react-native-kb/scripts/test-framing.sh b/rnmodules/react-native-kb/scripts/test-framing.sh new file mode 100755 index 000000000000..091b8817624b --- /dev/null +++ b/rnmodules/react-native-kb/scripts/test-framing.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Builds and runs the FrameParser unit tests +# (rnmodules/react-native-kb/cpp/tests/frame-parser-test.cpp). No JS runtime +# or JSI headers are needed -- FrameParser has no jsi/React dependency -- so +# this only needs a plain msgpack-cxx include path, the same one used for the +# react-native-kb.cpp syntax-only clang++ check. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +CPP_DIR="$ROOT/rnmodules/react-native-kb/cpp" +# Sets MSGPACK_INCLUDE, fetching the headers if yarn hasn't unpacked them. +# shellcheck source=./msgpack-include.sh +source "$(dirname "${BASH_SOURCE[0]}")/msgpack-include.sh" + +# clang++ locally / on the mac builders, g++ on the Linux CI image. +CXX="${CXX:-}" +if [ -z "$CXX" ]; then + if command -v clang++ >/dev/null 2>&1; then CXX=clang++; else CXX=g++; fi +fi + +BIN="$(mktemp -d)/frame-parser-test" +trap 'rm -rf "$(dirname "$BIN")"' EXIT + +"$CXX" -std=c++20 -O1 -g -DMSGPACK_NO_BOOST \ + -Wall -Wextra \ + -I "$MSGPACK_INCLUDE" \ + "$CPP_DIR/frame-parser.cpp" \ + "$CPP_DIR/tests/frame-parser-test.cpp" \ + -o "$BIN" + +"$BIN" diff --git a/rnmodules/react-native-kb/scripts/test-jsi-convert.sh b/rnmodules/react-native-kb/scripts/test-jsi-convert.sh new file mode 100755 index 000000000000..8905a3ed11e5 --- /dev/null +++ b/rnmodules/react-native-kb/scripts/test-jsi-convert.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Builds and runs the msgpack <-> JSI correctness tests +# (rnmodules/react-native-kb/cpp/tests/jsi-convert-test.cpp) against a real +# Hermes runtime. +# +# Hermes comes from the already-installed iOS pod, which ships a macOS slice, +# so nothing has to be built from source -- the same arrangement +# scripts/bench-jsi-convert.sh uses. Development-only; not part of the shipped +# module. +# +# Exits nonzero if any test case fails. +# +# Run it by hand on a mac before touching the conversion code: hermesvm.framework +# is a prebuilt Mach-O framework vendored by the hermes-engine CocoaPod, so this +# needs macOS + Xcode + `yarn ios:pod:install`. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +CPP_DIR="$ROOT/rnmodules/react-native-kb/cpp" +PODS="$ROOT/shared/ios/Pods" +HERMES="$PODS/hermes-engine/destroot" + +# Sets MSGPACK_INCLUDE, fetching the headers if yarn hasn't unpacked them. +# shellcheck source=./msgpack-include.sh +source "$(dirname "${BASH_SOURCE[0]}")/msgpack-include.sh" + +for p in "$MSGPACK_INCLUDE" "$HERMES/include" \ + "$PODS/Headers/Public/React-callinvoker"; do + test -d "$p" || { + echo "missing $p -- run yarn and yarn ios:pod:install first" >&2 + exit 1 + } +done + +FRAMEWORKS="$HERMES/Library/Frameworks/macosx" +test -d "$FRAMEWORKS/hermesvm.framework" || { + echo "missing $FRAMEWORKS/hermesvm.framework -- run yarn ios:pod:install" >&2 + exit 1 +} + +WORK="$(mktemp -d /tmp/kb-jsi-test.XXXXXX)" +trap 'rm -rf "$WORK"' EXIT + +# jsi-convert-test.cpp #includes react-native-kb.cpp (it needs at the +# anonymous-namespace packNumber), so that file must NOT also be compiled +# separately here or every symbol would be defined twice. +clang++ -std=c++20 -O1 -g -DMSGPACK_NO_BOOST \ + -Wall -Wextra -Wno-unused-function \ + -I "$MSGPACK_INCLUDE" \ + -I "$HERMES/include" \ + -I "$PODS/Headers/Public/React-callinvoker" \ + -I "$CPP_DIR" \ + -F "$FRAMEWORKS" -framework hermesvm \ + -Wl,-rpath,"$FRAMEWORKS" \ + "$CPP_DIR/frame-parser.cpp" \ + "$CPP_DIR/tests/jsi-convert-test.cpp" \ + -o "$WORK/jsi-convert-test" + +"$WORK/jsi-convert-test" diff --git a/shared/engine/index.platform.mobile.test.ts b/shared/engine/index.platform.mobile.test.ts new file mode 100644 index 000000000000..78d7f6fd8e48 --- /dev/null +++ b/shared/engine/index.platform.mobile.test.ts @@ -0,0 +1,157 @@ +/// + +import type {CreateClientType, IncomingRPCCallbackType, ConnectDisconnectCB} from './index.platform' +import {errors} from './rpc-transport' + +type IndexPlatformModule = { + createClient: ( + incomingRPCCallback: IncomingRPCCallbackType, + connectCallback: ConnectDisconnectCB, + disconnectCallback: ConnectDisconnectCB + ) => CreateClientType +} + +// isMobile is a bare global (see globals.d.ts / jest.setup.js), read once at +// module load time by index.platform.tsx. To exercise the isMobile branch +// (normally never taken in this desktop test env) each test here flips the +// global, resets the module registry so index.platform re-evaluates it, and +// mocks the native imports that branch touches. +const mockNativeModules = (onMetaEvent: (cb: (payload: string) => void) => void) => { + jest.doMock('react-native-kb', () => ({ + onMetaEvent, + notifyJSReady: () => {}, + })) +} + +const teardownMobileMocks = ( + originalIsMobile: boolean, + originalRpcOnGo: unknown, + originalRpcOnJs: unknown +) => { + jest.dontMock('react-native-kb') + jest.resetModules() + global.isMobile = originalIsMobile + global.rpcOnGo = originalRpcOnGo as typeof global.rpcOnGo + global.rpcOnJs = originalRpcOnJs as typeof global.rpcOnJs +} + +test('disconnectCallback throwing does not prevent connectCallback from running on kb-engine-reset', () => { + const originalIsMobile = global.isMobile + const originalRpcOnGo = global.rpcOnGo + const originalRpcOnJs = global.rpcOnJs + global.isMobile = true + + let capturedMetaCb: ((payload: string) => void) | undefined + mockNativeModules(cb => { + capturedMetaCb = cb + }) + jest.resetModules() + + try { + const {createClient} = require('./index.platform') as IndexPlatformModule + const connectCallback = jest.fn() + const disconnectCallback = jest.fn(() => { + throw new Error('disconnect handler blew up') + }) + const client = createClient(() => {}, connectCallback, disconnectCallback) + const resetSpy = jest.spyOn(client.transport, 'reset') + + if (!capturedMetaCb) { + throw new Error('kb-engine-reset handler was never registered') + } + capturedMetaCb('kb-engine-reset') + + // The reset must actually happen -- not just the two callbacks below -- + // or pre-reset in-flight invocations stay outstanding forever. + expect(resetSpy).toHaveBeenCalledTimes(1) + expect(disconnectCallback).toHaveBeenCalledTimes(1) + // The isolation fix: disconnectCallback throwing must not skip connectCallback, + // or the UI is stranded on the disconnect banner forever. + expect(connectCallback).toHaveBeenCalledTimes(1) + } finally { + teardownMobileMocks(originalIsMobile, originalRpcOnGo, originalRpcOnJs) + } +}) + +// Guard for a load-bearing asymmetry: on mobile, outstanding RPCs must be +// failed ONLY when Go drops the connection ('kb-engine-reset'). Engine.reset() +// early-returns on mobile precisely so an account switch does not fail them -- +// doing so EOFs login.login (proven on device). If a future refactor moves a +// reset() call into code shared with the account-switch path, the first half +// of this test starts failing instead of the bug shipping silently. +test('outstanding invocations survive everything except kb-engine-reset', () => { + const originalIsMobile = global.isMobile + const originalRpcOnGo = global.rpcOnGo + const originalRpcOnJs = global.rpcOnJs + global.isMobile = true + + let capturedMetaCb: ((payload: string) => void) | undefined + mockNativeModules(cb => { + capturedMetaCb = cb + }) + jest.resetModules() + + try { + const {createClient} = require('./index.platform') as IndexPlatformModule + global.rpcOnGo = () => true + const client = createClient( + () => {}, + () => {}, + () => {} + ) + + const cb = jest.fn() + client.invoke('keybase.1.login.login', [{}], cb) + expect(cb).not.toHaveBeenCalled() + + if (!capturedMetaCb) { + throw new Error('meta event handler was never registered') + } + + // Any other meta payload (and, by construction, everything else that runs + // on an account switch) must leave the invocation outstanding. + capturedMetaCb('kb-some-other-event') + expect(cb).not.toHaveBeenCalled() + + capturedMetaCb('kb-engine-reset') + expect(cb).toHaveBeenCalledTimes(1) + const [err] = cb.mock.calls[0] as [unknown, unknown] + expect((err as {code?: number}).code).toBe(errors.EOF) + } finally { + teardownMobileMocks(originalIsMobile, originalRpcOnGo, originalRpcOnJs) + } +}) + +test('NativeTransportMobile fails the invocation (not hang) when rpcOnGo reports failure', () => { + const originalIsMobile = global.isMobile + const originalRpcOnGo = global.rpcOnGo + const originalRpcOnJs = global.rpcOnJs + global.isMobile = true + + mockNativeModules(() => {}) + jest.resetModules() + + try { + const {createClient} = require('./index.platform') as IndexPlatformModule + const client = createClient( + () => {}, + () => {}, + () => {} + ) + global.rpcOnGo = () => false + + const cb = jest.fn() + client.invoke('keybase.1.test.hello', [{}], cb) + + expect(cb).toHaveBeenCalledTimes(1) + const [err] = cb.mock.calls[0] as [unknown, unknown] + // rpcOnGo is defined but returns false -- this must be the "native rpc + // write failed" throw site, not the "rpcOnGo send before rpcOnGo global" + // site that fires when rpcOnGo is undefined. The transport wraps the + // thrown Error into its code/desc shape; the message survives in desc. + expect((err as {code?: number; desc?: string}).code).toBe(errors.EOF) + expect((err as {code?: number; desc?: string}).desc).toBe('native rpc write failed') + } finally { + teardownMobileMocks(originalIsMobile, originalRpcOnGo, originalRpcOnJs) + } +}) diff --git a/shared/engine/index.platform.test.ts b/shared/engine/index.platform.test.ts new file mode 100644 index 000000000000..46708e9ee4c9 --- /dev/null +++ b/shared/engine/index.platform.test.ts @@ -0,0 +1,300 @@ +/// + +// jest.setup.js sets isMobile=false, isRenderer=true, and _fromPreload.functions +// with no engineSend -- exactly the "renderer up, preload never wired engineSend" +// case this test drives. +import {EventEmitter} from 'events' +import {createClient, resetClient, dispatchRpcBatch, makeDispatchOne} from './index.platform' +// Aliased: two tests below declare a local `errors` array for captured log +// messages. +import {errors as rpcErrors, type RPCMessage} from './rpc-transport' +import type {KB2} from '@/util/electron' + +// The non-renderer desktop transport opens a real unix socket via a lazy +// require('net') inside connectOnce(). Stand in for it so the transport's +// connect/write/close lifecycle can be driven deterministically. +class MockSocket extends EventEmitter { + written = new Array() + destroyed = false + write(b: Buffer) { + this.written.push(b) + return true + } + destroy() { + this.destroyed = true + } +} +const mockSockets = new Array() +jest.mock('net', () => ({ + connect: () => { + const socket = new MockSocket() + mockSockets.push(socket) + return socket + }, +})) + +const getPreload = () => globalThis._fromPreload as KB2 + +test('a missing engineSend fails the write instead of silently no-oping', () => { + const client = createClient( + () => {}, + () => {}, + () => {} + ) + + const ok = client.transport.send([1, 3, null, {}]) + + expect(ok).toBe(false) +}) + +test('ProxyNativeTransport.reset fails outstanding invocations so pre-switch callbacks cannot fire against post-switch state', () => { + const preload = getPreload() + const sent = new Array() + preload.functions.engineSend = m => { + sent.push(m) + } + + try { + const client = createClient( + () => {}, + () => {}, + () => {} + ) + const cb = jest.fn() + client.invoke('keybase.1.test.hello', [{}], cb) + + expect(sent).toHaveLength(1) + expect(cb).not.toHaveBeenCalled() + + // Desktop account switch: without this the pre-switch callback stays + // outstanding forever (or fires later against post-switch state). + client.transport.reset() + + expect(cb).toHaveBeenCalledTimes(1) + const [err] = cb.mock.calls[0] as [unknown, unknown] + expect((err as {code?: number}).code).toBe(rpcErrors.EOF) + + // A reply for the pre-switch seqid arriving after the reset must be + // dropped, not delivered to the already-failed callback. + const [, seqid] = sent[0] as [number, number, string, [object]] + client.transport.dispatchDecodedMessage([1, seqid, null, {ok: 'late'}]) + expect(cb).toHaveBeenCalledTimes(1) + } finally { + delete preload.functions.engineSend + } +}) + +test('resetClient outside the renderer closes the old transport and builds a fresh one', () => { + const preload = getPreload() + const originalIsRenderer = preload.constants.isRenderer + preload.constants.isRenderer = false + mockSockets.length = 0 + + try { + const client = createClient( + () => {}, + () => {}, + () => {} + ) + expect(mockSockets).toHaveLength(1) + const socket = mockSockets[0]! + socket.emit('connect') + + const cb = jest.fn() + client.invoke('keybase.1.test.hello', [{}], cb) + expect(socket.written).toHaveLength(1) + expect(cb).not.toHaveBeenCalled() + + const next = resetClient( + client, + () => {}, + () => {}, + () => {} + ) + + // close() must fail what was in flight -- same invariant the renderer + // branch gets from transport.reset(). + expect(cb).toHaveBeenCalledTimes(1) + const [err] = cb.mock.calls[0] as [unknown, unknown] + expect((err as {code?: number}).code).toBe(rpcErrors.EOF) + expect(socket.destroyed).toBe(true) + + // A brand new transport on a brand new socket, not the closed one. + expect(next.transport).not.toBe(client.transport) + expect(mockSockets).toHaveLength(2) + expect(mockSockets[1]).not.toBe(socket) + + next.transport.close() + } finally { + preload.constants.isRenderer = originalIsRenderer + } +}) + +// The reachable path for a non-empty pending queue at close() time: the +// non-renderer transport is created with needsConnect, so it starts +// disconnected and every invoke made before the socket connects is queued. +// resetClient then closes it. Dropping that queue silently would hang each +// caller forever. +test('resetClient outside the renderer fails invokes queued on a transport that never connected', () => { + const preload = getPreload() + const originalIsRenderer = preload.constants.isRenderer + preload.constants.isRenderer = false + mockSockets.length = 0 + + try { + const client = createClient( + () => {}, + () => {}, + () => {} + ) + // Socket never emits 'connect', so the transport stays disconnected. + expect(mockSockets).toHaveLength(1) + const socket = mockSockets[0]! + + const cbA = jest.fn() + const cbB = jest.fn() + client.invoke('keybase.1.test.a', [{}], cbA) + client.invoke('keybase.1.test.b', [{}], cbB) + expect(socket.written).toHaveLength(0) + expect(cbA).not.toHaveBeenCalled() + + const next = resetClient( + client, + () => {}, + () => {}, + () => {} + ) + + expect(cbA).toHaveBeenCalledTimes(1) + expect(cbB).toHaveBeenCalledTimes(1) + for (const cb of [cbA, cbB]) { + const [err] = cb.mock.calls[0] as [unknown, unknown] + expect((err as {code?: number}).code).toBe(rpcErrors.EOF) + } + + // A late connect on the abandoned socket must not flush the queue it + // already settled. + socket.emit('connect') + expect(socket.written).toHaveLength(0) + expect(cbA).toHaveBeenCalledTimes(1) + expect(cbB).toHaveBeenCalledTimes(1) + + next.transport.close() + } finally { + preload.constants.isRenderer = originalIsRenderer + } +}) + +// dispatchRpcBatch backs the mobile global.rpcOnJs batch dispatcher (only +// wired up inside createClient's isMobile branch, which this desktop test +// env never takes). Exercise it directly instead. +describe('dispatchRpcBatch', () => { + test('a normal multi-message array dispatches every element in order', () => { + const dispatched: Array = [] + dispatchRpcBatch(['a', 'b', 'c'], 3, obj => dispatched.push(obj), () => {}) + expect(dispatched).toEqual(['a', 'b', 'c']) + }) + + test('a single message (count === 1) dispatches directly, not as a wrapper', () => { + const dispatched: Array = [] + dispatchRpcBatch({solo: true}, 1, obj => dispatched.push(obj), () => {}) + expect(dispatched).toEqual([{solo: true}]) + }) + + test('count > 1 with a non-array logs an error and dispatches nothing', () => { + const dispatched: Array = [] + const errors: Array = [] + dispatchRpcBatch({not: 'an array'}, 2, obj => dispatched.push(obj), msg => errors.push(msg)) + expect(dispatched).toEqual([]) + expect(errors).toEqual(['rpcOnJs: count 2 but payload is not an array']) + }) + + test("one message's dispatch throwing does not stop the remaining messages", () => { + const dispatched: Array = [] + // Exercises production's own dispatchOne (via makeDispatchOne) rather + // than a re-implementation, so this test still fails if production ever + // loses its per-message try/catch. + const fakeClient = { + transport: { + dispatchDecodedMessage: (obj: unknown) => { + if (obj === 'bad') { + throw new Error('dispatch threw') + } + dispatched.push(obj) + }, + }, + } + const dispatchOne = makeDispatchOne(fakeClient) + const errors: Array = [] + dispatchRpcBatch(['a', 'bad', 'b'], 3, dispatchOne, msg => errors.push(msg)) + expect(dispatched).toEqual(['a', 'b']) + expect(errors).toEqual([]) + }) + + // The tests above route through production's makeDispatchOne, which swallows + // per-message throws -- so they never reach dispatchRpcBatch's own outer + // catch. These pass a RAW throwing dispatchOne instead. dispatchRpcBatch is + // called from native: a throw escaping it unwinds through JSI, which is + // undefined behavior rather than a catchable error. + describe('the outer batch guard', () => { + test('swallows and logs a raw dispatchOne throw on the multi-message path', () => { + const logged = new Array<[string, unknown]>() + expect(() => + dispatchRpcBatch( + ['a', 'b'], + 2, + () => { + throw new Error('raw dispatch threw') + }, + (msg, e) => logged.push([msg, e]) + ) + ).not.toThrow() + + expect(logged).toHaveLength(1) + expect(logged[0]?.[0]).toBe('rpcOnJs: batch guard threw') + expect((logged[0]?.[1] as Error).message).toBe('raw dispatch threw') + }) + + test('swallows and logs a raw dispatchOne throw on the single-message path', () => { + const logged = new Array<[string, unknown]>() + expect(() => + dispatchRpcBatch( + {solo: true}, + 1, + () => { + throw new Error('raw solo dispatch threw') + }, + (msg, e) => logged.push([msg, e]) + ) + ).not.toThrow() + + expect(logged).toHaveLength(1) + expect(logged[0]?.[0]).toBe('rpcOnJs: batch guard threw') + expect((logged[0]?.[1] as Error).message).toBe('raw solo dispatch threw') + }) + + test('swallows and logs a throw raised by iterating the batch itself', () => { + // Array.isArray() is true for a Proxy wrapping an array, so this gets + // past the count/array check and blows up inside the for..of instead -- + // outside any per-message try/catch. + const hostile = new Proxy(['a', 'b'], { + get(target, prop, receiver) { + if (prop === Symbol.iterator) { + throw new Error('iteration blew up') + } + return Reflect.get(target, prop, receiver) as unknown + }, + }) + const dispatched: Array = [] + const logged = new Array<[string, unknown]>() + + expect(() => + dispatchRpcBatch(hostile, 2, obj => dispatched.push(obj), (msg, e) => logged.push([msg, e])) + ).not.toThrow() + + expect(dispatched).toEqual([]) + expect(logged).toHaveLength(1) + expect(logged[0]?.[0]).toBe('rpcOnJs: batch guard threw') + }) + }) +}) diff --git a/shared/engine/index.platform.tsx b/shared/engine/index.platform.tsx index 1b8cee288d6a..f0cf37f0ba68 100644 --- a/shared/engine/index.platform.tsx +++ b/shared/engine/index.platform.tsx @@ -1,6 +1,6 @@ import logger from '@/logger' import {TransportShared, LocalTransport, sharedCreateClient, rpcLog} from './transport-shared' -import {makeEOFError, type RPCMessage} from './rpc-transport' +import type {RPCMessage} from './rpc-transport' import type {InvokeType, PayloadType, ConnectDisconnectCB, IncomingRPCCallbackType} from '@/engine/rpc-transport' export type {PayloadType, ConnectDisconnectCB, IncomingRPCCallbackType, InvokeType} @@ -143,25 +143,88 @@ class NativeTransport extends TransportShared { class ProxyNativeTransport extends LocalTransport { protected writeMessage(message: RPCMessage) { const {engineSend} = KB2.functions - engineSend?.(message) + if (!engineSend) { + // Silently no-oping here reports success upstream (send() returns true) + // while the invocation is never delivered, leaving it outstanding + // forever. Throwing lets the transport fail it, same as mobile. + throw new Error('engineSend missing') + } + engineSend(message) } // On account-switch reset fail outstanding invocations so pre-switch RPC // callbacks can't fire later against post-switch state override reset() { - this.failOutstanding(makeEOFError(), {}) + this.failAllOutstanding() } } // Mobile transport — only instantiated when isMobile class NativeTransportMobile extends LocalTransport { protected writeMessage(message: RPCMessage) { - try { - if (!global.rpcOnGo) { - logger.error('>>>> rpcOnGo send before rpcOnGo global?') + if (!global.rpcOnGo) { + throw new Error('rpcOnGo send before rpcOnGo global') + } + // Throwing rather than swallowing is load-bearing: the transport catches + // it and fails that invocation, instead of leaving the caller waiting on + // a reply that can never arrive. rpcOnGo returns false when the native + // write to Go failed. + if (!global.rpcOnGo(message)) { + throw new Error('native rpc write failed') + } + } + // Only reachable from the 'kb-engine-reset' meta event below: Go dropped + // the loopback connection (e.g. a stream desync detected natively), so + // nothing will answer the in-flight RPCs and hanging every caller is the + // alternative. Engine.reset() early-returns on mobile, so an account switch + // does NOT land here -- and must not: failing outstanding RPCs on a switch + // EOFs login.login, proven on device. Keep any new call site inside the + // meta-event handler, not in code shared with the account-switch path. + override reset() { + this.failAllOutstanding() + } +} + +// Expands a native rpcOnJs batch into individual dispatchOne calls. Exported +// so the mobile batch path (otherwise only reachable through the +// isMobile-gated global.rpcOnJs assignment inside createClient) can be +// exercised directly in tests. +export const dispatchRpcBatch = ( + objs: unknown, + count: number, + dispatchOne: (obj: unknown) => void, + logError: (msg: string, e?: unknown) => void +) => { + // Outer guard: this is called from native, so throwing here would abort the + // whole batch delivery and unwind into native code. + try { + if (count > 1) { + if (!Array.isArray(objs)) { + // Native always sends an array when it batches, so this means the + // two sides disagree -- and count-1 messages would vanish silently. + logError(`rpcOnJs: count ${count} but payload is not an array`) + return + } + for (const obj of objs) { + dispatchOne(obj) } - global.rpcOnGo?.(message) + } else { + dispatchOne(objs) + } + } catch (e) { + logError('rpcOnJs: batch guard threw', e) + } +} + +// Per-message try/catch: one bad message must not drop the rest of the batch +// the native side handed over. Exported (alongside dispatchRpcBatch) so the +// mobile-only wiring inside createClient's isMobile branch can be exercised +// directly in tests without a mobile jest environment. +export const makeDispatchOne = (client: {transport: {dispatchDecodedMessage: (obj: unknown) => void}}) => { + return (obj: unknown) => { + try { + client.transport.dispatchDecodedMessage(obj) } catch (e) { - logger.error('>>>> rpcOnGo JS thrown!', e) + logger.error('rpcOnJs: dispatch threw', e) } } } @@ -176,26 +239,36 @@ function createClient( new NativeTransportMobile(incomingRPCCallback, connectCallback, disconnectCallback) ) + const dispatchOne = makeDispatchOne(client) + global.rpcOnJs = (objs: unknown, count: number) => { - try { - if (count > 1) { - const arr = objs as Array - for (const obj of arr) { - client.transport.dispatchDecodedMessage(obj) - } - } else { - client.transport.dispatchDecodedMessage(objs) - } - } catch (e) { - logger.error('>>>> rpcOnJs JS thrown!', e) - } + dispatchRpcBatch(objs, count, dispatchOne, (msg, e) => logger.error(msg, e)) } onMetaEvent((payload: string) => { try { switch (payload) { case 'kb-engine-reset': - connectCallback() + // Go dropped the loopback connection; anything in flight is dead. + // Report the disconnect before the reconnect so the engine cancels + // its sessions and the UI shows the reconnect state -- the desktop + // socket path does the same pair. disconnectCallback and + // connectCallback are isolated in their own try/catch: a throw + // from a session cancel handler inside disconnectCallback must + // not strand the UI on the disconnect banner by skipping + // connectCallback (which synchronously clears the daemon error + // via startHandshake()). + client.transport.reset() + try { + disconnectCallback() + } catch (e) { + logger.error('>>>> meta engine event: disconnectCallback threw', e) + } + try { + connectCallback() + } catch (e) { + logger.error('>>>> meta engine event: connectCallback threw', e) + } } } catch (e) { logger.error('>>>> meta engine event JS thrown!', e) @@ -225,7 +298,7 @@ function createClient( try { client.transport.packetizeData(data as Uint8Array) } catch (e) { - logger.error('>>>> rpcOnJs JS thrown!', e) + logger.error('>>>> engineIncoming IPC JS thrown!', e) } }) diff --git a/shared/engine/kb-module-lifecycle.test.ts b/shared/engine/kb-module-lifecycle.test.ts new file mode 100644 index 000000000000..5874334e4ecb --- /dev/null +++ b/shared/engine/kb-module-lifecycle.test.ts @@ -0,0 +1,100 @@ +// Static guard for the Android "back button wedge" (HOTPOT-rpc-fixes, +// fixed in 15647bb4d4). On Android, onHostDestroy fires when the LAST +// ACTIVITY is destroyed while the ReactInstance/TurboModules survive +// (MainApplication holds the ReactHost in an application-scoped `by lazy`). +// nativeInvalidate() nulls the C++ g_bridge, which only the bindings +// installer repopulates -- and a surviving ReactInstance never re-runs +// that installer. If nativeInvalidate() is reachable from destroy()/ +// onHostDestroy() instead of from invalidate() (real TurboModule teardown, +// e.g. reload), every inbound RPC is silently dropped for the life of the +// process after pressing back to home and reopening the app. This was +// shipped past two per-task reviews; only a whole-branch review caught it. +// This test parses actual function bodies (brace-matched, not a whole-file +// substring search) so it survives reformatting but still catches the call +// moving to the wrong function. +import * as fs from 'fs' +import * as path from 'path' + +const kbModulePath = path.join( + __dirname, + '..', + '..', + 'rnmodules', + 'react-native-kb', + 'android', + 'src', + 'main', + 'java', + 'com', + 'reactnativekb', + 'KbModule.kt' +) + +// Extracts the body of a Kotlin function whose signature matches `namePattern`, +// via brace-depth counting from the first `{` after the signature -- not a +// line-based or whole-file match, so it scopes strictly to that one function +// even if other functions elsewhere also mention nativeInvalidate(). +function extractFunctionBody(source: string, namePattern: RegExp): string { + const sigMatch = namePattern.exec(source) + if (!sigMatch) { + throw new Error(`KbModule.kt: could not find a function matching ${namePattern}`) + } + const openBraceIdx = source.indexOf('{', sigMatch.index) + if (openBraceIdx === -1) { + throw new Error(`KbModule.kt: found signature for ${namePattern} but no function body`) + } + let depth = 0 + for (let i = openBraceIdx; i < source.length; i++) { + if (source[i] === '{') depth++ + else if (source[i] === '}') { + depth-- + if (depth === 0) return source.slice(openBraceIdx + 1, i) + } + } + throw new Error(`KbModule.kt: unterminated function body for ${namePattern}`) +} + +// Matches a real call `nativeInvalidate()`, not the `external fun +// nativeInvalidate()` declaration (which has no body / no call parens +// preceded by whitespace-only invocation context) -- both the declaration +// and a call end in the same text `nativeInvalidate()`, but only the +// declaration is preceded by `fun`. Since we already scope to a specific +// function body (which never contains the `external fun` declaration line), +// a plain substring check is safe here. +const CALLS_NATIVE_INVALIDATE = /\bnativeInvalidate\s*\(\s*\)/ + +describe('KbModule Android lifecycle wiring (back-button wedge guard)', () => { + const source = fs.readFileSync(kbModulePath, 'utf8') + + it('calls nativeInvalidate() from invalidate() (real TurboModule teardown)', () => { + const body = extractFunctionBody(source, /override\s+fun\s+invalidate\s*\(\s*\)/) + expect(CALLS_NATIVE_INVALIDATE.test(body)).toBe(true) + }) + + it('does NOT call nativeInvalidate() from destroy() (Activity death, not instance teardown)', () => { + const body = extractFunctionBody(source, /(? { + const body = extractFunctionBody(source, /override\s+fun\s+onHostDestroy\s*\(\s*\)/) + if (CALLS_NATIVE_INVALIDATE.test(body)) { + throw new Error( + 'onHostDestroy() calls nativeInvalidate() directly. onHostDestroy fires on Activity ' + + 'death, not ReactInstance teardown -- see the destroy() test above for the wedge this ' + + 'causes. nativeInvalidate() belongs only in invalidate().' + ) + } + }) +}) diff --git a/shared/engine/rpc-transport.test.ts b/shared/engine/rpc-transport.test.ts index c12fef0bf0d2..5bc757142671 100644 --- a/shared/engine/rpc-transport.test.ts +++ b/shared/engine/rpc-transport.test.ts @@ -1,5 +1,6 @@ /// +import logger from '@/logger' import { RPCTransport, encodeFrame, @@ -10,21 +11,38 @@ import { class TestTransport extends RPCTransport { private _connected = true + private _writeError: Error | undefined sent = new Array() + packetizeErrors = new Array() - constructor(p?: {connected?: boolean; incomingRPCCallback?: IncomingRPCCallbackType}) { + constructor(p?: {connected?: boolean; incomingRPCCallback?: IncomingRPCCallbackType; writeError?: Error}) { super({incomingRPCCallback: p?.incomingRPCCallback}) this._connected = p?.connected ?? true + this._writeError = p?.writeError } protected override isConnected() { return this._connected } + // Records rather than replaces: the base still console.error()s, so tests + // that only assert on that keep working. + protected override onPacketizeError(err: unknown) { + this.packetizeErrors.push(err) + super.onPacketizeError(err) + } + protected writeMessage(message: RPCMessage) { + if (this._writeError !== undefined) { + throw this._writeError + } this.sent.push(message) } + setWriteError(err: Error | undefined) { + this._writeError = err + } + setConnected(connected: boolean) { this._connected = connected } @@ -117,6 +135,532 @@ test('incoming invoke without handler returns unknown method error', () => { ]) }) +test('invoke fails the caller when the native write throws', () => { + const writeError = new Error('native write failed') + const transport = new TestTransport({writeError}) + const cb = jest.fn() + + transport.invoke('keybase.1.test.hello', [{}], cb) + + // The raw exception is wrapped into the transport error shape (code/desc) + // so convertToError produces an RPCError; the message survives in desc. + expect(cb).toHaveBeenCalledWith( + expect.objectContaining({code: errors.EOF, desc: writeError.message}), + {} + ) +}) + +test('a failed write leaves no outstanding invocation', () => { + const transport = new TestTransport({writeError: new Error('native write failed')}) + const cb = jest.fn() + + transport.invoke('keybase.1.test.hello', [{}], cb) + expect(cb).toHaveBeenCalledTimes(1) + + // If the seqid were still outstanding, this would fail the same callback + // a second time with EOF. + transport.failAllOutstanding() + expect(cb).toHaveBeenCalledTimes(1) +}) + +test('a failed write does not consume the response for a later invoke', () => { + const transport = new TestTransport({writeError: new Error('native write failed')}) + const failed = jest.fn() + transport.invoke('keybase.1.test.hello', [{}], failed) + + transport.setWriteError(undefined) + const ok = jest.fn() + transport.invoke('keybase.1.test.hello', [{}], ok) + + const [, seqid] = transport.sent[0] as [number, number, string, [object]] + transport.dispatchDecodedMessage([1, seqid, null, {ok: true}]) + + expect(ok).toHaveBeenCalledWith(null, {ok: true}) + expect(failed).toHaveBeenCalledTimes(1) +}) + +test('send reports failure when the native write throws', () => { + const transport = new TestTransport({writeError: new Error('native write failed')}) + + expect(transport.send([1, 3, null, {ok: true}])).toBe(false) + + transport.setWriteError(undefined) + expect(transport.send([1, 3, null, {ok: true}])).toBe(true) + expect(transport.sent).toEqual([[1, 3, null, {ok: true}]]) +}) + +test('failAllOutstanding fails every outstanding invocation exactly once', () => { + const transport = new TestTransport() + const calls: Array = [] + transport.invoke('keybase.1.test.a', [{}], err => calls.push(err)) + transport.invoke('keybase.1.test.b', [{}], err => calls.push(err)) + + transport.failAllOutstanding() + + expect(calls).toHaveLength(2) + expect(calls.every(e => (e as {code?: number} | undefined)?.code === errors.EOF)).toBe(true) + + // A second call must not re-fail anything already failed. + transport.failAllOutstanding() + expect(calls).toHaveLength(2) +}) + +test('a callback that re-enters with a new invoke during failAllOutstanding is not itself failed', () => { + const transport = new TestTransport() + const calls: Array = [] + const reentrant = jest.fn() + + transport.invoke('keybase.1.test.a', [{}], err => { + calls.push(err) + transport.invoke('keybase.1.test.c', [{}], reentrant) + }) + transport.invoke('keybase.1.test.b', [{}], err => calls.push(err)) + + transport.failAllOutstanding() + + expect(calls).toHaveLength(2) + expect(reentrant).not.toHaveBeenCalled() +}) + +test('a throwing callback does not stop the remaining outstanding invocations from being failed', () => { + const transport = new TestTransport() + const calls: Array = [] + + transport.invoke('keybase.1.test.a', [{}], () => { + calls.push(1) + throw new Error('boom') + }) + transport.invoke('keybase.1.test.b', [{}], () => calls.push(2)) + transport.invoke('keybase.1.test.c', [{}], () => calls.push(3)) + + expect(() => transport.failAllOutstanding()).not.toThrow() + + expect(calls).toEqual([1, 2, 3]) +}) + +test('a reset cycle fails outstanding invocations once and a post-reset invocation gets a fresh, non-colliding seqid that dispatches correctly', () => { + const transport = new TestTransport() + const preResetCb = jest.fn() + transport.invoke('keybase.1.test.old', [{}], preResetCb) + + transport.failAllOutstanding() + expect(preResetCb).toHaveBeenCalledTimes(1) + + const preResetSeqids = new Set( + transport.sent.map(m => (m as [number, number, string, [object]])[1]) + ) + + const postResetCb = jest.fn() + transport.invoke('keybase.1.test.new', [{}], postResetCb) + const [, postResetSeqid] = transport.sent[1] as [number, number, string, [object]] + + // The actual invariant: the post-reset seqid must not alias ANY seqid used + // before the reset. "greater than the last one" would also pass if the + // counter were reset and then advanced past a single stale value. + expect(preResetSeqids.has(postResetSeqid)).toBe(false) + + transport.dispatchDecodedMessage([1, postResetSeqid, null, {ok: 'post-reset'}]) + expect(postResetCb).toHaveBeenCalledWith(null, {ok: 'post-reset'}) +}) + +test('a response for a pre-reset seqid arriving after reset is ignored', () => { + const transport = new TestTransport() + const preResetCb = jest.fn() + transport.invoke('keybase.1.test.old', [{}], preResetCb) + const [, preResetSeqid] = transport.sent[0] as [number, number, string, [object]] + + transport.failAllOutstanding() + expect(preResetCb).toHaveBeenCalledTimes(1) + + // A stale response for the pre-reset seqid shows up late -- the seqid is + // gone from the invocations map, so this must be a silent no-op rather + // than re-firing the already-failed callback. + transport.dispatchDecodedMessage([1, preResetSeqid, null, {ok: 'stale'}]) + expect(preResetCb).toHaveBeenCalledTimes(1) +}) + +test('seqids keep advancing after outstanding invocations are failed, so a late reply cannot alias', () => { + const transport = new TestTransport() + // Several pre-reset invocations, so "not the last one" is genuinely weaker + // than "not any of them" and only the latter is asserted. + transport.invoke('keybase.1.test.a', [{}], () => {}) + transport.invoke('keybase.1.test.b', [{}], () => {}) + transport.invoke('keybase.1.test.c', [{}], () => {}) + const preResetSeqids = new Set(transport.sent.map(m => (m as [number, number, string, [object]])[1])) + expect(preResetSeqids.size).toBe(3) + + transport.failAllOutstanding() + + transport.invoke('keybase.1.test.d', [{}], () => {}) + transport.invoke('keybase.1.test.e', [{}], () => {}) + const postResetSeqids = transport.sent + .slice(3) + .map(m => (m as [number, number, string, [object]])[1]) + + // No post-reset seqid may be a member of the pre-reset set -- a late reply + // for a failed invocation would otherwise be delivered to a live callback. + for (const seqid of postResetSeqids) { + expect(preResetSeqids.has(seqid)).toBe(false) + } + expect(new Set(postResetSeqids).size).toBe(postResetSeqids.length) +}) + +test('a throwing incoming handler does not desync the packetizer', () => { + const delivered: Array = [] + let shouldThrow = true + const transport = new TestTransport({ + incomingRPCCallback: incoming => { + if (shouldThrow) { + shouldThrow = false + throw new Error('app handler blew up') + } + delivered.push(incoming) + }, + }) + + // Two well-formed frames arriving in a single chunk. The first frame's + // handler throws; the second frame must still be parsed and delivered. + const first = encodeFrame([2, 'keybase.1.test.first', [{}]]) + const second = encodeFrame([2, 'keybase.1.test.second', [{}]]) + const both = new Uint8Array(first.length + second.length) + both.set(first, 0) + both.set(second, first.length) + + transport.packetizeData(both) + + expect(delivered).toHaveLength(1) +}) + +test('a throwing invoke handler still answers the caller with an error', () => { + const transport = new TestTransport({ + incomingRPCCallback: () => { + throw new Error('handler blew up') + }, + }) + + transport.dispatchDecodedMessage([0, 7, 'keybase.1.test.hello', [{}]]) + + // Something must go back for seqid 7 -- otherwise the service waits forever. + expect(transport.sent).toEqual([ + [1, 7, {code: errors.UNKNOWN_METHOD, desc: 'No method available', name: 'UNKNOWN_METHOD'}, null], + ]) +}) + +test('a response cannot be settled twice: error() after result() is a no-op', () => { + let payload: Parameters[0] | undefined + const transport = new TestTransport({ + incomingRPCCallback: incoming => { + payload = incoming + }, + }) + + transport.dispatchDecodedMessage([0, 9, 'keybase.1.test.hello', [{}]]) + payload?.response?.result?.({ok: true}) + payload?.response?.error?.({code: errors.UNKNOWN_METHOD, desc: 'No method available', name: 'UNKNOWN_METHOD'}) + + expect(transport.sent).toEqual([[1, 9, null, {ok: true}]]) +}) + +test('an invoke handler that settles result() and then throws does not also send an error', () => { + const transport = new TestTransport({ + incomingRPCCallback: incoming => { + incoming.response?.result?.({ok: true}) + throw new Error('reducer blew up after acking') + }, + }) + + transport.dispatchDecodedMessage([0, 13, 'keybase.1.test.hello', [{}]]) + + expect(transport.sent).toEqual([[1, 13, null, {ok: true}]]) +}) + +test('auto-result then a throwing handler sends exactly one message and logs no double-settle error', () => { + const transport = new TestTransport({ + incomingRPCCallback: incoming => { + // Mirrors index.tsx: auto-result() for non-custom-response calls, + // then the next line (dispatching the action) throws. + incoming.response?.result?.({ok: true}) + throw new Error('reducer blew up after acking') + }, + }) + const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => {}) + + transport.dispatchDecodedMessage([0, 21, 'keybase.1.test.hello', [{}]]) + + expect(transport.sent).toEqual([[1, 21, null, {ok: true}]]) + // The handler-threw log is expected; a second, double-settle log is not -- + // this is an expected sequence (auto-result then a throwing dispatch), not + // a real control-flow bug. + expect(errorSpy).toHaveBeenCalledTimes(1) + expect(errorSpy).toHaveBeenCalledWith('incoming invoke handler threw', expect.any(Error)) + expect(errorSpy.mock.calls.some(args => args.some(a => typeof a === 'string' && a.includes('twice')))).toBe( + false + ) + errorSpy.mockRestore() +}) + +test('a response cannot be settled twice: result() after error() is a no-op (first settle wins)', () => { + let payload: Parameters[0] | undefined + const transport = new TestTransport({ + incomingRPCCallback: incoming => { + payload = incoming + }, + }) + + transport.dispatchDecodedMessage([0, 10, 'keybase.1.test.hello', [{}]]) + payload?.response?.error?.({code: errors.UNKNOWN_METHOD, desc: 'No method available', name: 'UNKNOWN_METHOD'}) + payload?.response?.result?.({ok: true}) + + expect(transport.sent).toEqual([ + [1, 10, {code: errors.UNKNOWN_METHOD, desc: 'No method available', name: 'UNKNOWN_METHOD'}, null], + ]) +}) + +test('a genuine double-settle -- result() then error() called directly -- still logs', () => { + let payload: Parameters[0] | undefined + const transport = new TestTransport({ + incomingRPCCallback: incoming => { + payload = incoming + }, + }) + const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => {}) + + transport.dispatchDecodedMessage([0, 22, 'keybase.1.test.hello', [{}]]) + payload?.response?.result?.({ok: true}) + payload?.response?.error?.({code: errors.UNKNOWN_METHOD, desc: 'No method available', name: 'UNKNOWN_METHOD'}) + + expect(transport.sent).toEqual([[1, 22, null, {ok: true}]]) + expect( + errorSpy.mock.calls.some(args => args.some(a => typeof a === 'string' && a.includes('twice'))) + ).toBe(true) + errorSpy.mockRestore() +}) + +test('a malformed frame resets the packetizer, and a subsequent well-formed frame still dispatches', () => { + const delivered: Array = [] + const transport = new TestTransport({ + incomingRPCCallback: incoming => { + delivered.push(incoming) + }, + }) + const consoleSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) + + // 0x80 (msgpack fixmap) is not a fixint (<0x80) nor one of the supported + // uint8/16/32 length-prefix bytes (0xcc/0xcd/0xce): frameHeaderLength + // returns 0 for it, so packetizeData throws "Bad frame header received". + transport.packetizeData(Uint8Array.of(0x80)) + expect(delivered).toHaveLength(0) + + const good = encodeFrame([2, 'keybase.1.test.recovered', [{}]]) + transport.packetizeData(good) + + expect(delivered).toHaveLength(1) + expect((delivered[0] as {method: string}).method).toBe('keybase.1.test.recovered') + consoleSpy.mockRestore() +}) + +test('a frame split at every possible byte boundary across two packetizeData calls still dispatches exactly once', () => { + // Seqid 1: every TestTransport starts its sequence at 1, the same value + // used directly elsewhere in this file. + const frame = encodeFrame([1, 1, null, {ok: 'sweep', pad: 'x'.repeat(40)}]) + + for (let splitAt = 1; splitAt < frame.length; splitAt++) { + const transport = new TestTransport() + const cb = jest.fn() + transport.invoke('keybase.1.test.hello', [{}], cb) + + transport.packetizeData(frame.slice(0, splitAt)) + expect(cb).not.toHaveBeenCalled() + + transport.packetizeData(frame.slice(splitAt)) + expect(cb).toHaveBeenCalledTimes(1) + expect(cb).toHaveBeenCalledWith(null, {ok: 'sweep', pad: 'x'.repeat(40)}) + } +}) + +test('failAllOutstanding clears buffered frame bytes', () => { + const delivered: Array = [] + const transport = new TestTransport({ + incomingRPCCallback: incoming => { + delivered.push(incoming) + }, + }) + + // Feed half a frame -- the packetizer buffers a partial header+payload. + const first = encodeFrame([2, 'keybase.1.test.first', [{}]]) + transport.packetizeData(first.slice(0, first.length - 2)) + transport.failAllOutstanding() + + // A complete, unrelated frame arrives next. If the stale half-frame was not + // dropped, it prefixes these bytes and corrupts the decode. + const second = encodeFrame([2, 'keybase.1.test.second', [{}]]) + transport.packetizeData(second) + + expect(delivered).toHaveLength(1) + expect((delivered[0] as {method: string}).method).toBe('keybase.1.test.second') +}) + +test('a failed response write is reported, not swallowed', () => { + let payload: Parameters[0] | undefined + const transport = new TestTransport({ + incomingRPCCallback: incoming => { + payload = incoming + }, + writeError: new Error('native write failed'), + }) + const errorSpy = jest.spyOn(logger, 'error').mockImplementation(() => {}) + + transport.dispatchDecodedMessage([0, 11, 'keybase.1.test.hello', [{}]]) + payload?.response?.result?.({ok: true}) + + // send() itself already logs the generic write failure; makeResponse must + // add a second, seqid-specific log or the service-side hang is invisible. + expect(errorSpy).toHaveBeenCalledTimes(2) + + // The response is marked settled before the write is attempted, so a + // failed write must not leave it settleable again -- there is no + // connection left to retry on. Fixing the write error and calling + // result() again must not silently open a retry path: it should trip + // the double-settle guard and send nothing. + transport.setWriteError(undefined) + payload?.response?.result?.({ok: 'retry'}) + expect(transport.sent).toEqual([]) + errorSpy.mockRestore() +}) + +// maxFrameSize (64MB) is a JS-side sanity limit: a corrupt/desynced length +// prefix would otherwise make the packetizer sit on a multi-hundred-MB +// allocation intent and buffer forever, never dispatching again. +const maxFrameSize = 64 * 1024 * 1024 + +// 0xce is the msgpack uint32 length prefix; the four bytes after it are the +// big-endian payload length. Only this prefix can express a length above +// maxFrameSize (0xcd tops out at 65535). +const oversizedHeader = (payloadLen: number) => + Uint8Array.of( + 0xce, + (payloadLen >>> 24) & 0xff, + (payloadLen >>> 16) & 0xff, + (payloadLen >>> 8) & 0xff, + payloadLen & 0xff + ) + +test('a frame header declaring more than 64MB is rejected, resets the packetizer, and the next valid frame still dispatches', () => { + const delivered: Array = [] + const transport = new TestTransport({ + incomingRPCCallback: incoming => { + delivered.push(incoming) + }, + }) + const consoleSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) + + // Header only -- no payload bytes follow. The size check must fire before + // anything tries to buffer 128MB. + transport.packetizeData(oversizedHeader(128 * 1024 * 1024)) + + expect(transport.packetizeErrors).toHaveLength(1) + expect((transport.packetizeErrors[0] as Error).message).toBe(`Frame too large: ${128 * 1024 * 1024} bytes`) + expect(delivered).toHaveLength(0) + + // The packetizer was reset, so the stale 5 header bytes are gone and this + // frame parses from its own first byte. + const good = encodeFrame([2, 'keybase.1.test.after-oversized', [{}]]) + transport.packetizeData(good) + + expect(delivered).toHaveLength(1) + expect((delivered[0] as {method: string}).method).toBe('keybase.1.test.after-oversized') + consoleSpy.mockRestore() +}) + +test('the 64MB frame limit is exclusive: exactly maxFrameSize is accepted and only one byte over is rejected', () => { + const consoleSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) + + // Exactly at the limit: legal, so the packetizer just waits for the payload. + const atLimit = new TestTransport() + atLimit.packetizeData(oversizedHeader(maxFrameSize)) + expect(atLimit.packetizeErrors).toEqual([]) + + const overLimit = new TestTransport() + overLimit.packetizeData(oversizedHeader(maxFrameSize + 1)) + expect(overLimit.packetizeErrors).toHaveLength(1) + + consoleSpy.mockRestore() +}) + +test('the 1001st queued invoke is failed, not silently swallowed', () => { + const transport = new TestTransport({connected: false}) + const callbacks = new Array() + + // queueMax is 1000: the first 1000 are queued with no callback yet. + for (let i = 0; i < 1000; i++) { + const cb = jest.fn() + callbacks.push(cb) + transport.invoke(`keybase.1.test.queued${i}`, [{}], cb) + } + expect(callbacks.every(cb => cb.mock.calls.length === 0)).toBe(true) + + // A silently dropped invoke is a permanent hang for its caller, so the + // overflow branch must answer rather than discard. + const overflowCb = jest.fn() + transport.invoke('keybase.1.test.overflow', [{}], overflowCb) + expect(overflowCb).toHaveBeenCalledTimes(1) + const [err] = overflowCb.mock.calls[0] as [unknown, unknown] + expect((err as Error).message).toBe('Queue overflow for keybase.1.test.overflow') + + // The overflowed invoke was never queued, so connecting must flush exactly + // the 1000 that were accepted -- and must not re-answer the overflowed one. + transport.flushConnected() + expect(transport.sent).toHaveLength(1000) + expect(callbacks.every(cb => cb.mock.calls.length === 0)).toBe(true) + expect(overflowCb).toHaveBeenCalledTimes(1) +}) + +test('send reports failure once the pending queue is full instead of growing without bound', () => { + const transport = new TestTransport({connected: false}) + const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {}) + + for (let i = 0; i < 1000; i++) { + expect(transport.send([1, i, null, {}])).toBe(true) + } + + expect(transport.send([1, 1000, null, {}])).toBe(false) + expect(warnSpy).toHaveBeenCalledWith('Queue overflow for raw RPC message') + + transport.flushConnected() + expect(transport.sent).toHaveLength(1000) + warnSpy.mockRestore() +}) + +test('flushPending terminates and settles every queued item exactly once when the write fails on connect', () => { + const transport = new TestTransport({connected: false}) + const calls = new Array>() + + for (let i = 0; i < 5; i++) { + transport.invoke(`keybase.1.test.q${i}`, [{}], (err, data) => calls.push([err, data])) + } + transport.send([1, 99, null, {}]) + + // Connected, but every write throws. flushPending() swaps _pending and + // re-enters invoke()/send(); if a failing write could put items back on the + // queue this would spin forever or double-settle a callback. + const writeError = new Error('write failed right after connect') + transport.setWriteError(writeError) + transport.flushConnected() + + expect(calls).toHaveLength(5) + expect( + calls.every(([err]) => (err as {code?: number; desc?: string}).desc === writeError.message) + ).toBe(true) + expect(transport.sent).toEqual([]) + + // Nothing was re-queued: a second flush with a working write sends nothing + // and settles nothing again. + transport.setWriteError(undefined) + transport.flushConnected() + expect(calls).toHaveLength(5) + expect(transport.sent).toEqual([]) +}) + test('cancel packets surface a cancelled response payload', () => { const incoming = jest.fn() const transport = new TestTransport({incomingRPCCallback: incoming}) @@ -131,3 +675,65 @@ test('cancel packets surface a cancelled response payload', () => { }) ) }) + +test('close settles every queued invoke exactly once with EOF and drops queued sends', () => { + const transport = new TestTransport({connected: false}) + const calls: Array> = [] + const cbA = jest.fn((err: unknown, data: unknown) => calls.push([err, data])) + const cbB = jest.fn((err: unknown, data: unknown) => calls.push([err, data])) + + transport.invoke('keybase.1.test.a', [{}], cbA) + transport.invoke('keybase.1.test.b', [{}], cbB) + // A queued raw send() has no callback and nothing waiting on it; it must + // simply be dropped, not replayed later. + expect(transport.send([1, 5, null, {ok: true}])).toBe(true) + + transport.close() + + expect(cbA).toHaveBeenCalledTimes(1) + expect(cbB).toHaveBeenCalledTimes(1) + expect(calls.every(([err]) => (err as {code?: number} | undefined)?.code === errors.EOF)).toBe(true) + expect(transport.sent).toEqual([]) + + // Reconnecting must not replay or re-settle anything the close already + // settled -- the queue was detached, not just marked. + transport.flushConnected() + expect(cbA).toHaveBeenCalledTimes(1) + expect(cbB).toHaveBeenCalledTimes(1) + expect(transport.sent).toEqual([]) +}) + +test('close settles queued invokes even when one callback throws', () => { + const transport = new TestTransport({connected: false}) + const calls: Array = [] + + transport.invoke('keybase.1.test.a', [{}], () => { + calls.push(1) + throw new Error('boom') + }) + transport.invoke('keybase.1.test.b', [{}], () => calls.push(2)) + transport.invoke('keybase.1.test.c', [{}], () => calls.push(3)) + + expect(() => transport.close()).not.toThrow() + expect(calls).toEqual([1, 2, 3]) +}) + +test('close does not re-queue an invoke made from a settling callback', () => { + const transport = new TestTransport({connected: false}) + const reentrant = jest.fn() + + transport.invoke('keybase.1.test.a', [{}], () => { + // Post-close invokes hit the explicit-close branch and fail immediately + // rather than sitting in a queue nothing will ever flush. + transport.invoke('keybase.1.test.reentrant', [{}], reentrant) + }) + + transport.close() + + expect(reentrant).toHaveBeenCalledTimes(1) + expect(reentrant).toHaveBeenCalledWith(expect.objectContaining({code: errors.EOF}), {}) + + transport.flushConnected() + expect(reentrant).toHaveBeenCalledTimes(1) + expect(transport.sent).toEqual([]) +}) diff --git a/shared/engine/rpc-transport.tsx b/shared/engine/rpc-transport.tsx index 6caea764c5a0..f3df3ca549e5 100644 --- a/shared/engine/rpc-transport.tsx +++ b/shared/engine/rpc-transport.tsx @@ -1,4 +1,5 @@ import {decode, encode} from '@msgpack/msgpack' +import logger from '@/logger' const MESSAGE_TYPE_INVOKE = 0 const MESSAGE_TYPE_RESPONSE = 1 @@ -30,6 +31,11 @@ export type ResponseType = { seqid: number error?: (e?: ErrorType) => void result?: (r?: unknown) => void + // Read-only: true once error()/result() has settled this response. Lets a + // caller that catches after an intentional settle (e.g. a throwing + // reducer that runs after the auto-result() call) skip re-settling instead + // of tripping the double-settle guard below and logging a false alarm. + readonly settled?: boolean } export type PayloadType = { @@ -272,10 +278,35 @@ export abstract class RPCTransport { } } + // Settles everything queued while disconnected. Detaching the array first is + // what makes this once-only: a later flushPending()/onConnected() sees an + // empty queue, so nothing is re-sent or settled twice. Queued raw send()s + // have no callback and nothing waiting on them, so they're just dropped. + protected failPending(err: unknown, data: unknown) { + const pending = this._pending + this._pending = [] + for (const item of pending) { + if (item.type !== 'invoke') { + continue + } + try { + item.cb(err, data) + } catch (e) { + logger.error('failPending callback threw', e) + } + } + } + protected failOutstanding(err: unknown, data: unknown) { const invocations = this._invocations this._invocations = new Map() - invocations.forEach(cb => cb(err, data)) + invocations.forEach(cb => { + try { + cb(err, data) + } catch (e) { + logger.error('failOutstanding callback threw', e) + } + }) } packetizeData(data: Uint8Array) { @@ -326,7 +357,17 @@ export abstract class RPCTransport { } const payload = decode(payloadBytes) p.consumeBytes(payloadLen) - this.dispatchDecodedMessage(payload) + + // Dispatch outside the framing try: an app-side handler that throws must + // not reach the catch below, which resets the packetizer and discards + // every buffered byte. That leaves parsing to resume at an arbitrary + // offset -- and on the renderer transport there is no socket to + // reconnect, so it never recovers. + try { + this.dispatchDecodedMessage(payload) + } catch (e) { + logger.error('dispatchDecodedMessage threw', e) + } } } catch (err) { p.reset() @@ -354,7 +395,20 @@ export abstract class RPCTransport { response: this.makeResponse(seqid), } if (this._incomingRPCCallback) { - this._incomingRPCCallback(payload) + try { + this._incomingRPCCallback(payload) + } catch (e) { + logger.error('incoming invoke handler threw', e) + // If the handler already settled the response (e.g. an + // auto-result() call followed by a throwing reducer on the next + // line) it has already answered the service; settling again + // would only trip the double-settle guard and log a false + // alarm. Only settle here when nothing has answered yet -- the + // service side would otherwise wait forever on this seqid. + if (!payload.response.settled) { + payload.response.error?.(makeTransportError('UNKNOWN_METHOD')) + } + } } else { payload.response.error?.(makeTransportError('UNKNOWN_METHOD')) } @@ -411,7 +465,12 @@ export abstract class RPCTransport { } if (this.isConnected()) { - this.writeMessage(message) + try { + this.writeMessage(message) + } catch (err) { + logger.error('Failed to write RPC message', err) + return false + } return true } if (this._explicitClose) { @@ -450,7 +509,7 @@ export abstract class RPCTransport { close() { this.markExplicitClose() - this._pending = [] + this.failPending(makeEOFError(), {}) this._packetizer.reset() this.failOutstanding(makeEOFError(), {}) } @@ -459,26 +518,70 @@ export abstract class RPCTransport { return encodeFrame(message) } + // Fails every outstanding invocation. Transports that sit on a connection + // which can die underneath them (mobile JSI, renderer IPC) call this when + // the engine resets; without it the callbacks are never invoked and every + // in-flight RPC hangs forever. + failAllOutstanding(err: unknown = makeEOFError()) { + // Also drop any partial frame: on the renderer transport this runs on the + // account switch, and a half-delivered pre-switch frame would otherwise + // concatenate with post-switch bytes into one corrupt decode. + this._packetizer.reset() + this.failOutstanding(err, {}) + } + private invokeNow(method: string, args: [object], cb: InvocationCallback) { const seqid = this._seqid this._seqid += 1 this._invocations.set(seqid, cb) - this.writeMessage([MESSAGE_TYPE_INVOKE, seqid, method, args]) + try { + this.writeMessage([MESSAGE_TYPE_INVOKE, seqid, method, args]) + } catch (err) { + // The message never left, so no response is coming. Fail the caller + // rather than leaving the seqid outstanding for the rest of the session. + // Shaped like every other transport-level failure (code/desc, not the + // raw exception) so downstream convertToError yields an RPCError with a + // code; the original message survives in desc. + this._invocations.delete(seqid) + cb({code: errors.EOF, desc: err instanceof Error ? err.message : String(err), name: 'EOF'}, {}) + } } private makeResponse(seqid: number): ResponseType { + let settled = false return { cancelled: false, + get settled() { + return settled + }, error: err => { - this.send([MESSAGE_TYPE_RESPONSE, seqid, err, null]) + if (settled) { + logger.error(`Attempted to settle response for seqid ${seqid} twice (error after already settled)`) + return + } + settled = true + if (!this.send([MESSAGE_TYPE_RESPONSE, seqid, err, null])) { + // The service is waiting on this reply and nothing else will tell + // it. The write already failed (send() logged that), so there's no + // connection left to retry on -- surface which seqid was lost. + logger.error(`failed to write error response for seqid ${seqid}`) + } }, result: result => { - this.send([MESSAGE_TYPE_RESPONSE, seqid, null, result]) + if (settled) { + logger.error(`Attempted to settle response for seqid ${seqid} twice (result after already settled)`) + return + } + settled = true + if (!this.send([MESSAGE_TYPE_RESPONSE, seqid, null, result])) { + // Same as above: the write failed, the connection is gone, and + // nothing will retry this seqid. + logger.error(`failed to write response for seqid ${seqid}`) + } }, seqid, } } - } export {encodeFrame, makeEOFError, makeTransportError} diff --git a/shared/globals.d.ts b/shared/globals.d.ts index 083297167dd5..7e55cb944897 100644 --- a/shared/globals.d.ts +++ b/shared/globals.d.ts @@ -37,7 +37,8 @@ declare global { var __VERSION__: string var __FILE_SUFFIX__: string var __PROFILE__: boolean - var rpcOnGo: undefined | ((msg: unknown) => void) + // Returns false if the native write to Go failed + var rpcOnGo: undefined | ((msg: unknown) => boolean) var rpcOnJs: undefined | ((objs: unknown, count: number) => void) // RN var __turboModuleProxy: unknown diff --git a/shared/ios/Podfile.lock b/shared/ios/Podfile.lock index e4be7a7c6475..ca7316898f83 100644 --- a/shared/ios/Podfile.lock +++ b/shared/ios/Podfile.lock @@ -2965,7 +2965,7 @@ SPEC CHECKSUMS: React-Mapbuffer: 1a42eda3f9d415ba9f7dcd7f1fe19d4f27925751 React-microtasksnativemodule: 354ab193a7ed66e42869c8a0c2c0d844e8e22f67 React-mutationobservernativemodule: f53ad06cbe4b44d2a33df6aa3f01ca8e9b57e83b - react-native-kb: 71289be80d93678d5abff5805cb638cd36d9e08e + react-native-kb: d3ef6bac088ef6794bcc6c7ab8d533d046c5675e react-native-keyboard-controller: 8305b19b7841bab139be15cde9f424a502abaa6e react-native-netinfo: a05f9b897e76ad24b53f615fed1cbb731932363d react-native-safe-area-context: 0bbcdfba5c4a1b3203276f1ed9128e3239fdf199 diff --git a/shared/test/mocks/react-native.js b/shared/test/mocks/react-native.js index d5772bd2a6e1..2b9f178aca3d 100644 --- a/shared/test/mocks/react-native.js +++ b/shared/test/mocks/react-native.js @@ -16,6 +16,7 @@ exports.StyleSheet = { } exports.NativeModules = {} +exports.LogBox = {ignoreAllLogs: () => {}} exports.TurboModuleRegistry = {get: () => null, getEnforcing: () => ({})} exports.findNodeHandle = () => null diff --git a/shared/tests/e2e/ios-appium/all.test.ts b/shared/tests/e2e/ios-appium/all.test.ts index c4cd6631ab00..8b895b9bcd8a 100644 --- a/shared/tests/e2e/ios-appium/all.test.ts +++ b/shared/tests/e2e/ios-appium/all.test.ts @@ -2,6 +2,7 @@ // every flow here (side-effect imports register their describe/it blocks) runs // the whole suite in ONE session — no per-file session teardown/recreate (~800ms // each) and the app stays warm between flows. +import './flows/android-activity-restart.test' import './flows/chat-conversation.test' import './flows/chat-send-message.test' import './flows/crypto-outputs.test' diff --git a/shared/tests/e2e/ios-appium/flows/android-activity-restart.test.ts b/shared/tests/e2e/ios-appium/flows/android-activity-restart.test.ts new file mode 100644 index 000000000000..86e8eff71ad7 --- /dev/null +++ b/shared/tests/e2e/ios-appium/flows/android-activity-restart.test.ts @@ -0,0 +1,146 @@ +import {expect} from '@wdio/globals' +import {requireSmokeUser} from '../helpers/app' +import {escapeToTabs, navigateToChat} from '../helpers/navigate' +import {anyExist, byText, el, els, waitForTestID, enterText} from '../helpers/elements' +import * as T from '../../shared/test-ids' + +// Regression coverage for the Android "back-button wedge" (fixed in +// 15647bb4d4, HOTPOT-rpc-fixes). onHostDestroy fires when the LAST ACTIVITY +// is destroyed while the ReactInstance/TurboModules SURVIVE (MainApplication +// holds the ReactHost in an application-scoped `by lazy`) — pressing back +// from the root exits to the launcher this way, without killing the process. +// The bug: nativeInvalidate() was reachable from that path and nulled the +// C++ g_bridge, which only the JSI bindings installer repopulates — and a +// surviving ReactInstance never re-runs that installer. Outbound RPC still +// "works" (writeArr doesn't check g_bridge), so every inbound reply is +// silently dropped for the life of the process; the app looks alive and +// never gets a response. Reopening the app (without a process restart) must +// still be able to complete a full RPC round trip. +const ANDROID_PACKAGE = 'io.keybase.ossifrage' + +describe('android activity restart (back-button RPC wedge)', () => { + // Non-arrow body so `this.skip()` is available: returning early from an + // async test body marks it PASSED, which is exactly the false green this + // regression test must not produce. + it('keeps inbound RPC alive after the Activity (not the process) is destroyed and reopened', async function () { + if (!browser.isAndroid) { + this.skip() + } + + // Proving the process survived needs `mobile: shell`, which needs the + // appium service running with relaxedSecurity. That widens what the local + // appium server will execute, so it is opt-in. Without it this test cannot + // tell a surviving process from a fresh one, and a fresh one reinstalls the + // bridge and passes while proving nothing — so skip instead of pretending. + if (!process.env['KB_E2E_RELAXED_SECURITY']) { + console.warn( + 'android-activity-restart: SKIPPED — set KB_E2E_RELAXED_SECURITY=1 to run it (needs appium relaxedSecurity for pidof)' + ) + this.skip() + } + + requireSmokeUser() + await escapeToTabs() + await navigateToChat() + + // Baseline: inbound RPC works before we touch the Activity lifecycle at + // all. If there are no conversations to prove this with, the rest of the + // test can't prove anything either. + if (!(await anyExist(T.CHAT_INBOX_ROW))) { + throw new Error('android-activity-restart: no chat conversations to establish an RPC baseline with') + } + + // Capture the process PID BEFORE destroying the Activity. `mobile: shell` + // needs the appium service started with relaxedSecurity (wdio.android.conf.ts). + const pidBefore = await getAppPid() + if (!pidBefore) { + throw new Error( + 'android-activity-restart: could not read a PID via `mobile: shell pidof` — cannot prove ' + + 'process continuity, which is the entire point of this test. Refusing to continue rather ' + + 'than risk a false green (see comment above getAppPid).' + ) + } + + // Destroy the Activity WITHOUT killing the process: press back all the + // way out to the launcher. Do NOT use terminateApp/closeApp — killing the + // process resets init{} and the bindings installer, which would make the + // wedge impossible to reproduce and this test a false green. + for (let i = 0; i < 12; i++) { + const pkg = await browser.getCurrentPackage().catch(() => '') + if (pkg !== ANDROID_PACKAGE) break + await browser.back() + await browser.pause(500) + } + const pkgAfterBack = await browser.getCurrentPackage().catch(() => '') + if (pkgAfterBack === ANDROID_PACKAGE) { + throw new Error('android-activity-restart: never left the app after 12 back presses') + } + + // Give onHostDestroy a moment to actually run before reopening. + await browser.pause(1000) + + // Reopen via activateApp, which brings the existing process's task back to + // the foreground and creates a NEW Activity — it does not relaunch the + // process. (This mirrors escapeToTabs' own recovery path.) + await browser.activateApp(ANDROID_PACKAGE) + await browser.waitUntil(async () => (await browser.getCurrentPackage().catch(() => '')) === ANDROID_PACKAGE, { + timeout: 10000, + interval: 250, + }) + + // The critical guard: if the PID changed, Android killed the process + // during step 3 (low memory, aggressive OEM task killer, etc). That means + // activateApp started a FRESH process — init{} reran, the bridge was + // reinstalled, and any pass below would prove nothing about the wedge. + // Fail loudly rather than silently pass on a meaningless run. + const pidAfter = await getAppPid() + if (pidAfter !== pidBefore) { + throw new Error( + `android-activity-restart: process PID changed (${pidBefore} -> ${pidAfter}) — Android killed ` + + 'the process instead of just the Activity, so this run cannot exercise the back-button wedge ' + + '(a fresh process reinstalls the native bridge via init{}, masking the bug). Not a real failure ' + + 'of the fix; the environment did not hold up its end. Re-run on a device/emulator with more ' + + 'headroom, or investigate why the process was killed.' + ) + } + + await escapeToTabs() + await navigateToChat() + await waitForTestID(T.CHAT_INBOX_ROW, 5000) + await els(T.CHAT_INBOX_ROW)[0]!.click() + await waitForTestID(T.CHAT_MESSAGE_LIST, 5000) + + // Strongest available proof of a live inbound RPC path: a full round trip. + const testMessage = `e2e-restart-${Date.now()}` + await waitForTestID(T.CHAT_INPUT, 5000) + await enterText(T.CHAT_INPUT, testMessage) + await waitForTestID(T.CHAT_SEND_BUTTON, 3000) + await el(T.CHAT_SEND_BUTTON).click() + + const sent = byText(testMessage) + await sent.waitForDisplayed({ + timeout: 5000, + timeoutMsg: `sent message "${testMessage}" never appeared after Activity restart — inbound RPC is likely wedged`, + }) + await expect(sent).toBeDisplayed() + }) +}) + +// `pidof` via `mobile: shell` is the process-continuity signal: it requires +// the appium service to run with relaxedSecurity (see wdio.android.conf.ts). +// Returns undefined if the command isn't available or the app isn't running, +// in which case the caller must refuse to proceed rather than assume +// continuity it can't actually prove. +async function getAppPid(): Promise { + try { + const result = (await browser.execute('mobile: shell', { + command: 'pidof', + args: [ANDROID_PACKAGE], + })) as string | {stdout?: string} | undefined + const raw = typeof result === 'string' ? result : (result?.stdout ?? '') + const pid = raw.trim().split(/\s+/)[0] + return pid || undefined + } catch { + return undefined + } +} diff --git a/shared/tests/e2e/ios-appium/wdio.android.conf.ts b/shared/tests/e2e/ios-appium/wdio.android.conf.ts index 394504e37484..ae9980a1862e 100644 --- a/shared/tests/e2e/ios-appium/wdio.android.conf.ts +++ b/shared/tests/e2e/ios-appium/wdio.android.conf.ts @@ -32,7 +32,17 @@ export const config: WebdriverIO.Config = { // nav/list flake without masking real failures. Retries run ONLY on failure. mochaOpts: {ui: 'bdd', timeout: 120000, retries: 2}, reporters: ['spec'], - services: [['appium', {args: {basePath: '/', port}}]], + // relaxedSecurity lets Appium run privileged commands such as `mobile: shell`. + // android-activity-restart.test.ts needs it (pidof) to prove the app process + // survived an Activity restart, but it applies server-wide, so it stays + // opt-in: set KB_E2E_RELAXED_SECURITY=1 for that run. Without it that one + // test skips itself rather than reporting a result it cannot substantiate. + services: [ + [ + 'appium', + {args: {basePath: '/', port, ...(process.env['KB_E2E_RELAXED_SECURITY'] ? {relaxedSecurity: true} : {})}}, + ], + ], // The app restores its last screen on launch and screens leak between specs, // so reset to the root tab bar before each test by climbing out of any stack. beforeTest: async test => {