From 2983d45632b6bfd38344b039947d49b3fc6089b2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 16 Aug 2026 12:33:57 -0500 Subject: [PATCH 1/5] feat(ffi): add retained callbacks - Add format 4 retained registration and explicit release descriptors with pointer-identity validation. - Pin callbacks in runtime registration tables and emit matching C/LLVM adapters with deferred exception checks. - Cover lifecycle and sanitizer cases, and document the same-thread callback boundary. Co-authored-by: DemonGPT <127974011+DemonGPT@users.noreply.github.com> --- CHANGELOG.md | 1 + docs/src/app/ffi/page.mdx | 68 ++++++- docs/src/app/limitations/page.mdx | 2 +- packages/compiler/src/backend/cc.ts | 6 +- .../src/backend/emission/emit-exprs.ts | 59 +++++- .../compiler/src/backend/emission/emitter.ts | 45 ++++- .../src/backend/emission/may-throw.ts | 16 +- .../compiler/src/backend/ffi-callbacks.ts | 29 ++- packages/compiler/src/backend/llvm/emitter.ts | 90 ++++++++- .../compiler/src/diagnostics/diagnostic.ts | 1 + packages/compiler/src/ffi/profile.ts | 180 +++++++++++++++--- .../src/frontend/lowering/lower-calls.ts | 51 ++++- packages/compiler/src/ir/nodes.ts | 36 +++- packages/compiler/src/ir/validate.ts | 45 ++++- packages/runtime/src/scr_async.c | 5 + packages/runtime/src/scr_ffi.c | 62 ++++++ packages/runtime/src/scr_runtime.h | 19 ++ packages/runtime/test/bytes.test.ts | 1 + packages/runtime/test/runtime.test.ts | 2 +- tests/ffi/main.ts | 83 ++++++++ tests/ffi/native.c | 59 ++++++ tests/ffi/profile.json | 68 ++++++- tests/harness/ffi.test.ts | 180 +++++++++++++++++- 23 files changed, 1025 insertions(+), 83 deletions(-) create mode 100644 packages/runtime/src/scr_ffi.c diff --git a/CHANGELOG.md b/CHANGELOG.md index efa6e8cc6..d32bfef13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to scriptc will be documented in this file. ### Features +- **Native callbacks can be retained and explicitly released.** FFI format 4 adds `lifetime: "retained"` registrations and paired `release` descriptors that reuse the original function-pointer trampoline. Registrations pin captured closures until the same function value is released, count duplicate registrations, support multiple context-bearing registrations and raw single-slot replacement, defer callback throws through later FFI pump calls, and clean up live registrations at process exit across both backends. - **Native callbacks copy in strings and byte spans.** FFI format 3 adds callback-only `cstring` parameters plus length-delimited `string` and `bytes` parameters. Trampolines in both backends copy native memory into owned scriptc values, decode malformed UTF-8 with U+FFFD replacement, preserve embedded NUL bytes in spans, and trap precise invalid null pointers before invoking the closure. diff --git a/docs/src/app/ffi/page.mdx b/docs/src/app/ffi/page.mdx index 38455a11d..fc2fe407d 100644 --- a/docs/src/app/ffi/page.mdx +++ b/docs/src/app/ffi/page.mdx @@ -104,21 +104,21 @@ The manifest is the native ABI authority. TypeScript has only `number`, so its d cstring string const char *; callback input only, copied through lossy UTF-8 decoding - callback only (format 3) + callback only (formats 3–4) no string string const uint8_t *, size_t; UTF-8 bytes, length-delimited - yes; callback input in format 3 + yes; callback input in formats 3–4 no bytes Uint8Array or Buffer const uint8_t *, size_t; raw bytes, length-delimited - yes; callback input in format 3 + yes; callback input in formats 3–4 no @@ -189,13 +189,63 @@ The callback id connects the two independently positioned context entries. Both } ``` -A callback descriptor consumes one TypeScript function parameter and one native function-pointer slot. A context entry consumes no TypeScript parameter and one native `void *` slot. Formats 2 and 3 accept `f64`, `bool`, `u8`, `u32`, and `i32` callback parameters plus at most one context entry. Format 3 additionally accepts `cstring`, `string`, and `bytes`; callback returns remain scalar or `void`. +A callback descriptor consumes one TypeScript function parameter and one native function-pointer slot. A context entry consumes no TypeScript parameter and one native `void *` slot. Formats 2 through 4 accept `f64`, `bool`, `u8`, `u32`, and `i32` callback parameters plus at most one context entry. Formats 3 and 4 additionally accept `cstring`, `string`, and `bytes`; callback returns remain scalar or `void`. Format 3 string-bearing callback parameters copy native data before the closure runs. `cstring` reads one non-null, NUL-terminated `const char *`. `string` and `bytes` each consume a `const uint8_t *, size_t` pair; a null pointer is valid only when its length is zero. Text is decoded as UTF-8 with malformed sequences replaced by U+FFFD, matching `Buffer.toString("utf8")`. The resulting string or `Uint8Array` is freshly owned scriptc storage, so the closure may retain it without depending on the native buffer's lifetime. An unexpected null `cstring`, or a null non-empty span, traps at the boundary instead of being treated as empty. -For a raw C callback type with no userdata, omit the context entry from both parameter lists. scriptc installs that closure in a binding-specific thread-local slot around the native call, so captures and nested calls still work. This remains call-scoped: the native function must invoke it synchronously on the thread that entered the native call. +For a raw call-scoped C callback type with no userdata, omit the context entry from both parameter lists. scriptc installs that closure in a binding-specific thread-local slot around the native call, so captures and nested calls still work. -`lifetime` must currently be `"call"`. Native code must not retain a callback or context, invoke it after the outer function returns, or invoke it from another thread. As with a bad native pointer or mismatched signature, violating that contract is outside scriptc's memory-safety guarantees. Retained callbacks will require an explicit registration/unregistration ownership model; foreign-thread callbacks will additionally require runtime scheduling and synchronization. +Format 4 adds `lifetime: "retained"` for same-thread native APIs that store a callback and invoke it from a later FFI call. Registration pins the closure and its captures. A paired release binding passes the same trampoline and closure context back to native code, then unpins one matching registration after the native call returns: + +```ts:main.ts +declare function timerAdd(interval: number, tick: () => void): void; +declare function timerRemove(tick: () => void): void; + +const tick = () => console.log("tick"); +timerAdd(100, tick); +// A later native pump call may invoke tick here. +timerRemove(tick); +``` + +```json:ffi.json +{ + "ffi_format": 4, + "functions": [ + { + "name": "timerAdd", + "symbol": "timer_add", + "params": [ + "u32", + { + "callback": { + "id": "tick", + "params": [{ "context": "tick" }], + "returns": "void", + "lifetime": "retained" + } + }, + { "context": "tick" } + ], + "returns": "void" + }, + { + "name": "timerRemove", + "symbol": "timer_remove", + "params": [ + { "callback": { "release": "timerAdd:tick" } }, + { "context": "timerAdd:tick" } + ], + "returns": "void" + } + ] +} +``` + +The release argument must be the same function value used for registration. Registrations are counted: registering the same closure twice requires two releases. Releasing an unregistered value traps because native code may still hold the original pointer. The callback's function type must be exact at retained call sites; an implicit wrapper would create a different pointer and make release identity unsound. + +Context-bearing descriptors support multiple concurrent closures. A raw retained descriptor has no context pointer, so it has one process-global slot with replace semantics: registering a new closure releases the old slot. Retained registrations do not keep the event loop alive and remaining registrations are released at process exit. + +Both `"call"` and `"retained"` callbacks must run synchronously on the script thread. Direct foreign-thread invocation, value-returning foreign-thread callbacks, and real-time callback delivery are unsupported: scriptc's reference counting and exception cell are thread-confined, and a future foreign-thread surface must enqueue fire-and-forget delivery onto the script loop. Dereferenceable struct callback parameters are also unsupported; use an opaque native handle with accessor functions when the API permits it. If a callback throws, the adapter returns zero (or `void`) to native code and suppresses further script callback execution while the exception is pending. When the outer native function returns, the original exception resumes through scriptc's ordinary catchable unwind path. Native work performed between the callback's return and the outer function's return is not rolled back. @@ -203,10 +253,10 @@ If a callback throws, the adapter returns zero (or `void`) to native code and su
ffi_format
-
Required. Format 1 supports value parameters; format 2 preserves them and adds callback/context entries; format 3 adds copy-in cstring, string-span, and byte-span callback parameters.
+
Required. Format 1 supports value parameters; format 2 preserves them and adds callback/context entries; format 3 adds copy-in cstring, string-span, and byte-span callback parameters; format 4 adds retained registrations and release references.
functions
-
Required array. Every entry has exactly name, symbol, params, and returns. Binding names and symbols must be unique. In formats 2 and 3, callback ids must be unique within a function and every context must match exactly one callback.
+
Required array. Every entry has exactly name, symbol, params, and returns. Binding names and symbols must be unique. Callback ids must be unique within a function and every context must match exactly one callback or release. A release references a retained <binding>:<callback-id> in the same manifest and inherits its callback ABI.
libraries
Optional array of archive or object paths. Relative paths are resolved from the manifest directory and appended after the generated program at link time.
@@ -221,7 +271,7 @@ Unknown fields, invalid ABI classes, duplicate names, and signature mismatches f - Native calls are synchronous and must return normally. Do not unwind C++ exceptions or `longjmp` across the boundary. - Native code is outside scriptc's exception, reference-counting, and sanitizer contracts. A bad pointer or mismatched C signature can still corrupt the process. -- Callbacks are synchronous, call-scoped, and same-thread only. Retained callbacks and foreign-thread invocation are not supported yet. +- Callbacks are same-thread only. Format 4 supports retained callbacks with explicit release; foreign-thread invocation is not supported. - There are no variadic calls, struct-by-value arguments, owned pointer returns, or runtime `dlopen`/`dlsym` handles yet. - The archive or object must match the build target. Cross-compilation does not translate native inputs. - Outbound FFI is currently available for executable builds, not `scriptc build --lib`. diff --git a/docs/src/app/limitations/page.mdx b/docs/src/app/limitations/page.mdx index 90333063e..00e230ea3 100644 --- a/docs/src/app/limitations/page.mdx +++ b/docs/src/app/limitations/page.mdx @@ -90,5 +90,5 @@ The production wasm32-wasi target supports the complete executable ## Tooling gaps - `scriptc run` does not forward extra CLI arguments to the program — `build` and invoke the binary directly. -- Native FFI is a direct, manifest-declared C ABI link surface. Callback parameters are synchronous, call-scoped, and same-thread; retained or foreign-thread callbacks are not supported yet. Variadic calls, structs by value, owned pointer/string/byte returns, runtime dynamic-library loading, and library-mode builds also remain unsupported. See [Native FFI](/ffi). +- Native FFI is a direct, manifest-declared C ABI link surface. Callback invocation is same-thread only; format 4 supports explicitly released retained callbacks, but foreign-thread callbacks remain unsupported. Variadic calls, structs by value, owned pointer/string/byte returns, runtime dynamic-library loading, and library-mode builds also remain unsupported. See [Native FFI](/ffi). - Numbers are JS-exact f64 everywhere. Integer inference and ownership analysis — the systems-language performance ceiling — are roadmap, not shipped. diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index 82f81b612..e4e3fa512 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -37,7 +37,7 @@ function stableTestMemo( return pending; } -const RUNTIME_SOURCES = ["scr_number.c", "scr_string.c", "scr_array.c", "scr_bytes.c", "scr_bytes_io.c", "scr_map.c", "scr_closure.c", "scr_object.c", "scr_union.c", "scr_exception.c", "scr_error.c", "scr_console.c", "scr_lib.c", "scr_path.c", "scr_url.c", "scr_json.c", "scr_async.c", "scr_child.c", "scr_cycle.c"]; +const RUNTIME_SOURCES = ["scr_number.c", "scr_string.c", "scr_array.c", "scr_bytes.c", "scr_bytes_io.c", "scr_map.c", "scr_closure.c", "scr_ffi.c", "scr_object.c", "scr_union.c", "scr_exception.c", "scr_error.c", "scr_console.c", "scr_lib.c", "scr_path.c", "scr_url.c", "scr_json.c", "scr_async.c", "scr_child.c", "scr_cycle.c"]; /** The pinned quickjs-ng snapshot under packages/runtime/vendor/quickjs-ng * (see vendor/README.md — update both together). Keys the archive cache so @@ -1385,7 +1385,9 @@ async function ensureTlsArchive( /** The library base: the executable lane's unconditional sources minus the * fiber/loop and child-process units, plus the library-mode TU. */ const LIB_RUNTIME_SOURCES = [ - ...RUNTIME_SOURCES.filter((f) => f !== "scr_async.c" && f !== "scr_child.c"), + ...RUNTIME_SOURCES.filter( + (f) => f !== "scr_async.c" && f !== "scr_child.c" && f !== "scr_ffi.c", + ), "scr_library.c", ]; diff --git a/packages/compiler/src/backend/emission/emit-exprs.ts b/packages/compiler/src/backend/emission/emit-exprs.ts index 6093c0a6d..97bca6805 100644 --- a/packages/compiler/src/backend/emission/emit-exprs.ts +++ b/packages/compiler/src/backend/emission/emit-exprs.ts @@ -2,7 +2,7 @@ * expression lands in a fresh C temp, with RC ownership tracked on the * emitter's frames (see the discipline comment in emitter core). */ import type { CEmitter, Temp } from "./emitter.js"; -import { arrayOf, BOOL, BYTES_U8, bytesOf, canMarshalFuncIntoIsland, CHILDSTREAM_T, DYN, F64, IrExpr, IrRecordShape, IrType, islandPromisePayloadTag, isFfiCallbackParam, isFfiContextParam, isRefCounted, isUnitType, MAY_THROW_LIB_FNS, RUNTIME_ERROR_CLASSES, STRING, typeEquals, typeKey } from "../../ir/nodes.js"; +import { arrayOf, BOOL, BYTES_U8, bytesOf, canMarshalFuncIntoIsland, CHILDSTREAM_T, DYN, F64, IrExpr, IrRecordShape, IrType, islandPromisePayloadTag, isFfiCallbackParam, isFfiContextParam, isFfiReleaseParam, isRefCounted, isUnitType, MAY_THROW_LIB_FNS, RUNTIME_ERROR_CLASSES, STRING, typeEquals, typeKey } from "../../ir/nodes.js"; import { boxAccess, BYTES_NUM_KIND_C, BYTES_NUM_VAR_C, bytesElemKindC, cDecl, cFnPtrCast, cNumberLiteral, cStringLiteral, cType, DV_GET_KIND_C, DV_SET_KIND_C, elemAccess, mapKeyAccess, mapKeyKindC, mapValKindC, releaseCallC, retainCallC, vAdapters } from "./emit-types.js"; import { mangleClassNew, mangleClassRetain, mangleClassStruct, mangleField, mangleFnClosure, mangleFunction, mangleGlobal, mangleLocal, mangleRecordNew, mangleRecordStruct, mangleVtStruct } from "../mangle.js"; import { OVERFLOW_MEMBER } from "./emit-shapes.js"; @@ -2131,8 +2131,40 @@ export function emitExpr(E: CEmitter, e: IrExpr): Temp { const arg = args[sourceIndex++]!; sourceArgs.set(abiIndex, arg); if (isFfiCallbackParam(param)) callbackArgs.set(param.callback.id, arg); + if (isFfiReleaseParam(param)) callbackArgs.set(param.callback.release, arg); }); + const retainedRegistrations: { table: string; callback: Temp; global: string | null }[] = []; + const retainedReleases: { table: string; callback: Temp; global: string | null }[] = []; + for (const param of entry.params) { + if (isFfiCallbackParam(param) && param.callback.lifetime === "retained") { + const adapter = E.ffiCallbackAdapter(entry.name, param.callback.id); + const callback = callbackArgs.get(param.callback.id)!; + if (adapter.table === null) throw new Error("emitter bug: retained callback has no table"); + retainedRegistrations.push({ table: adapter.table, callback, global: adapter.global }); + } else if (isFfiReleaseParam(param)) { + const split = param.callback.release.lastIndexOf(":"); + const adapter = E.ffiCallbackAdapter( + param.callback.release.slice(0, split), + param.callback.release.slice(split + 1), + ); + const callback = callbackArgs.get(param.callback.release)!; + if (adapter.table === null) throw new Error("emitter bug: retained release has no table"); + retainedReleases.push({ table: adapter.table, callback, global: adapter.global }); + } + } + + // Pin before registration. Raw retained descriptors are native + // singletons: replacing them drops the previous table entry and + // points the global trampoline slot at the new closure. + for (const registration of retainedRegistrations) { + if (registration.global !== null) { + E.line(`scr_ffi_teardown(&${registration.table});`); + E.line(`${registration.global} = ${registration.callback.name};`); + } + E.line(`scr_ffi_retain(&${registration.table}, ${registration.callback.name});`); + } + // Raw C callback pointers carry no userdata. For the documented // call-scoped/same-thread policy, lend each one a distinct TLS // slot for the dynamic extent of this native call. Save/restore @@ -2154,6 +2186,15 @@ export function emitExpr(E: CEmitter, e: IrExpr): Temp { nativeArgs.push(`&${adapter.symbol}`); return; } + if (isFfiReleaseParam(param)) { + const split = param.callback.release.lastIndexOf(":"); + const adapter = E.ffiCallbackAdapter( + param.callback.release.slice(0, split), + param.callback.release.slice(split + 1), + ); + nativeArgs.push(`&${adapter.symbol}`); + return; + } if (isFfiContextParam(param)) { const callback = callbackArgs.get(param.context); if (!callback) throw new Error(`emitter bug: FFI context '${param.context}' has no callback arg`); @@ -2192,22 +2233,35 @@ export function emitExpr(E: CEmitter, e: IrExpr): Temp { E.line(`${saved.tls} = ${saved.previous.name};`); } }; - const callbacksMayThrow = callbackArgs.size > 0; + const finishRetainedReleases = (): void => { + for (const release of retainedReleases) { + E.line(`scr_ffi_release(&${release.table}, ${release.callback.name});`); + if (release.global !== null) E.line(`${release.global} = NULL;`); + } + }; + const callbacksMayThrow = callbackArgs.size > 0 || (E.mod.ffiImports ?? []).some( + (candidate) => candidate.params.some( + (param) => isFfiCallbackParam(param) && param.callback.lifetime === "retained", + ), + ); switch (entry.returns) { case "void": E.line(`${call};${E.srcComment(e.loc)}`); restoreRawContexts(); + finishRetainedReleases(); if (callbacksMayThrow) E.emitPendingCheck(); return { name: "", type: e.type }; case "f64": { const result = E.newTemp(e.type, call); restoreRawContexts(); + finishRetainedReleases(); if (callbacksMayThrow) E.emitPendingCheck(); return result; } case "bool": { const result = E.newTemp(e.type, `(${call} != 0)`); restoreRawContexts(); + finishRetainedReleases(); if (callbacksMayThrow) E.emitPendingCheck(); return result; } @@ -2216,6 +2270,7 @@ export function emitExpr(E: CEmitter, e: IrExpr): Temp { case "i32": { const result = E.newTemp(e.type, `(double)${call}`); restoreRawContexts(); + finishRetainedReleases(); if (callbacksMayThrow) E.emitPendingCheck(); return result; } diff --git a/packages/compiler/src/backend/emission/emitter.ts b/packages/compiler/src/backend/emission/emitter.ts index a1c505e37..1cfff66a8 100644 --- a/packages/compiler/src/backend/emission/emitter.ts +++ b/packages/compiler/src/backend/emission/emitter.ts @@ -28,6 +28,7 @@ import type { IrFfiCallbackParamClass, IrFfiCallbackParam, IrFfiImport, + IrFfiReleaseParam, IrFfiReturnClass, IrFfiValueParamClass, IrFunction, @@ -38,7 +39,7 @@ import type { IrUnionDef, SrcLoc, } from "../../ir/nodes.js"; -import { ffiCallbackType, funcOf, isFfiCallbackParam, isFfiContextParam, isRefCounted, isUnitType, mapOf, moduleEmbedsCompressedNpm, moduleUsesDgram, moduleUsesDynInvoke, moduleEmbedsBuiltin, moduleUsesFetch, moduleUsesFsWatch, moduleUsesHttp2, moduleUsesHttpServer, moduleUsesNet, moduleUsesNodeTest, moduleUsesProcessEvents, moduleUsesStream, moduleUsesTls, moduleUsesTlsCa, RUNTIME_EMITTER_CLASS, STRING, VOID } from "../../ir/nodes.js"; +import { ffiCallbackType, funcOf, isFfiCallbackParam, isFfiContextParam, isFfiReleaseParam, isRefCounted, isUnitType, mapOf, moduleEmbedsCompressedNpm, moduleUsesDgram, moduleUsesDynInvoke, moduleEmbedsBuiltin, moduleUsesFetch, moduleUsesFsWatch, moduleUsesHttp2, moduleUsesHttpServer, moduleUsesNet, moduleUsesNodeTest, moduleUsesProcessEvents, moduleUsesStream, moduleUsesTls, moduleUsesTlsCa, RUNTIME_EMITTER_CLASS, STRING, VOID } from "../../ir/nodes.js"; import { allocateFfiCallbackAdapters, type FfiCallbackAdapter } from "../ffi-callbacks.js"; import { mangleAsyncSpawn, @@ -106,7 +107,10 @@ export function ffiNativeTypeC( } } -function ffiCallbackNativeParamsC(callback: IrFfiCallbackParam["callback"], named: boolean): string[] { +function ffiCallbackNativeParamsC( + callback: IrFfiCallbackParam["callback"] | IrFfiReleaseParam["callback"], + named: boolean, +): string[] { return callback.params.flatMap((param, i): string[] => { if (isFfiContextParam(param)) return [`void *${named ? "sc_ctx" : ""}`.trim()]; if (param === "string" || param === "bytes") { @@ -119,7 +123,9 @@ function ffiCallbackNativeParamsC(callback: IrFfiCallbackParam["callback"], name }); } -function ffiCallbackPointerTypeC(callback: IrFfiCallbackParam["callback"]): string { +function ffiCallbackPointerTypeC( + callback: IrFfiCallbackParam["callback"] | IrFfiReleaseParam["callback"], +): string { const ret = ffiNativeTypeC(callback.returns); const params = ffiCallbackNativeParamsC(callback, false); return `${ret} (*)(${params.length > 0 ? params.join(", ") : "void"})`; @@ -719,7 +725,9 @@ export class CEmitter { ); for (const entry of directFfiImports) { const params = entry.params.flatMap((param): string[] => { - if (isFfiCallbackParam(param)) return [ffiCallbackPointerTypeC(param.callback)]; + if (isFfiCallbackParam(param) || isFfiReleaseParam(param)) { + return [ffiCallbackPointerTypeC(param.callback)]; + } if (isFfiContextParam(param)) return ["void *"]; switch (param) { case "f64": @@ -1740,12 +1748,12 @@ export class CEmitter { return adapter; } - /** C-callable trampolines for format-2/3 callbacks. The external callback + /** C-callable trampolines for format-2/3/4 callbacks. The external callback * ABI is scalar C, format-3 copy-in string/byte slots, plus an optional * exact-position context pointer; the internal side is scriptc's * (ScrClosure *env, params...) convention. A raw callback borrows its * closure through a distinct TLS slot for the dynamic extent of the outer - * call. */ + * call, or a process-global slot for retained replace semantics. */ emitFfiCallbackDefs(out: string[]): void { if (this.ffiCallbackAdapters.size === 0) return; for (const adapter of this.ffiCallbackAdapters.values()) { @@ -1753,13 +1761,19 @@ export class CEmitter { if (adapter.tls !== null) { out.push(`static _Thread_local ScrClosure *${adapter.tls};`); } + if (adapter.global !== null) { + out.push(`static ScrClosure *${adapter.global};`); + } + if (adapter.table !== null) { + out.push(`static ScrFfiTable ${adapter.table};`); + } const nativeParams = ffiCallbackNativeParamsC(cb, true); const ret = ffiNativeTypeC(cb.returns); const contextParam = cb.params.findIndex(isFfiContextParam); out.push( `static ${ret} ${adapter.symbol}(${nativeParams.length > 0 ? nativeParams.join(", ") : "void"}) {`, - ` ScrClosure *sc_cb = ${contextParam >= 0 ? `(ScrClosure *)sc_ctx` : adapter.tls};`, - ` if (sc_cb == NULL) scr_trap("scriptc: native callback invoked outside its call-scoped lifetime\\n");`, + ` ScrClosure *sc_cb = ${contextParam >= 0 ? `(ScrClosure *)sc_ctx` : adapter.tls ?? adapter.global};`, + ` if (sc_cb == NULL) scr_trap("scriptc: native callback invoked outside its ${adapter.callback.lifetime === "call" ? "call-scoped" : "retained"} lifetime\\n");`, ); const dummy = ffiCallbackDummyC(cb); out.push(` if (scr_exc_pending()) ${cb.returns === "void" ? "return;" : `return ${dummy};`}`); @@ -1816,11 +1830,24 @@ export class CEmitter { } }); const call = `(${cFnPtrCast(ft)}sc_cb->fn)(sc_cb${scriptArgs.length ? `, ${scriptArgs.join(", ")}` : ""})`; + if (cb.lifetime === "retained") { + // The callback may unregister or replace its own descriptor. Hold + // one invocation reference so that dropping the table's last pin + // cannot free the executing closure or its captures mid-call. + out.push(` scr_closure_retain(sc_cb);`); + } if (cb.returns === "void") { - out.push(` ${call};`, ` return;`, `}`, ``); + out.push( + ` ${call};`, + ...(cb.lifetime === "retained" ? [` scr_closure_release(sc_cb);`] : []), + ` return;`, + `}`, + ``, + ); continue; } out.push(` ${cDecl(ft.ret, "sc_result")} = ${call};`); + if (cb.lifetime === "retained") out.push(` scr_closure_release(sc_cb);`); switch (cb.returns) { case "f64": out.push(` return sc_result;`); diff --git a/packages/compiler/src/backend/emission/may-throw.ts b/packages/compiler/src/backend/emission/may-throw.ts index 4ec560730..6a01c950a 100644 --- a/packages/compiler/src/backend/emission/may-throw.ts +++ b/packages/compiler/src/backend/emission/may-throw.ts @@ -34,6 +34,11 @@ export function computeMayThrow(mod: IrModule): { fns: Set; indirect: bo .filter((entry) => entry.params.some(isFfiCallbackParam)) .map((entry) => entry.name), ); + const manifestHasRetainedCallback = (mod.ffiImports ?? []).some( + (entry) => entry.params.some( + (param) => isFfiCallbackParam(param) && param.callback.lifetime === "retained", + ), + ); // Method name → every class's implementation of it (virtualCall callees). const methodImpls = new Map(); for (const cls of mod.classes ?? []) { @@ -169,10 +174,13 @@ export function computeMayThrow(mod: IrModule): { fns: Set; indirect: bo if (MAY_THROW_LIB_FNS.has(rec["fn"] as IrLibFn)) f.throws = true; break; case "ffiCall": - // A call-scoped native callback may run arbitrary scriptc code. - // Its exception stays pending until the outer native call - // returns, where the emitter checks and unwinds normally. - if (callbackFfiImports.has(rec["import"] as string)) f.throws = true; + // A native callback may run arbitrary scriptc code. With retained + // descriptors any manifest binding may pump a previously stored + // callback, so every FFI call is conservatively a checkpoint. + if ( + callbackFfiImports.has(rec["import"] as string) || + manifestHasRetainedCallback + ) f.throws = true; break; case "bytesNew": { // The size form (`new Uint8Array(n)`) throws Node's "Invalid diff --git a/packages/compiler/src/backend/ffi-callbacks.ts b/packages/compiler/src/backend/ffi-callbacks.ts index 997ec6f84..4bed076f1 100644 --- a/packages/compiler/src/backend/ffi-callbacks.ts +++ b/packages/compiler/src/backend/ffi-callbacks.ts @@ -3,8 +3,12 @@ import { isFfiCallbackParam, isFfiContextParam } from "../ir/nodes.js"; export interface FfiCallbackAdapter { symbol: string; - /** Raw callbacks have no ABI context parameter and borrow this TLS slot. */ + /** Raw call-scoped callbacks borrow this thread-local slot. */ tls: string | null; + /** Raw retained callbacks occupy this process-global replaceable slot. */ + global: string | null; + /** Every retained descriptor owns one counted registration table. */ + table: string | null; callback: IrFfiCallbackParam["callback"]; } @@ -24,16 +28,35 @@ export function allocateFfiCallbackAdapters( const hasContext = param.callback.params.some(isFfiContextParam); let symbol: string; let tls: string | null; + let global: string | null; + let table: string | null; do { const index = suffix++; symbol = `sc_ffi_cb_${index}`; - tls = hasContext ? null : `sc_ffi_cb_ctx_${index}`; - } while (reserved.has(symbol) || (tls !== null && reserved.has(tls))); + tls = !hasContext && param.callback.lifetime === "call" + ? `sc_ffi_cb_ctx_${index}` + : null; + global = !hasContext && param.callback.lifetime === "retained" + ? `sc_ffi_cb_retained_${index}` + : null; + table = param.callback.lifetime === "retained" + ? `sc_ffi_cb_table_${index}` + : null; + } while ( + reserved.has(symbol) || + (tls !== null && reserved.has(tls)) || + (global !== null && reserved.has(global)) || + (table !== null && reserved.has(table)) + ); reserved.add(symbol); if (tls !== null) reserved.add(tls); + if (global !== null) reserved.add(global); + if (table !== null) reserved.add(table); adapters.set(`${entry.name}:${param.callback.id}`, { symbol, tls, + global, + table, callback: param.callback, }); } diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 34180a90c..56a7e03ff 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -67,6 +67,7 @@ import type { IrFfiCallbackParam, IrFfiCallbackParamClass, IrFfiImport, + IrFfiReleaseParam, IrFfiReturnClass, IrFfiValueParamClass, IrFunction, @@ -79,7 +80,7 @@ import type { IrUnionDef, SrcLoc, } from "../../ir/nodes.js"; -import { canMarshalFuncIntoIsland, CAUGHT, DYN, F64, ffiCallbackType, islandCallbackRet, islandPromisePayloadTag, isFfiCallbackParam, isFfiContextParam, isRefCounted, isUnitType, MAY_THROW_LIB_FNS, moduleEmbedsBuiltin, moduleEmbedsCompressedNpm, moduleUsesDynInvoke, moduleUsesFetch, moduleUsesFsWatch, moduleUsesHttpServer, moduleUsesNet, moduleUsesNodeTest, moduleUsesProcessEvents, moduleUsesStream, moduleUsesTls, moduleUsesTlsCa, NPM_COMPRESS_MIN, RUNTIME_EMITTER_CLASS, RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, STRING, typeEquals, typeKey, VOID } from "../../ir/nodes.js"; +import { canMarshalFuncIntoIsland, CAUGHT, DYN, F64, ffiCallbackType, islandCallbackRet, islandPromisePayloadTag, isFfiCallbackParam, isFfiContextParam, isFfiReleaseParam, isRefCounted, isUnitType, MAY_THROW_LIB_FNS, moduleEmbedsBuiltin, moduleEmbedsCompressedNpm, moduleUsesDynInvoke, moduleUsesFetch, moduleUsesFsWatch, moduleUsesHttpServer, moduleUsesNet, moduleUsesNodeTest, moduleUsesProcessEvents, moduleUsesStream, moduleUsesTls, moduleUsesTlsCa, NPM_COMPRESS_MIN, RUNTIME_EMITTER_CLASS, RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, STRING, typeEquals, typeKey, VOID } from "../../ir/nodes.js"; import { matchIntegerBytesForLoop } from "../../ir/integer-loops.js"; import { allocateFfiCallbackAdapters, type FfiCallbackAdapter } from "../ffi-callbacks.js"; import { computeMayThrow } from "../emission/may-throw.js"; @@ -1289,9 +1290,14 @@ class LlEmitter { this.declare(`declare zeroext i1 @scr_exc_pending()`); this.declare(`declare void @scr_trap(ptr)`); const expired = this.cstr("scriptc: native callback invoked outside its call-scoped lifetime\n"); + const released = this.cstr("scriptc: native callback invoked outside its retained lifetime\n"); for (const adapter of this.ffiCallbackAdapters.values()) { const cb = adapter.callback; if (adapter.tls !== null) globals.push(`@${adapter.tls} = internal thread_local global ptr null`); + if (adapter.global !== null) globals.push(`@${adapter.global} = internal global ptr null`); + if (adapter.table !== null) { + globals.push(`@${adapter.table} = internal global %ScrFfiTable zeroinitializer`); + } const params = cb.params.flatMap((param, i): string[] => { if (isFfiContextParam(param)) return [`ptr %ctx`]; if (param === "string" || param === "bytes") { @@ -1303,11 +1309,15 @@ class LlEmitter { defs.push( `define internal ${ret} @${adapter.symbol}(${params.join(", ")}) ${FN_ATTRS} {`, `entry:`, - adapter.tls === null ? ` %cb = getelementptr inbounds i8, ptr %ctx, i64 0` : ` %cb = load ptr, ptr @${adapter.tls}`, + adapter.tls !== null + ? ` %cb = load ptr, ptr @${adapter.tls}` + : adapter.global !== null + ? ` %cb = load ptr, ptr @${adapter.global}` + : ` %cb = getelementptr inbounds i8, ptr %ctx, i64 0`, ` %missing = icmp eq ptr %cb, null`, ` br i1 %missing, label %expired, label %ready`, `expired:`, - ` call void @scr_trap(ptr ${expired})`, + ` call void @scr_trap(ptr ${adapter.callback.lifetime === "call" ? expired : released})`, ` unreachable`, `ready:`, ` %pending = call zeroext i1 @scr_exc_pending()`, @@ -1398,11 +1408,25 @@ class LlEmitter { const ft = ffiCallbackType(cb); const internalRet = this.llType(ft.ret); const callArgs = [`ptr %cb`, ...scriptArgs].join(", "); + if (cb.lifetime === "retained") { + this.declare(`declare ptr @scr_closure_retain_v(ptr)`); + this.declare(`declare void @scr_closure_release_v(ptr)`); + defs.push(` %invoke_pin = call ptr @scr_closure_retain_v(ptr %cb)`); + } if (cb.returns === "void") { - defs.push(` call void %fn(${callArgs})`, ` ret void`, `}`, ``); + defs.push( + ` call void %fn(${callArgs})`, + ...(cb.lifetime === "retained" ? [` call void @scr_closure_release_v(ptr %invoke_pin)`] : []), + ` ret void`, + `}`, + ``, + ); continue; } defs.push(` %result = call ${internalRet} %fn(${callArgs})`); + if (cb.lifetime === "retained") { + defs.push(` call void @scr_closure_release_v(ptr %invoke_pin)`); + } switch (cb.returns) { case "f64": defs.push(` ret double %result`); @@ -1714,6 +1738,7 @@ class LlEmitter { `%ScrVt = type { ${this.sizeType}, ${this.sizeType}, ptr }`, `%ScrUnion = type { ${this.sizeType}, i32, ptr, ptr, ptr, i64 }`, `%ScrClosure = type { ${this.sizeType}, ptr, ${this.sizeType}, ptr }`, + `%ScrFfiTable = type { ptr, ${this.sizeType}, ${this.sizeType}, ptr, i8 }`, `%ScrRegex = type { ${this.sizeType}, ptr, ptr, ptr }`, // ScrArr mirror { rc, len, cap, elem(i32+pad), elem_retain, // elem_release, elem_trace, data } — the immortal tagged-template @@ -6010,8 +6035,40 @@ class LlEmitter { const arg = args[sourceIndex++]!; sourceArgs.set(abiIndex, arg); if (isFfiCallbackParam(param)) callbackArgs.set(param.callback.id, arg); + if (isFfiReleaseParam(param)) callbackArgs.set(param.callback.release, arg); }); + const retainedRegistrations: { table: string; callback: LlValue; global: string | null }[] = []; + const retainedReleases: { table: string; callback: LlValue; global: string | null }[] = []; + for (const param of entry.params) { + if (isFfiCallbackParam(param) && param.callback.lifetime === "retained") { + const adapter = this.ffiCallbackAdapter(entry.name, param.callback.id); + const callback = callbackArgs.get(param.callback.id)!; + if (adapter.table === null) throw new Error("llvm emitter bug: retained callback has no table"); + retainedRegistrations.push({ table: adapter.table, callback, global: adapter.global }); + } else if (isFfiReleaseParam(param)) { + const split = param.callback.release.lastIndexOf(":"); + const adapter = this.ffiCallbackAdapter( + param.callback.release.slice(0, split), + param.callback.release.slice(split + 1), + ); + const callback = callbackArgs.get(param.callback.release)!; + if (adapter.table === null) throw new Error("llvm emitter bug: retained release has no table"); + retainedReleases.push({ table: adapter.table, callback, global: adapter.global }); + } + } + if (retainedRegistrations.length > 0) { + this.declare(`declare void @scr_ffi_retain(ptr, ptr)`); + this.declare(`declare void @scr_ffi_teardown(ptr)`); + } + for (const registration of retainedRegistrations) { + if (registration.global !== null) { + B.line(`call void @scr_ffi_teardown(ptr @${registration.table})`); + B.line(`store ptr ${registration.callback.name}, ptr @${registration.global}`); + } + B.line(`call void @scr_ffi_retain(ptr @${registration.table}, ptr ${registration.callback.name})`); + } + const rawContexts: { tls: string; previous: string }[] = []; for (const param of entry.params) { if (!isFfiCallbackParam(param)) continue; @@ -6032,6 +6089,16 @@ class LlEmitter { nativeArgs.push(`ptr @${adapter.symbol}`); return; } + if (isFfiReleaseParam(param)) { + const split = param.callback.release.lastIndexOf(":"); + const adapter = this.ffiCallbackAdapter( + param.callback.release.slice(0, split), + param.callback.release.slice(split + 1), + ); + nativeParamTypes.push("ptr"); + nativeArgs.push(`ptr @${adapter.symbol}`); + return; + } if (isFfiContextParam(param)) { const callback = callbackArgs.get(param.context); if (!callback) throw new Error(`llvm emitter bug: FFI context '${param.context}' has no callback arg`); @@ -6117,16 +6184,29 @@ class LlEmitter { B.line(`store ptr ${saved.previous}, ptr @${saved.tls}`); } }; - const callbacksMayThrow = callbackArgs.size > 0; + const finishRetainedReleases = (): void => { + if (retainedReleases.length > 0) this.declare(`declare void @scr_ffi_release(ptr, ptr)`); + for (const release of retainedReleases) { + B.line(`call void @scr_ffi_release(ptr @${release.table}, ptr ${release.callback.name})`); + if (release.global !== null) B.line(`store ptr null, ptr @${release.global}`); + } + }; + const callbacksMayThrow = callbackArgs.size > 0 || (this.mod.ffiImports ?? []).some( + (candidate) => candidate.params.some( + (param) => isFfiCallbackParam(param) && param.callback.lifetime === "retained", + ), + ); if (entry.returns === "void") { B.line(call); restoreRawContexts(); + finishRetainedReleases(); if (callbacksMayThrow) this.emitPendingCheck(); return { name: "", type: e.type }; } const raw = B.tmp(); B.line(`${raw} = ${call}`); restoreRawContexts(); + finishRetainedReleases(); if (entry.returns === "f64") { const result = { name: raw, type: e.type }; if (callbacksMayThrow) this.emitPendingCheck(); diff --git a/packages/compiler/src/diagnostics/diagnostic.ts b/packages/compiler/src/diagnostics/diagnostic.ts index b1e99643b..3f54d207d 100644 --- a/packages/compiler/src/diagnostics/diagnostic.ts +++ b/packages/compiler/src/diagnostics/diagnostic.ts @@ -107,6 +107,7 @@ export function ffiSignatureDiag(name: string, detail: string, loc: SrcLoc): Scr "FFI parameter classes: f64/u8/u32/i32 (TypeScript number), bool, string, and bytes " + "(Uint8Array/Buffer); format 2 also accepts call-scoped callback descriptors with explicit context slots; " + "format 3 callback parameters additionally accept cstring/string (TypeScript string) and bytes (Uint8Array); " + + "format 4 adds retained callback descriptors and explicit release references; " + "return classes: f64/u8/u32/i32, bool, and void", }; } diff --git a/packages/compiler/src/ffi/profile.ts b/packages/compiler/src/ffi/profile.ts index 0e7669759..840e24932 100644 --- a/packages/compiler/src/ffi/profile.ts +++ b/packages/compiler/src/ffi/profile.ts @@ -32,14 +32,26 @@ * ] * * Context entries are compiler-supplied and consume no TypeScript - * parameter. `lifetime: "call"` is deliberately the only policy today: - * native code may invoke the callback synchronously on the calling thread - * and must not retain either pointer after the outer call returns. + * parameter. `lifetime: "call"` lets native code invoke the callback only + * during the outer call. Format 4 adds `lifetime: "retained"` plus release + * descriptors that reuse a registration's callback ABI and trampoline. * * Format 3 preserves format 2 and adds copy-in callback parameters: * `cstring` is one NUL-terminated pointer, while `string` and `bytes` are * pointer+length spans. The copies have ordinary scriptc ownership and no - * lifetime relationship to the native storage. */ + * lifetime relationship to the native storage. + * + * Format 4 preserves format 3 and adds retained callback registration: + * + * { "callback": { "id": "tick", "params": [{ "context": "tick" }], + * "returns": "void", "lifetime": "retained" } } + * + * A paired release parameter names that descriptor. Its resolved profile + * node carries the inherited ABI, but those fields are never accepted from + * JSON: + * + * { "callback": { "release": "timerAdd:tick" } } + * { "context": "timerAdd:tick" } */ import { readFileSync, statSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { ffiProfileDiag, type ScrDiagnostic } from "../diagnostics/diagnostic.js"; @@ -83,12 +95,27 @@ export interface FfiCallbackParam { /** Exact native callback ABI; a context entry consumes no TS argument. */ params: (FfiCallbackParamClass | FfiContextParam)[]; returns: FfiReturnClass; - /** Callbacks are valid only for the dynamic extent of the outer call. */ - lifetime: "call"; + /** Dynamic-extent borrow, or explicitly paired retained registration. */ + lifetime: "call" | "retained"; + }; +} + +export interface FfiReleaseParam { + callback: { + /** `:` of the retained descriptor being released. */ + release: string; + /** Inherited from the target after whole-manifest cross-checking. */ + params: (FfiCallbackParamClass | FfiContextParam)[]; + /** Inherited from the target after whole-manifest cross-checking. */ + returns: FfiReturnClass; }; } -export type FfiParamClass = FfiValueParamClass | FfiCallbackParam | FfiContextParam; +export type FfiParamClass = + | FfiValueParamClass + | FfiCallbackParam + | FfiReleaseParam + | FfiContextParam; export interface FfiFunction { /** The signature-only TypeScript function declaration's binding name. */ @@ -100,7 +127,7 @@ export interface FfiFunction { } export interface FfiProfile { - ffiFormat: 1 | 2 | 3; + ffiFormat: 1 | 2 | 3 | 4; functions: FfiFunction[]; /** Absolute paths, resolved relative to the manifest. */ libraries: string[]; @@ -152,7 +179,17 @@ function contextParam(value: unknown, path: string): FfiContextParam | null { return { context: stringField(rec["context"], `${path}.context`) }; } -function callbackParam(value: unknown, path: string, format: 2 | 3): FfiCallbackParam | null { +interface UnresolvedFfiReleaseParam { + callback: { release: string }; +} + +type UnresolvedFfiParamClass = FfiParamClass | UnresolvedFfiReleaseParam; + +function callbackParam( + value: unknown, + path: string, + format: 2 | 3 | 4, +): FfiCallbackParam | UnresolvedFfiReleaseParam | null { if (value === null || typeof value !== "object" || Array.isArray(value)) return null; const rec = value as Record; if (!("callback" in rec)) return null; @@ -162,6 +199,15 @@ function callbackParam(value: unknown, path: string, format: 2 | 3): FfiCallback throw new FfiProfileError(`'${path}.callback' must be an object`); } const callback = raw as Record; + if ("release" in callback) { + if (format < 4) { + throw new FfiProfileError(`'${path}.callback.release' requires ffi_format 4`); + } + rejectUnknownKeys(callback, `${path}.callback`, ["release"]); + return { + callback: { release: stringField(callback["release"], `${path}.callback.release`) }, + }; + } rejectUnknownKeys(callback, `${path}.callback`, ["id", "params", "returns", "lifetime"]); const id = stringField(callback["id"], `${path}.callback.id`); if (!TS_IDENT.test(id)) { @@ -172,7 +218,7 @@ function callbackParam(value: unknown, path: string, format: 2 | 3): FfiCallback } const params = callback["params"].map((entry, i): FfiCallbackParamClass | FfiContextParam => { const entryPath = `${path}.callback.params[${i}]`; - const allowed = format === 3 + const allowed = format >= 3 ? FFI_CALLBACK_PARAM_CLASSES : FFI_FORMAT_2_CALLBACK_PARAM_CLASSES; if ( @@ -199,15 +245,19 @@ function callbackParam(value: unknown, path: string, format: 2 | 3): FfiCallback `'${path}.callback.returns' must be one of ${FFI_RETURN_CLASSES.join("/")}, got '${returns}'`, ); } - if (callback["lifetime"] !== "call") { - throw new FfiProfileError(`'${path}.callback.lifetime' must be 'call'`); + const lifetime = callback["lifetime"]; + if (lifetime !== "call" && lifetime !== "retained") { + throw new FfiProfileError(`'${path}.callback.lifetime' must be 'call' or 'retained'`); + } + if (lifetime === "retained" && format < 4) { + throw new FfiProfileError(`'${path}.callback.lifetime' value 'retained' requires ffi_format 4`); } return { callback: { id, params, returns: returns as FfiReturnClass, - lifetime: "call", + lifetime, }, }; } @@ -238,11 +288,11 @@ export function loadFfiProfile( } const root = raw as Record; const format = root["ffi_format"]; - if (format !== 1 && format !== 2 && format !== 3) { + if (format !== 1 && format !== 2 && format !== 3 && format !== 4) { throw new FfiProfileError( typeof format === "number" - ? `unsupported ffi_format ${format} (this scriptc reads formats 1, 2, and 3)` - : "'ffi_format' must be the number 1, 2, or 3", + ? `unsupported ffi_format ${format} (this scriptc reads formats 1, 2, 3, and 4)` + : "'ffi_format' must be the number 1, 2, 3, or 4", ); } rejectUnknownKeys(root, "", [ @@ -258,7 +308,7 @@ export function loadFfiProfile( } const names = new Set(); const symbols = new Set(); - const functions = functionsRaw.map((entry, i): FfiFunction => { + const functions = functionsRaw.map((entry, i): Omit & { params: UnresolvedFfiParamClass[] } => { const path = `functions[${i}]`; if (entry === null || typeof entry !== "object" || Array.isArray(entry)) { throw new FfiProfileError(`'${path}' must be an object`); @@ -286,14 +336,14 @@ export function loadFfiProfile( if (!Array.isArray(row["params"])) { throw new FfiProfileError(`'${path}.params' must be an array`); } - const params = row["params"].map((value, j): FfiParamClass => { + const params = row["params"].map((value, j): UnresolvedFfiParamClass => { const paramPath = `${path}.params[${j}]`; if ( typeof value !== "string" || !(FFI_PARAM_CLASSES as readonly string[]).includes(value) ) { if (format >= 2) { - const callback = callbackParam(value, paramPath, format === 2 ? 2 : 3); + const callback = callbackParam(value, paramPath, format as 2 | 3 | 4); if (callback !== null) return callback; const context = contextParam(value, paramPath); if (context !== null) return context; @@ -308,20 +358,30 @@ export function loadFfiProfile( }); if (format >= 2) { const callbacks = new Map(); + const releases = new Set(); const outerContexts = new Map(); for (const param of params) { if (typeof param === "object" && "callback" in param) { const cb = param.callback; - if (callbacks.has(cb.id)) { - throw new FfiProfileError(`callback id '${cb.id}' is declared twice in '${path}.params'`); + if ("release" in cb) { + if (releases.has(cb.release)) { + throw new FfiProfileError( + `callback release '${cb.release}' is declared twice in '${path}.params'`, + ); + } + releases.add(cb.release); + } else { + if (callbacks.has(cb.id)) { + throw new FfiProfileError(`callback id '${cb.id}' is declared twice in '${path}.params'`); + } + callbacks.set(cb.id, cb); } - callbacks.set(cb.id, cb); } else if (typeof param === "object") { outerContexts.set(param.context, (outerContexts.get(param.context) ?? 0) + 1); } } for (const [id, count] of outerContexts) { - if (!callbacks.has(id)) { + if (!callbacks.has(id) && !releases.has(id)) { throw new FfiProfileError(`context '${id}' in '${path}.params' has no matching callback`); } if (count !== 1) { @@ -361,6 +421,78 @@ export function loadFfiProfile( return { name, symbol, params, returns: returns as FfiReturnClass }; }); + if (format >= 4) { + const retained = new Map(); + for (const fn of functions) { + for (const param of fn.params) { + if ( + typeof param === "object" && + "callback" in param && + !("release" in param.callback) && + param.callback.lifetime === "retained" + ) { + retained.set(`${fn.name}:${param.callback.id}`, param.callback); + } + } + } + for (const [i, fn] of functions.entries()) { + for (const [j, param] of fn.params.entries()) { + if ( + typeof param !== "object" || + !("callback" in param) || + !("release" in param.callback) + ) continue; + const target = param.callback.release; + const descriptor = retained.get(target); + if (descriptor === undefined) { + const [binding, id, ...extra] = target.split(":"); + const candidate = extra.length === 0 && binding !== undefined && id !== undefined + ? functions.find((entry) => entry.name === binding)?.params.find( + (entry) => + typeof entry === "object" && + "callback" in entry && + !("release" in entry.callback) && + entry.callback.id === id, + ) + : undefined; + if ( + candidate !== undefined && + typeof candidate === "object" && + "callback" in candidate && + !("release" in candidate.callback) + ) { + throw new FfiProfileError( + `release '${target}' in 'functions[${i}].params[${j}]' targets a non-retained callback`, + ); + } + throw new FfiProfileError( + `release '${target}' in 'functions[${i}].params[${j}]' has no matching retained callback`, + ); + } + const inheritedContext = descriptor.params.some( + (entry) => typeof entry === "object", + ); + const contextCount = fn.params.filter( + (entry) => typeof entry === "object" && "context" in entry && entry.context === target, + ).length; + if (inheritedContext !== (contextCount === 1)) { + throw new FfiProfileError( + `release '${target}' must declare its context exactly once in the native function parameter list when the retained callback has a context`, + ); + } + fn.params[j] = { + callback: { + release: target, + params: descriptor.params, + returns: descriptor.returns, + }, + }; + } + } + } + + const resolvedFunctions = functions as FfiFunction[]; + const libraries = stringArray(root["libraries"], "libraries").map((path) => resolve(dirname(profilePath), path) ); @@ -399,7 +531,7 @@ export function loadFfiProfile( ok: true, profile: { ffiFormat: format, - functions, + functions: resolvedFunctions, libraries, systemLibraries, }, diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 31861a8b2..f6bbfdb3c 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -5,8 +5,8 @@ import * as ts from "../ts7/adapter.js"; import type { Lowerer } from "./lowerer.js"; import { lowerGenMethodCall } from "./lower-generators.js"; -import { BOOL, CAUGHT, DYN, F64, IrExpr, IrFunction, IrLocal, IrParam, IrStmt, IrType, JSVAL, STRING, SYMBOL_T, SrcLoc, UNDEFINED_T, VOID, arrayOf, canBoxFuncIntoDyn, canConvertToDyn, canDynCheckTo, canMarshalTypedFuncIntoIsland, ffiClassType, ffiSourceParamTypes, funcOf, isFfiCallbackParam, isFfiContextParam, isUnitType, shapeHasAccessorSlots, typeEquals } from "../../ir/nodes.js"; -import type { IrFfiCallbackParam, IrFfiCallbackParamClass, IrFfiImport } from "../../ir/nodes.js"; +import { BOOL, CAUGHT, DYN, F64, IrExpr, IrFunction, IrLocal, IrParam, IrStmt, IrType, JSVAL, STRING, SYMBOL_T, SrcLoc, UNDEFINED_T, VOID, arrayOf, canBoxFuncIntoDyn, canConvertToDyn, canDynCheckTo, canMarshalTypedFuncIntoIsland, ffiClassType, ffiSourceParamTypes, funcOf, isFfiCallbackParam, isFfiContextParam, isFfiReleaseParam, isUnitType, shapeHasAccessorSlots, typeEquals } from "../../ir/nodes.js"; +import type { IrFfiCallbackParam, IrFfiCallbackParamClass, IrFfiImport, IrFfiReleaseParam } from "../../ir/nodes.js"; import { isJsSourceFile, locOf } from "../program.js"; import { isGenericCallableMemberType, typeKey } from "../types.js"; import { PoisonError, dynFallbackType, dynUndefinedExpr, importCallHandleType, jsFuncNameOf, newFnCtx, nodeThrowExpr } from "./lowerer.js"; @@ -2479,7 +2479,11 @@ function ffiSourceParams(binding: IrFfiImport): Exclude[number]): string { - return isFfiCallbackParam(param) ? `callback '${param.callback.id}'` : `class '${param}'`; + return isFfiCallbackParam(param) + ? `callback '${param.callback.id}'` + : isFfiReleaseParam(param) + ? `release callback '${param.callback.release}'` + : `class '${param}'`; } /** Callback arguments flow from native code into TypeScript. Their source @@ -2488,7 +2492,7 @@ function ffiParamDisplay(param: ReturnType[number]): str * enum, or `never` from an unrestricted `number`. */ function ffiCallbackInputDiagnostic( L: Lowerer, - descriptor: IrFfiCallbackParam, + descriptor: IrFfiCallbackParam | IrFfiReleaseParam, callbackType: ts.Type, ): string | null { const signatures = L.checker.getCallSignatures(callbackType); @@ -2527,8 +2531,11 @@ function ffiCallbackInputDiagnostic( ? (paramType.flags & ts.TypeFlags.String) !== 0 : (paramType.flags & ts.TypeFlags.Number) !== 0; if (!coversDomain) { + const descriptorName = isFfiCallbackParam(descriptor) + ? descriptor.callback.id + : descriptor.callback.release; return ( - `callback '${descriptor.callback.id}' parameter ${i + 1} is '${L.checker.typeToString(paramType)}', ` + + `callback '${descriptorName}' parameter ${i + 1} is '${L.checker.typeToString(paramType)}', ` + `but native class '${nativeClass}' may supply any ${domain}; declare it as '${domain}'` ); } @@ -2638,7 +2645,7 @@ function ffiDeclarationDiagnostic( loc, ); } - if (isFfiCallbackParam(sourceParam)) { + if (isFfiCallbackParam(sourceParam) || isFfiReleaseParam(sourceParam)) { const callbackDiagnostic = ffiCallbackInputDiagnostic(L, sourceParam, paramType); if (callbackDiagnostic !== null) { return signatureDiag(binding.name, callbackDiagnostic, loc); @@ -2905,9 +2912,35 @@ export function lowerFfiCall(L: Lowerer, expr: ts.CallExpression): IrExpr | null ); } const expectedReturn = ffiClassType(binding.returns); - const args = expr.arguments.map((arg, i) => - L.lowerExprExpecting(arg, expectedParams[i]!) - ); + const sourceParams = ffiSourceParams(binding); + const args = expr.arguments.map((arg, i) => { + const sourceParam = sourceParams[i]!; + const expected = expectedParams[i]!; + const lowered = L.lowerExprExpecting(arg, expected); + // Retained identity is the runtime closure pointer. A coercion adapter + // would be freshly allocated at registration and release sites, so an + // assignable-but-different function shape (notably `() => number` into + // `() => void`) cannot honestly participate in explicit release. + if ( + ( + isFfiReleaseParam(sourceParam) || + (isFfiCallbackParam(sourceParam) && sourceParam.callback.lifetime === "retained") + ) && + ( + lowered.kind === "dynCheck" || + (lowered.kind === "call" && + (lowered.callee.startsWith("%fn.adapt.") || + lowered.callee.startsWith("%fn.width.") || + lowered.callee.startsWith("%fnval."))) + ) + ) { + signatureError( + `retained callback argument ${i + 1} must have the exact manifest function type; ` + + `an implicit function adapter would change its release identity`, + ); + } + return lowered; + }); return { kind: "ffiCall", import: binding.name, diff --git a/packages/compiler/src/ir/nodes.ts b/packages/compiler/src/ir/nodes.ts index 20fb380b3..c3471f36b 100644 --- a/packages/compiler/src/ir/nodes.ts +++ b/packages/compiler/src/ir/nodes.ts @@ -785,14 +785,33 @@ export interface IrFfiCallbackParam { /** Exact callback ABI order; context entries consume no TS argument. */ params: (IrFfiCallbackParamClass | IrFfiContextParam)[]; returns: IrFfiReturnClass; - lifetime: "call"; + lifetime: "call" | "retained"; }; } -export type IrFfiParam = IrFfiValueParamClass | IrFfiCallbackParam | IrFfiContextParam; +export interface IrFfiReleaseParam { + callback: { + /** `:` of the retained registration descriptor. */ + release: string; + /** Resolved/inherited callback ABI. */ + params: (IrFfiCallbackParamClass | IrFfiContextParam)[]; + /** Resolved/inherited callback return ABI. */ + returns: IrFfiReturnClass; + }; +} + +export type IrFfiParam = + | IrFfiValueParamClass + | IrFfiCallbackParam + | IrFfiReleaseParam + | IrFfiContextParam; export function isFfiCallbackParam(param: IrFfiParam): param is IrFfiCallbackParam { - return typeof param === "object" && "callback" in param; + return typeof param === "object" && "callback" in param && "id" in param.callback; +} + +export function isFfiReleaseParam(param: IrFfiParam): param is IrFfiReleaseParam { + return typeof param === "object" && "callback" in param && "release" in param.callback; } export function isFfiContextParam( @@ -822,7 +841,7 @@ export function ffiClassType( /** The ordinary TypeScript function type a native callback descriptor consumes. */ export function ffiCallbackType( - callback: IrFfiCallbackParam["callback"], + callback: IrFfiCallbackParam["callback"] | IrFfiReleaseParam["callback"], ): IrType & { kind: "func" } { return { kind: "func", @@ -838,16 +857,17 @@ export function ffiSourceParamTypes(params: readonly IrFfiParam[]): IrType[] { return params.flatMap((param): IrType[] => { if (isFfiContextParam(param)) return []; if (isFfiCallbackParam(param)) return [ffiCallbackType(param.callback)]; + if (isFfiReleaseParam(param)) return [ffiCallbackType(param.callback)]; return [ffiClassType(param)]; }); } /** One outbound native FFI declaration. Format 1 contains only value * classes. Format 2 additionally carries exact-position callback/context - * entries. Format 3 adds callback copy-in cstrings and spans. Outer - * string/bytes values still expand to pointer+length pairs; callbacks and - * contexts are each one native pointer slot. Callback lifetimes are - * call-scoped and their closure/context storage is borrowed. */ + * entries. Format 3 adds callback copy-in cstrings and spans. Format 4 adds + * retained callbacks and resolved release entries. Outer string/bytes + * values still expand to pointer+length pairs; callbacks, releases, and + * contexts are each one native pointer slot. */ export interface IrFfiImport { /** The signature-only ambient TypeScript binding name. */ name: string; diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index f7f99e815..d3216ee70 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -18,7 +18,7 @@ import type { IrUnionDef, SrcLoc, } from "./nodes.js"; -import { arrayOf, BOOL, BYTES_U8, bytesOf, canAdaptDynFuncTo, canConvertToDyn, canExitIslandToType, canMarshalIntoIsland, canMarshalTypedFuncIntoIsland, CHILD_T, CHILDSTREAM_T, DATE_T, DGRAMSOCK_T, DYN, DYN_HANDLE_KINDS, F64, ffiClassType, ffiSourceParamTypes, FILEHANDLE_T, FSWATCHER_T, HTTP2SESSION_T, HTTP2STREAM_T, HTTPCLIENTREQ_T, HTTPREQ_T, HTTPRES_T, islandPromisePayloadTag, isJsonSafeType, isRefCounted, isSupportedIndexValue, isSupportedMapKey, isSupportedMapValue, isSupportedSetElem, isUnitType, jsOpResultKind, JSVAL, NETSERVER_T, NETSOCKET_T, PROCSTREAM_T, REF_TRUTHY_KINDS, REGEX, RUNTIME_EMITTER_CLASS, RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, SEARCH_PARAMS_T, SECURECTX_T, SPAWNRES_T, STATS_T, STRING, SYMBOL_T, TESTCTX_T, typeEquals, typeKey, unionFuncSetArmsOk, URL_T, VOID } from "./nodes.js"; +import { arrayOf, BOOL, BYTES_U8, bytesOf, canAdaptDynFuncTo, canConvertToDyn, canExitIslandToType, canMarshalIntoIsland, canMarshalTypedFuncIntoIsland, CHILD_T, CHILDSTREAM_T, DATE_T, DGRAMSOCK_T, DYN, DYN_HANDLE_KINDS, F64, ffiClassType, ffiSourceParamTypes, FILEHANDLE_T, FSWATCHER_T, HTTP2SESSION_T, HTTP2STREAM_T, HTTPCLIENTREQ_T, HTTPREQ_T, HTTPRES_T, islandPromisePayloadTag, isFfiCallbackParam, isFfiContextParam, isFfiReleaseParam, isJsonSafeType, isRefCounted, isSupportedIndexValue, isSupportedMapKey, isSupportedMapValue, isSupportedSetElem, isUnitType, jsOpResultKind, JSVAL, NETSERVER_T, NETSOCKET_T, PROCSTREAM_T, REF_TRUTHY_KINDS, REGEX, RUNTIME_EMITTER_CLASS, RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, SEARCH_PARAMS_T, SECURECTX_T, SPAWNRES_T, STATS_T, STRING, SYMBOL_T, TESTCTX_T, typeEquals, typeKey, unionFuncSetArmsOk, URL_T, VOID } from "./nodes.js"; /** Per-method signature for strIntrinsic: `argTypes` lists every argument * position (optional ones included); `minArgs` is how many may be omitted @@ -1182,6 +1182,49 @@ export function validateModule(mod: IrModule): IrValidationError[] { ffiByName.set(entry.name, entry); ffiSymbols.add(entry.symbol); } + const retainedFfiCallbacks = new Map[number]["params"][number], { callback: { id: string } }>["callback"]>(); + for (const entry of mod.ffiImports ?? []) { + const ids = new Set(); + for (const param of entry.params) { + if (!isFfiCallbackParam(param)) continue; + if (ids.has(param.callback.id)) { + errors.push({ message: `FFI binding "${entry.name}" has duplicate callback id "${param.callback.id}"`, loc: moduleLoc }); + } + ids.add(param.callback.id); + if (param.callback.lifetime === "retained") { + retainedFfiCallbacks.set(`${entry.name}:${param.callback.id}`, param.callback); + } + const hasInnerContext = param.callback.params.some(isFfiContextParam); + const outerContexts = entry.params.filter( + (candidate) => isFfiContextParam(candidate) && candidate.context === param.callback.id, + ).length; + if (hasInnerContext !== (outerContexts === 1)) { + errors.push({ message: `FFI callback "${entry.name}:${param.callback.id}" has inconsistent context slots`, loc: moduleLoc }); + } + } + } + for (const entry of mod.ffiImports ?? []) { + for (const param of entry.params) { + if (!isFfiReleaseParam(param)) continue; + const target = retainedFfiCallbacks.get(param.callback.release); + if (target === undefined) { + errors.push({ message: `FFI release "${entry.name}:${param.callback.release}" has no retained target`, loc: moduleLoc }); + continue; + } + const inherited = JSON.stringify(param.callback.params) === JSON.stringify(target.params) && + param.callback.returns === target.returns; + if (!inherited) { + errors.push({ message: `FFI release "${entry.name}:${param.callback.release}" does not inherit its target ABI`, loc: moduleLoc }); + } + const hasInnerContext = target.params.some(isFfiContextParam); + const outerContexts = entry.params.filter( + (candidate) => isFfiContextParam(candidate) && candidate.context === param.callback.release, + ).length; + if (hasInnerContext !== (outerContexts === 1)) { + errors.push({ message: `FFI release "${entry.name}:${param.callback.release}" has inconsistent context slots`, loc: moduleLoc }); + } + } + } // The lib section (library mode): every mapped function exists, is // synchronous, and its IR signature fits the declared marshalling // classes — the SC4xxx refusals ran before this landed on the module, so diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index f95c67b2a..5bea3e0bf 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -2618,6 +2618,11 @@ bool scr_loop_run(ScrPromise *top_level) { * them as leaks. Uncaught callback throws still return above for main * to report through the existing exceptional teardown path. */ scr_timers_teardown(); + /* Retained native registrations carry no fd and do not hold liveness. + * Once the loop is exhausted the process cannot meaningfully pump their + * libraries again, so drop the ledger's closure references here. The + * idempotent atexit sweep covers synchronous and exceptional exits. */ + scr_ffi_teardown_all(); /* Same story for unref'd children the loop never reaped: release the * registry's references (their listeners never fire — the process is * exiting, Node's behavior; the OS reparents the children). */ diff --git a/packages/runtime/src/scr_ffi.c b/packages/runtime/src/scr_ffi.c new file mode 100644 index 000000000..6804d3c75 --- /dev/null +++ b/packages/runtime/src/scr_ffi.c @@ -0,0 +1,62 @@ +#include "scr_runtime.h" + +#include + +/* Retained callbacks are same-thread only in format 4, so the registration + * ledger deliberately has no synchronization. The global list exists only + * to make every still-live registration release before the executable RC + * audit. A table remains linked after an explicit teardown so a later + * registration can reuse it without mutating the list twice. */ +static ScrFfiTable *scr_ffi_tables; +static bool scr_ffi_exit_registered; + +static void scr_ffi_oom(void) { scr_trap("scriptc: out of memory\n"); } + +void scr_ffi_teardown(ScrFfiTable *table) { + for (size_t i = 0; i < table->len; i++) { + scr_closure_release(table->entries[i]); + } + free(table->entries); + table->entries = NULL; + table->len = 0; + table->cap = 0; +} + +void scr_ffi_teardown_all(void) { + for (ScrFfiTable *table = scr_ffi_tables; table != NULL; table = table->next) { + scr_ffi_teardown(table); + } +} + +void scr_ffi_retain(ScrFfiTable *table, ScrClosure *callback) { + if (!table->linked) { + table->linked = true; + table->next = scr_ffi_tables; + scr_ffi_tables = table; + } + if (!scr_ffi_exit_registered) { + scr_ffi_exit_registered = true; + scr_atexit(scr_ffi_teardown_all); + } + if (table->len == table->cap) { + size_t cap = table->cap == 0 ? 4 : table->cap * 2; + if (cap < table->cap || cap > SIZE_MAX / sizeof *table->entries) scr_ffi_oom(); + ScrClosure **entries = realloc(table->entries, cap * sizeof *entries); + if (entries == NULL) scr_ffi_oom(); + table->entries = entries; + table->cap = cap; + } + table->entries[table->len++] = scr_closure_retain(callback); +} + +void scr_ffi_release(ScrFfiTable *table, ScrClosure *callback) { + for (size_t i = 0; i < table->len; i++) { + if (table->entries[i] != callback) continue; + ScrClosure *owned = table->entries[i]; + table->len--; + if (i != table->len) table->entries[i] = table->entries[table->len]; + scr_closure_release(owned); + return; + } + scr_trap("scriptc: releasing a native callback registration that does not exist\n"); +} diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index 6678e75bc..bf617a52d 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -1319,6 +1319,25 @@ static inline ScrClosure *scr_closure_retain(ScrClosure *c) { void scr_closure_release(ScrClosure *c); /* releases the boxes; NULL-tolerant */ +/* ── outbound FFI retained callbacks (scr_ffi.c) ───────────────────── + * One compiler-emitted table per retained callback descriptor. Entries are + * counted rather than deduplicated: registering the same closure twice + * requires two matching releases. The table owns one closure reference per + * entry and joins a process-global teardown list on first use. Retained + * callbacks do not contribute event-loop liveness. */ +typedef struct ScrFfiTable { + ScrClosure **entries; + size_t len; + size_t cap; + struct ScrFfiTable *next; + bool linked; +} ScrFfiTable; + +void scr_ffi_retain(ScrFfiTable *table, ScrClosure *callback); +void scr_ffi_release(ScrFfiTable *table, ScrClosure *callback); +void scr_ffi_teardown(ScrFfiTable *table); +void scr_ffi_teardown_all(void); + /* ── unions ───────────────────────────────────────────────────────── * A union value (`A | B`) is an IMMUTABLE tagged box: a refcounted header, * the arm's tag (its index in the compiler's canonical arm order), and one diff --git a/packages/runtime/test/bytes.test.ts b/packages/runtime/test/bytes.test.ts index b302594f2..305d21929 100644 --- a/packages/runtime/test/bytes.test.ts +++ b/packages/runtime/test/bytes.test.ts @@ -36,6 +36,7 @@ beforeAll(async () => { join(testDir, "../src/scr_number.c"), join(testDir, "../src/scr_console.c"), join(testDir, "../src/scr_closure.c"), + join(testDir, "../src/scr_ffi.c"), join(testDir, "../src/scr_object.c"), join(testDir, "../src/scr_union.c"), join(testDir, "../src/scr_cycle.c"), diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts index 2e6190bba..f20040548 100644 --- a/packages/runtime/test/runtime.test.ts +++ b/packages/runtime/test/runtime.test.ts @@ -8,7 +8,7 @@ const execFileAsync = promisify(execFile); const testDir = import.meta.dirname; const srcDir = join(testDir, "../src"); -const RUNTIME_SOURCES = ["scr_number.c", "scr_string.c", "scr_array.c", "scr_bytes.c", "scr_map.c", "scr_closure.c", "scr_object.c", "scr_union.c", "scr_exception.c", "scr_error.c", "scr_console.c", "scr_lib.c", "scr_json.c", "scr_async.c", "scr_child.c", "scr_cycle.c"].map( +const RUNTIME_SOURCES = ["scr_number.c", "scr_string.c", "scr_array.c", "scr_bytes.c", "scr_map.c", "scr_closure.c", "scr_ffi.c", "scr_object.c", "scr_union.c", "scr_exception.c", "scr_error.c", "scr_console.c", "scr_lib.c", "scr_json.c", "scr_async.c", "scr_child.c", "scr_cycle.c"].map( (f) => join(srcDir, f), ); diff --git a/tests/ffi/main.ts b/tests/ffi/main.ts index 6d925f232..3948f9dcc 100644 --- a/tests/ffi/main.ts +++ b/tests/ffi/main.ts @@ -31,6 +31,13 @@ declare function nativeCallbackSpans( ): void; declare function nativeCallbackStringThrow(callback: (value: string) => void): void; declare function nativeNullCString(callback: (value: string) => void): void; +declare function nativeRetainedAdd(callback: (value: number) => void): void; +declare function nativeRetainedRemove(callback: (value: number) => void): void; +declare function nativeRetainedPump(value: number): void; +declare function nativeRetainedFireFirst(value: number): void; +declare function nativeRetainedRawSet(callback: (value: number) => void): void; +declare function nativeRetainedRawRemove(callback: (value: number) => void): void; +declare function nativeRetainedRawPump(value: number): void; console.log(nativeScale(21)); console.log(nativeInvert(false), nativeInvert(true)); @@ -86,6 +93,82 @@ try { console.log("caught", (error as Error).message); } +const retainedEvents: string[] = []; +const retainedOffset = 10; +const retainedFirst = (value: number) => { + retainedEvents.push(`first:${value + retainedOffset}`); +}; +let retainedSecondTotal = 0; +const retainedSecond = (value: number) => { + retainedSecondTotal += value; + retainedEvents.push(`second:${retainedSecondTotal}`); +}; +nativeRetainedAdd(retainedFirst); +nativeRetainedAdd(retainedSecond); +nativeRetainedPump(1); +nativeRetainedRemove(retainedFirst); +nativeRetainedPump(2); +nativeRetainedRemove(retainedSecond); +console.log(retainedEvents.join("|")); + +let retainedDuplicateTotal = 0; +const retainedDuplicate = (value: number) => { + retainedDuplicateTotal += value; +}; +nativeRetainedAdd(retainedDuplicate); +nativeRetainedAdd(retainedDuplicate); +nativeRetainedPump(2); +nativeRetainedRemove(retainedDuplicate); +nativeRetainedPump(3); +nativeRetainedRemove(retainedDuplicate); +nativeRetainedPump(4); +console.log(retainedDuplicateTotal); + +const retainedThrow = (value: number) => { + throw new Error(`retained boom ${value}`); +}; +nativeRetainedAdd(retainedThrow); +try { + nativeRetainedPump(9); +} catch (error) { + console.log("caught", (error as Error).message); +} +nativeRetainedRemove(retainedThrow); + +let selfReleaseTotal = 0; +const selfRelease = (value: number) => { + selfReleaseTotal += value; + nativeRetainedRemove(selfRelease); +}; +nativeRetainedAdd(selfRelease); +nativeRetainedFireFirst(4); +nativeRetainedPump(5); +console.log(selfReleaseTotal); + +const rawEvents: number[] = []; +const rawOffset = 5; +const rawFirst = (value: number) => { + rawEvents.push(value + rawOffset); +}; +const rawSecond = (value: number) => { + rawEvents.push(value * 10); +}; +nativeRetainedRawSet(rawFirst); +nativeRetainedRawPump(1); +nativeRetainedRawSet(rawSecond); +nativeRetainedRawPump(2); +nativeRetainedRawRemove(rawSecond); +nativeRetainedRawPump(3); +console.log(rawEvents.join(" ")); + +// A still-live registration at normal process exit exercises the runtime's +// teardown path (the sanitized lane checks that its captured closure leaks +// neither the closure nor its capture box). +const exitCapture = "live-at-exit"; +nativeRetainedAdd((_value: number) => { + if (exitCapture.length === 0) console.log("unreachable"); +}); + try { nativeCallbackStringThrow((value) => { throw new Error(`string callback boom: ${value}`); diff --git a/tests/ffi/native.c b/tests/ffi/native.c index 584090780..4cee98991 100644 --- a/tests/ffi/native.c +++ b/tests/ffi/native.c @@ -120,3 +120,62 @@ void sf_callback_string_throw(sf_cstring_cb callback, void *context) { void sf_null_cstring(sf_cstring_cb callback, void *context) { callback(NULL, context); } + +/* Format-4 retained callbacks. This fixture deliberately stores the exact + * function/context pair and fires it only from a later pump call. Duplicate + * registrations are distinct native entries, matching the runtime ledger. */ +typedef void (*sf_retained_cb)(double value, void *context); + +typedef struct { + sf_retained_cb callback; + void *context; +} sf_retained_entry; + +static sf_retained_entry retained_entries[16]; +static size_t retained_len; + +void sf_retained_add(sf_retained_cb callback, void *context) { + if (retained_len < 16) { + retained_entries[retained_len++] = (sf_retained_entry){callback, context}; + } +} + +void sf_retained_remove(sf_retained_cb callback, void *context) { + for (size_t i = 0; i < retained_len; i++) { + if (retained_entries[i].callback != callback || + retained_entries[i].context != context) continue; + retained_len--; + for (size_t j = i; j < retained_len; j++) { + retained_entries[j] = retained_entries[j + 1]; + } + return; + } +} + +void sf_retained_pump(double value) { + size_t end = retained_len; + for (size_t i = 0; i < end; i++) { + retained_entries[i].callback(value, retained_entries[i].context); + } +} + +void sf_retained_fire_first(double value) { + if (retained_len != 0) { + retained_entries[0].callback(value, retained_entries[0].context); + } +} + +typedef void (*sf_retained_raw_cb)(double value); +static sf_retained_raw_cb retained_raw; + +void sf_retained_raw_set(sf_retained_raw_cb callback) { + retained_raw = callback; +} + +void sf_retained_raw_remove(sf_retained_raw_cb callback) { + if (retained_raw == callback) retained_raw = NULL; +} + +void sf_retained_raw_pump(double value) { + if (retained_raw != NULL) retained_raw(value); +} diff --git a/tests/ffi/profile.json b/tests/ffi/profile.json index 21a5bc9b4..1fc087591 100644 --- a/tests/ffi/profile.json +++ b/tests/ffi/profile.json @@ -1,5 +1,5 @@ { - "ffi_format": 3, + "ffi_format": 4, "functions": [ { "name": "nativeScale", "symbol": "sf_scale", "params": ["f64"], "returns": "f64" }, { "name": "nativeInvert", "symbol": "sf_invert", "params": ["bool"], "returns": "bool" }, @@ -167,6 +167,72 @@ { "context": "nullCString" } ], "returns": "void" + }, + { + "name": "nativeRetainedAdd", + "symbol": "sf_retained_add", + "params": [ + { + "callback": { + "id": "tick", + "params": ["f64", { "context": "tick" }], + "returns": "void", + "lifetime": "retained" + } + }, + { "context": "tick" } + ], + "returns": "void" + }, + { + "name": "nativeRetainedRemove", + "symbol": "sf_retained_remove", + "params": [ + { "callback": { "release": "nativeRetainedAdd:tick" } }, + { "context": "nativeRetainedAdd:tick" } + ], + "returns": "void" + }, + { + "name": "nativeRetainedPump", + "symbol": "sf_retained_pump", + "params": ["f64"], + "returns": "void" + }, + { + "name": "nativeRetainedFireFirst", + "symbol": "sf_retained_fire_first", + "params": ["f64"], + "returns": "void" + }, + { + "name": "nativeRetainedRawSet", + "symbol": "sf_retained_raw_set", + "params": [ + { + "callback": { + "id": "raw", + "params": ["f64"], + "returns": "void", + "lifetime": "retained" + } + } + ], + "returns": "void" + }, + { + "name": "nativeRetainedRawRemove", + "symbol": "sf_retained_raw_remove", + "params": [ + { "callback": { "release": "nativeRetainedRawSet:raw" } } + ], + "returns": "void" + }, + { + "name": "nativeRetainedRawPump", + "symbol": "sf_retained_raw_pump", + "params": ["f64"], + "returns": "void" } ], "libraries": [], diff --git a/tests/harness/ffi.test.ts b/tests/harness/ffi.test.ts index 0165bba31..70dc7bac0 100644 --- a/tests/harness/ffi.test.ts +++ b/tests/harness/ffi.test.ts @@ -3,8 +3,9 @@ * TypeScript and native archive run through BOTH scriptc backends, and the * result bytes must match. The fixture covers every value ABI class, * integer coercion, embedded-NUL/UTF-8 string spans, byte spans, format-2 - * raw/context callbacks, and format-3 callback cstring/span copies (lossy - * UTF-8, exact bytes, empty spans, ownership, and catchable throws). */ + * raw/context callbacks, format-3 callback cstring/span copies (lossy UTF-8, + * exact bytes, empty spans, ownership, and catchable throws), and format-4 + * retained registration/release ownership. */ import { execFileSync, spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; @@ -69,12 +70,17 @@ const expected = [ "0 NaN ", "4 0 Bé 0,255,1", "caught callback boom", + "first:11|second:1|second:3", + "7", + "caught retained boom 9", + "4", + "6 20", "caught string callback boom: materialized", "", ].join("\n"); describe.each(["c", "llvm"] as const)("outbound native FFI, %s backend", (backend) => { - test("calls the manifest-bound archive across every v1 ABI class plus v2/v3 callback ABI classes", async () => { + test("calls the manifest-bound archive across every v1 ABI class plus v2/v3/v4 callback ABI classes", async () => { const outDir = join(cacheRoot, backend); mkdirSync(outDir, { recursive: true }); const result = await compile(join(fixtureRoot, "main.ts"), { @@ -237,7 +243,7 @@ test.each([ returns: "void", }], }, - message: "lifetime' must be 'call", + message: "value 'retained' requires ffi_format 4", }, { name: "a format 3 callback class in format 2", @@ -277,6 +283,63 @@ test.each([ }, message: "must be one of", }, + { + name: "a release descriptor before format 4", + profile: { + ffi_format: 3, + functions: [{ + name: "remove", + symbol: "sf_remove", + params: [{ callback: { release: "add:tick" } }], + returns: "void", + }], + }, + message: "callback.release' requires ffi_format 4", + }, + { + name: "a dangling retained release", + profile: { + ffi_format: 4, + functions: [{ + name: "remove", + symbol: "sf_remove", + params: [{ callback: { release: "missing:tick" } }], + returns: "void", + }], + }, + message: "has no matching retained callback", + }, + { + name: "a release targeting a call-scoped callback", + profile: { + ffi_format: 4, + functions: [{ + name: "add", + symbol: "sf_add", + params: [{ callback: { id: "tick", params: [], returns: "void", lifetime: "call" } }], + returns: "void", + }, { + name: "remove", + symbol: "sf_remove", + params: [{ callback: { release: "add:tick" } }], + returns: "void", + }], + }, + message: "targets a non-retained callback", + }, + { + name: "a release carrying its own signature", + profile: { + ffi_format: 4, + functions: [{ + name: "remove", + symbol: "sf_remove", + params: [{ callback: { release: "add:tick", params: [] } }], + returns: "void", + }], + }, + message: "unknown field 'functions[0].params[0].callback.params'", + }, ])("manifest validation rejects $name", ({ name, profile, message }) => { const path = join(cacheRoot, `invalid-callback-${name.replaceAll(" ", "-")}.json`); mkdirSync(cacheRoot, { recursive: true }); @@ -533,6 +596,115 @@ describe.each(["c", "llvm"] as const)("FFI binding identity, %s backend", (backe }); }); +describe.each(["c", "llvm"] as const)("retained FFI release traps, %s backend", (backend) => { + test("releasing a closure that was never registered traps precisely", async () => { + const outDir = join(cacheRoot, `retained-missing-${backend}`); + mkdirSync(outDir, { recursive: true }); + const entry = join(outDir, "main.ts"); + const profilePath = join(outDir, "profile.json"); + writeFileSync( + entry, + [ + "declare function nativeRetainedAdd(callback: (value: number) => void): void;", + "declare function nativeRetainedRemove(callback: (value: number) => void): void;", + "const registered = (_value: number) => {};", + "const missing = (_value: number) => {};", + "nativeRetainedAdd(registered);", + "nativeRetainedRemove(missing);", + "", + ].join("\n"), + ); + writeFileSync( + profilePath, + JSON.stringify({ + ffi_format: 4, + functions: [{ + name: "nativeRetainedAdd", + symbol: "sf_retained_add", + params: [{ + callback: { + id: "tick", + params: ["f64", { context: "tick" }], + returns: "void", + lifetime: "retained", + }, + }, { context: "tick" }], + returns: "void", + }, { + name: "nativeRetainedRemove", + symbol: "sf_retained_remove", + params: [{ callback: { release: "nativeRetainedAdd:tick" } }, { + context: "nativeRetainedAdd:tick", + }], + returns: "void", + }], + libraries: [nativeArchive()], + }), + ); + const result = await compile(entry, { + outDir, + outPath: join(outDir, "program"), + backend, + sanitize, + ffiProfilePath: profilePath, + }); + if (!result.ok) { + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + } + const run = spawnSync(result.binaryPath, [], { encoding: "utf8" }); + expect(run.status).not.toBe(0); + expect(run.stderr).toContain( + "scriptc: releasing a native callback registration that does not exist", + ); + }); +}); + +test("retained callback calls reject function adapters that would change identity", async () => { + const outDir = join(cacheRoot, "retained-adapter-identity"); + mkdirSync(outDir, { recursive: true }); + const entry = join(outDir, "main.ts"); + const profilePath = join(outDir, "profile.json"); + writeFileSync( + entry, + [ + "declare function nativeRetainedRawSet(callback: (value: number) => void): void;", + "const returnsNumber = (value: number) => value;", + "nativeRetainedRawSet(returnsNumber);", + "", + ].join("\n"), + ); + writeFileSync( + profilePath, + JSON.stringify({ + ffi_format: 4, + functions: [{ + name: "nativeRetainedRawSet", + symbol: "sf_retained_raw_set", + params: [{ + callback: { + id: "raw", + params: ["f64"], + returns: "void", + lifetime: "retained", + }, + }], + returns: "void", + }], + libraries: [nativeArchive()], + }), + ); + const result = await compile(entry, { + outDir, + outPath: join(outDir, "program"), + ffiProfilePath: profilePath, + }); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.diagnostics[0]?.code).toBe("SC5003"); + expect(result.diagnostics[0]?.message).toContain("would change its release identity"); + } +}); + test("a missing FFI symbol is an SC5004 diagnostic, not a rejected compile", async () => { const outDir = join(cacheRoot, "missing-symbol"); mkdirSync(outDir, { recursive: true }); From 56de7ccac9b578a436fc3ecd9dafff7ba60130b2 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 16 Aug 2026 16:04:44 -0500 Subject: [PATCH 2/5] fix(ffi): harden retained callback lifecycle --- docs/src/app/ffi/page.mdx | 4 +- .../src/backend/emission/emit-exprs.ts | 64 ++++++------- .../compiler/src/backend/emission/emitter.ts | 34 ++++--- .../src/backend/emission/may-throw.ts | 7 +- .../compiler/src/backend/ffi-callbacks.ts | 66 ++++++++++++- packages/compiler/src/backend/llvm/emitter.ts | 96 +++++++++++-------- packages/compiler/src/ffi/profile.ts | 4 +- .../src/frontend/lowering/lower-calls.ts | 10 +- .../compiler/src/frontend/lowering/lowerer.ts | 16 ++++ packages/compiler/src/ir/validate.ts | 12 ++- packages/runtime/src/scr_async.c | 14 +-- packages/runtime/src/scr_ffi.c | 70 ++++++++++++++ packages/runtime/src/scr_runtime.h | 14 +++ tests/ffi/main.ts | 41 ++++++++ tests/ffi/native.c | 28 +++++- tests/ffi/profile.json | 15 +++ tests/harness/ffi.test.ts | 70 ++++++++++++++ 17 files changed, 453 insertions(+), 112 deletions(-) diff --git a/docs/src/app/ffi/page.mdx b/docs/src/app/ffi/page.mdx index fc2fe407d..5fc8f8c8f 100644 --- a/docs/src/app/ffi/page.mdx +++ b/docs/src/app/ffi/page.mdx @@ -241,9 +241,9 @@ timerRemove(tick); } ``` -The release argument must be the same function value used for registration. Registrations are counted: registering the same closure twice requires two releases. Releasing an unregistered value traps because native code may still hold the original pointer. The callback's function type must be exact at retained call sites; an implicit wrapper would create a different pointer and make release identity unsound. +The release argument must be the same function value used for registration. Registrations are counted: registering the same closure twice requires two releases. Releasing an unregistered value traps because native code may still hold the original pointer — the trap fires before the native release call runs, so native code never observes the invalid release. The callback's function type must be exact at retained call sites; an implicit wrapper would create a different pointer and make release identity unsound. -Context-bearing descriptors support multiple concurrent closures. A raw retained descriptor has no context pointer, so it has one process-global slot with replace semantics: registering a new closure releases the old slot. Retained registrations do not keep the event loop alive and remaining registrations are released at process exit. +Context-bearing descriptors support multiple concurrent closures. A raw retained descriptor has no context pointer, so it has one process-global slot with replace semantics: the previous registration stays live and dispatching until the replacing set call returns (a native setter that flushes the outgoing callback mid-replace still reaches the old closure), then it is released and the slot commits to the new closure. Retained registrations do not keep the event loop alive. At process exit, `process` `'exit'` listeners run first — they may still release or pump registrations on every exit path — then the runtime drops the remaining registrations and disarms raw slots, so a native invocation after teardown traps instead of reaching a freed closure. Both `"call"` and `"retained"` callbacks must run synchronously on the script thread. Direct foreign-thread invocation, value-returning foreign-thread callbacks, and real-time callback delivery are unsupported: scriptc's reference counting and exception cell are thread-confined, and a future foreign-thread surface must enqueue fire-and-forget delivery onto the script loop. Dereferenceable struct callback parameters are also unsupported; use an opaque native handle with accessor functions when the API permits it. diff --git a/packages/compiler/src/backend/emission/emit-exprs.ts b/packages/compiler/src/backend/emission/emit-exprs.ts index 97bca6805..871f6c817 100644 --- a/packages/compiler/src/backend/emission/emit-exprs.ts +++ b/packages/compiler/src/backend/emission/emit-exprs.ts @@ -7,6 +7,7 @@ import { boxAccess, BYTES_NUM_KIND_C, BYTES_NUM_VAR_C, bytesElemKindC, cDecl, cF import { mangleClassNew, mangleClassRetain, mangleClassStruct, mangleField, mangleFnClosure, mangleFunction, mangleGlobal, mangleLocal, mangleRecordNew, mangleRecordStruct, mangleVtStruct } from "../mangle.js"; import { OVERFLOW_MEMBER } from "./emit-shapes.js"; import { dynDestrCheckHelper, dynIterNHelper, dynKeyGetHelper } from "./emit-walkers.js"; +import { collectFfiRetainedOps, parseFfiCallbackKey } from "../ffi-callbacks.js"; import { genResultThunkFor } from "./emit-async.js"; function streamTypedRefCommitAdapter( @@ -2134,35 +2135,28 @@ export function emitExpr(E: CEmitter, e: IrExpr): Temp { if (isFfiReleaseParam(param)) callbackArgs.set(param.callback.release, arg); }); - const retainedRegistrations: { table: string; callback: Temp; global: string | null }[] = []; - const retainedReleases: { table: string; callback: Temp; global: string | null }[] = []; - for (const param of entry.params) { - if (isFfiCallbackParam(param) && param.callback.lifetime === "retained") { - const adapter = E.ffiCallbackAdapter(entry.name, param.callback.id); - const callback = callbackArgs.get(param.callback.id)!; - if (adapter.table === null) throw new Error("emitter bug: retained callback has no table"); - retainedRegistrations.push({ table: adapter.table, callback, global: adapter.global }); - } else if (isFfiReleaseParam(param)) { - const split = param.callback.release.lastIndexOf(":"); - const adapter = E.ffiCallbackAdapter( - param.callback.release.slice(0, split), - param.callback.release.slice(split + 1), - ); - const callback = callbackArgs.get(param.callback.release)!; - if (adapter.table === null) throw new Error("emitter bug: retained release has no table"); - retainedReleases.push({ table: adapter.table, callback, global: adapter.global }); - } - } + const { registrations: retainedRegistrations, releases: retainedReleases } = + collectFfiRetainedOps(entry, callbackArgs, (binding, id) => E.ffiCallbackAdapter(binding, id)); // Pin before registration. Raw retained descriptors are native - // singletons: replacing them drops the previous table entry and - // points the global trampoline slot at the new closure. + // singletons: the incoming closure is pinned (and an EMPTY slot + // armed) before the native set call, but a replaced registration + // stays live and dispatching until the call returns — a native + // setter may flush the outgoing callback one last time mid-replace. + // scr_ffi_commit_slot below repoints the slot and retires the + // superseded pins after the call. for (const registration of retainedRegistrations) { if (registration.global !== null) { - E.line(`scr_ffi_teardown(&${registration.table});`); - E.line(`${registration.global} = ${registration.callback.name};`); + E.line(`scr_ffi_retain_slot(&${registration.table}, &${registration.global}, ${registration.callback.name});`); + } else { + E.line(`scr_ffi_retain(&${registration.table}, ${registration.callback.name});`); } - E.line(`scr_ffi_retain(&${registration.table}, ${registration.callback.name});`); + } + // Validate releases BEFORE the native removal call runs: a bogus + // release traps without native code observing any side effect. The + // registration itself is unpinned only after the call returns. + for (const release of retainedReleases) { + E.line(`scr_ffi_require(&${release.table}, ${release.callback.name});`); } // Raw C callback pointers carry no userdata. For the documented @@ -2187,11 +2181,8 @@ export function emitExpr(E: CEmitter, e: IrExpr): Temp { return; } if (isFfiReleaseParam(param)) { - const split = param.callback.release.lastIndexOf(":"); - const adapter = E.ffiCallbackAdapter( - param.callback.release.slice(0, split), - param.callback.release.slice(split + 1), - ); + const { binding, id } = parseFfiCallbackKey(param.callback.release); + const adapter = E.ffiCallbackAdapter(binding, id); nativeArgs.push(`&${adapter.symbol}`); return; } @@ -2234,16 +2225,19 @@ export function emitExpr(E: CEmitter, e: IrExpr): Temp { } }; const finishRetainedReleases = (): void => { + // Commit raw replacements first (repoint the slot, retire the + // superseded pins), then unpin explicit releases — the runtime + // disarms the slot itself when the released closure holds it. + for (const registration of retainedRegistrations) { + if (registration.global !== null) { + E.line(`scr_ffi_commit_slot(&${registration.table}, ${registration.callback.name});`); + } + } for (const release of retainedReleases) { E.line(`scr_ffi_release(&${release.table}, ${release.callback.name});`); - if (release.global !== null) E.line(`${release.global} = NULL;`); } }; - const callbacksMayThrow = callbackArgs.size > 0 || (E.mod.ffiImports ?? []).some( - (candidate) => candidate.params.some( - (param) => isFfiCallbackParam(param) && param.callback.lifetime === "retained", - ), - ); + const callbacksMayThrow = callbackArgs.size > 0 || E.ffiHasRetainedCallback; switch (entry.returns) { case "void": E.line(`${call};${E.srcComment(e.loc)}`); diff --git a/packages/compiler/src/backend/emission/emitter.ts b/packages/compiler/src/backend/emission/emitter.ts index 1cfff66a8..088f4f31d 100644 --- a/packages/compiler/src/backend/emission/emitter.ts +++ b/packages/compiler/src/backend/emission/emitter.ts @@ -40,7 +40,7 @@ import type { SrcLoc, } from "../../ir/nodes.js"; import { ffiCallbackType, funcOf, isFfiCallbackParam, isFfiContextParam, isFfiReleaseParam, isRefCounted, isUnitType, mapOf, moduleEmbedsCompressedNpm, moduleUsesDgram, moduleUsesDynInvoke, moduleEmbedsBuiltin, moduleUsesFetch, moduleUsesFsWatch, moduleUsesHttp2, moduleUsesHttpServer, moduleUsesNet, moduleUsesNodeTest, moduleUsesProcessEvents, moduleUsesStream, moduleUsesTls, moduleUsesTlsCa, RUNTIME_EMITTER_CLASS, STRING, VOID } from "../../ir/nodes.js"; -import { allocateFfiCallbackAdapters, type FfiCallbackAdapter } from "../ffi-callbacks.js"; +import { allocateFfiCallbackAdapters, hasRetainedFfiCallback, type FfiCallbackAdapter } from "../ffi-callbacks.js"; import { mangleAsyncSpawn, mangleGenSpawn, @@ -242,6 +242,11 @@ export class CEmitter { * pointers additionally get a distinct TLS context slot, so two * callbacks with the same signature never alias each other's closure. */ readonly ffiCallbackAdapters: Map; + /** Module-level constant consulted per ffiCall: with a retained + * descriptor anywhere in the manifest, every native call is a + * pending-exception checkpoint (may-throw derives the same fact from + * the same helper). */ + readonly ffiHasRetainedCallback: boolean; readonly globalsById = new Map(); readonly unionsById = new Map(); /** Active optional-chain bind temps, by chain id (chainRecv reads). */ @@ -407,6 +412,7 @@ export class CEmitter { sourceText?: string, ) { this.ffiCallbackAdapters = allocateFfiCallbackAdapters(mod.ffiImports ?? []); + this.ffiHasRetainedCallback = hasRetainedFfiCallback(mod.ffiImports ?? []); for (const fn of mod.functions) { this.returnTypeByFn.set(fn.name, fn.returnType); this.fnByName.set(fn.name, fn); @@ -855,20 +861,26 @@ export class CEmitter { // main freed them (observed use-after-free). scr_run_exit_listeners // is idempotent (scr_exit_ran), so the atexit becomes a no-op; the // code argument is the hint the failure reporters maintain — exactly - // what the atexit path would have passed. + // what the atexit path would have passed. Emitted whenever the module + // touches process events, even with nothing to release: listeners must + // also beat the ATEXIT teardowns (the retained-FFI ledger sweep above + // all — a listener may legitimately release or pump a registration), + // and only the inline call orders ahead of every atexit handler. const needsRelease = refGlobals.length > 0 || fnValueProps.length > 0; - const runExitListeners = - moduleUsesProcessEvents(this.mod) && needsRelease - ? "scr_run_exit_listeners((double)scr_exit_code_hint_get()); " - : ""; + const runExitListeners = moduleUsesProcessEvents(this.mod) + ? "scr_run_exit_listeners((double)scr_exit_code_hint_get()); " + : ""; + const exitCleanup = `${runExitListeners}${needsRelease ? "sc_release_globals(); " : ""}`; const releaseGlobals = needsRelease ? ` ${runExitListeners}sc_release_globals();` - : ` /* no refcounted globals */`; + : runExitListeners !== "" + ? ` ${runExitListeners.trim()}` + : ` /* no refcounted globals */`; const uncaught = (indent: string, releaseTop = false) => [ `${indent}if (scr_exc_pending()) {`, `${indent} scr_exc_print_uncaught();`, `${indent} ${releaseTop ? "scr_promise_release(sc_top); " : ""}` + - `${needsRelease ? `${runExitListeners}sc_release_globals(); ` : ""}return 1;`, + `${exitCleanup}return 1;`, `${indent}}`, ]; out.push( @@ -973,7 +985,7 @@ export class CEmitter { ` if (sc_loop_rejection) {`, ` scr_discard_unhandled_rejections();`, ...(asyncEntry ? [` scr_promise_release(sc_top);`] : []), - ` ${needsRelease ? `${runExitListeners}sc_release_globals(); ` : ""}return 1;`, + ` ${exitCleanup}return 1;`, ` }`, ...(asyncEntry ? [ @@ -986,13 +998,13 @@ export class CEmitter { ` scr_promise_rethrow_top_level(sc_top);`, ` scr_promise_release(sc_top);`, ` scr_exc_print_uncaught();`, - ` ${needsRelease ? `${runExitListeners}sc_release_globals(); ` : ""}return 1;`, + ` ${exitCleanup}return 1;`, ` }`, ` scr_promise_release(sc_top);`, ] : []), ` if (scr_report_unhandled_rejections()) {`, - ` ${needsRelease ? `${runExitListeners}sc_release_globals(); ` : ""}return 1;`, + ` ${exitCleanup}return 1;`, ` }`, ...(asyncEntry ? [ diff --git a/packages/compiler/src/backend/emission/may-throw.ts b/packages/compiler/src/backend/emission/may-throw.ts index 6a01c950a..03ad40704 100644 --- a/packages/compiler/src/backend/emission/may-throw.ts +++ b/packages/compiler/src/backend/emission/may-throw.ts @@ -2,6 +2,7 @@ * of the IR module; the emitter consults the result to place unwind checks. */ import type { IrArrIntrinsicMethod, IrBytesIntrinsicMethod, IrLibFn, IrModule } from "../../ir/nodes.js"; import { isFfiCallbackParam, MAY_THROW_ARR_METHODS, MAY_THROW_BYTES_METHODS, MAY_THROW_LIB_FNS } from "../../ir/nodes.js"; +import { hasRetainedFfiCallback } from "../ffi-callbacks.js"; /** Cheap may-throw analysis (cost discipline: functions that transitively * CANNOT throw pay for no pending-exception checks). A function may throw @@ -34,11 +35,7 @@ export function computeMayThrow(mod: IrModule): { fns: Set; indirect: bo .filter((entry) => entry.params.some(isFfiCallbackParam)) .map((entry) => entry.name), ); - const manifestHasRetainedCallback = (mod.ffiImports ?? []).some( - (entry) => entry.params.some( - (param) => isFfiCallbackParam(param) && param.callback.lifetime === "retained", - ), - ); + const manifestHasRetainedCallback = hasRetainedFfiCallback(mod.ffiImports ?? []); // Method name → every class's implementation of it (virtualCall callees). const methodImpls = new Map(); for (const cls of mod.classes ?? []) { diff --git a/packages/compiler/src/backend/ffi-callbacks.ts b/packages/compiler/src/backend/ffi-callbacks.ts index 4bed076f1..1b9a90154 100644 --- a/packages/compiler/src/backend/ffi-callbacks.ts +++ b/packages/compiler/src/backend/ffi-callbacks.ts @@ -1,5 +1,5 @@ import type { IrFfiCallbackParam, IrFfiImport } from "../ir/nodes.js"; -import { isFfiCallbackParam, isFfiContextParam } from "../ir/nodes.js"; +import { isFfiCallbackParam, isFfiContextParam, isFfiReleaseParam } from "../ir/nodes.js"; export interface FfiCallbackAdapter { symbol: string; @@ -64,3 +64,67 @@ export function allocateFfiCallbackAdapters( return adapters; } + +/** The adapter-map key is `:`; a release descriptor + * carries the same key as its target. Binding names may themselves contain + * `:`; callback ids may not — split on the LAST separator. The single + * parser for every consumer (both emitters, twice each). */ +export function parseFfiCallbackKey(key: string): { binding: string; id: string } { + const split = key.lastIndexOf(":"); + return { binding: key.slice(0, split), id: key.slice(split + 1) }; +} + +/** Whether any manifest binding declares a retained callback — the + * throw-checkpoint policy predicate (with retained descriptors ANY binding + * may pump a stored callback). Computed once per module by may-throw and + * each emitter; keep every consumer on this helper so the policy cannot + * drift between analysis and emission. */ +export function hasRetainedFfiCallback(imports: readonly IrFfiImport[]): boolean { + return imports.some((entry) => + entry.params.some( + (param) => isFfiCallbackParam(param) && param.callback.lifetime === "retained", + ), + ); +} + +/** One retained lifecycle operation of an FFI call: the registration table, + * the raw singleton trampoline slot (null for context-bearing descriptors), + * and the backend's value for the closure argument. */ +export interface FfiRetainedOp { + table: string; + global: string | null; + callback: V; +} + +/** Collect a call's retained registrations and releases in manifest order — + * the lifecycle-policy walk shared by the C and LLVM emitters, so ordering + * fixes apply to both backends at once. */ +export function collectFfiRetainedOps( + entry: IrFfiImport, + callbackArgs: ReadonlyMap, + adapterFor: (binding: string, id: string) => FfiCallbackAdapter, +): { registrations: FfiRetainedOp[]; releases: FfiRetainedOp[] } { + const registrations: FfiRetainedOp[] = []; + const releases: FfiRetainedOp[] = []; + for (const param of entry.params) { + if (isFfiCallbackParam(param) && param.callback.lifetime === "retained") { + const adapter = adapterFor(entry.name, param.callback.id); + if (adapter.table === null) throw new Error("emitter bug: retained callback has no table"); + registrations.push({ + table: adapter.table, + global: adapter.global, + callback: callbackArgs.get(param.callback.id)!, + }); + } else if (isFfiReleaseParam(param)) { + const { binding, id } = parseFfiCallbackKey(param.callback.release); + const adapter = adapterFor(binding, id); + if (adapter.table === null) throw new Error("emitter bug: retained release has no table"); + releases.push({ + table: adapter.table, + global: adapter.global, + callback: callbackArgs.get(param.callback.release)!, + }); + } + } + return { registrations, releases }; +} diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 56a7e03ff..ba72c59d9 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -67,7 +67,6 @@ import type { IrFfiCallbackParam, IrFfiCallbackParamClass, IrFfiImport, - IrFfiReleaseParam, IrFfiReturnClass, IrFfiValueParamClass, IrFunction, @@ -82,7 +81,7 @@ import type { } from "../../ir/nodes.js"; import { canMarshalFuncIntoIsland, CAUGHT, DYN, F64, ffiCallbackType, islandCallbackRet, islandPromisePayloadTag, isFfiCallbackParam, isFfiContextParam, isFfiReleaseParam, isRefCounted, isUnitType, MAY_THROW_LIB_FNS, moduleEmbedsBuiltin, moduleEmbedsCompressedNpm, moduleUsesDynInvoke, moduleUsesFetch, moduleUsesFsWatch, moduleUsesHttpServer, moduleUsesNet, moduleUsesNodeTest, moduleUsesProcessEvents, moduleUsesStream, moduleUsesTls, moduleUsesTlsCa, NPM_COMPRESS_MIN, RUNTIME_EMITTER_CLASS, RUNTIME_ERROR_CLASSES, RUNTIME_STREAM_CLASSES, STRING, typeEquals, typeKey, VOID } from "../../ir/nodes.js"; import { matchIntegerBytesForLoop } from "../../ir/integer-loops.js"; -import { allocateFfiCallbackAdapters, type FfiCallbackAdapter } from "../ffi-callbacks.js"; +import { allocateFfiCallbackAdapters, collectFfiRetainedOps, hasRetainedFfiCallback, parseFfiCallbackKey, type FfiCallbackAdapter } from "../ffi-callbacks.js"; import { computeMayThrow } from "../emission/may-throw.js"; import { mangleArgPack, mangleAsyncSpawn, mangleClassNew, mangleClassObj, mangleClassRetain, mangleFnClosure, mangleFunction, mangleGenDrop, mangleGenResThunk, mangleGenSpawn, mangleGlobal, mangleLocal, mangleRecordNew, mangleRecordStruct, mangleResolveThunk, mangleTrampoline, mangleVtStruct, mangleWrapper } from "../mangle.js"; import { BlockBuilder } from "./blocks.js"; @@ -1019,6 +1018,11 @@ class LlEmitter { /** C-ABI callback trampolines and (for raw/no-userdata callbacks) their * distinct call-scoped TLS closure slots. */ private readonly ffiCallbackAdapters: Map; + /** Module-level constant consulted per ffiCall: with a retained + * descriptor anywhere in the manifest, every native call is a + * pending-exception checkpoint (may-throw derives the same fact from + * the same helper). */ + private readonly ffiHasRetainedCallback: boolean; private readonly globalTypes = new Map(); /** May-throw analysis (the C emitter's computeMayThrow, shared): pending * checks are emitted only after calls that can actually raise. */ @@ -1140,6 +1144,7 @@ class LlEmitter { // bytes behind a wasm32 object and 16 bytes behind a 64-bit object. this.cycleColorOffset = options.pointerBits === 32 ? 12 : 16; this.ffiCallbackAdapters = allocateFfiCallbackAdapters(mod.ffiImports ?? []); + this.ffiHasRetainedCallback = hasRetainedFfiCallback(mod.ffiImports ?? []); for (const fn of mod.functions) this.fnByName.set(fn.name, fn); for (const entry of mod.ffiImports ?? []) { this.ffiByName.set(entry.name, entry); @@ -1562,7 +1567,6 @@ class LlEmitter { moduleUsesTls(this.mod) || moduleUsesTlsCa(this.mod) || moduleEmbedsBuiltin(this.mod, "node:https") || moduleEmbedsBuiltin(this.mod, "node:tls"); - const hasRefGlobals = globals.some((g) => isRefCounted(g.type)) || fnValueProps.length > 0; // The process verdict has the same precedence as the C reference // emitter: node:test owns the final status when present; otherwise an // embedded process.exitCode owns it; ordinary programs return zero. @@ -1592,12 +1596,18 @@ class LlEmitter { if (snapshotsTlsCa) { this.declare(`declare void @scr_tls_ca_install()`); } - if (usesEvents && hasRefGlobals) { + // Emitted whenever the module touches process events, even with no + // refcounted globals: listeners must beat the ATEXIT teardowns (the + // retained-FFI ledger sweep above all — a listener may legitimately + // release or pump a registration), and only the inline call orders + // ahead of every atexit handler (the C emitter's runExitListeners + // stance). + if (usesEvents) { this.declare(`declare void @scr_run_exit_listeners(double)`); this.declare(`declare i32 @scr_exit_code_hint_get()`); } const exitListenerLines = (prefix: string): string[] => { - if (!usesEvents || !hasRefGlobals) return []; + if (!usesEvents) return []; return [ ` %${prefix}h = call i32 @scr_exit_code_hint_get()`, ` %${prefix}hd = sitofp i32 %${prefix}h to double`, @@ -1637,7 +1647,7 @@ class LlEmitter { this.declare(`declare void @scr_promise_rethrow_top_level(ptr)`); this.declare(`declare void @scr_promise_release(ptr)`); this.declare(`declare void @scr_exit_code_note(i32)`); - if (programExitUsesIsland && usesEvents && hasRefGlobals) { + if (programExitUsesIsland && usesEvents) { this.declare(`declare ${this.sizeType} @scr_island_exit_code_version()`); } } @@ -1656,7 +1666,7 @@ class LlEmitter { ); } const exitStatus = usesNodeTest || usesIsland ? "%tla_exit_status" : "%tla_status"; - const tracksIslandExit = programExitUsesIsland && usesEvents && hasRefGlobals; + const tracksIslandExit = programExitUsesIsland && usesEvents; if (tracksIslandExit) { lines.push(` %tla_exit_version = call ${this.sizeType} @scr_island_exit_code_version()`); } @@ -1738,7 +1748,7 @@ class LlEmitter { `%ScrVt = type { ${this.sizeType}, ${this.sizeType}, ptr }`, `%ScrUnion = type { ${this.sizeType}, i32, ptr, ptr, ptr, i64 }`, `%ScrClosure = type { ${this.sizeType}, ptr, ${this.sizeType}, ptr }`, - `%ScrFfiTable = type { ptr, ${this.sizeType}, ${this.sizeType}, ptr, i8 }`, + `%ScrFfiTable = type { ptr, ${this.sizeType}, ${this.sizeType}, ptr, i8, ptr }`, `%ScrRegex = type { ${this.sizeType}, ptr, ptr, ptr }`, // ScrArr mirror { rc, len, cap, elem(i32+pad), elem_retain, // elem_release, elem_trace, data } — the immortal tagged-template @@ -6038,35 +6048,37 @@ class LlEmitter { if (isFfiReleaseParam(param)) callbackArgs.set(param.callback.release, arg); }); - const retainedRegistrations: { table: string; callback: LlValue; global: string | null }[] = []; - const retainedReleases: { table: string; callback: LlValue; global: string | null }[] = []; - for (const param of entry.params) { - if (isFfiCallbackParam(param) && param.callback.lifetime === "retained") { - const adapter = this.ffiCallbackAdapter(entry.name, param.callback.id); - const callback = callbackArgs.get(param.callback.id)!; - if (adapter.table === null) throw new Error("llvm emitter bug: retained callback has no table"); - retainedRegistrations.push({ table: adapter.table, callback, global: adapter.global }); - } else if (isFfiReleaseParam(param)) { - const split = param.callback.release.lastIndexOf(":"); - const adapter = this.ffiCallbackAdapter( - param.callback.release.slice(0, split), - param.callback.release.slice(split + 1), - ); - const callback = callbackArgs.get(param.callback.release)!; - if (adapter.table === null) throw new Error("llvm emitter bug: retained release has no table"); - retainedReleases.push({ table: adapter.table, callback, global: adapter.global }); - } - } + const { registrations: retainedRegistrations, releases: retainedReleases } = + collectFfiRetainedOps(entry, callbackArgs, (binding, id) => this.ffiCallbackAdapter(binding, id)); if (retainedRegistrations.length > 0) { this.declare(`declare void @scr_ffi_retain(ptr, ptr)`); - this.declare(`declare void @scr_ffi_teardown(ptr)`); + if (retainedRegistrations.some((registration) => registration.global !== null)) { + this.declare(`declare void @scr_ffi_retain_slot(ptr, ptr, ptr)`); + this.declare(`declare void @scr_ffi_commit_slot(ptr, ptr)`); + } + } + if (retainedReleases.length > 0) { + this.declare(`declare void @scr_ffi_require(ptr, ptr)`); } + // Pin before registration. Raw retained descriptors are native + // singletons: the incoming closure is pinned (and an EMPTY slot + // armed) before the native set call, but a replaced registration + // stays live and dispatching until the call returns — a native + // setter may flush the outgoing callback one last time mid-replace. + // scr_ffi_commit_slot below repoints the slot and retires the + // superseded pins after the call. for (const registration of retainedRegistrations) { if (registration.global !== null) { - B.line(`call void @scr_ffi_teardown(ptr @${registration.table})`); - B.line(`store ptr ${registration.callback.name}, ptr @${registration.global}`); + B.line(`call void @scr_ffi_retain_slot(ptr @${registration.table}, ptr @${registration.global}, ptr ${registration.callback.name})`); + } else { + B.line(`call void @scr_ffi_retain(ptr @${registration.table}, ptr ${registration.callback.name})`); } - B.line(`call void @scr_ffi_retain(ptr @${registration.table}, ptr ${registration.callback.name})`); + } + // Validate releases BEFORE the native removal call runs: a bogus + // release traps without native code observing any side effect. The + // registration itself is unpinned only after the call returns. + for (const release of retainedReleases) { + B.line(`call void @scr_ffi_require(ptr @${release.table}, ptr ${release.callback.name})`); } const rawContexts: { tls: string; previous: string }[] = []; @@ -6090,11 +6102,8 @@ class LlEmitter { return; } if (isFfiReleaseParam(param)) { - const split = param.callback.release.lastIndexOf(":"); - const adapter = this.ffiCallbackAdapter( - param.callback.release.slice(0, split), - param.callback.release.slice(split + 1), - ); + const { binding, id } = parseFfiCallbackKey(param.callback.release); + const adapter = this.ffiCallbackAdapter(binding, id); nativeParamTypes.push("ptr"); nativeArgs.push(`ptr @${adapter.symbol}`); return; @@ -6185,17 +6194,20 @@ class LlEmitter { } }; const finishRetainedReleases = (): void => { + // Commit raw replacements first (repoint the slot, retire the + // superseded pins), then unpin explicit releases — the runtime + // disarms the slot itself when the released closure holds it. + for (const registration of retainedRegistrations) { + if (registration.global !== null) { + B.line(`call void @scr_ffi_commit_slot(ptr @${registration.table}, ptr ${registration.callback.name})`); + } + } if (retainedReleases.length > 0) this.declare(`declare void @scr_ffi_release(ptr, ptr)`); for (const release of retainedReleases) { B.line(`call void @scr_ffi_release(ptr @${release.table}, ptr ${release.callback.name})`); - if (release.global !== null) B.line(`store ptr null, ptr @${release.global}`); } }; - const callbacksMayThrow = callbackArgs.size > 0 || (this.mod.ffiImports ?? []).some( - (candidate) => candidate.params.some( - (param) => isFfiCallbackParam(param) && param.callback.lifetime === "retained", - ), - ); + const callbacksMayThrow = callbackArgs.size > 0 || this.ffiHasRetainedCallback; if (entry.returns === "void") { B.line(call); restoreRawContexts(); diff --git a/packages/compiler/src/ffi/profile.ts b/packages/compiler/src/ffi/profile.ts index 840e24932..e78a4d46b 100644 --- a/packages/compiler/src/ffi/profile.ts +++ b/packages/compiler/src/ffi/profile.ts @@ -477,7 +477,9 @@ export function loadFfiProfile( ).length; if (inheritedContext !== (contextCount === 1)) { throw new FfiProfileError( - `release '${target}' must declare its context exactly once in the native function parameter list when the retained callback has a context`, + inheritedContext + ? `release '${target}' must declare its context exactly once in the native function parameter list because the retained callback has a context` + : `release '${target}' must not declare a context in the native function parameter list because the retained callback has none`, ); } fn.params[j] = { diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index f6bbfdb3c..411ba4813 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -2920,7 +2920,10 @@ export function lowerFfiCall(L: Lowerer, expr: ts.CallExpression): IrExpr | null // Retained identity is the runtime closure pointer. A coercion adapter // would be freshly allocated at registration and release sites, so an // assignable-but-different function shape (notably `() => number` into - // `() => void`) cannot honestly participate in explicit release. + // `() => void`) cannot honestly participate in explicit release. The + // adapter set comes from the mint sites themselves (Lowerer's + // freshClosureAdapters), not name-prefix matching, so a new coercion + // helper cannot silently slip past this guard. if ( ( isFfiReleaseParam(sourceParam) || @@ -2928,10 +2931,7 @@ export function lowerFfiCall(L: Lowerer, expr: ts.CallExpression): IrExpr | null ) && ( lowered.kind === "dynCheck" || - (lowered.kind === "call" && - (lowered.callee.startsWith("%fn.adapt.") || - lowered.callee.startsWith("%fn.width.") || - lowered.callee.startsWith("%fnval."))) + (lowered.kind === "call" && L.freshClosureAdapters.has(lowered.callee)) ) ) { signatureError( diff --git a/packages/compiler/src/frontend/lowering/lowerer.ts b/packages/compiler/src/frontend/lowering/lowerer.ts index cf81ea92a..db04882d3 100644 --- a/packages/compiler/src/frontend/lowering/lowerer.ts +++ b/packages/compiler/src/frontend/lowering/lowerer.ts @@ -964,6 +964,16 @@ export class Lowerer { /** Union re-tag helpers (%union.retag.N), interned per (from, to) * unionId pair — see unionRetagHelper. */ readonly retagHelpers = new Map(); + /** Callee names of every interned coercion helper whose CALL mints a + * FRESH closure per evaluation (%fn.width.*, %fn.adapt.*, + * %fnval.spawnres.*). Registered at the mint site — NOT recovered by + * name-prefix matching — because retained-FFI release identity is the + * runtime closure pointer: a coercion adapter allocates a different + * closure at the registration and release sites, so lowerFfiCall must + * refuse these forms at compile time (SC5003). Any new closure-minting + * adapter helper MUST add its name here, or the identity guard silently + * reopens and the mismatch surfaces as a runtime release trap instead. */ + readonly freshClosureAdapters = new Set(); /** Symbols bound by `const x = promisify(execFile)` — the one lowered * util.promisify shape. Declarations register here and emit nothing; * calls through the binding lower (lowerExecFileAsyncCall) and value @@ -4697,6 +4707,8 @@ export class Lowerer { if (existing) return existing; const name = `%fn.width.${this.widthHelpers.size}`; this.widthHelpers.set(key, name); + this.freshClosureAdapters.add(name); // wraps `f` in a new closure per call + const impl = `${name}.impl`; // The returned closure's body: call the captured original, width-map. this.liftedFns.push({ @@ -4865,6 +4877,8 @@ export class Lowerer { if (existing) return existing; const name = `%fn.adapt.${this.retagHelpers.size}`; this.retagHelpers.set(key, name); + this.freshClosureAdapters.add(name); // wraps `f` in a new closure per call + const impl = `${name}.impl`; const params: IrParam[] = toT.params.map((t, i) => ({ localId: `a.${i}`, name: `a${i}`, type: t })); const strandThrow = (why: string): IrStmt => ({ @@ -5027,6 +5041,8 @@ export class Lowerer { if (existing) return existing; const name = `%fnval.spawnres.${this.widthHelpers.size}`; this.widthHelpers.set(key, name); + this.freshClosureAdapters.add(name); // wraps `f` in a new closure per call + const impl = `${name}.impl`; const params: IrParam[] = toT.params.map((p, i) => ({ localId: `p${i}.0`, name: `p${i}`, type: p })); const rRef: IrExpr = { kind: "varRef", localId: "r.0", type: fromT.ret, loc }; diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index d3216ee70..a85719365 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -1211,7 +1211,17 @@ export function validateModule(mod: IrModule): IrValidationError[] { errors.push({ message: `FFI release "${entry.name}:${param.callback.release}" has no retained target`, loc: moduleLoc }); continue; } - const inherited = JSON.stringify(param.callback.params) === JSON.stringify(target.params) && + // Structural ABI comparison: a params entry is a value-class string + // or a {context} object. Key order and incidental object shape must + // not matter — a producer that rebuilds these arrays (deserialized + // IR, a second frontend) still validates. + const inherited = param.callback.params.length === target.params.length && + param.callback.params.every((entry, i) => { + const other = target.params[i]!; + return isFfiContextParam(entry) + ? isFfiContextParam(other) && entry.context === other.context + : entry === other; + }) && param.callback.returns === target.returns; if (!inherited) { errors.push({ message: `FFI release "${entry.name}:${param.callback.release}" does not inherit its target ABI`, loc: moduleLoc }); diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index 5bea3e0bf..c93c092f0 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -2618,12 +2618,14 @@ bool scr_loop_run(ScrPromise *top_level) { * them as leaks. Uncaught callback throws still return above for main * to report through the existing exceptional teardown path. */ scr_timers_teardown(); - /* Retained native registrations carry no fd and do not hold liveness. - * Once the loop is exhausted the process cannot meaningfully pump their - * libraries again, so drop the ledger's closure references here. The - * idempotent atexit sweep covers synchronous and exceptional exits. */ - scr_ffi_teardown_all(); - /* Same story for unref'd children the loop never reaped: release the + /* Retained native FFI registrations are deliberately NOT torn down + * here: process 'exit' listeners run after the loop returns (inline in + * main, before any atexit handler) and may legitimately release a + * registration or pump a native library one last time — exactly like + * the process.exit() path, where the ledger is also still intact. The + * atexit sweep scr_ffi_retain registered (LIFO before the RC audit) + * drops the ledger's references on every exit path. */ + /* Unref'd children the loop never reaped: release the * registry's references (their listeners never fire — the process is * exiting, Node's behavior; the OS reparents the children). */ scr_children_teardown(); diff --git a/packages/runtime/src/scr_ffi.c b/packages/runtime/src/scr_ffi.c index 6804d3c75..6750cb78e 100644 --- a/packages/runtime/src/scr_ffi.c +++ b/packages/runtime/src/scr_ffi.c @@ -13,6 +13,10 @@ static bool scr_ffi_exit_registered; static void scr_ffi_oom(void) { scr_trap("scriptc: out of memory\n"); } void scr_ffi_teardown(ScrFfiTable *table) { + /* Disarm the raw trampoline slot FIRST: a closure release below cannot + * run script code today, but the slot must never dangle over freed + * entries — a native exit-path invocation takes the NULL trap instead. */ + if (table->slot != NULL) *table->slot = NULL; for (size_t i = 0; i < table->len; i++) { scr_closure_release(table->entries[i]); } @@ -49,12 +53,78 @@ void scr_ffi_retain(ScrFfiTable *table, ScrClosure *callback) { table->entries[table->len++] = scr_closure_retain(callback); } +/* Raw singleton registration runs in two halves around the native set + * call. The first half pins the incoming closure and records the slot, + * but leaves the CURRENT registration untouched: a native setter may + * flush the outgoing callback one last time mid-replace, and it must + * dispatch a live closure. An empty slot arms immediately so a native + * fire-on-subscribe during the set call reaches the new closure. */ +void scr_ffi_retain_slot(ScrFfiTable *table, ScrClosure **slot, ScrClosure *callback) { + table->slot = slot; + scr_ffi_retain(table, callback); + if (*slot == NULL) *slot = callback; +} + +/* The second half, after the native call returns: point the slot at the + * new registration and retire every pin it superseded. A callback pumped + * during the native call may have released or replaced this registration + * already (nested rawSet/rawRemove) — commit only a still-live entry. */ +void scr_ffi_commit_slot(ScrFfiTable *table, ScrClosure *callback) { + bool live = false; + for (size_t i = 0; i < table->len; i++) { + if (table->entries[i] == callback) { + live = true; + break; + } + } + if (!live) return; + *table->slot = callback; + bool kept = false; + size_t i = 0; + while (i < table->len) { + ScrClosure *entry = table->entries[i]; + if (entry == callback && !kept) { + kept = true; + i++; + continue; + } + /* Swap-remove BEFORE releasing so the table stays consistent while + * the release runs; the swapped-in tail entry is re-examined at i. */ + table->len--; + table->entries[i] = table->entries[table->len]; + scr_closure_release(entry); + } +} + +/* Pre-call validation for an explicit release: emitted before the native + * removal call, so releasing an unregistered value traps before native + * code can act on the bogus pointer pair. */ +void scr_ffi_require(ScrFfiTable *table, ScrClosure *callback) { + for (size_t i = 0; i < table->len; i++) { + if (table->entries[i] == callback) return; + } + scr_trap("scriptc: releasing a native callback registration that does not exist\n"); +} + void scr_ffi_release(ScrFfiTable *table, ScrClosure *callback) { for (size_t i = 0; i < table->len; i++) { if (table->entries[i] != callback) continue; ScrClosure *owned = table->entries[i]; table->len--; if (i != table->len) table->entries[i] = table->entries[table->len]; + /* If the raw slot dispatched to this registration and no duplicate + * pin remains, disarm it — the trampoline must not reach a released + * closure. */ + if (table->slot != NULL && *table->slot == callback) { + bool remaining = false; + for (size_t j = 0; j < table->len; j++) { + if (table->entries[j] == callback) { + remaining = true; + break; + } + } + if (!remaining) *table->slot = NULL; + } scr_closure_release(owned); return; } diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index bf617a52d..d747262dc 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -1331,9 +1331,23 @@ typedef struct ScrFfiTable { size_t cap; struct ScrFfiTable *next; bool linked; + /* Raw retained singletons only: the compiler-emitted global the + * trampoline dispatches through. Teardown and release disarm it so a + * late native invocation hits the trampoline's NULL trap instead of a + * freed closure. NULL for context-bearing descriptors. */ + ScrClosure **slot; } ScrFfiTable; void scr_ffi_retain(ScrFfiTable *table, ScrClosure *callback); +/* Raw singleton registration, split around the native set call: + * retain_slot pins the incoming closure BEFORE the call without touching + * the current registration; commit_slot repoints the slot and retires the + * superseded pins after the call returns. */ +void scr_ffi_retain_slot(ScrFfiTable *table, ScrClosure **slot, ScrClosure *callback); +void scr_ffi_commit_slot(ScrFfiTable *table, ScrClosure *callback); +/* Traps unless the registration exists — emitted BEFORE a native release + * call so a bogus release cannot reach native code. */ +void scr_ffi_require(ScrFfiTable *table, ScrClosure *callback); void scr_ffi_release(ScrFfiTable *table, ScrClosure *callback); void scr_ffi_teardown(ScrFfiTable *table); void scr_ffi_teardown_all(void); diff --git a/tests/ffi/main.ts b/tests/ffi/main.ts index 3948f9dcc..c6a9c0ec5 100644 --- a/tests/ffi/main.ts +++ b/tests/ffi/main.ts @@ -38,6 +38,7 @@ declare function nativeRetainedFireFirst(value: number): void; declare function nativeRetainedRawSet(callback: (value: number) => void): void; declare function nativeRetainedRawRemove(callback: (value: number) => void): void; declare function nativeRetainedRawPump(value: number): void; +declare function nativeRetainedRawSetFlush(callback: (value: number) => void): void; console.log(nativeScale(21)); console.log(nativeInvert(false), nativeInvert(true)); @@ -161,6 +162,46 @@ nativeRetainedRawRemove(rawSecond); nativeRetainedRawPump(3); console.log(rawEvents.join(" ")); +// A pumped callback removing a LATER registration mid-pump: the fixture +// pump must neither double-fire the shifted entry nor invoke the released +// closure (the sanitized lane checks the latter). +const midPumpEvents: string[] = []; +const midPumpTrailing = (value: number) => { + midPumpEvents.push(`trail:${value}`); +}; +let midPumpRemoved = false; +const midPumpLead = (value: number) => { + midPumpEvents.push(`lead:${value}`); + if (!midPumpRemoved) { + midPumpRemoved = true; + nativeRetainedRemove(midPumpTrailing); + } +}; +nativeRetainedAdd(midPumpLead); +nativeRetainedAdd(midPumpTrailing); +nativeRetainedPump(6); +nativeRetainedPump(7); +nativeRetainedRemove(midPumpLead); +console.log(midPumpEvents.join("|")); + +// Flush-on-replace: a raw setter that fires the OUTGOING callback while +// replacing it must still reach the OLD closure — the replacement commits +// (slot repointed, previous pin dropped) only after the set call returns. +const flushEvents: string[] = []; +const flushFirst = (value: number) => { + flushEvents.push(`first:${value}`); +}; +const flushSecond = (value: number) => { + flushEvents.push(`second:${value}`); +}; +nativeRetainedRawSetFlush(flushFirst); +nativeRetainedRawPump(11); +nativeRetainedRawSetFlush(flushSecond); +nativeRetainedRawPump(12); +console.log(flushEvents.join("|")); +// flushSecond stays registered at exit: teardown must disarm the raw slot +// (a post-teardown native pump takes the NULL trap, not a use-after-free). + // A still-live registration at normal process exit exercises the runtime's // teardown path (the sanitized lane checks that its captured closure leaks // neither the closure nor its capture box). diff --git a/tests/ffi/native.c b/tests/ffi/native.c index 4cee98991..741e1589e 100644 --- a/tests/ffi/native.c +++ b/tests/ffi/native.c @@ -153,9 +153,23 @@ void sf_retained_remove(sf_retained_cb callback, void *context) { } void sf_retained_pump(double value) { - size_t end = retained_len; - for (size_t i = 0; i < end; i++) { - retained_entries[i].callback(value, retained_entries[i].context); + /* A pumped callback may remove entries mid-pump (sf_retained_remove + * shifts the array left). Walk a snapshot and fire only entries still + * registered, so a removal never double-fires the former last entry or + * invokes a just-released callback. */ + sf_retained_entry snapshot[16]; + size_t count = retained_len; + for (size_t i = 0; i < count; i++) snapshot[i] = retained_entries[i]; + for (size_t i = 0; i < count; i++) { + int live = 0; + for (size_t j = 0; j < retained_len; j++) { + if (retained_entries[j].callback == snapshot[i].callback && + retained_entries[j].context == snapshot[i].context) { + live = 1; + break; + } + } + if (live) snapshot[i].callback(value, snapshot[i].context); } } @@ -172,6 +186,14 @@ void sf_retained_raw_set(sf_retained_raw_cb callback) { retained_raw = callback; } +void sf_retained_raw_set_flush(sf_retained_raw_cb callback) { + /* Flush-on-replace: fire the OUTGOING callback one last time before + * storing the new one — the runtime must keep the previous registration + * live and dispatching until this call returns. */ + if (retained_raw != NULL) retained_raw(-1); + retained_raw = callback; +} + void sf_retained_raw_remove(sf_retained_raw_cb callback) { if (retained_raw == callback) retained_raw = NULL; } diff --git a/tests/ffi/profile.json b/tests/ffi/profile.json index 1fc087591..59993c502 100644 --- a/tests/ffi/profile.json +++ b/tests/ffi/profile.json @@ -228,6 +228,21 @@ ], "returns": "void" }, + { + "name": "nativeRetainedRawSetFlush", + "symbol": "sf_retained_raw_set_flush", + "params": [ + { + "callback": { + "id": "raw", + "params": ["f64"], + "returns": "void", + "lifetime": "retained" + } + } + ], + "returns": "void" + }, { "name": "nativeRetainedRawPump", "symbol": "sf_retained_raw_pump", diff --git a/tests/harness/ffi.test.ts b/tests/harness/ffi.test.ts index 70dc7bac0..31245c6b1 100644 --- a/tests/harness/ffi.test.ts +++ b/tests/harness/ffi.test.ts @@ -75,6 +75,8 @@ const expected = [ "caught retained boom 9", "4", "6 20", + "lead:6|lead:7", + "first:11|first:-1|second:12", "caught string callback boom: materialized", "", ].join("\n"); @@ -596,6 +598,74 @@ describe.each(["c", "llvm"] as const)("FFI binding identity, %s backend", (backe }); }); +describe.each(["c", "llvm"] as const)("retained FFI at process exit, %s backend", (backend) => { + test("process 'exit' listeners can release retained registrations after a normal loop drain", async () => { + const outDir = join(cacheRoot, `retained-exit-listener-${backend}`); + mkdirSync(outDir, { recursive: true }); + const entry = join(outDir, "main.ts"); + const profilePath = join(outDir, "profile.json"); + writeFileSync( + entry, + [ + "declare function nativeRetainedAdd(callback: (value: number) => void): void;", + "declare function nativeRetainedRemove(callback: (value: number) => void): void;", + "const tick = (_value: number) => {};", + "nativeRetainedAdd(tick);", + "process.on('exit', () => {", + " nativeRetainedRemove(tick);", + " console.log('released-at-exit');", + "});", + // Force the event loop to run: the loop-drain exit path must leave + // the retained ledger intact for the listener, exactly like the + // process.exit() path. + "await Promise.resolve();", + "", + ].join("\n"), + ); + writeFileSync( + profilePath, + JSON.stringify({ + ffi_format: 4, + functions: [{ + name: "nativeRetainedAdd", + symbol: "sf_retained_add", + params: [{ + callback: { + id: "tick", + params: ["f64", { context: "tick" }], + returns: "void", + lifetime: "retained", + }, + }, { context: "tick" }], + returns: "void", + }, { + name: "nativeRetainedRemove", + symbol: "sf_retained_remove", + params: [{ callback: { release: "nativeRetainedAdd:tick" } }, { + context: "nativeRetainedAdd:tick", + }], + returns: "void", + }], + libraries: [nativeArchive()], + }), + ); + const result = await compile(entry, { + outDir, + outPath: join(outDir, "program"), + backend, + sanitize, + ffiProfilePath: profilePath, + }); + if (!result.ok) { + throw new Error(result.diagnostics.map((d) => `${d.code}: ${d.message}`).join("\n")); + } + const run = spawnSync(result.binaryPath, [], { encoding: "utf8" }); + expect(run.stderr).toBe(""); + expect(run.status).toBe(0); + expect(run.stdout).toBe("released-at-exit\n"); + }); +}); + describe.each(["c", "llvm"] as const)("retained FFI release traps, %s backend", (backend) => { test("releasing a closure that was never registered traps precisely", async () => { const outDir = join(cacheRoot, `retained-missing-${backend}`); From 3a7c94cd2317081c2fdf9052196f8c63a2ae8655 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 16 Aug 2026 18:01:15 -0500 Subject: [PATCH 3/5] fix(ffi): reject same-call retained releases --- docs/src/app/ffi/page.mdx | 4 ++-- packages/compiler/src/ffi/profile.ts | 17 +++++++++++++++++ packages/compiler/src/ir/validate.ts | 11 +++++++++++ tests/harness/ffi.test.ts | 16 ++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/docs/src/app/ffi/page.mdx b/docs/src/app/ffi/page.mdx index 5fc8f8c8f..4ca461cee 100644 --- a/docs/src/app/ffi/page.mdx +++ b/docs/src/app/ffi/page.mdx @@ -241,9 +241,9 @@ timerRemove(tick); } ``` -The release argument must be the same function value used for registration. Registrations are counted: registering the same closure twice requires two releases. Releasing an unregistered value traps because native code may still hold the original pointer — the trap fires before the native release call runs, so native code never observes the invalid release. The callback's function type must be exact at retained call sites; an implicit wrapper would create a different pointer and make release identity unsound. +The release argument must be the same function value used for registration. Registrations are counted: registering the same closure twice requires two releases. Releasing an unregistered value traps because native code may still hold the original pointer — the trap fires before the native release call runs, so native code never observes the invalid release. The callback's function type must be exact at retained call sites; an implicit wrapper would create a different pointer and make release identity unsound. A single binding cannot both register and release the same descriptor — the manifest loader rejects a `release` targeting a retained callback declared in the same function's parameter list, because the register-then-release ordering within one call would defeat the pre-call release validation. -Context-bearing descriptors support multiple concurrent closures. A raw retained descriptor has no context pointer, so it has one process-global slot with replace semantics: the previous registration stays live and dispatching until the replacing set call returns (a native setter that flushes the outgoing callback mid-replace still reaches the old closure), then it is released and the slot commits to the new closure. Retained registrations do not keep the event loop alive. At process exit, `process` `'exit'` listeners run first — they may still release or pump registrations on every exit path — then the runtime drops the remaining registrations and disarms raw slots, so a native invocation after teardown traps instead of reaching a freed closure. +Context-bearing descriptors support multiple concurrent closures. A raw retained descriptor has no context pointer, so it has one process-global slot with replace semantics: the previous registration stays live and dispatching until the replacing set call returns (a native setter that flushes the outgoing callback mid-replace still reaches the old closure), then it is released and the slot commits to the new closure. Retained registrations do not keep the event loop alive. At process exit, `process` `'exit'` listeners run first — they may still release or pump registrations on every exit path — then the runtime drops the remaining registrations and disarms raw slots. A raw-slot invocation after teardown traps instead of reaching a freed closure. A context-bearing registration has no slot to disarm — its trampoline and context pointer dangle once teardown frees the closure — so native code must not invoke one after exit; a library that can fire on its own exit path should have its registrations released from a `process` `'exit'` listener. Both `"call"` and `"retained"` callbacks must run synchronously on the script thread. Direct foreign-thread invocation, value-returning foreign-thread callbacks, and real-time callback delivery are unsupported: scriptc's reference counting and exception cell are thread-confined, and a future foreign-thread surface must enqueue fire-and-forget delivery onto the script loop. Dereferenceable struct callback parameters are also unsupported; use an opaque native handle with accessor functions when the API permits it. diff --git a/packages/compiler/src/ffi/profile.ts b/packages/compiler/src/ffi/profile.ts index e78a4d46b..daf73a1ff 100644 --- a/packages/compiler/src/ffi/profile.ts +++ b/packages/compiler/src/ffi/profile.ts @@ -469,6 +469,23 @@ export function loadFfiProfile( `release '${target}' in 'functions[${i}].params[${j}]' has no matching retained callback`, ); } + /* The emitted lifecycle is pin -> require -> native call -> commit -> + * release. A call that registers its own release target either + * retires the released pin during the commit sweep or satisfies the + * pre-call require with the pin it just created, so the + * release-validation trap cannot hold. */ + const registeredBySameCall = fn.params.some( + (entry) => + typeof entry === "object" && + "callback" in entry && + !("release" in entry.callback) && + `${fn.name}:${entry.callback.id}` === target, + ); + if (registeredBySameCall) { + throw new FfiProfileError( + `release '${target}' in 'functions[${i}].params[${j}]' targets a retained callback registered by the same call; registration and release must be separate bindings`, + ); + } const inheritedContext = descriptor.params.some( (entry) => typeof entry === "object", ); diff --git a/packages/compiler/src/ir/validate.ts b/packages/compiler/src/ir/validate.ts index a85719365..875269a78 100644 --- a/packages/compiler/src/ir/validate.ts +++ b/packages/compiler/src/ir/validate.ts @@ -1211,6 +1211,17 @@ export function validateModule(mod: IrModule): IrValidationError[] { errors.push({ message: `FFI release "${entry.name}:${param.callback.release}" has no retained target`, loc: moduleLoc }); continue; } + // A call registering its own release target defeats the emitted + // pin -> require -> call -> commit -> release ordering (the loader + // rejects this shape; mirrored here for deserialized IR). + const registeredBySameCall = entry.params.some( + (candidate) => + isFfiCallbackParam(candidate) && + `${entry.name}:${candidate.callback.id}` === param.callback.release, + ); + if (registeredBySameCall) { + errors.push({ message: `FFI release "${entry.name}:${param.callback.release}" targets a retained callback registered by the same call`, loc: moduleLoc }); + } // Structural ABI comparison: a params entry is a value-class string // or a {context} object. Key order and incidental object shape must // not matter — a producer that rebuilds these arrays (deserialized diff --git a/tests/harness/ffi.test.ts b/tests/harness/ffi.test.ts index 31245c6b1..a41921754 100644 --- a/tests/harness/ffi.test.ts +++ b/tests/harness/ffi.test.ts @@ -329,6 +329,22 @@ test.each([ }, message: "targets a non-retained callback", }, + { + name: "a release registered by the same call", + profile: { + ffi_format: 4, + functions: [{ + name: "swap", + symbol: "sf_swap", + params: [ + { callback: { id: "tick", params: [], returns: "void", lifetime: "retained" } }, + { callback: { release: "swap:tick" } }, + ], + returns: "void", + }], + }, + message: "registered by the same call", + }, { name: "a release carrying its own signature", profile: { From 5c65ea3d668273e9fb1ceb148bbd0bd4c7c4ad09 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 16 Aug 2026 19:14:01 -0500 Subject: [PATCH 4/5] fix(ffi): reject inline retained releases --- docs/src/app/ffi/page.mdx | 4 +- .../src/frontend/lowering/lower-calls.ts | 42 ++++++--- packages/runtime/src/scr_async.c | 9 +- packages/runtime/src/scr_runtime.h | 14 ++- tests/harness/ffi.test.ts | 93 +++++++++++++++++-- 5 files changed, 131 insertions(+), 31 deletions(-) diff --git a/docs/src/app/ffi/page.mdx b/docs/src/app/ffi/page.mdx index 4ca461cee..36f7d3562 100644 --- a/docs/src/app/ffi/page.mdx +++ b/docs/src/app/ffi/page.mdx @@ -241,9 +241,9 @@ timerRemove(tick); } ``` -The release argument must be the same function value used for registration. Registrations are counted: registering the same closure twice requires two releases. Releasing an unregistered value traps because native code may still hold the original pointer — the trap fires before the native release call runs, so native code never observes the invalid release. The callback's function type must be exact at retained call sites; an implicit wrapper would create a different pointer and make release identity unsound. A single binding cannot both register and release the same descriptor — the manifest loader rejects a `release` targeting a retained callback declared in the same function's parameter list, because the register-then-release ordering within one call would defeat the pre-call release validation. +The release argument must be the same function value used for registration. Registrations to a context-bearing descriptor are counted: registering the same closure twice requires two releases. A raw descriptor's slot has replace semantics instead — every set call supersedes the previous registration, including one that passes the already-registered closure, so exactly one release is ever pending. Releasing an unregistered value traps because native code may still hold the original pointer — the trap fires before the native release call runs, so native code never observes the invalid release. The callback's function type must be exact at retained call sites; an implicit wrapper would create a different pointer and make release identity unsound. An inline function literal as a release argument is rejected for the same reason: it creates a fresh closure at every evaluation, a pointer no registration holds — pass the same named value used to register. Registering an inline literal remains legal; such a registration is simply permanent and is dropped by the exit teardown. A single binding cannot both register and release the same descriptor — the manifest loader rejects a `release` targeting a retained callback declared in the same function's parameter list, because the register-then-release ordering within one call would defeat the pre-call release validation. -Context-bearing descriptors support multiple concurrent closures. A raw retained descriptor has no context pointer, so it has one process-global slot with replace semantics: the previous registration stays live and dispatching until the replacing set call returns (a native setter that flushes the outgoing callback mid-replace still reaches the old closure), then it is released and the slot commits to the new closure. Retained registrations do not keep the event loop alive. At process exit, `process` `'exit'` listeners run first — they may still release or pump registrations on every exit path — then the runtime drops the remaining registrations and disarms raw slots. A raw-slot invocation after teardown traps instead of reaching a freed closure. A context-bearing registration has no slot to disarm — its trampoline and context pointer dangle once teardown frees the closure — so native code must not invoke one after exit; a library that can fire on its own exit path should have its registrations released from a `process` `'exit'` listener. +Context-bearing descriptors support multiple concurrent closures. A raw retained descriptor has no context pointer, so it has one process-global slot with replace semantics: the previous registration stays live and dispatching until the replacing set call returns (a native setter that flushes the outgoing callback mid-replace still reaches the old closure), then it is released and the slot commits to the new closure. Retained registrations do not keep the event loop alive. At process exit, `process` `'exit'` listeners run first — they may still release or pump registrations on every exit path. On exits that run atexit handlers, the runtime then drops the remaining registrations and disarms raw slots; `process.exit()` terminates immediately after its listeners and skips that sweep, leaving remaining registrations to the operating system. A raw-slot invocation after teardown traps instead of reaching a freed closure. A context-bearing registration has no slot to disarm — its trampoline and context pointer dangle once teardown frees the closure — so native code must not invoke one after exit; a library that can fire on its own exit path should have its registrations released from a `process` `'exit'` listener. Both `"call"` and `"retained"` callbacks must run synchronously on the script thread. Direct foreign-thread invocation, value-returning foreign-thread callbacks, and real-time callback delivery are unsupported: scriptc's reference counting and exception cell are thread-confined, and a future foreign-thread surface must enqueue fire-and-forget delivery onto the script loop. Dereferenceable struct callback parameters are also unsupported; use an opaque native handle with accessor functions when the API permits it. diff --git a/packages/compiler/src/frontend/lowering/lower-calls.ts b/packages/compiler/src/frontend/lowering/lower-calls.ts index 411ba4813..b6b6c3c88 100644 --- a/packages/compiler/src/frontend/lowering/lower-calls.ts +++ b/packages/compiler/src/frontend/lowering/lower-calls.ts @@ -2925,19 +2925,39 @@ export function lowerFfiCall(L: Lowerer, expr: ts.CallExpression): IrExpr | null // freshClosureAdapters), not name-prefix matching, so a new coercion // helper cannot silently slip past this guard. if ( - ( - isFfiReleaseParam(sourceParam) || - (isFfiCallbackParam(sourceParam) && sourceParam.callback.lifetime === "retained") - ) && - ( + isFfiReleaseParam(sourceParam) || + (isFfiCallbackParam(sourceParam) && sourceParam.callback.lifetime === "retained") + ) { + if ( lowered.kind === "dynCheck" || (lowered.kind === "call" && L.freshClosureAdapters.has(lowered.callee)) - ) - ) { - signatureError( - `retained callback argument ${i + 1} must have the exact manifest function type; ` + - `an implicit function adapter would change its release identity`, - ); + ) { + signatureError( + `retained callback argument ${i + 1} must have the exact manifest function type; ` + + `an implicit function adapter would change its release identity`, + ); + } + // An inline function value at a RELEASE site can never match: + // lifted lambdas always carry a captures list (even an empty one), + // so both backends mint a fresh closure per evaluation of the + // expression — the release argument is a pointer no registration + // holds, a guaranteed runtime trap. Declared functions stay valid + // here — their value is the interned immortal closure (captures + // undefined), one pointer for every mention. Registration sites + // still accept literals: an unnameable registration is simply + // permanent, released by the exit teardown (the live-at-exit + // fixture shape), and hides no matching failure. + if ( + isFfiReleaseParam(sourceParam) && + lowered.kind === "closure" && + L.liftedFns.some((f) => f.name === lowered.fnName && f.captures !== undefined) + ) { + signatureError( + `retained callback argument ${i + 1} cannot be an inline function value; ` + + `each evaluation creates a fresh closure no registration holds — ` + + `pass the same named value used to register`, + ); + } } return lowered; }); diff --git a/packages/runtime/src/scr_async.c b/packages/runtime/src/scr_async.c index c93c092f0..c67aef4b7 100644 --- a/packages/runtime/src/scr_async.c +++ b/packages/runtime/src/scr_async.c @@ -2622,9 +2622,12 @@ bool scr_loop_run(ScrPromise *top_level) { * here: process 'exit' listeners run after the loop returns (inline in * main, before any atexit handler) and may legitimately release a * registration or pump a native library one last time — exactly like - * the process.exit() path, where the ledger is also still intact. The - * atexit sweep scr_ffi_retain registered (LIFO before the RC audit) - * drops the ledger's references on every exit path. */ + * the process.exit() path, where the ledger is also still intact. On + * returns that reach atexit, the sweep scr_ffi_retain registered (LIFO + * before the RC audit) then drops the ledger's references. It does NOT + * run on process.exit(): scr_process_exit ends in _Exit, skipping every + * atexit handler — the sweep and the RC audit alike — and leaves the + * ledger to the OS. */ /* Unref'd children the loop never reaped: release the * registry's references (their listeners never fire — the process is * exiting, Node's behavior; the OS reparents the children). */ diff --git a/packages/runtime/src/scr_runtime.h b/packages/runtime/src/scr_runtime.h index d747262dc..08b6e0b55 100644 --- a/packages/runtime/src/scr_runtime.h +++ b/packages/runtime/src/scr_runtime.h @@ -1320,11 +1320,15 @@ static inline ScrClosure *scr_closure_retain(ScrClosure *c) { void scr_closure_release(ScrClosure *c); /* releases the boxes; NULL-tolerant */ /* ── outbound FFI retained callbacks (scr_ffi.c) ───────────────────── - * One compiler-emitted table per retained callback descriptor. Entries are - * counted rather than deduplicated: registering the same closure twice - * requires two matching releases. The table owns one closure reference per - * entry and joins a process-global teardown list on first use. Retained - * callbacks do not contribute event-loop liveness. */ + * One compiler-emitted table per retained callback descriptor. For + * context-bearing descriptors, entries are counted rather than + * deduplicated: registering the same closure twice requires two matching + * releases. Raw singleton slots replace instead: commit_slot retires + * every pin the new registration superseded — a duplicate of the same + * closure included — so after any number of set calls exactly one + * release is pending. The table owns one closure reference per entry and + * joins a process-global teardown list on first use. Retained callbacks + * do not contribute event-loop liveness. */ typedef struct ScrFfiTable { ScrClosure **entries; size_t len; diff --git a/tests/harness/ffi.test.ts b/tests/harness/ffi.test.ts index a41921754..57fab7c9c 100644 --- a/tests/harness/ffi.test.ts +++ b/tests/harness/ffi.test.ts @@ -625,16 +625,29 @@ describe.each(["c", "llvm"] as const)("retained FFI at process exit, %s backend" [ "declare function nativeRetainedAdd(callback: (value: number) => void): void;", "declare function nativeRetainedRemove(callback: (value: number) => void): void;", - "const tick = (_value: number) => {};", - "nativeRetainedAdd(tick);", - "process.on('exit', () => {", - " nativeRetainedRemove(tick);", - " console.log('released-at-exit');", - "});", - // Force the event loop to run: the loop-drain exit path must leave - // the retained ledger intact for the listener, exactly like the - // process.exit() path. - "await Promise.resolve();", + // The callback deliberately lives in a FUNCTION-LOCAL const, not at + // module scope: a module-level const arrow is a refcounted func + // global and a declared function used as a value is an interned fn + // value — either makes needsRelease true, and the old + // `usesEvents && needsRelease` gate ran exit listeners inline + // whenever globals needed releasing, passing this test by + // accident. A local closure captured by the listener leaves the + // program with NOTHING to release, pinning the actual fix: process + // events alone must run 'exit' listeners inline, before the atexit + // FFI ledger sweep drops the registration the listener releases. + "function main() {", + " const tick = (_value: number) => {};", + " nativeRetainedAdd(tick);", + " process.on('exit', () => {", + " nativeRetainedRemove(tick);", + " console.log('released-at-exit');", + " });", + // Force the event loop to run via a timer, NOT a top-level await: + // an async module init caches its promise in a refcounted global, + // which would flip needsRelease back to true and un-pin the gate. + " setTimeout(() => {}, 0);", + "}", + "main();", "", ].join("\n"), ); @@ -791,6 +804,66 @@ test("retained callback calls reject function adapters that would change identit } }); +// An inline function literal mints a fresh closure at every evaluation, +// so at a RELEASE site it is a pointer no registration holds — a +// guaranteed runtime trap, refused at compile time. Registration sites +// still accept literals (a permanent registration the exit teardown +// releases — the live-at-exit shape in tests/ffi/main.ts). +test("retained release calls reject an inline function literal", async () => { + const outDir = join(cacheRoot, "retained-inline-literal-release"); + mkdirSync(outDir, { recursive: true }); + const entry = join(outDir, "main.ts"); + const profilePath = join(outDir, "profile.json"); + writeFileSync( + entry, + [ + "declare function nativeRetainedAdd(callback: (value: number) => void): void;", + "declare function nativeRetainedRemove(callback: (value: number) => void): void;", + "function tick(_value: number) {}", + "nativeRetainedAdd(tick);", + "nativeRetainedRemove((_value: number) => {});", + "", + ].join("\n"), + ); + writeFileSync( + profilePath, + JSON.stringify({ + ffi_format: 4, + functions: [{ + name: "nativeRetainedAdd", + symbol: "sf_retained_add", + params: [{ + callback: { + id: "tick", + params: ["f64", { context: "tick" }], + returns: "void", + lifetime: "retained", + }, + }, { context: "tick" }], + returns: "void", + }, { + name: "nativeRetainedRemove", + symbol: "sf_retained_remove", + params: [{ callback: { release: "nativeRetainedAdd:tick" } }, { + context: "nativeRetainedAdd:tick", + }], + returns: "void", + }], + libraries: [nativeArchive()], + }), + ); + const result = await compile(entry, { + outDir, + outPath: join(outDir, "program"), + ffiProfilePath: profilePath, + }); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.diagnostics[0]?.code).toBe("SC5003"); + expect(result.diagnostics[0]?.message).toContain("cannot be an inline function value"); + } +}); + test("a missing FFI symbol is an SC5004 diagnostic, not a rejected compile", async () => { const outDir = join(cacheRoot, "missing-symbol"); mkdirSync(outDir, { recursive: true }); From 5505b54b6080b633b6a5ef706c4bd407f3ad2860 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Sun, 16 Aug 2026 20:27:16 -0500 Subject: [PATCH 5/5] fix(ffi): preserve plain exit listener timing --- docs/src/app/ffi/page.mdx | 4 +++- .../compiler/src/backend/emission/emitter.ts | 19 ++++++++------- packages/compiler/src/backend/llvm/emitter.ts | 23 +++++++++++-------- tests/harness/ffi.test.ts | 8 ++++--- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/docs/src/app/ffi/page.mdx b/docs/src/app/ffi/page.mdx index 36f7d3562..ad224036a 100644 --- a/docs/src/app/ffi/page.mdx +++ b/docs/src/app/ffi/page.mdx @@ -241,10 +241,12 @@ timerRemove(tick); } ``` -The release argument must be the same function value used for registration. Registrations to a context-bearing descriptor are counted: registering the same closure twice requires two releases. A raw descriptor's slot has replace semantics instead — every set call supersedes the previous registration, including one that passes the already-registered closure, so exactly one release is ever pending. Releasing an unregistered value traps because native code may still hold the original pointer — the trap fires before the native release call runs, so native code never observes the invalid release. The callback's function type must be exact at retained call sites; an implicit wrapper would create a different pointer and make release identity unsound. An inline function literal as a release argument is rejected for the same reason: it creates a fresh closure at every evaluation, a pointer no registration holds — pass the same named value used to register. Registering an inline literal remains legal; such a registration is simply permanent and is dropped by the exit teardown. A single binding cannot both register and release the same descriptor — the manifest loader rejects a `release` targeting a retained callback declared in the same function's parameter list, because the register-then-release ordering within one call would defeat the pre-call release validation. +The release argument must be the same function value used for registration. Registrations to a context-bearing descriptor are counted: registering the same closure twice requires two releases. A raw descriptor's slot has replace semantics instead — every set call supersedes the previous registration, including one that passes the already-registered closure, so exactly one release is ever pending for that descriptor. Releasing an unregistered value traps because native code may still hold the original pointer — the trap fires before the native release call runs, so native code never observes the invalid release. The callback's function type must be exact at retained call sites; an implicit wrapper would create a different pointer and make release identity unsound. An inline function literal as a release argument is rejected for the same reason: it creates a fresh closure at every evaluation, a pointer no registration holds — pass the same named value used to register. Registering an inline literal remains legal; such a registration is simply permanent and is dropped by the exit teardown. A single binding cannot both register and release the same descriptor — the manifest loader rejects a `release` targeting a retained callback declared in the same function's parameter list, because the register-then-release ordering within one call would defeat the pre-call release validation. Context-bearing descriptors support multiple concurrent closures. A raw retained descriptor has no context pointer, so it has one process-global slot with replace semantics: the previous registration stays live and dispatching until the replacing set call returns (a native setter that flushes the outgoing callback mid-replace still reaches the old closure), then it is released and the slot commits to the new closure. Retained registrations do not keep the event loop alive. At process exit, `process` `'exit'` listeners run first — they may still release or pump registrations on every exit path. On exits that run atexit handlers, the runtime then drops the remaining registrations and disarms raw slots; `process.exit()` terminates immediately after its listeners and skips that sweep, leaving remaining registrations to the operating system. A raw-slot invocation after teardown traps instead of reaching a freed closure. A context-bearing registration has no slot to disarm — its trampoline and context pointer dangle once teardown frees the closure — so native code must not invoke one after exit; a library that can fire on its own exit path should have its registrations released from a `process` `'exit'` listener. +Retained identity is scoped to the declaring binding. Every retained callback parameter is its own descriptor: the `:` pair names one registration ledger, one generated trampoline, and (for a raw descriptor) one slot, and a release binding validates and unpins only registrations made through the binding its `release` reference targets. Two bindings that store into the same native state — a plain setter and a flush-on-replace setter for one native slot, say — are therefore independent descriptors that pass native code two different function pointers. Registering the same function value through both and then releasing it through one is unsound: the release unpins in its own descriptor's ledger, but native code compares stored pointers against the other descriptor's trampoline, so the surviving registration stays armed and keeps dispatching — nothing traps, and the callback keeps firing after the program believes it released it. Keep a function value registered with one such native registration point through exactly one binding at a time, and release it through that binding's paired release. + Both `"call"` and `"retained"` callbacks must run synchronously on the script thread. Direct foreign-thread invocation, value-returning foreign-thread callbacks, and real-time callback delivery are unsupported: scriptc's reference counting and exception cell are thread-confined, and a future foreign-thread surface must enqueue fire-and-forget delivery onto the script loop. Dereferenceable struct callback parameters are also unsupported; use an opaque native handle with accessor functions when the API permits it. If a callback throws, the adapter returns zero (or `void`) to native code and suppresses further script callback execution while the exception is pending. When the outer native function returns, the original exception resumes through scriptc's ordinary catchable unwind path. Native work performed between the callback's return and the outer function's return is not rolled back. diff --git a/packages/compiler/src/backend/emission/emitter.ts b/packages/compiler/src/backend/emission/emitter.ts index 088f4f31d..223153c69 100644 --- a/packages/compiler/src/backend/emission/emitter.ts +++ b/packages/compiler/src/backend/emission/emitter.ts @@ -861,15 +861,18 @@ export class CEmitter { // main freed them (observed use-after-free). scr_run_exit_listeners // is idempotent (scr_exit_ran), so the atexit becomes a no-op; the // code argument is the hint the failure reporters maintain — exactly - // what the atexit path would have passed. Emitted whenever the module - // touches process events, even with nothing to release: listeners must - // also beat the ATEXIT teardowns (the retained-FFI ledger sweep above - // all — a listener may legitimately release or pump a registration), - // and only the inline call orders ahead of every atexit handler. + // what the atexit path would have passed. With a retained FFI + // descriptor the inline call is required even with nothing to + // release: listeners must beat the atexit FFI ledger sweep (a + // listener may legitimately release or pump a registration), and + // only the inline call orders ahead of every atexit handler. Plain + // event programs with neither keep the atexit path, so their + // listener timing is unchanged. const needsRelease = refGlobals.length > 0 || fnValueProps.length > 0; - const runExitListeners = moduleUsesProcessEvents(this.mod) - ? "scr_run_exit_listeners((double)scr_exit_code_hint_get()); " - : ""; + const runExitListeners = + moduleUsesProcessEvents(this.mod) && (needsRelease || this.ffiHasRetainedCallback) + ? "scr_run_exit_listeners((double)scr_exit_code_hint_get()); " + : ""; const exitCleanup = `${runExitListeners}${needsRelease ? "sc_release_globals(); " : ""}`; const releaseGlobals = needsRelease ? ` ${runExitListeners}sc_release_globals();` diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index ba72c59d9..e4b24414a 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -1596,18 +1596,21 @@ class LlEmitter { if (snapshotsTlsCa) { this.declare(`declare void @scr_tls_ca_install()`); } - // Emitted whenever the module touches process events, even with no - // refcounted globals: listeners must beat the ATEXIT teardowns (the - // retained-FFI ledger sweep above all — a listener may legitimately - // release or pump a registration), and only the inline call orders - // ahead of every atexit handler (the C emitter's runExitListeners - // stance). - if (usesEvents) { + // Inline exit listeners run when something they must beat exists: + // the refcounted-global releases, or the retained-FFI atexit ledger + // sweep (a listener may legitimately release or pump a registration, + // and only the inline call orders ahead of every atexit handler — + // the C emitter's runExitListeners stance). Plain event programs + // with neither keep the atexit path, so their listener timing is + // unchanged. + const hasRefGlobals = globals.some((g) => isRefCounted(g.type)) || fnValueProps.length > 0; + const inlineExitListeners = usesEvents && (hasRefGlobals || this.ffiHasRetainedCallback); + if (inlineExitListeners) { this.declare(`declare void @scr_run_exit_listeners(double)`); this.declare(`declare i32 @scr_exit_code_hint_get()`); } const exitListenerLines = (prefix: string): string[] => { - if (!usesEvents) return []; + if (!inlineExitListeners) return []; return [ ` %${prefix}h = call i32 @scr_exit_code_hint_get()`, ` %${prefix}hd = sitofp i32 %${prefix}h to double`, @@ -1647,7 +1650,7 @@ class LlEmitter { this.declare(`declare void @scr_promise_rethrow_top_level(ptr)`); this.declare(`declare void @scr_promise_release(ptr)`); this.declare(`declare void @scr_exit_code_note(i32)`); - if (programExitUsesIsland && usesEvents) { + if (programExitUsesIsland && inlineExitListeners) { this.declare(`declare ${this.sizeType} @scr_island_exit_code_version()`); } } @@ -1666,7 +1669,7 @@ class LlEmitter { ); } const exitStatus = usesNodeTest || usesIsland ? "%tla_exit_status" : "%tla_status"; - const tracksIslandExit = programExitUsesIsland && usesEvents; + const tracksIslandExit = programExitUsesIsland && inlineExitListeners; if (tracksIslandExit) { lines.push(` %tla_exit_version = call ${this.sizeType} @scr_island_exit_code_version()`); } diff --git a/tests/harness/ffi.test.ts b/tests/harness/ffi.test.ts index 57fab7c9c..58f6ba5ec 100644 --- a/tests/harness/ffi.test.ts +++ b/tests/harness/ffi.test.ts @@ -632,9 +632,11 @@ describe.each(["c", "llvm"] as const)("retained FFI at process exit, %s backend" // `usesEvents && needsRelease` gate ran exit listeners inline // whenever globals needed releasing, passing this test by // accident. A local closure captured by the listener leaves the - // program with NOTHING to release, pinning the actual fix: process - // events alone must run 'exit' listeners inline, before the atexit - // FFI ledger sweep drops the registration the listener releases. + // program with NOTHING to release, pinning the actual fix: a + // retained descriptor alone must run 'exit' listeners inline, + // before the atexit FFI ledger sweep drops the registration the + // listener releases. (Event programs with no refcounted globals + // and no retained FFI keep the atexit listener path.) "function main() {", " const tick = (_value: number) => {};", " nativeRetainedAdd(tick);",