diff --git a/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt b/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt index 4695c0eeb4..0021d96683 100644 --- a/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt +++ b/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt @@ -9864,6 +9864,53 @@ SOFTWARE. ================================================================================ +Package: hash-wasm@4.12.0 +Declared license: MIT +Selected license: MIT +Repository: git+https://github.com/Daninet/hash-wasm.git + +--- LICENSE --- +MIT License + +Copyright (c) 2020 Dani Biró + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Embedded C implementations might use other, similarly permissive licenses. +Check the beginning of the files from the /src directory. + +Special thank you to the authors of original C algorithms: +- Alexander Peslyak +- Aleksey Kravchenko +- Colin Percival +- Stephan Brumme +- Steve Reid +- Samuel Neves +- Solar Designer +- Project Nayuki +- ARM Limited +- Yanbo Li dreamfly281@gmail.com, goldboar@163.comYanbo Li +- Mark Adler +- Yann Collet + +================================================================================ + Package: hasown@2.0.4 Declared license: MIT Selected license: MIT @@ -14581,3 +14628,704 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================================ + +Embedded component: Go crypto Argon2 implementation basis +Embedded by: hash-wasm@4.12.0 +Selected license: BSD-3-Clause +Repository: https://go.googlesource.com/crypto + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================================ + +Embedded component: Stephan Brumme CRC32/CRC64/xxHash32/xxHash64 implementations +Embedded by: hash-wasm@4.12.0 +Selected license: Zlib-like +Repository: https://create.stephan-brumme.com + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +Copyright (c) Stephan Brumme + +This software is provided 'as-is', without any express or implied warranty. In no event will the author be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. +2. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. +3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. + +================================================================================ + +Embedded component: Yanbo Li SM3 implementation +Embedded by: hash-wasm@4.12.0 +Selected license: MIT +Repository: https://github.com/Daninet/hash-wasm + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +MIT License + +Copyright 2016 Yanbo Li dreamfly281@gmail.com, goldboar@163.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================================ + +Embedded source notice: adler32.c +Embedded by: hash-wasm@4.12.0 +Source: src/adler32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + adler32.c -- compute the Adler-32 checksum of a data stream + Copyright (C) 1995-2011, 2016 Mark Adler + + Licensed under the zlib license: + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + + Modified for hash-wasm by Nicholas Sherlock and Dani Biro, 2021 +*/ + +================================================================================ + +Embedded source notice: argon2.c +Embedded by: hash-wasm@4.12.0 +Source: src/argon2.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + Based on Golang's Argon2 implementation from crypto package + + Written for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: bcrypt.c +Embedded by: hash-wasm@4.12.0 +Source: src/bcrypt.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * The crypt_blowfish homepage is: + * + * http://www.openwall.com/crypt/ + * + * This code comes from John the Ripper password cracker, with reentrant + * and crypt(3) interfaces added, but optimizations specific to password + * cracking removed. + * + * Written by Solar Designer in 1998-2014. + * No copyright is claimed, and the software is hereby placed in the public + * domain. In case this attempt to disclaim copyright and place the software + * in the public domain is deemed null and void, then the software is + * Copyright (c) 1998-2014 Solar Designer and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * It is my intent that you should be able to use this on your system, + * as part of a software package, or anywhere else to improve security, + * ensure compatibility, or for any other purpose. I would appreciate + * it if you give credit where it is due and keep your modifications in + * the public domain as well, but I don't require that in order to let + * you place this code and any modifications you make under a license + * of your choice. + * + * This implementation is fully compatible with OpenBSD's bcrypt.c for prefix + * "$2b$", originally by Niels Provos , and it uses + * some of his ideas. The password hashing algorithm was designed by David + * Mazieres . For information on the level of + * compatibility for bcrypt hash prefixes other than "$2b$", please refer to + * the comments in BF_set_key() below and to the included crypt(3) man page. + * + * There's a paper on the algorithm that explains its design decisions: + * + * http://www.usenix.org/events/usenix99/provos.html + * + * Some of the tricks in BF_ROUND might be inspired by Eric Young's + * Blowfish library (I can't be sure if I would think of something if I + * hadn't seen his code). + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: blake2b.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake2b.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE2 reference source code package - reference C implementations + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: blake2s.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake2s.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE2 reference source code package - reference C implementations + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: blake3.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE3 - reference C implementation + https://github.com/BLAKE3-team/BLAKE3 + This work is released into the public domain with CC0 1.0. Alternatively, it + is licensed under the Apache License 2.0. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: crc32.c +Embedded by: hash-wasm@4.12.0 +Source: src/crc32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// Crc32.cpp +// Copyright (c) 2011-2019 Stephan Brumme. All rights reserved. +// Slicing-by-16 contributed by Bulat Ziganshin +// Tableless bytewise CRC contributed by Hagai Gold +// see http://create.stephan-brumme.com/disclaimer.html +// +// Modified for hash-wasm by Dani Biró +// + +================================================================================ + +Embedded source notice: crc64.c +Embedded by: hash-wasm@4.12.0 +Source: src/crc64.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// Based on crc32.c implementation of Stephan Brumme +// Modified for hash-wasm by Dani Biró + +================================================================================ + +Embedded source notice: md4.c +Embedded by: hash-wasm@4.12.0 +Source: src/md4.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. + * MD4 Message-Digest Algorithm (RFC 1320). + * + * Homepage: + * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md4 + * + * Author: + * Alexander Peslyak, better known as Solar Designer + * + * This software was written by Alexander Peslyak in 2001. No copyright is + * claimed, and the software is hereby placed in the public domain. + * In case this attempt to disclaim copyright and place the software in the + * public domain is deemed null and void, then the software is + * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * (This is a heavily cut-down "BSD license".) + * + * This differs from Colin Plumb's older public domain implementation in that + * no exactly 32-bit integer data type is required (any 32-bit or wider + * unsigned integer data type will do), there's no compile-time endianness + * configuration, and the function prototypes match OpenSSL's. No code from + * Colin Plumb's implementation has been reused; this comment merely compares + * the properties of the two independent implementations. + * + * The primary goals of this implementation are portability and ease of use. + * It is meant to be fast, but not as fast as possible. Some known + * optimizations are not included to reduce source code size and avoid + * compile-time configuration. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: md5.c +Embedded by: hash-wasm@4.12.0 +Source: src/md5.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. + * MD5 Message-Digest Algorithm (RFC 1321). + * + * Homepage: + * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 + * + * Author: + * Alexander Peslyak, better known as Solar Designer + * + * This software was written by Alexander Peslyak in 2001. No copyright is + * claimed, and the software is hereby placed in the public domain. + * In case this attempt to disclaim copyright and place the software in the + * public domain is deemed null and void, then the software is + * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * (This is a heavily cut-down "BSD license".) + * + * This differs from Colin Plumb's older public domain implementation in that + * no exactly 32-bit integer data type is required (any 32-bit or wider + * unsigned integer data type will do), there's no compile-time endianness + * configuration, and the function prototypes match OpenSSL's. No code from + * Colin Plumb's implementation has been reused; this comment merely compares + * the properties of the two independent implementations. + * + * The primary goals of this implementation are portability and ease of use. + * It is meant to be fast, but not as fast as possible. Some known + * optimizations are not included to reduce source code size and avoid + * compile-time configuration. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: ripemd160.c +Embedded by: hash-wasm@4.12.0 +Source: src/ripemd160.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * RIPE MD-160 implementation + * + * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file is part of mbed TLS (https://tls.mbed.org) + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: scrypt.c +Embedded by: hash-wasm@4.12.0 +Source: src/scrypt.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * Copyright 2009 Colin Percival + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This file was originally written by Colin Percival as part of the Tarsnap + * online backup system. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha1.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha1.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* +SHA-1 in C +By Steve Reid +100% Public Domain + +Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: sha256.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha256.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha256.c - an implementation of SHA-256/224 hash functions + * based on FIPS 180-3 (Federal Information Processing Standart). + * + * Copyright (c) 2010, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha3.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha3.c - an implementation of Secure Hash Algorithm 3 (Keccak). + * based on the + * The Keccak SHA-3 submission. Submission to NIST (Round 3), 2011 + * by Guido Bertoni, Joan Daemen, Michaël Peeters and Gilles Van Assche + * + * Copyright (c) 2013, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha512.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha512.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha512.c - an implementation of SHA-384/512 hash functions + * based on FIPS 180-3 (Federal Information Processing Standart). + * + * Copyright (c) 2010, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sm3.c +Embedded by: hash-wasm@4.12.0 +Source: src/sm3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/******************************************************************************* + * SM3 function implementation + * Copyright 2016 Yanbo Li dreamfly281@gmail.com, goldboar@163.com + * MIT License + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: whirlpool.c +Embedded by: hash-wasm@4.12.0 +Source: src/whirlpool.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/** + * Whirlpool hash in C + * + * Copyright (c) 2017 Project Nayuki. (MIT License) + * https://www.nayuki.io/page/fast-whirlpool-hash-in-x86-assembly + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * - The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * - The Software is provided "as is", without warranty of any kind, express or + * implied, including but not limited to the warranties of merchantability, + * fitness for a particular purpose and noninfringement. In no event shall the + * authors or copyright holders be liable for any claim, damages or other + * liability, whether in an action of contract, tort or otherwise, arising from, + * out of or in connection with the Software or the use or other dealings in the + * Software. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash128.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash128.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash3.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash32.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// xxhash32.h +// Copyright (c) 2016 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// +// XXHash (32 bit), based on Yann Collet's descriptions, see +// http://cyan4973.github.io/xxHash/ +// +// Modified for hash-wasm by Dani Biró +// + +================================================================================ + +Embedded source notice: xxhash64.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash64.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// xxhash64.h +// Copyright (c) 2016 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// +// XXHash (64 bit), based on Yann Collet's descriptions, see +// http://cyan4973.github.io/xxHash/ +// +// Modified for hash-wasm by Dani Biró +// diff --git a/apps/desktop/src/main/__tests__/runtime-host-client.test.ts b/apps/desktop/src/main/__tests__/runtime-host-client.test.ts index 0b82aef122..a1e8cd817a 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-client.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-client.test.ts @@ -194,6 +194,15 @@ function subscription( loadTranscriptPage: async () => { throw new Error('Fake subscription does not expose transcript pages'); }, + queryTranscriptPositions: async () => { + throw new Error('Fake subscription does not expose semantic transcript positions'); + }, + loadTranscriptTurnWindow: async () => { + throw new Error('Fake subscription does not expose semantic transcript windows'); + }, + decodeTranscriptTurnWindowPage: async () => { + throw new Error('Fake subscription does not expose semantic transcript windows'); + }, close: async () => { lifecycle.push(`${sessionId}:close`); }, diff --git a/package-lock.json b/package-lock.json index c0c1f26999..c5a7f8a2aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8724,6 +8724,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hash-wasm": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.12.0.tgz", + "integrity": "sha512-+/2B2rYLb48I/evdOIhP+K/DD2ca2fgBjp6O+GBEnCDk2e4rpeXIK8GvIyRPjTezgmWn9gmKwkQjjx6BtqDHVQ==", + "license": "MIT" + }, "node_modules/hasown": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", @@ -14480,7 +14486,8 @@ "license": "Apache-2.0", "dependencies": { "@maka/core": "0.1.0", - "fs-native-extensions": "^1.5.0" + "fs-native-extensions": "^1.5.0", + "hash-wasm": "4.12.0" } }, "packages/ui": { diff --git a/packages/cli/THIRD_PARTY_NOTICES.txt b/packages/cli/THIRD_PARTY_NOTICES.txt index d1b9143775..5e26ecf32b 100644 --- a/packages/cli/THIRD_PARTY_NOTICES.txt +++ b/packages/cli/THIRD_PARTY_NOTICES.txt @@ -4587,6 +4587,53 @@ SOFTWARE. ================================================================================ +Package: hash-wasm@4.12.0 +Declared license: MIT +Selected license: MIT +Repository: git+https://github.com/Daninet/hash-wasm.git + +--- LICENSE --- +MIT License + +Copyright (c) 2020 Dani Biró + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Embedded C implementations might use other, similarly permissive licenses. +Check the beginning of the files from the /src directory. + +Special thank you to the authors of original C algorithms: +- Alexander Peslyak +- Aleksey Kravchenko +- Colin Percival +- Stephan Brumme +- Steve Reid +- Samuel Neves +- Solar Designer +- Project Nayuki +- ARM Limited +- Yanbo Li dreamfly281@gmail.com, goldboar@163.comYanbo Li +- Mark Adler +- Yann Collet + +================================================================================ + Package: hasown@2.0.4 Declared license: MIT Selected license: MIT @@ -7707,3 +7754,704 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================================ + +Embedded component: Go crypto Argon2 implementation basis +Embedded by: hash-wasm@4.12.0 +Selected license: BSD-3-Clause +Repository: https://go.googlesource.com/crypto + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================================ + +Embedded component: Stephan Brumme CRC32/CRC64/xxHash32/xxHash64 implementations +Embedded by: hash-wasm@4.12.0 +Selected license: Zlib-like +Repository: https://create.stephan-brumme.com + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +Copyright (c) Stephan Brumme + +This software is provided 'as-is', without any express or implied warranty. In no event will the author be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. +2. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. +3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. + +================================================================================ + +Embedded component: Yanbo Li SM3 implementation +Embedded by: hash-wasm@4.12.0 +Selected license: MIT +Repository: https://github.com/Daninet/hash-wasm + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +MIT License + +Copyright 2016 Yanbo Li dreamfly281@gmail.com, goldboar@163.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================================ + +Embedded source notice: adler32.c +Embedded by: hash-wasm@4.12.0 +Source: src/adler32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + adler32.c -- compute the Adler-32 checksum of a data stream + Copyright (C) 1995-2011, 2016 Mark Adler + + Licensed under the zlib license: + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + + Modified for hash-wasm by Nicholas Sherlock and Dani Biro, 2021 +*/ + +================================================================================ + +Embedded source notice: argon2.c +Embedded by: hash-wasm@4.12.0 +Source: src/argon2.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + Based on Golang's Argon2 implementation from crypto package + + Written for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: bcrypt.c +Embedded by: hash-wasm@4.12.0 +Source: src/bcrypt.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * The crypt_blowfish homepage is: + * + * http://www.openwall.com/crypt/ + * + * This code comes from John the Ripper password cracker, with reentrant + * and crypt(3) interfaces added, but optimizations specific to password + * cracking removed. + * + * Written by Solar Designer in 1998-2014. + * No copyright is claimed, and the software is hereby placed in the public + * domain. In case this attempt to disclaim copyright and place the software + * in the public domain is deemed null and void, then the software is + * Copyright (c) 1998-2014 Solar Designer and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * It is my intent that you should be able to use this on your system, + * as part of a software package, or anywhere else to improve security, + * ensure compatibility, or for any other purpose. I would appreciate + * it if you give credit where it is due and keep your modifications in + * the public domain as well, but I don't require that in order to let + * you place this code and any modifications you make under a license + * of your choice. + * + * This implementation is fully compatible with OpenBSD's bcrypt.c for prefix + * "$2b$", originally by Niels Provos , and it uses + * some of his ideas. The password hashing algorithm was designed by David + * Mazieres . For information on the level of + * compatibility for bcrypt hash prefixes other than "$2b$", please refer to + * the comments in BF_set_key() below and to the included crypt(3) man page. + * + * There's a paper on the algorithm that explains its design decisions: + * + * http://www.usenix.org/events/usenix99/provos.html + * + * Some of the tricks in BF_ROUND might be inspired by Eric Young's + * Blowfish library (I can't be sure if I would think of something if I + * hadn't seen his code). + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: blake2b.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake2b.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE2 reference source code package - reference C implementations + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: blake2s.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake2s.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE2 reference source code package - reference C implementations + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: blake3.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE3 - reference C implementation + https://github.com/BLAKE3-team/BLAKE3 + This work is released into the public domain with CC0 1.0. Alternatively, it + is licensed under the Apache License 2.0. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: crc32.c +Embedded by: hash-wasm@4.12.0 +Source: src/crc32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// Crc32.cpp +// Copyright (c) 2011-2019 Stephan Brumme. All rights reserved. +// Slicing-by-16 contributed by Bulat Ziganshin +// Tableless bytewise CRC contributed by Hagai Gold +// see http://create.stephan-brumme.com/disclaimer.html +// +// Modified for hash-wasm by Dani Biró +// + +================================================================================ + +Embedded source notice: crc64.c +Embedded by: hash-wasm@4.12.0 +Source: src/crc64.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// Based on crc32.c implementation of Stephan Brumme +// Modified for hash-wasm by Dani Biró + +================================================================================ + +Embedded source notice: md4.c +Embedded by: hash-wasm@4.12.0 +Source: src/md4.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. + * MD4 Message-Digest Algorithm (RFC 1320). + * + * Homepage: + * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md4 + * + * Author: + * Alexander Peslyak, better known as Solar Designer + * + * This software was written by Alexander Peslyak in 2001. No copyright is + * claimed, and the software is hereby placed in the public domain. + * In case this attempt to disclaim copyright and place the software in the + * public domain is deemed null and void, then the software is + * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * (This is a heavily cut-down "BSD license".) + * + * This differs from Colin Plumb's older public domain implementation in that + * no exactly 32-bit integer data type is required (any 32-bit or wider + * unsigned integer data type will do), there's no compile-time endianness + * configuration, and the function prototypes match OpenSSL's. No code from + * Colin Plumb's implementation has been reused; this comment merely compares + * the properties of the two independent implementations. + * + * The primary goals of this implementation are portability and ease of use. + * It is meant to be fast, but not as fast as possible. Some known + * optimizations are not included to reduce source code size and avoid + * compile-time configuration. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: md5.c +Embedded by: hash-wasm@4.12.0 +Source: src/md5.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. + * MD5 Message-Digest Algorithm (RFC 1321). + * + * Homepage: + * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 + * + * Author: + * Alexander Peslyak, better known as Solar Designer + * + * This software was written by Alexander Peslyak in 2001. No copyright is + * claimed, and the software is hereby placed in the public domain. + * In case this attempt to disclaim copyright and place the software in the + * public domain is deemed null and void, then the software is + * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * (This is a heavily cut-down "BSD license".) + * + * This differs from Colin Plumb's older public domain implementation in that + * no exactly 32-bit integer data type is required (any 32-bit or wider + * unsigned integer data type will do), there's no compile-time endianness + * configuration, and the function prototypes match OpenSSL's. No code from + * Colin Plumb's implementation has been reused; this comment merely compares + * the properties of the two independent implementations. + * + * The primary goals of this implementation are portability and ease of use. + * It is meant to be fast, but not as fast as possible. Some known + * optimizations are not included to reduce source code size and avoid + * compile-time configuration. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: ripemd160.c +Embedded by: hash-wasm@4.12.0 +Source: src/ripemd160.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * RIPE MD-160 implementation + * + * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file is part of mbed TLS (https://tls.mbed.org) + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: scrypt.c +Embedded by: hash-wasm@4.12.0 +Source: src/scrypt.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * Copyright 2009 Colin Percival + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This file was originally written by Colin Percival as part of the Tarsnap + * online backup system. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha1.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha1.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* +SHA-1 in C +By Steve Reid +100% Public Domain + +Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: sha256.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha256.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha256.c - an implementation of SHA-256/224 hash functions + * based on FIPS 180-3 (Federal Information Processing Standart). + * + * Copyright (c) 2010, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha3.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha3.c - an implementation of Secure Hash Algorithm 3 (Keccak). + * based on the + * The Keccak SHA-3 submission. Submission to NIST (Round 3), 2011 + * by Guido Bertoni, Joan Daemen, Michaël Peeters and Gilles Van Assche + * + * Copyright (c) 2013, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha512.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha512.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha512.c - an implementation of SHA-384/512 hash functions + * based on FIPS 180-3 (Federal Information Processing Standart). + * + * Copyright (c) 2010, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sm3.c +Embedded by: hash-wasm@4.12.0 +Source: src/sm3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/******************************************************************************* + * SM3 function implementation + * Copyright 2016 Yanbo Li dreamfly281@gmail.com, goldboar@163.com + * MIT License + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: whirlpool.c +Embedded by: hash-wasm@4.12.0 +Source: src/whirlpool.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/** + * Whirlpool hash in C + * + * Copyright (c) 2017 Project Nayuki. (MIT License) + * https://www.nayuki.io/page/fast-whirlpool-hash-in-x86-assembly + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * - The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * - The Software is provided "as is", without warranty of any kind, express or + * implied, including but not limited to the warranties of merchantability, + * fitness for a particular purpose and noninfringement. In no event shall the + * authors or copyright holders be liable for any claim, damages or other + * liability, whether in an action of contract, tort or otherwise, arising from, + * out of or in connection with the Software or the use or other dealings in the + * Software. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash128.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash128.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash3.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash32.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// xxhash32.h +// Copyright (c) 2016 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// +// XXHash (32 bit), based on Yann Collet's descriptions, see +// http://cyan4973.github.io/xxHash/ +// +// Modified for hash-wasm by Dani Biró +// + +================================================================================ + +Embedded source notice: xxhash64.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash64.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// xxhash64.h +// Copyright (c) 2016 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// +// XXHash (64 bit), based on Yann Collet's descriptions, see +// http://cyan4973.github.io/xxHash/ +// +// Modified for hash-wasm by Dani Biró +// diff --git a/packages/cli/src/__tests__/runtime-host-session-driver.test.ts b/packages/cli/src/__tests__/runtime-host-session-driver.test.ts index 571afe359e..7230dfe774 100644 --- a/packages/cli/src/__tests__/runtime-host-session-driver.test.ts +++ b/packages/cli/src/__tests__/runtime-host-session-driver.test.ts @@ -2816,6 +2816,18 @@ class FakeSubscription implements RuntimeHostSessionSubscription, AsyncIterator< throw new Error('Fake subscription does not expose transcript pages'); } + async queryTranscriptPositions(): Promise { + throw new Error('Fake subscription does not expose semantic transcript positions'); + } + + async loadTranscriptTurnWindow(): Promise { + throw new Error('Fake subscription does not expose semantic transcript windows'); + } + + async decodeTranscriptTurnWindowPage(): Promise { + throw new Error('Fake subscription does not expose semantic transcript windows'); + } + async close(): Promise { this.#closed = true; for (const waiter of this.#waiters.splice(0)) { diff --git a/packages/core/src/__tests__/shared-session-transcript-visibility.test.ts b/packages/core/src/__tests__/shared-session-transcript-visibility.test.ts new file mode 100644 index 0000000000..5c7d896f0f --- /dev/null +++ b/packages/core/src/__tests__/shared-session-transcript-visibility.test.ts @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + classifySharedSessionTranscriptVisibility, + SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION, + type StoredMessage, + type SystemNoteMessage, +} from '../session.js'; + +test('shared transcript visibility is closed over every StoredMessage type', () => { + const expected = { + user: 'visible', + assistant: 'visible', + tool_call: 'visible', + tool_result: 'visible', + turn_state: 'visible', + token_usage: 'visible', + permission_decision: 'hidden', + workhub_coordination: 'hidden', + } as const satisfies Record, 'visible' | 'hidden'>; + + for (const [type, visibility] of Object.entries(expected)) { + assert.equal( + classifySharedSessionTranscriptVisibility({ + type: type as Exclude, + }), + visibility, + ); + } + assert.equal(SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION, 1); +}); + +test('shared transcript visibility makes an explicit decision for every system-note kind', () => { + const expected = { + session_start: 'hidden', + session_resume: 'hidden', + mode_change: 'hidden', + model_change: 'hidden', + context_compacted: 'visible', + context_compaction_failed_open: 'visible', + step_limit: 'visible', + error: 'hidden', + abort: 'hidden', + } as const satisfies Record; + + for (const [kind, visibility] of Object.entries(expected)) { + assert.equal( + classifySharedSessionTranscriptVisibility({ + type: 'system_note', + kind: kind as SystemNoteMessage['kind'], + }), + visibility, + ); + } +}); diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index cbfc7efec3..e60c37ff35 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -780,17 +780,6 @@ export function userFacingText(message: Pick; + }; + +const SHARED_SESSION_SYSTEM_NOTE_VISIBILITY = { + session_start: 'hidden', + session_resume: 'hidden', + mode_change: 'hidden', + model_change: 'hidden', + context_compacted: 'visible', + context_compaction_failed_open: 'visible', + step_limit: 'visible', + error: 'hidden', + abort: 'hidden', +} as const satisfies Record; + +/** Closed scalar policy for records admitted to a shared transcript projection. */ +export function classifySharedSessionTranscriptVisibility( + envelope: SharedSessionTranscriptVisibilityEnvelope, +): SharedSessionTranscriptVisibility { + switch (envelope.type) { + case 'user': + case 'assistant': + case 'tool_call': + case 'tool_result': + case 'turn_state': + case 'token_usage': + return 'visible'; + case 'permission_decision': + case 'workhub_coordination': + return 'hidden'; + case 'system_note': + return SHARED_SESSION_SYSTEM_NOTE_VISIBILITY[envelope.kind]; + default: { + const unreachable: never = envelope; + return unreachable; + } + } +} + +export function isSessionSystemNoteKind(value: unknown): value is SystemNoteMessage['kind'] { + return typeof value === 'string' && Object.hasOwn(SHARED_SESSION_SYSTEM_NOTE_VISIBILITY, value); +} + +export function isStoredMessageType(value: unknown): value is StoredMessage['type'] { + switch (value) { + case 'user': + case 'assistant': + case 'tool_call': + case 'tool_result': + case 'permission_decision': + case 'token_usage': + case 'turn_state': + case 'workhub_coordination': + case 'system_note': + return true; + default: + return false; + } +} + +/** Closed policy for system notes that are part of the user-visible transcript. */ +export function isUserVisibleSessionSystemNote(kind: string): boolean { + return isSessionSystemNoteKind(kind) && SHARED_SESSION_SYSTEM_NOTE_VISIBILITY[kind] === 'visible'; +} + const USER_MESSAGE_SHAPE = defineObjectShape()( ['type', 'id', 'turnId', 'ts', 'text'], ['displayText', 'attachments', 'quotes', 'inlineReferences', 'steeringEventId', 'origin'], @@ -1135,18 +1199,6 @@ const ASSISTANT_THINKING_SHAPE = defineObjectShape()( ['text'], ['signature', 'providerOptions', 'parts'], ); -const SYSTEM_NOTE_KINDS = new Set([ - 'session_start', - 'session_resume', - 'mode_change', - 'model_change', - 'context_compacted', - 'context_compaction_failed_open', - 'step_limit', - 'error', - 'abort', -]); - export function decodeCanonicalMessage(value: unknown): StoredMessage { return decodeMessage(value, decodeCanonicalToolResultContent); } @@ -1280,7 +1332,7 @@ function decodeMessage( hasExactShape(message, SYSTEM_NOTE_MESSAGE_SHAPE) && hasMessageEnvelope(message, false) && isOptionalString(message.turnId) && - SYSTEM_NOTE_KINDS.has(message.kind as string) + isSessionSystemNoteKind(message.kind) ) return message as unknown as SystemNoteMessage; break; diff --git a/packages/runtime-host/protocol-compatible-changes/session-semantic-transcript-operations.json b/packages/runtime-host/protocol-compatible-changes/session-semantic-transcript-operations.json new file mode 100644 index 0000000000..c48fcbe369 --- /dev/null +++ b/packages/runtime-host/protocol-compatible-changes/session-semantic-transcript-operations.json @@ -0,0 +1,8 @@ +{ + "epoch": 73, + "files": [ + "packages/runtime-host/src/protocol/operations.ts", + "packages/runtime-host/src/protocol/session-transcript.ts" + ], + "reason": "Adds dormant subscription-scoped semantic transcript query operations without changing existing operation or subscription frame shapes; rollout remains disabled until the planned compatibility activation" +} diff --git a/packages/runtime-host/scripts/transcript-data-plane-benchmark.mjs b/packages/runtime-host/scripts/transcript-data-plane-benchmark.mjs index 5832696c60..01efa8b0d8 100644 --- a/packages/runtime-host/scripts/transcript-data-plane-benchmark.mjs +++ b/packages/runtime-host/scripts/transcript-data-plane-benchmark.mjs @@ -18,9 +18,12 @@ */ import { performance } from 'node:perf_hooks'; -import { mkdtemp, rm } from 'node:fs/promises'; +import { randomBytes } from 'node:crypto'; +import { mkdtemp, rm, stat } from 'node:fs/promises'; +import { DatabaseSync } from 'node:sqlite'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { OPERATIONAL_STATE_DATABASE_NAME } from '@maka/storage/operational-state-store'; import { createSessionStore } from '@maka/storage/session-store'; import { ClientSessionSubscription } from '../dist/client/session-subscription.js'; import { SESSION_CONTINUITY_SCHEMA_VERSION } from '../dist/protocol/index.js'; @@ -28,12 +31,28 @@ import { createSessionTranscriptBootstrap, readSessionTranscriptPage, } from '../dist/server/session-transcript-pager.js'; +import { + confirmSemanticTranscriptWatermark, + createSubscriberSemanticTranscriptState, + querySemanticTranscriptPositions, + readSemanticTranscriptTurnWindow, + releaseSubscriberSemanticTranscript, +} from '../dist/server/semantic-session-transcript-pager.js'; const BOOTSTRAP_BYTES = 16 * 1024; const RTT_MS = Number.parseInt(process.env.MAKA_TRANSCRIPT_BENCHMARK_RTT_MS ?? '20', 10); +const FULL_SEMANTIC = process.env.MAKA_TRANSCRIPT_BENCHMARK_FULL_SEMANTIC === '1'; const cases = [ { name: '5k-small-messages', messages: 5_000, textBytes: 96 }, - { name: '15MiB-single-message', messages: 1, textBytes: 15 * 1024 * 1024 }, + { name: '10k-semantic-positions', messages: 10_000, textBytes: 96, alternatingVisibility: true }, + { name: '15MiB-single-message', messages: 1, textBytes: 15 * 1024 * 1024, semantic: true }, + { + name: '17MiB-single-message', + messages: 1, + textBytes: 17 * 1024 * 1024, + semantic: true, + semanticOnly: true, + }, { name: '17MiB-transcript', totalBytes: 17 * 1024 * 1024, textBytes: 4 * 1024 }, { name: '64MiB-transcript', totalBytes: 64 * 1024 * 1024, textBytes: 4 * 1024 }, ]; @@ -60,6 +79,34 @@ async function runFixture(fixture) { const setupMs = performance.now() - setupAt; const reader = sqliteReader(store); try { + if (fixture.semanticOnly) { + const throughSequence = await reader.readDurableHighWater(session.id); + const semantic = await measureSemantic({ + reader, + sessionId: session.id, + throughSequence, + fullWindows: true, + databasePath: join(root, OPERATIONAL_STATE_DATABASE_NAME), + }); + return { + fixture: fixture.name, + messages: messages.length, + rawMiB: decimalMiB( + messages.reduce( + (total, message) => total + Buffer.byteLength(JSON.stringify(message), 'utf8'), + 0, + ), + ), + bootstrapKiB: '-', + wireRequests: '-', + pageRequests: '-', + setupMs: setupMs.toFixed(1), + bootstrapCpuMs: '-', + materializeCpuMs: '-', + modeledRttFloorMs: '-', + ...semanticMetrics(semantic), + }; + } const openedAt = performance.now(); const throughSequence = await reader.readDurableHighWater(session.id); const { bootstrap, state } = await createSessionTranscriptBootstrap({ @@ -70,6 +117,7 @@ async function runFixture(fixture) { rootTurn: null, activeAssistantStreams: [], maxBytes: BOOTSTRAP_BYTES, + projection: 'owner', }); const bootstrapCpuMs = performance.now() - openedAt; let pageRequests = 0; @@ -119,6 +167,17 @@ async function runFixture(fixture) { ); } const wireRequests = 1 + pageRequests; + const semantic = + fixture.semantic || fixture.alternatingVisibility || FULL_SEMANTIC + ? await measureSemantic({ + reader, + store, + sessionId: session.id, + throughSequence, + fullWindows: FULL_SEMANTIC, + databasePath: join(root, OPERATIONAL_STATE_DATABASE_NAME), + }) + : undefined; return { fixture: fixture.name, messages: messages.length, @@ -130,6 +189,7 @@ async function runFixture(fixture) { bootstrapCpuMs: bootstrapCpuMs.toFixed(1), materializeCpuMs: materializeCpuMs.toFixed(1), modeledRttFloorMs: wireRequests * RTT_MS, + ...semanticMetrics(semantic), }; } finally { await store.close?.(); @@ -146,25 +206,301 @@ function buildMessages(fixture) { (fixture.totalBytes === undefined || encodedBytes < fixture.totalBytes) ) { const index = messages.length; - const message = { - type: 'user', - id: `message-${index}`, - turnId: `turn-${index}`, - ts: index + 1, - text: 'x'.repeat(fixture.textBytes), - }; + const message = + fixture.alternatingVisibility && index % 2 === 1 + ? { + type: 'permission_decision', + id: `message-${index}`, + turnId: `turn-${index}`, + ts: index + 1, + toolUseId: `tool-${index}`, + toolName: 'benchmark', + decision: 'allow', + } + : { + type: 'user', + id: `message-${index}`, + turnId: `turn-${index}`, + ts: index + 1, + text: 'x'.repeat(fixture.textBytes), + }; messages.push(message); encodedBytes += Buffer.byteLength(JSON.stringify(message), 'utf8'); } return messages; } +async function measureSemantic({ + reader, + store, + sessionId, + throughSequence, + fullWindows, + databasePath, +}) { + let storageCalls = 0; + let positionPageCalls = 0; + let bodyCalls = 0; + let releaseCalls = 0; + let rawBytes = 0; + let responses = 0; + let encodedBytes = 0; + let retainedBytes = 0; + let peakRetainedBytes = 0; + let preparingBytes = 0; + let peakPreparationBytes = 0; + const measuredReader = { + ...reader, + readPositionPage: async (request) => { + storageCalls += 1; + positionPageCalls += 1; + return reader.readPositionPage(request); + }, + readPositionRecords: async (request) => { + storageCalls += 1; + bodyCalls += 1; + const result = await reader.readPositionRecords(request); + rawBytes += result.rawBytes; + return result; + }, + releasePositionSnapshot: async (request) => { + storageCalls += 1; + releaseCalls += 1; + return reader.releasePositionSnapshot(request); + }, + }; + const accounting = { + retain: (bytes) => { + if (retainedBytes + bytes > 64 * 1024 * 1024) return false; + retainedBytes += bytes; + peakRetainedBytes = Math.max(peakRetainedBytes, retainedBytes); + return true; + }, + release: (bytes) => { + retainedBytes -= bytes; + if (retainedBytes < 0) throw new Error('semantic benchmark accounting underflow'); + }, + }; + const owner = await acquireSemanticSnapshot(measuredReader, sessionId, 'owner', throughSequence); + const shared = await acquireSemanticSnapshot( + measuredReader, + sessionId, + 'shared', + throughSequence, + ); + let buildSteps = owner.buildSteps + shared.buildSteps; + let secondOwner; + if (owner.total === 10_000 && shared.total === 5_000) { + await store.appendMessage(sessionId, { + type: 'system_note', + id: 'benchmark-hidden-generation-note', + ts: throughSequence + 2, + kind: 'session_start', + }); + secondOwner = await acquireSemanticSnapshot( + measuredReader, + sessionId, + 'owner', + throughSequence + 1, + ); + buildSteps += secondOwner.buildSteps; + if (secondOwner.total !== owner.total) { + throw new Error('hidden generation marker changed semantic position cardinality'); + } + } + const inspector = new DatabaseSync(databasePath); + inspector.exec('PRAGMA wal_checkpoint(TRUNCATE)'); + const cardinality = inspector + .prepare( + `SELECT + (SELECT COUNT(*) FROM session_turn_position_snapshots WHERE session_id = ?) AS snapshot_rows, + (SELECT COUNT(*) FROM session_turn_snapshot_positions WHERE session_id = ?) AS position_rows, + (SELECT COUNT(*) FROM session_turn_snapshot_leases WHERE session_id = ?) AS lease_rows`, + ) + .get(sessionId, sessionId, sessionId); + const dataVersionBefore = inspector.prepare('PRAGMA data_version').get().data_version; + const warmWalBefore = await fileSize(`${databasePath}-wal`); + const cpuSamples = []; + const warmSnapshotKey = JSON.stringify(owner.state.currentSnapshot?.key); + const warmAnchors = [ + { kind: 'tail' }, + { kind: 'ordinal', ordinal: Math.floor(owner.total / 2) }, + { kind: 'turn', turnId: `turn-${Math.floor(owner.total / 2)}` }, + ]; + for (let iteration = 0; iteration < 100; iteration += 1) { + const started = performance.now(); + const page = await querySemanticTranscriptPositions({ + reader: measuredReader, + state: owner.state, + request: { + kind: 'page', + subscriptionId: owner.state.subscriptionId, + snapshotToken: owner.token, + anchor: warmAnchors[iteration % warmAnchors.length], + maxPositions: 10, + }, + }); + if (page.kind !== 'page') throw new Error(`warm semantic read returned ${page.kind}`); + if (iteration >= 2) cpuSamples.push(performance.now() - started); + } + if (JSON.stringify(owner.state.currentSnapshot?.key) !== warmSnapshotKey) { + throw new Error('warm semantic reads changed their exact snapshot identity'); + } + const warmWalAfter = await fileSize(`${databasePath}-wal`); + const dataVersionAfter = inspector.prepare('PRAGMA data_version').get().data_version; + inspector.close(); + const windowPositionReadsBefore = positionPageCalls; + const windowBodyReadsBefore = bodyCalls; + const windowLimit = fullWindows ? Number.POSITIVE_INFINITY : 1; + let startOrdinal = 0; + let windows = 0; + let acceptedPositions = 0; + let oversizedPositions = 0; + for ( + let windowIndex = 0; + startOrdinal < owner.total && windowIndex < windowLimit; + windowIndex += 1 + ) { + preparingBytes += 48 * 1024 * 1024; + peakPreparationBytes = Math.max(peakPreparationBytes, preparingBytes); + let result; + try { + result = await readSemanticTranscriptTurnWindow({ + reader: measuredReader, + state: owner.state, + request: { + kind: 'open', + subscriptionId: owner.state.subscriptionId, + snapshotToken: owner.token, + startOrdinal, + maxPositions: 10, + replaceCursor: null, + }, + accounting, + }); + } finally { + preparingBytes -= 48 * 1024 * 1024; + } + if (result.kind === 'position_too_large') { + oversizedPositions += 1; + break; + } + if (result.kind !== 'page') throw new Error(`semantic window returned ${result.kind}`); + const fragments = []; + while (result.kind === 'page') { + responses += 1; + encodedBytes += Buffer.byteLength(JSON.stringify(result), 'utf8'); + fragments.push(Buffer.from(result.data, 'base64')); + if (result.nextCursor === null) break; + result = await readSemanticTranscriptTurnWindow({ + reader: measuredReader, + state: owner.state, + request: { + kind: 'continue', + subscriptionId: owner.state.subscriptionId, + cursor: result.nextCursor, + }, + accounting, + }); + } + if (result.kind !== 'page') throw new Error(`semantic continuation returned ${result.kind}`); + const decoded = JSON.parse(Buffer.concat(fragments).toString('utf8')); + if (decoded.startOrdinal !== startOrdinal || decoded.endOrdinalExclusive <= startOrdinal) { + throw new Error('semantic benchmark window did not advance'); + } + windows += 1; + acceptedPositions += decoded.positions.length; + startOrdinal = decoded.endOrdinalExclusive; + } + const ownerTotal = owner.total; + const sharedTotal = shared.total; + if (secondOwner) { + await releaseSubscriberSemanticTranscript(measuredReader, secondOwner.state, accounting); + } + await releaseSubscriberSemanticTranscript(measuredReader, shared.state, accounting); + await releaseSubscriberSemanticTranscript(measuredReader, owner.state, accounting); + if (retainedBytes !== 0) throw new Error('semantic benchmark retained accounting leaked'); + if (preparingBytes !== 0) throw new Error('semantic benchmark preparation accounting leaked'); + if (fullWindows && startOrdinal < owner.total && owner.total !== 1) { + throw new Error('semantic benchmark did not cover every position'); + } + cpuSamples.sort((left, right) => left - right); + const rawForAmplification = Math.max(1, rawBytes); + return { + ownerTotal, + sharedTotal, + storageCalls, + buildSteps, + windowPositionReads: positionPageCalls - windowPositionReadsBefore, + windowBodyReads: bodyCalls - windowBodyReadsBefore, + releaseCalls, + windows, + acceptedPositions, + oversizedPositions, + rawBytes, + responses, + encodedBytes, + peakRetainedBytes, + peakPreparationBytes, + snapshotRows: Number(cardinality.snapshot_rows), + positionRows: Number(cardinality.position_rows), + leaseRows: Number(cardinality.lease_rows), + warmWalDeltaBytes: warmWalAfter - warmWalBefore, + warmDataVersionDelta: Number(dataVersionAfter) - Number(dataVersionBefore), + wireAmplification: (encodedBytes / rawForAmplification).toFixed(3), + cpuMedianMs: cpuSamples[Math.floor(cpuSamples.length / 2)].toFixed(3), + cpuP95Ms: cpuSamples[Math.ceil(cpuSamples.length * 0.95) - 1].toFixed(3), + }; +} + +async function acquireSemanticSnapshot(reader, sessionId, projection, throughSequence) { + const state = createSubscriberSemanticTranscriptState({ + sessionId, + subscriptionId: `benchmark-${projection}-${sessionId}`, + projection, + cursorSecret: randomBytes(32), + }); + confirmSemanticTranscriptWatermark(state, throughSequence); + let result = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: state.subscriptionId, + anchor: { kind: 'tail' }, + maxPositions: 10, + }, + }); + let buildSteps = result.kind === 'building' ? 1 : 0; + for (let steps = 0; result.kind === 'building' && steps < 100_000; steps += 1) { + result = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'page', + subscriptionId: state.subscriptionId, + snapshotToken: result.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 10, + }, + }); + if (result.kind === 'building') buildSteps += 1; + } + if (result.kind !== 'page') throw new Error(`semantic snapshot returned ${result.kind}`); + return { state, token: result.snapshotToken, total: result.totalPositions, buildSteps }; +} + function sqliteReader(store) { return { readDurableHighWater: (sessionId) => store.readTranscriptHighWaterSnapshot(sessionId), readDurablePage: (sessionId, request) => store.readTranscriptPageSnapshot(sessionId, request), - readDurableMessagesById: (sessionId, messageIds, throughSequence) => - store.readTranscriptMessagesSnapshot(sessionId, messageIds, throughSequence), + readDurableRecords: (sessionId, request) => + store.readTranscriptRecordsSnapshot(sessionId, request), + readDurableMessagesById: (sessionId, request) => + store.readTranscriptMessagesSnapshot(sessionId, request), + readPositionPage: (request) => store.readTurnPositionPageSnapshot(request), + readPositionRecords: (request) => store.readTranscriptRecordsByPositionKeysSnapshot(request), + releasePositionSnapshot: (request) => store.releaseTurnPositionSnapshot(request), readActiveOverlay: async () => [], }; } @@ -176,3 +512,40 @@ function decimalMiB(bytes) { function decimalKiB(bytes) { return (bytes / 1024).toFixed(2); } + +function semanticMetrics(semantic) { + return { + semanticOwner: semantic?.ownerTotal ?? '-', + semanticShared: semantic?.sharedTotal ?? '-', + semanticStorageCalls: semantic?.storageCalls ?? '-', + semanticBuildSteps: semantic?.buildSteps ?? '-', + semanticPositionReads: semantic?.windowPositionReads ?? '-', + semanticBodyReads: semantic?.windowBodyReads ?? '-', + semanticReleaseCalls: semantic?.releaseCalls ?? '-', + semanticWindows: semantic?.windows ?? '-', + semanticAcceptedPositions: semantic?.acceptedPositions ?? '-', + semanticOversizedPositions: semantic?.oversizedPositions ?? '-', + semanticResponses: semantic?.responses ?? '-', + semanticRawMiB: semantic ? decimalMiB(semantic.rawBytes) : '-', + semanticEncodedMiB: semantic ? decimalMiB(semantic.encodedBytes) : '-', + semanticPeakMiB: semantic ? decimalMiB(semantic.peakRetainedBytes) : '-', + semanticPeakPreparationMiB: semantic ? decimalMiB(semantic.peakPreparationBytes) : '-', + semanticSnapshotRows: semantic?.snapshotRows ?? '-', + semanticPositionRows: semantic?.positionRows ?? '-', + semanticLeaseRows: semantic?.leaseRows ?? '-', + semanticWarmWalDelta: semantic?.warmWalDeltaBytes ?? '-', + semanticWarmDataVersionDelta: semantic?.warmDataVersionDelta ?? '-', + semanticWireAmplification: semantic?.wireAmplification ?? '-', + semanticCpuMedianMs: semantic?.cpuMedianMs ?? '-', + semanticCpuP95Ms: semantic?.cpuP95Ms ?? '-', + }; +} + +async function fileSize(path) { + try { + return (await stat(path)).size; + } catch (error) { + if (error && typeof error === 'object' && error.code === 'ENOENT') return 0; + throw error; + } +} diff --git a/packages/runtime-host/src/__tests__/authenticated-websocket.test.ts b/packages/runtime-host/src/__tests__/authenticated-websocket.test.ts index 99986ea8b2..36fa139d7f 100644 --- a/packages/runtime-host/src/__tests__/authenticated-websocket.test.ts +++ b/packages/runtime-host/src/__tests__/authenticated-websocket.test.ts @@ -1141,6 +1141,43 @@ test('migrates the released transcript query grant when opening an existing acce 'host.status', 'session.transcript.page', 'session.transcript.overlay.release', + 'session.transcript.positions.query', + 'session.transcript.turn_window.page', + ]); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('does not expand a modern transcript-page-only credential to semantic operations', async () => { + const directory = await mkdtemp(join(tmpdir(), 'maka-access-authority-transcript-modern-')); + const credential = 'maka_rh_existing_transcript_page_client'; + try { + await writeFile( + join(directory, 'runtime-host-access.json'), + `${JSON.stringify({ + schemaVersion: 1, + credentials: [ + { + credentialId: 'existing-transcript-page-client', + credentialHash: createHash('sha256').update(credential).digest('hex'), + principalId: 'existing-transcript-page-client', + principalKind: 'remote_owner', + status: 'active', + operationGrants: ['host.status', 'session.transcript.page'], + canPublishClientCapabilities: false, + canUseHostPaths: false, + createdAt: '2026-01-01T00:00:00.000Z', + }, + ], + })}\n`, + { mode: 0o600 }, + ); + + const authority = await openRuntimeHostAccessAuthority(directory); + assert.deepEqual(authority.authenticate(credential)?.operationGrants, [ + 'host.status', + 'session.transcript.page', ]); } finally { await rm(directory, { recursive: true, force: true }); diff --git a/packages/runtime-host/src/__tests__/connection-session.test.ts b/packages/runtime-host/src/__tests__/connection-session.test.ts index 3abca22312..658b0455d7 100644 --- a/packages/runtime-host/src/__tests__/connection-session.test.ts +++ b/packages/runtime-host/src/__tests__/connection-session.test.ts @@ -134,13 +134,22 @@ test('concurrent responses remain framed and correlated in reverse completion or ); }); -test('transcript pages are serialized per connection before their responses are retained', async () => { +test('all transcript data-plane operations are serialized per connection', async () => { const pair = await openTransportPair(); const entered = Array.from({ length: 3 }, () => deferred()); const release = Array.from({ length: 3 }, () => deferred()); let calls = 0; let active = 0; let maxActive = 0; + const enter = async () => { + const index = calls; + calls += 1; + active += 1; + maxActive = Math.max(maxActive, active); + entered[index]?.resolve(); + await release[index]?.promise; + active -= 1; + }; const handlers: OperationHandlerMap = { 'host.status': async () => ({ ok: true, @@ -161,13 +170,7 @@ test('transcript pages are serialized per connection before their responses are result: runningSnapshot(input.sessionId, input.turnId), })), 'session.transcript.page': async (input) => { - const index = calls; - calls += 1; - active += 1; - maxActive = Math.max(maxActive, active); - entered[index]?.resolve(); - await release[index]?.promise; - active -= 1; + await enter(); return { ok: true, result: { @@ -184,6 +187,27 @@ test('transcript pages are serialized per connection before their responses are }, }; }, + 'session.transcript.positions.query': async (input) => { + await enter(); + return { + ok: true, + result: { + kind: 'released', + subscriptionId: input.subscriptionId, + }, + }; + }, + 'session.transcript.turn_window.page': async (input) => { + await enter(); + return { + ok: true, + result: { + kind: 'snapshot_stale', + subscriptionId: input.subscriptionId, + snapshotToken: input.kind === 'open' ? input.snapshotToken : 'snapshot-token', + }, + }; + }, }; const session = new RuntimeHostConnectionSession({ transport: pair.serverTransport, @@ -199,21 +223,40 @@ test('transcript pages are serialized per connection before their responses are }); const run = session.run(); try { - for (let index = 0; index < 3; index += 1) { - await writeProtocolFrame(pair.clientTransport, { - requestId: `transcript-page-${index}`, - operation: 'session.transcript.page', - input: { - subscriptionId: 'subscription-1', - source: 'durable', - direction: 'older', - throughSequence: null, - cursor: null, - anchorSequence: null, - maxBytes: 512 * 1024, - }, - }); - } + await writeProtocolFrame(pair.clientTransport, { + requestId: 'transcript-data-plane-0', + operation: 'session.transcript.page', + input: { + subscriptionId: 'subscription-1', + source: 'durable', + direction: 'older', + throughSequence: null, + cursor: null, + anchorSequence: null, + maxBytes: 512 * 1024, + }, + }); + await writeProtocolFrame(pair.clientTransport, { + requestId: 'transcript-data-plane-1', + operation: 'session.transcript.positions.query', + input: { + kind: 'release', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + }, + }); + await writeProtocolFrame(pair.clientTransport, { + requestId: 'transcript-data-plane-2', + operation: 'session.transcript.turn_window.page', + input: { + kind: 'open', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + startOrdinal: 0, + maxPositions: 1, + replaceCursor: null, + }, + }); await withTimeout(entered[0]!.promise, 1_000, 'first transcript page was not admitted'); await new Promise((resolve) => setImmediate(resolve)); assert.equal(calls, 1); @@ -222,7 +265,9 @@ test('transcript pages are serialized per connection before their responses are release[index]!.resolve(); const response = decodeHostFrame(await pair.clientTransport.read(1_000)); assert.equal('kind' in response, false); - if (!('kind' in response)) assert.equal(response.requestId, `transcript-page-${index}`); + if (!('kind' in response)) { + assert.equal(response.requestId, `transcript-data-plane-${index}`); + } if (index < 2) { await withTimeout( entered[index + 1]!.promise, @@ -460,6 +505,14 @@ test('flushes concurrent subscription opens before activating their live frame s ok: false, error: { code: 'operation_unavailable', message: 'not used' }, }), + 'session.transcript.positions.query': async () => ({ + ok: false, + error: { code: 'operation_unavailable', message: 'not used' }, + }), + 'session.transcript.turn_window.page': async () => ({ + ok: false, + error: { code: 'operation_unavailable', message: 'not used' }, + }), }, attachConnection: (_connectionId, attachedSink) => { sink = attachedSink; @@ -477,7 +530,7 @@ test('flushes concurrent subscription opens before activating their live frame s .catch(() => undefined); }, abort() {}, - close() {}, + async close() {}, }; }, }; diff --git a/packages/runtime-host/src/__tests__/execution-composition.test.ts b/packages/runtime-host/src/__tests__/execution-composition.test.ts index 4eb2376355..25894e7d46 100644 --- a/packages/runtime-host/src/__tests__/execution-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-composition.test.ts @@ -30,6 +30,8 @@ import type { AgentGraphIntentClaimRequest, } from '@maka/core/agent-graph-control'; import type { ShellRunRecord } from '@maka/core/shell-run'; +import { decodeStoredMessage } from '@maka/core/session'; +import { markPersisted } from '@maka/core/persisted-value'; import { FAKE_ASK_USER_QUESTION_PROMPT, FakeBackend } from '@maka/runtime/test-only/fake-backend'; import { LOCAL_READ_AGENT_DEFINITION } from '@maka/runtime/agent-catalog'; import { SessionManager } from '@maka/runtime/session-manager'; @@ -41,6 +43,8 @@ import { LONG_TERM_MEMORY_DATABASE_NAME, openInteractiveLongTermMemoryStoreForWrite, } from '@maka/storage/long-term-memory-store'; +import { OPERATIONAL_STATE_DATABASE_NAME } from '@maka/storage/operational-state-store'; +import { createSessionStore } from '@maka/storage/session-store'; import { resolveStorageRoot, tryAcquireInteractiveRootOwner, @@ -50,6 +54,7 @@ import { openInteractiveUsageStoresForWrite } from '@maka/storage/usage-stores'; import { openInteractiveShellRunStoreForWrite } from '@maka/storage/shell-run-authority'; import { openInteractiveRuntimePolicyStoresForWrite } from '@maka/storage/runtime-policy-stores'; import { HostResidencyRegistry } from '../server/host-residency-registry.js'; +import { ClientSessionSubscription } from '../client/session-subscription.js'; import { createExecutionRuntimeHostComposition, runtimeHostFilesystemWorkerRuntime, @@ -159,6 +164,281 @@ test('production composition closes long-term memory after a later startup failu }); }); +test('real SQLite semantic positions reach the production subscription client', async () => { + await withCompositionRoot(async ({ root, owner }) => { + const stores = await openInteractiveExecutionStoresForWrite(owner.lease); + const session = await stores.sessionStore.create({ + cwd: root, + llmConnectionId: FAKE_CONNECTION_ID, + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode: 'ask', + }); + await stores.sessionStore.appendMessage(session.id, { + type: 'user', + id: 'semantic-user-1', + turnId: 'semantic-turn-1', + ts: 1, + text: 'semantic transcript', + }); + const composition = await createExecutionRuntimeHostComposition(compositionContext(owner)); + try { + await composition.recover(); + const continuity = composition.continuity; + assert.ok(continuity); + if (!continuity) return; + const connectionId = 'semantic-composition-client'; + let client: ClientSessionSubscription | undefined; + const attached = continuity.attachConnection(connectionId, { + send: async (frame) => client?.accept(frame), + }); + const context = { + hostEpoch: 'execution-composition-test', + connectionId, + principal: 'local_os_user' as const, + principalKind: 'local_owner' as const, + acquireResidency: () => ({ release() {} }), + }; + const opened = await composition.handlers['subscription.open']( + { + sessionId: session.id, + transcript: { kind: 'tail', maxBytes: 16 * 1024 }, + }, + context, + ); + assert.equal(opened.ok, true); + if (!opened.ok) return; + client = new ClientSessionSubscription( + opened.result, + async () => { + await composition.handlers['subscription.close']( + { subscriptionId: opened.result.subscriptionId }, + context, + ); + }, + async (input) => { + const outcome = await composition.handlers['session.transcript.page'](input, context); + if (!outcome.ok) throw new Error(outcome.error.message); + return outcome.result; + }, + async () => undefined, + { + queryPositions: async (input) => { + const outcome = await composition.handlers['session.transcript.positions.query']( + input, + context, + ); + if (!outcome.ok) throw new Error(outcome.error.message); + return outcome.result; + }, + readTurnWindow: async (input) => { + const outcome = await composition.handlers['session.transcript.turn_window.page']( + input, + context, + ); + if (!outcome.ok) throw new Error(outcome.error.message); + return outcome.result; + }, + }, + ); + attached.activate(opened.result.subscriptionId); + + let positions = await client.queryTranscriptPositions({ + kind: 'acquire', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + for (let step = 0; positions.kind === 'building' && step < 16; step += 1) { + positions = await client.queryTranscriptPositions({ + kind: 'page', + snapshotToken: positions.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(positions.kind, 'page'); + if (positions.kind !== 'page') assert.fail('semantic snapshot did not become ready'); + assert.deepEqual(positions.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'semantic-turn-1' } }, + ]); + const window = await client.loadTranscriptTurnWindow( + { + snapshotToken: positions.snapshotToken, + startOrdinal: 0, + maxPositions: 1, + replaceCursor: null, + }, + (value) => decodeStoredMessage(markPersisted(value)), + ); + assert.ok(!('kind' in window)); + if ('kind' in window) return; + assert.deepEqual(window.positions[0]?.messages, [ + { + type: 'user', + id: 'semantic-user-1', + turnId: 'semantic-turn-1', + ts: 1, + text: 'semantic transcript', + }, + ]); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + const leaseCount = () => + Number( + database + .prepare( + 'SELECT COUNT(*) AS count FROM session_turn_snapshot_leases WHERE session_id = ?', + ) + .get(session.id)?.count, + ); + assert.equal(leaseCount(), 1); + await client.close(); + await attached.close(); + assert.equal(leaseCount(), 0); + } finally { + database.close(); + } + } finally { + await composition.close(); + } + }); +}); + +test('production semantic positions resume persisted v34 chunk recovery after Host reopen', async () => { + const base = await mkdtemp(join(tmpdir(), 'maka-semantic-v34-composition-')); + const root = join(base, 'interactive'); + const setupStore = createSessionStore(root); + const recovery = await setupStore.create({ + cwd: root, + llmConnectionId: FAKE_CONNECTION_ID, + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode: 'ask', + }); + await setupStore.appendMessages(recovery.id, [ + { + type: 'user', + id: 'legacy-large-user', + turnId: 'legacy-large-turn', + ts: 1, + text: 'x'.repeat(4 * 1024 * 1024 + 1_024), + }, + { + type: 'system_note', + id: 'legacy-visible-note', + ts: 2, + kind: 'context_compacted', + }, + { type: 'system_note', id: 'legacy-hidden-note', ts: 3, kind: 'session_start' }, + ]); + await setupStore.close?.(); + downgradeSemanticProjectionToV34(root); + const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); + let owner = await tryAcquireInteractiveRootOwner(capability); + assert.ok(owner); + if (!owner) throw new Error('Unable to acquire semantic recovery root'); + let composition: Awaited> | undefined; + try { + composition = await createExecutionRuntimeHostComposition(compositionContext(owner)); + await composition.recover(); + const first = await openCompositionSemanticClient( + composition, + recovery.id, + 'semantic-v34-before-reopen', + ); + let positions = await first.client.queryTranscriptPositions({ + kind: 'acquire', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + let partial = readSemanticRecoveryPartial(root, recovery.id); + for (let step = 0; positions.kind === 'building' && !partial && step < 4; step += 1) { + positions = await first.client.queryTranscriptPositions({ + kind: 'page', + snapshotToken: positions.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + partial = readSemanticRecoveryPartial(root, recovery.id); + } + assert.equal(positions.kind, 'building'); + assert.ok(partial); + if (!partial) assert.fail('expected persisted partial scalar recovery'); + assert.ok(partial.byte_offset > 0 && partial.byte_offset <= 4 * 1024 * 1024); + assert.ok(partial.byte_offset < partial.record_bytes); + assert.equal(partial.hash_implementation, 'hash-wasm@4.12.0'); + assert.ok(partial.hash_state_bytes > 0 && partial.scanner_state_bytes > 0); + const generationBeforeReopen = readNextSemanticGeneration(root, recovery.id); + + await first.client.close(); + await first.attached.close(); + await composition.close(); + composition = undefined; + await owner.close(); + owner = undefined; + assert.deepEqual(readSemanticRecoveryPartial(root, recovery.id), partial); + + owner = await tryAcquireInteractiveRootOwner(capability); + assert.ok(owner); + if (!owner) throw new Error('Unable to reopen semantic recovery root'); + composition = await createExecutionRuntimeHostComposition(compositionContext(owner)); + await composition.recover(); + const reopened = await openCompositionSemanticClient( + composition, + recovery.id, + 'semantic-v34-after-reopen', + ); + let ready = await reopened.client.queryTranscriptPositions({ + kind: 'acquire', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + for (let step = 0; ready.kind === 'building' && step < 32; step += 1) { + ready = await reopened.client.queryTranscriptPositions({ + kind: 'page', + snapshotToken: ready.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(ready.kind, 'page'); + if (ready.kind !== 'page') assert.fail('expected resumed semantic recovery to finish'); + assert.deepEqual(ready.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'legacy-large-turn' } }, + { ordinal: 1, key: { kind: 'note', id: 'legacy-visible-note' } }, + ]); + assert.equal(readSemanticRecoveryPartial(root, recovery.id), undefined); + assert.ok(readNextSemanticGeneration(root, recovery.id) > generationBeforeReopen); + const visibleNoteWindow = await reopened.client.loadTranscriptTurnWindow( + { + snapshotToken: ready.snapshotToken, + startOrdinal: 1, + maxPositions: 1, + }, + (value) => decodeStoredMessage(markPersisted(value)), + ); + assert.ok(!('kind' in visibleNoteWindow)); + if ('kind' in visibleNoteWindow) assert.fail('expected a complete visible-note window'); + assert.deepEqual(visibleNoteWindow.positions[0]?.messages, [ + { + type: 'system_note', + id: 'legacy-visible-note', + ts: 2, + kind: 'context_compacted', + }, + ]); + + await reopened.client.close(); + await reopened.attached.close(); + } finally { + await composition?.close(); + await owner?.close(); + await rm(base, { recursive: true, force: true }); + } +}); + test('production recovery preserves legacy Automation history and closes an orphaned admission', async () => { await withCompositionRoot(async ({ root, owner }) => { const stores = await openInteractiveExecutionStoresForWrite(owner.lease); @@ -996,6 +1276,133 @@ function compositionContext(owner: InteractiveRootOwner) { }; } +async function openCompositionSemanticClient( + composition: Awaited>, + sessionId: string, + connectionId: string, +) { + const continuity = composition.continuity; + assert.ok(continuity); + if (!continuity) throw new Error('Semantic composition has no continuity coordinator'); + let client: ClientSessionSubscription | undefined; + const attached = continuity.attachConnection(connectionId, { + send: async (frame) => client?.accept(frame), + }); + const context = { + hostEpoch: 'execution-composition-test', + connectionId, + principal: 'local_os_user' as const, + principalKind: 'local_owner' as const, + acquireResidency: () => ({ release() {} }), + }; + const opened = await composition.handlers['subscription.open']( + { sessionId, transcript: { kind: 'tail', maxBytes: 16 * 1024 } }, + context, + ); + if (!opened.ok) throw new Error(opened.error.message); + assert.equal(opened.ok, true); + client = new ClientSessionSubscription( + opened.result, + async () => { + await composition.handlers['subscription.close']( + { subscriptionId: opened.result.subscriptionId }, + context, + ); + }, + async (input) => { + const outcome = await composition.handlers['session.transcript.page'](input, context); + if (!outcome.ok) throw new Error(outcome.error.message); + return outcome.result; + }, + async () => undefined, + { + queryPositions: async (input) => { + const outcome = await composition.handlers['session.transcript.positions.query']( + input, + context, + ); + if (!outcome.ok) throw new Error(`${outcome.error.code}: ${outcome.error.message}`); + return outcome.result; + }, + readTurnWindow: async (input) => { + const outcome = await composition.handlers['session.transcript.turn_window.page']( + input, + context, + ); + if (!outcome.ok) throw new Error(`${outcome.error.code}: ${outcome.error.message}`); + return outcome.result; + }, + }, + ); + attached.activate(opened.result.subscriptionId); + await new Promise((resolve) => setImmediate(resolve)); + return { client, attached }; +} + +function downgradeSemanticProjectionToV34(root: string): void { + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec(` + PRAGMA foreign_keys = OFF; + DROP TABLE session_turn_snapshot_leases; + DROP TABLE session_turn_snapshot_positions; + DROP TABLE session_turn_position_snapshots; + DROP TABLE session_turn_identity_recovery; + DROP TABLE session_turn_memberships; + DROP TABLE session_turn_metadata; + DROP TABLE session_turn_index_state; + DROP TABLE session_turn_authority_revisions; + UPDATE session_metadata_schema SET version = 34 WHERE scope = 'session_metadata'; + `); + } finally { + database.close(); + } +} + +function readSemanticRecoveryPartial( + root: string, + sessionId: string, +): + | { + readonly byte_offset: number; + readonly record_bytes: number; + readonly hash_implementation: string; + readonly hash_state_bytes: number; + readonly scanner_state_bytes: number; + } + | undefined { + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + const row = database + .prepare(`SELECT byte_offset, record_bytes, hash_implementation, + length(hash_state) AS hash_state_bytes, + length(CAST(scanner_state AS BLOB)) AS scanner_state_bytes + FROM session_turn_identity_recovery WHERE session_id = ?`) + .get(sessionId); + return row ? ({ ...row } as ReturnType) : undefined; + } finally { + database.close(); + } +} + +function readNextSemanticGeneration(root: string, sessionId: string): number { + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + return Number( + database + .prepare(`SELECT next_snapshot_generation FROM session_turn_authority_revisions + WHERE session_id = ?`) + .get(sessionId)?.next_snapshot_generation, + ); + } finally { + database.close(); + } +} + function shellRunRecord( sessionId: string, shellRunId: string, diff --git a/packages/runtime-host/src/__tests__/fixtures/session-transcript-reader.ts b/packages/runtime-host/src/__tests__/fixtures/session-transcript-reader.ts index 8d2dbecb96..e9e850ebb8 100644 --- a/packages/runtime-host/src/__tests__/fixtures/session-transcript-reader.ts +++ b/packages/runtime-host/src/__tests__/fixtures/session-transcript-reader.ts @@ -142,6 +142,13 @@ export function transcriptReader( (message, sequence) => sequence <= request.throughSequence! && request.messageIds.includes(message.id), ), + readPositionPage: async () => { + throw new Error('Semantic transcript fixture is not configured'); + }, + readPositionRecords: async () => { + throw new Error('Semantic transcript fixture is not configured'); + }, + releasePositionSnapshot: async () => undefined, readActiveOverlay: async () => overlay, }; } diff --git a/packages/runtime-host/src/__tests__/semantic-session-transcript-pager.test.ts b/packages/runtime-host/src/__tests__/semantic-session-transcript-pager.test.ts new file mode 100644 index 0000000000..ccdcfe5c9b --- /dev/null +++ b/packages/runtime-host/src/__tests__/semantic-session-transcript-pager.test.ts @@ -0,0 +1,1252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import type { StoredMessage } from '@maka/core/session'; +import { createSqliteAgentRunStore } from '@maka/storage/agent-run-store'; +import { + SessionTurnPositionLimitError, + type SessionTurnPositionSnapshotKey, +} from '@maka/storage/execution-stores'; +import { createSessionStore } from '@maka/storage/session-store'; +import { + confirmSemanticTranscriptWatermark, + createSubscriberSemanticTranscriptState, + querySemanticTranscriptPositions, + readSemanticTranscriptTurnWindow, + releaseSubscriberSemanticTranscript, + SEMANTIC_TRANSCRIPT_WINDOW_IDLE_TTL_MS, + SemanticTranscriptRequestError, +} from '../server/semantic-session-transcript-pager.js'; +import type { SessionTranscriptReader } from '../server/session-transcript-reader.js'; + +const snapshotKey: SessionTurnPositionSnapshotKey = { + throughSequence: 5, + authorityRevision: 2, + snapshotGeneration: 7, +}; + +test('acquires, continues, and releases an opaque exact semantic snapshot', async () => { + const calls: Array<{ kind: string; value: unknown }> = []; + const reader = semanticReader({ + readPositionPage: async (request) => { + calls.push({ kind: 'positions', value: request }); + return { + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 2, + positions: [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-1' }, firstSequence: 0 }, + { ordinal: 1, key: { kind: 'note', id: 'note-1' }, firstSequence: 5 }, + ], + hasOlder: false, + hasNewer: false, + }; + }, + releasePositionSnapshot: async (request) => { + calls.push({ kind: 'release', value: request }); + }, + }); + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x41), + }); + confirmSemanticTranscriptWatermark(state, 5); + const acquired = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: 'subscription-1', + anchor: { kind: 'tail' }, + maxPositions: 2, + }, + }); + assert.equal(acquired.kind, 'page'); + if (acquired.kind !== 'page') assert.fail('expected a position page'); + const publicHandle = JSON.parse( + Buffer.from(acquired.snapshotToken.split('.')[0]!, 'base64url').toString('utf8'), + ) as Record; + assert.deepEqual(Object.keys(publicHandle).sort(), ['handleId', 'incarnation', 'version']); + assert.equal(JSON.stringify(publicHandle).includes('session-1'), false); + assert.equal(JSON.stringify(publicHandle).includes('owner'), false); + assert.deepEqual(acquired.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-1' } }, + { ordinal: 1, key: { kind: 'note', id: 'note-1' } }, + ]); + assert.equal(JSON.stringify(acquired).includes('snapshotGeneration'), false); + const firstRequest = calls[0]?.value as Record; + assert.equal(firstRequest.throughSequence, 5); + assert.equal(firstRequest.projection, 'owner'); + + await assert.rejects( + querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'page', + subscriptionId: 'subscription-1', + snapshotToken: `${acquired.snapshotToken}x`, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }), + SemanticTranscriptRequestError, + ); + const otherState = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-2', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x51), + }); + confirmSemanticTranscriptWatermark(otherState, 5); + await assert.rejects( + querySemanticTranscriptPositions({ + reader, + state: otherState, + request: { + kind: 'page', + subscriptionId: 'subscription-2', + snapshotToken: acquired.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }), + SemanticTranscriptRequestError, + ); + + const released = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'release', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + }, + }); + assert.deepEqual(released, { kind: 'released', subscriptionId: 'subscription-1' }); + assert.deepEqual( + await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'release', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + }, + }), + released, + ); + await releaseSubscriberSemanticTranscript(reader, state); + assert.equal(calls.filter((call) => call.kind === 'release').length, 1); +}); + +test('uses a reduced older-page limit near the transcript head to avoid overlap', async () => { + const requests: Array<{ anchor: unknown; maxPositions: number }> = []; + const reader = semanticReader({ + readPositionPage: async (request) => { + requests.push({ anchor: request.anchor, maxPositions: request.maxPositions }); + const startOrdinal = requests.length === 1 ? 3 : 0; + const count = requests.length === 1 ? 5 : 3; + return { + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal, + totalPositions: 8, + positions: Array.from({ length: count }, (_, index) => ({ + ordinal: startOrdinal + index, + key: { kind: 'turn' as const, id: `turn-${startOrdinal + index}` }, + firstSequence: startOrdinal + index, + })), + hasOlder: startOrdinal > 0, + hasNewer: startOrdinal + count < 8, + }; + }, + }); + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x61), + }); + confirmSemanticTranscriptWatermark(state, 5); + const acquired = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: state.subscriptionId, + anchor: { kind: 'ordinal', ordinal: 3 }, + maxPositions: 5, + }, + }); + assert.equal(acquired.kind, 'page'); + if (acquired.kind !== 'page' || acquired.olderCursor === null) { + assert.fail('expected an older positions cursor'); + } + const older = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'continue', + subscriptionId: state.subscriptionId, + cursor: acquired.olderCursor, + }, + }); + assert.equal(older.kind, 'page'); + if (older.kind !== 'page') assert.fail('expected an older position page'); + assert.deepEqual( + older.positions.map((position) => position.ordinal), + [0, 1, 2], + ); + assert.deepEqual(requests[1], { + anchor: { kind: 'ordinal', ordinal: 0 }, + maxPositions: 3, + }); +}); + +test('encodes a complete shared semantic window once and continues by slicing only', async () => { + let positionReads = 0; + let bodyReads = 0; + const reader = semanticReader({ + readPositionPage: async (request) => { + positionReads += 1; + return { + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'turn', id: 'turn-1' }, firstSequence: 0 }], + hasOlder: false, + hasNewer: false, + }; + }, + readPositionRecords: async (request) => { + bodyReads += 1; + return { + projection: request.projection, + snapshotKey, + rawBytes: 200, + records: [ + { + positionKey: { kind: 'turn', id: 'turn-1' }, + sequence: 0, + message: { + type: 'permission_decision', + id: 'hidden-1', + turnId: 'turn-1', + ts: 1, + toolUseId: 'tool-use-1', + toolName: 'shell', + decision: 'allow', + }, + }, + { + positionKey: { kind: 'turn', id: 'turn-1' }, + sequence: 1, + message: { + type: 'user', + id: 'user-1', + turnId: 'turn-1', + ts: 2, + text: 'visible', + }, + }, + ], + }; + }, + }); + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'shared', + cursorSecret: Buffer.alloc(32, 0x42), + }); + confirmSemanticTranscriptWatermark(state, 5); + const acquired = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: 'subscription-1', + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(acquired.kind, 'page'); + if (acquired.kind !== 'page') assert.fail('expected a position page'); + const beforeWindow = positionReads; + const first = await readSemanticTranscriptTurnWindow({ + reader, + state, + request: { + kind: 'open', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + startOrdinal: 0, + maxPositions: 1, + replaceCursor: null, + }, + fragmentBytes: 32, + }); + assert.equal(first.kind, 'page'); + if (first.kind !== 'page') assert.fail('expected a window page'); + const fragments = [Buffer.from(first.data, 'base64')]; + let cursor = first.nextCursor; + while (cursor) { + const continued = await readSemanticTranscriptTurnWindow({ + reader, + state, + request: { kind: 'continue', subscriptionId: 'subscription-1', cursor }, + fragmentBytes: 32, + }); + assert.equal(continued.kind, 'page'); + if (continued.kind !== 'page') assert.fail('expected a window continuation'); + fragments.push(Buffer.from(continued.data, 'base64')); + cursor = continued.nextCursor; + } + assert.equal(positionReads, beforeWindow + 1); + assert.equal(bodyReads, 1); + const decoded = JSON.parse(Buffer.concat(fragments).toString('utf8')) as { + positions: Array<{ messages: StoredMessage[] }>; + }; + assert.deepEqual(decoded.positions[0]?.messages, [ + { type: 'user', id: 'user-1', turnId: 'turn-1', ts: 2, text: 'visible' }, + ]); +}); + +test('refreshes the semantic window idle TTL after every accepted continuation', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const reader = semanticReader({ + readPositionPage: async (request) => ({ + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'turn', id: 'turn-1' }, firstSequence: 0 }], + hasOlder: false, + hasNewer: false, + }), + readPositionRecords: async (request) => ({ + projection: request.projection, + snapshotKey, + rawBytes: 1_024, + records: [ + { + positionKey: { kind: 'turn', id: 'turn-1' }, + sequence: 0, + message: { + type: 'user', + id: 'user-1', + turnId: 'turn-1', + ts: 1, + text: 'x'.repeat(1_024), + }, + }, + ], + }), + }); + let retainedBytes = 0; + const accounting = { + retain: (bytes: number) => { + retainedBytes += bytes; + return true; + }, + release: (bytes: number) => { + retainedBytes -= bytes; + }, + }; + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x62), + }); + confirmSemanticTranscriptWatermark(state, 5); + const acquired = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: state.subscriptionId, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + if (acquired.kind !== 'page') assert.fail('expected a position page'); + const first = await readSemanticTranscriptTurnWindow({ + reader, + state, + request: { + kind: 'open', + subscriptionId: state.subscriptionId, + snapshotToken: acquired.snapshotToken, + startOrdinal: 0, + maxPositions: 1, + }, + fragmentBytes: 16, + accounting, + }); + if (first.kind !== 'page' || first.nextCursor === null) { + assert.fail('expected a retained semantic window'); + } + t.mock.timers.tick(SEMANTIC_TRANSCRIPT_WINDOW_IDLE_TTL_MS - 1); + const continued = await readSemanticTranscriptTurnWindow({ + reader, + state, + request: { + kind: 'continue', + subscriptionId: state.subscriptionId, + cursor: first.nextCursor, + }, + fragmentBytes: 16, + accounting, + }); + assert.equal(continued.kind, 'page'); + t.mock.timers.tick(2); + assert.ok(retainedBytes > 0); + t.mock.timers.tick(SEMANTIC_TRANSCRIPT_WINDOW_IDLE_TTL_MS); + assert.equal(retainedBytes, 0); + await releaseSubscriberSemanticTranscript(reader, state, accounting); +}); + +test('maps an oversized first semantic position to a typed zero-fragment result', async () => { + const reader = semanticReader({ + readPositionPage: async (request) => ({ + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'turn', id: 'turn-1' }, firstSequence: 0 }], + hasOlder: false, + hasNewer: false, + }), + readPositionRecords: async () => { + throw new SessionTurnPositionLimitError('session-1', 'transcript_record_bytes'); + }, + }); + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x43), + }); + confirmSemanticTranscriptWatermark(state, 5); + const acquired = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: 'subscription-1', + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(acquired.kind, 'page'); + if (acquired.kind !== 'page') assert.fail('expected a position page'); + const result = await readSemanticTranscriptTurnWindow({ + reader, + state, + request: { + kind: 'open', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + startOrdinal: 0, + maxPositions: 1, + replaceCursor: null, + }, + }); + assert.deepEqual(result, { + kind: 'position_too_large', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + }); +}); + +test('reuses one exact lease and key across bounded building retries', async () => { + const requests: Array<{ snapshotLeaseId: string; snapshotKey?: SessionTurnPositionSnapshotKey }> = + []; + let building = true; + const reader = semanticReader({ + readPositionPage: async (request) => { + requests.push(request); + if (building) { + building = false; + return { + kind: 'building', + projection: request.projection, + snapshotKey, + progress: { + phase: 'recovering', + nextSequence: 1, + currentByteOffset: 0, + sourceRecords: 1, + sourceBytes: 10, + builtPositions: 0, + lastStepRecords: 1, + lastStepBytes: 10, + lastStepPositions: 0, + }, + }; + } + return { + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'empty' }, firstSequence: null }], + hasOlder: false, + hasNewer: false, + }; + }, + }); + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x44), + }); + confirmSemanticTranscriptWatermark(state, 5); + const first = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: 'subscription-1', + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(first.kind, 'building'); + if (first.kind !== 'building') assert.fail('expected building'); + const second = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'page', + subscriptionId: 'subscription-1', + snapshotToken: first.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(second.kind, 'page'); + assert.equal(requests[0]?.snapshotLeaseId, requests[1]?.snapshotLeaseId); + assert.equal(requests[0]?.snapshotKey, undefined); + assert.deepEqual(requests[1]?.snapshotKey, snapshotKey); +}); + +test('keeps the maximal complete prefix when a later projected position exceeds the encoded budget', async () => { + const reader = semanticReader({ + readPositionPage: async (request) => ({ + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 2, + positions: [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-1' }, firstSequence: 0 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-2' }, firstSequence: 1 }, + ], + hasOlder: false, + hasNewer: false, + }), + readPositionRecords: async (request) => { + const key = request.positionKeys[0]!; + const large = key.id === 'turn-2'; + return { + projection: request.projection, + snapshotKey, + rawBytes: large ? 2_800_000 : 32, + records: [ + { + positionKey: key, + sequence: large ? 1 : 0, + message: { + type: 'user', + id: large ? 'user-2' : 'user-1', + turnId: key.id, + ts: large ? 2 : 1, + text: large ? '\u0000'.repeat(2_800_000) : 'fits', + }, + }, + ], + }; + }, + }); + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x45), + }); + confirmSemanticTranscriptWatermark(state, 5); + const acquired = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: 'subscription-1', + anchor: { kind: 'tail' }, + maxPositions: 2, + }, + }); + assert.equal(acquired.kind, 'page'); + if (acquired.kind !== 'page') assert.fail('expected a position page'); + const window = await readSemanticTranscriptTurnWindow({ + reader, + state, + request: { + kind: 'open', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + startOrdinal: 0, + maxPositions: 2, + replaceCursor: null, + }, + fragmentBytes: 16 * 1024 * 1024, + }); + assert.equal(window.kind, 'page'); + if (window.kind !== 'page') assert.fail('expected a complete-prefix window'); + const decoded = JSON.parse(Buffer.from(window.data, 'base64').toString('utf8')) as { + endOrdinalExclusive: number; + positions: unknown[]; + hasNewer: boolean; + }; + assert.equal(decoded.endOrdinalExclusive, 1); + assert.equal(decoded.positions.length, 1); + assert.equal(decoded.hasNewer, true); +}); + +test('keeps a snapshot releasable when the Storage release fails', async () => { + let releases = 0; + const reader = semanticReader({ + readPositionPage: async (request) => ({ + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'empty' }, firstSequence: null }], + hasOlder: false, + hasNewer: false, + }), + releasePositionSnapshot: async () => { + releases += 1; + if (releases === 1) throw new Error('transient release failure'); + }, + }); + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x46), + }); + confirmSemanticTranscriptWatermark(state, 5); + const acquired = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: 'subscription-1', + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(acquired.kind, 'page'); + if (acquired.kind !== 'page') assert.fail('expected a position page'); + const release = () => + querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'release', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + }, + }); + await assert.rejects(release(), /transient release failure/); + assert.deepEqual(await release(), { kind: 'released', subscriptionId: 'subscription-1' }); + assert.equal(releases, 2); +}); + +test('maps every previously valid replaced snapshot token to stale without an ABA window', async () => { + let generation = 6; + const reader = semanticReader({ + readPositionPage: async (request) => ({ + kind: 'page', + projection: request.projection, + snapshotKey: { ...snapshotKey, snapshotGeneration: ++generation }, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'empty' }, firstSequence: null }], + hasOlder: false, + hasNewer: false, + }), + }); + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x47), + }); + confirmSemanticTranscriptWatermark(state, 5); + const first = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: 'subscription-1', + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(first.kind, 'page'); + if (first.kind !== 'page') assert.fail('expected a position page'); + let currentToken = first.snapshotToken; + for (let index = 0; index < 6; index += 1) { + const replaced = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'replace', + subscriptionId: 'subscription-1', + snapshotToken: currentToken, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(replaced.kind, 'page'); + if (replaced.kind !== 'page') assert.fail('expected a replacement page'); + currentToken = replaced.snapshotToken; + } + assert.deepEqual( + await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'page', + subscriptionId: 'subscription-1', + snapshotToken: first.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }), + { + kind: 'snapshot_stale', + subscriptionId: 'subscription-1', + snapshotToken: first.snapshotToken, + }, + ); +}); + +test('uses the current signed window cursor as CAS and retains the old window on preparation failure', async () => { + let failPreparation = false; + let retainedBytes = 0; + const reader = semanticReader({ + readPositionPage: async (request) => ({ + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'turn', id: 'turn-1' }, firstSequence: 0 }], + hasOlder: false, + hasNewer: false, + }), + readPositionRecords: async (request) => { + if (failPreparation) throw new Error('projection failed'); + return { + projection: request.projection, + snapshotKey, + rawBytes: 64, + records: [ + { + positionKey: { kind: 'turn', id: 'turn-1' }, + sequence: 0, + message: { + type: 'user', + id: 'user-1', + turnId: 'turn-1', + ts: 1, + text: 'fragmented body', + }, + }, + ], + }; + }, + }); + const accounting = { + retain: (bytes: number) => { + retainedBytes += bytes; + return true; + }, + release: (bytes: number) => { + retainedBytes -= bytes; + }, + }; + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x48), + }); + confirmSemanticTranscriptWatermark(state, 5); + const acquired = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: 'subscription-1', + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(acquired.kind, 'page'); + if (acquired.kind !== 'page') assert.fail('expected a position page'); + const first = await readSemanticTranscriptTurnWindow({ + reader, + state, + request: { + kind: 'open', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + startOrdinal: 0, + maxPositions: 1, + replaceCursor: null, + }, + fragmentBytes: 16, + accounting, + }); + assert.equal(first.kind, 'page'); + if (first.kind !== 'page' || first.nextCursor === null) assert.fail('expected a retained window'); + const retainedBeforeFailure = retainedBytes; + await assert.rejects( + readSemanticTranscriptTurnWindow({ + reader, + state, + request: { + kind: 'open', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + startOrdinal: 0, + maxPositions: 1, + replaceCursor: `${first.nextCursor}x`, + }, + fragmentBytes: 16, + accounting, + }), + SemanticTranscriptRequestError, + ); + failPreparation = true; + await assert.rejects( + readSemanticTranscriptTurnWindow({ + reader, + state, + request: { + kind: 'open', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + startOrdinal: 0, + maxPositions: 1, + replaceCursor: first.nextCursor, + }, + fragmentBytes: 16, + accounting, + }), + /projection failed/, + ); + assert.equal(retainedBytes, retainedBeforeFailure); + failPreparation = false; + const continued = await readSemanticTranscriptTurnWindow({ + reader, + state, + request: { + kind: 'continue', + subscriptionId: 'subscription-1', + cursor: first.nextCursor, + }, + fragmentBytes: 16, + accounting, + }); + assert.equal(continued.kind, 'page'); + if (continued.kind !== 'page') assert.fail('expected the retained old window'); +}); + +test('keeps the current snapshot active while a replacement receives typed capacity', async () => { + let capacity = false; + const reader = semanticReader({ + readPositionPage: async (request) => + capacity + ? { + kind: 'capacity', + projection: request.projection, + throughSequence: 5, + authorityRevision: 2, + retainedSnapshots: 2, + } + : { + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'empty' }, firstSequence: null }], + hasOlder: false, + hasNewer: false, + }, + }); + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x49), + }); + confirmSemanticTranscriptWatermark(state, 5); + const acquired = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: 'subscription-1', + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(acquired.kind, 'page'); + if (acquired.kind !== 'page') assert.fail('expected a position page'); + capacity = true; + const replacement = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'replace', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(replacement.kind, 'capacity'); + capacity = false; + const current = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'page', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(current.kind, 'page'); + if (current.kind !== 'page') assert.fail('expected the original snapshot page'); + assert.equal(current.snapshotToken, acquired.snapshotToken); +}); + +test('releasing the current snapshot also releases a pending replacement lease', async () => { + let generation = 0; + let replacement = false; + const releasedLeases: string[] = []; + const reader = semanticReader({ + readPositionPage: async (request) => { + const key = { ...snapshotKey, snapshotGeneration: ++generation }; + return replacement + ? { + kind: 'building', + projection: request.projection, + snapshotKey: key, + progress: { + phase: 'legacy', + nextSequence: 1, + currentByteOffset: 0, + sourceRecords: 1, + sourceBytes: 10, + builtPositions: 0, + lastStepRecords: 0, + lastStepBytes: 0, + lastStepPositions: 0, + }, + } + : { + kind: 'page', + projection: request.projection, + snapshotKey: key, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'empty' }, firstSequence: null }], + hasOlder: false, + hasNewer: false, + }; + }, + releasePositionSnapshot: async (request) => { + releasedLeases.push(request.snapshotLeaseId); + }, + }); + const state = createSubscriberSemanticTranscriptState({ + sessionId: 'session-1', + subscriptionId: 'subscription-1', + projection: 'owner', + cursorSecret: Buffer.alloc(32, 0x4a), + }); + confirmSemanticTranscriptWatermark(state, 5); + const acquired = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: 'subscription-1', + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(acquired.kind, 'page'); + if (acquired.kind !== 'page') assert.fail('expected a position page'); + replacement = true; + const building = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'replace', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + }); + assert.equal(building.kind, 'building'); + const released = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'release', + subscriptionId: 'subscription-1', + snapshotToken: acquired.snapshotToken, + }, + }); + assert.equal(released.kind, 'released'); + assert.equal(releasedLeases.length, 2); + assert.notEqual(releasedLeases[0], releasedLeases[1]); +}); + +test('real SQLite snapshots preserve shared privacy, two-slot leases, ABA, and bodyless Turns', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-runtime-semantic-sqlite-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + const reader = semanticReader({ + readPositionPage: (request) => store.readTurnPositionPageSnapshot(request), + readPositionRecords: (request) => store.readTranscriptRecordsByPositionKeysSnapshot(request), + releasePositionSnapshot: (request) => store.releaseTurnPositionSnapshot(request), + }); + const makeState = (sessionId: string, subscriptionId: string, projection: 'owner' | 'shared') => + createSubscriberSemanticTranscriptState({ + sessionId, + subscriptionId, + projection, + cursorSecret: Buffer.alloc(32, subscriptionId.length), + }); + const ready = async (state: ReturnType, throughSequence: number | null) => { + confirmSemanticTranscriptWatermark(state, throughSequence); + let result = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'acquire', + subscriptionId: state.subscriptionId, + anchor: { kind: 'tail' }, + maxPositions: 8, + }, + }); + for (let step = 0; result.kind === 'building' && step < 32; step += 1) { + result = await querySemanticTranscriptPositions({ + reader, + state, + request: { + kind: 'page', + subscriptionId: state.subscriptionId, + snapshotToken: result.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 8, + }, + }); + } + return result; + }; + try { + const hidden = await store.create({ + cwd: root, + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode: 'ask', + }); + await store.appendMessages(hidden.id, [ + { + type: 'permission_decision', + id: 'hidden-permission', + turnId: 'hidden-turn', + ts: 1, + toolUseId: 'hidden-tool', + toolName: 'Read', + decision: 'deny', + }, + { type: 'system_note', id: 'hidden-note', ts: 2, kind: 'session_start' }, + ]); + const sharedFirst = makeState(hidden.id, 'shared-first', 'shared'); + const ownerFirst = makeState(hidden.id, 'owner-first', 'owner'); + const sharedPage = await ready(sharedFirst, 1); + const ownerPage = await ready(ownerFirst, 1); + assert.equal(sharedPage.kind, 'page'); + assert.equal(ownerPage.kind, 'page'); + if (sharedPage.kind !== 'page' || ownerPage.kind !== 'page') { + assert.fail('expected ready owner and shared snapshots'); + } + assert.deepEqual(sharedPage.positions, [{ ordinal: 0, key: { kind: 'empty' } }]); + assert.deepEqual(ownerPage.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'hidden-turn' } }, + ]); + assert.equal( + sharedFirst.currentSnapshot?.key?.snapshotGeneration, + ownerFirst.currentSnapshot?.key?.snapshotGeneration, + ); + + await store.appendMessage(hidden.id, { + type: 'system_note', + id: 'visible-note', + ts: 3, + kind: 'context_compacted', + }); + const sharedSecond = makeState(hidden.id, 'shared-second', 'shared'); + const secondPage = await ready(sharedSecond, 2); + assert.equal(secondPage.kind, 'page'); + if (secondPage.kind !== 'page') assert.fail('expected a second exact generation'); + assert.deepEqual(secondPage.positions, [ + { ordinal: 0, key: { kind: 'note', id: 'visible-note' } }, + ]); + const secondGeneration = sharedSecond.currentSnapshot?.key?.snapshotGeneration; + + await store.appendMessage(hidden.id, { + type: 'system_note', + id: 'visible-note-2', + ts: 4, + kind: 'step_limit', + }); + const sharedThird = makeState(hidden.id, 'shared-third', 'shared'); + assert.equal((await ready(sharedThird, 3)).kind, 'capacity'); + await releaseSubscriberSemanticTranscript(reader, sharedFirst); + assert.equal((await ready(sharedThird, 3)).kind, 'capacity'); + await releaseSubscriberSemanticTranscript(reader, ownerFirst); + const thirdPage = await ready(sharedThird, 3); + assert.equal(thirdPage.kind, 'page'); + if (thirdPage.kind !== 'page') assert.fail('expected capacity release to admit a generation'); + assert.ok( + (sharedThird.currentSnapshot?.key?.snapshotGeneration ?? 0) > (secondGeneration ?? 0), + ); + + const bodyless = await store.create({ + cwd: root, + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode: 'ask', + }); + await runs.admitRootTurn({ + sessionId: bodyless.id, + turnId: 'bodyless-turn', + proposedRunId: 'bodyless-run', + proposedUserMessageId: 'bodyless-user', + execution: { kind: 'external_message' }, + previousRootTurnId: null, + normalizedInput: { text: 'future body' }, + sourceMessages: [], + admittedAt: 10, + }); + const bodylessOwner = makeState(bodyless.id, 'bodyless-owner', 'owner'); + const bodylessPage = await ready(bodylessOwner, null); + assert.equal(bodylessPage.kind, 'page'); + if (bodylessPage.kind !== 'page') assert.fail('expected a bodyless Turn position'); + assert.deepEqual(bodylessPage.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'bodyless-turn' } }, + ]); + const bodylessWindow = await readSemanticTranscriptTurnWindow({ + reader, + state: bodylessOwner, + request: { + kind: 'open', + subscriptionId: bodylessOwner.subscriptionId, + snapshotToken: bodylessPage.snapshotToken, + startOrdinal: 0, + maxPositions: 1, + }, + }); + assert.equal(bodylessWindow.kind, 'page'); + if (bodylessWindow.kind !== 'page') assert.fail('expected a bodyless semantic window'); + assert.deepEqual( + JSON.parse(Buffer.from(bodylessWindow.data, 'base64').toString('utf8')).positions, + [{ position: { ordinal: 0, key: { kind: 'turn', id: 'bodyless-turn' } }, messages: [] }], + ); + + await releaseSubscriberSemanticTranscript(reader, sharedSecond); + await releaseSubscriberSemanticTranscript(reader, sharedThird); + await releaseSubscriberSemanticTranscript(reader, bodylessOwner); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } +}); + +function semanticReader(overrides: Partial): SessionTranscriptReader { + return { + readDurableHighWater: async () => 5, + readDurablePage: async () => ({ + throughSequence: 5, + fragments: [], + rawBytes: 0, + next: null, + }), + readDurableRecords: async () => ({ + throughSequence: 5, + records: [], + nextPosition: null, + }), + readDurableMessagesById: async () => [], + readActiveOverlay: async () => [], + readPositionPage: async () => { + throw new Error('not configured'); + }, + readPositionRecords: async () => { + throw new Error('not configured'); + }, + releasePositionSnapshot: async () => undefined, + ...overrides, + }; +} diff --git a/packages/runtime-host/src/__tests__/session-collaboration-authority.test.ts b/packages/runtime-host/src/__tests__/session-collaboration-authority.test.ts index 7faa139454..8b106a10ba 100644 --- a/packages/runtime-host/src/__tests__/session-collaboration-authority.test.ts +++ b/packages/runtime-host/src/__tests__/session-collaboration-authority.test.ts @@ -78,6 +78,8 @@ test('Session Guest invitation, grants, and revocation form one durable authorit 'subscription.close', 'session.transcript.page', 'session.transcript.overlay.release', + 'session.transcript.positions.query', + 'session.transcript.turn_window.page', ]); assert.ok(activeGuest); const unidentifiedQuery = queryCollaborationTurnRequests( diff --git a/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts b/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts index 04b4852b31..95df67ec13 100644 --- a/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/session-continuity-coordinator.test.ts @@ -53,6 +53,10 @@ const TEST_OWNER_IDENTITY = { } as const; type TestIdentity = | typeof TEST_OWNER_IDENTITY + | { + readonly principalId: string; + readonly principalKind: 'remote_owner'; + } | { readonly principalId: string; readonly principalKind: 'session_guest'; @@ -1850,6 +1854,344 @@ test('an in-flight transcript page cannot outlive its owning connection', async coordinator.close(); }); +test('semantic positions bind to delivered subscription watermark and release on close', async () => { + const snapshotKey = { throughSequence: 0, authorityRevision: 0, snapshotGeneration: 1 }; + const reads: unknown[] = []; + const releases: unknown[] = []; + const baseReader = transcriptReader([ + { type: 'user', id: 'user-1', turnId: 'turn-1', ts: 1, text: 'hello' }, + ]); + const reader: SessionTranscriptReader = { + ...baseReader, + readPositionPage: async (request) => { + reads.push(request); + return { + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'turn', id: 'turn-1' }, firstSequence: 0 }], + hasOlder: false, + hasNewer: false, + }; + }, + releasePositionSnapshot: async (request) => { + releases.push(request); + }, + }; + const coordinator = new SessionContinuityCoordinator( + HOST_EPOCH, + async () => canonical(), + new SessionAdmissionGate(), + undefined, + reader, + ); + const connection = coordinator.attachConnection('connection-semantic', new RecordingSink()); + const opened = await open(coordinator, 'connection-semantic', { + kind: 'tail', + maxBytes: SESSION_TRANSCRIPT_BOOTSTRAP_MAX_BYTES, + }); + const beforeActivation = await coordinator.handlers['session.transcript.positions.query']( + { + kind: 'acquire', + subscriptionId: opened.subscriptionId, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + connectionContext('connection-semantic'), + ); + assert.equal(beforeActivation.ok, false); + connection.activate(opened.subscriptionId); + + const acquired = await coordinator.handlers['session.transcript.positions.query']( + { + kind: 'acquire', + subscriptionId: opened.subscriptionId, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + connectionContext('connection-semantic'), + ); + assert.equal(acquired.ok, true); + if (!acquired.ok || acquired.result.kind !== 'page') assert.fail('expected semantic page'); + assert.equal((reads[0] as { throughSequence: number }).throughSequence, 0); + + await coordinator.handlers['subscription.close']( + { subscriptionId: opened.subscriptionId }, + connectionContext('connection-semantic'), + ); + await coordinator.settled(); + assert.equal(releases.length, 1); + await coordinator.close(); +}); + +test('explicit remote owners read owner semantics while active Guests read dense shared semantics', async () => { + const projections: string[] = []; + const baseReader = transcriptReader([ + { type: 'user', id: 'user-1', turnId: 'turn-1', ts: 1, text: 'hello' }, + ]); + const reader: SessionTranscriptReader = { + ...baseReader, + readPositionPage: async (request) => { + projections.push(request.projection); + return { + kind: 'page', + projection: request.projection, + snapshotKey: { throughSequence: 0, authorityRevision: 0, snapshotGeneration: 1 }, + startOrdinal: 0, + totalPositions: 1, + positions: [ + request.projection === 'owner' + ? { ordinal: 0, key: { kind: 'turn', id: 'owner-turn' }, firstSequence: 0 } + : { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ], + hasOlder: false, + hasNewer: false, + }; + }, + releasePositionSnapshot: async () => undefined, + }; + const grant = { + kind: 'session_observation' as const, + grantId: 'active-semantic-grant', + principalId: 'active-semantic-guest', + sessionId: SESSION_ID, + createdAt: '2026-08-31T00:00:00.000Z', + }; + const coordinator = new SessionContinuityCoordinator( + HOST_EPOCH, + async () => canonical(), + new SessionAdmissionGate(), + undefined, + reader, + undefined, + { + activeSessionGrant: () => grant, + subscribeGrantRevocations: () => () => undefined, + }, + ); + for (const identity of [ + { principalId: 'explicit-remote-owner', principalKind: 'remote_owner' as const }, + { principalId: grant.principalId, principalKind: 'session_guest' as const }, + ]) { + const connectionId = `connection-${identity.principalKind}`; + const connection = coordinator.attachConnection(connectionId, new RecordingSink()); + const opened = await open( + coordinator, + connectionId, + { kind: 'tail', maxBytes: SESSION_TRANSCRIPT_BOOTSTRAP_MAX_BYTES }, + identity, + ); + connection.activate(opened.subscriptionId); + const result = await coordinator.handlers['session.transcript.positions.query']( + { + kind: 'acquire', + subscriptionId: opened.subscriptionId, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + connectionContext(connectionId, identity), + ); + assert.equal(result.ok, true); + if (!result.ok || result.result.kind !== 'page') assert.fail('expected semantic positions'); + assert.deepEqual( + result.result.positions, + identity.principalKind === 'remote_owner' + ? [{ ordinal: 0, key: { kind: 'turn', id: 'owner-turn' } }] + : [{ ordinal: 0, key: { kind: 'empty' } }], + ); + await connection.close(); + } + assert.deepEqual(projections, ['owner', 'shared']); + await coordinator.close(); +}); + +test('semantic replacement waits for transcript_advanced delivery before using its watermark', async () => { + const durable: StoredMessage[] = [ + { type: 'user', id: 'user-1', turnId: 'turn-1', ts: 1, text: 'one' }, + ]; + const baseReader = transcriptReader(durable); + const capturedWatermarks: Array = []; + let generation = 0; + const reader: SessionTranscriptReader = { + ...baseReader, + readPositionPage: async (request) => { + capturedWatermarks.push(request.snapshotKey?.throughSequence ?? request.throughSequence); + const throughSequence = + request.snapshotKey?.throughSequence ?? request.throughSequence ?? null; + const snapshotKey = { + throughSequence, + authorityRevision: 0, + snapshotGeneration: request.snapshotKey?.snapshotGeneration ?? ++generation, + }; + return { + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'turn', id: 'turn-1' }, firstSequence: 0 }], + hasOlder: false, + hasNewer: false, + }; + }, + releasePositionSnapshot: async () => undefined, + }; + const advancedFlushed = deferred(); + const advancedSent = deferred(); + const sink: SessionContinuityFrameSink = { + send: async (frame) => { + if (frame.kind === 'subscription.transcript_advanced') { + advancedSent.resolve(); + await advancedFlushed.promise; + } + }, + }; + const coordinator = new SessionContinuityCoordinator( + HOST_EPOCH, + async () => canonical(), + new SessionAdmissionGate(), + undefined, + reader, + ); + const connection = coordinator.attachConnection('connection-watermark', sink); + const opened = await open(coordinator, 'connection-watermark', { + kind: 'tail', + maxBytes: SESSION_TRANSCRIPT_BOOTSTRAP_MAX_BYTES, + }); + connection.activate(opened.subscriptionId); + const first = await coordinator.handlers['session.transcript.positions.query']( + { + kind: 'acquire', + subscriptionId: opened.subscriptionId, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + connectionContext('connection-watermark'), + ); + assert.equal(first.ok, true); + if (!first.ok || first.result.kind !== 'page') assert.fail('expected first snapshot'); + + durable.push({ type: 'user', id: 'user-2', turnId: 'turn-2', ts: 2, text: 'two' }); + await coordinator.refreshCanonical(SESSION_ID); + await advancedSent.promise; + const beforeFlush = await coordinator.handlers['session.transcript.positions.query']( + { + kind: 'replace', + subscriptionId: opened.subscriptionId, + snapshotToken: first.result.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + connectionContext('connection-watermark'), + ); + assert.equal(beforeFlush.ok, true); + if (!beforeFlush.ok || beforeFlush.result.kind !== 'page') assert.fail('expected replacement'); + assert.equal(capturedWatermarks.at(-1), 0); + + advancedFlushed.resolve(); + await delayImmediate(); + await coordinator.handlers['session.transcript.positions.query']( + { + kind: 'replace', + subscriptionId: opened.subscriptionId, + snapshotToken: beforeFlush.result.snapshotToken, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + connectionContext('connection-watermark'), + ); + assert.equal(capturedWatermarks.at(-1), 1); + await connection.close(); + await coordinator.close(); +}); + +test('revoking a Guest during first semantic I/O releases the late-bound snapshot lease', async () => { + const snapshotKey = { throughSequence: 0, authorityRevision: 0, snapshotGeneration: 1 }; + const continued = deferred(); + const entered = deferred(); + let releases = 0; + const baseReader = transcriptReader([ + { type: 'user', id: 'user-1', turnId: 'turn-1', ts: 1, text: 'hello' }, + ]); + const reader: SessionTranscriptReader = { + ...baseReader, + readPositionPage: async (request) => { + entered.resolve(); + await continued.promise; + return { + kind: 'page', + projection: request.projection, + snapshotKey, + startOrdinal: 0, + totalPositions: 1, + positions: [{ ordinal: 0, key: { kind: 'turn', id: 'turn-1' }, firstSequence: 0 }], + hasOlder: false, + hasNewer: false, + }; + }, + releasePositionSnapshot: async () => { + releases += 1; + }, + }; + const grant = { + kind: 'session_observation' as const, + grantId: 'semantic-grant-1', + principalId: 'semantic-guest-1', + sessionId: SESSION_ID, + createdAt: '2026-08-31T00:00:00.000Z', + }; + let active = true; + let publishRevocation: ((revoked: typeof grant) => void) | undefined; + const coordinator = new SessionContinuityCoordinator( + HOST_EPOCH, + async () => canonical(), + new SessionAdmissionGate(), + undefined, + reader, + undefined, + { + activeSessionGrant: () => (active ? grant : undefined), + subscribeGrantRevocations: (listener) => { + publishRevocation = listener; + return () => undefined; + }, + }, + ); + const connection = coordinator.attachConnection('semantic-guest-connection', new RecordingSink()); + const opened = await open( + coordinator, + 'semantic-guest-connection', + { kind: 'tail', maxBytes: SESSION_TRANSCRIPT_BOOTSTRAP_MAX_BYTES }, + { principalId: grant.principalId, principalKind: 'session_guest' }, + ); + connection.activate(opened.subscriptionId); + const reading = coordinator.handlers['session.transcript.positions.query']( + { + kind: 'acquire', + subscriptionId: opened.subscriptionId, + anchor: { kind: 'tail' }, + maxPositions: 1, + }, + connectionContext('semantic-guest-connection', { + principalId: grant.principalId, + principalKind: 'session_guest', + }), + ); + await entered.promise; + active = false; + publishRevocation?.(grant); + continued.resolve(); + assert.deepEqual(await reading, { + ok: false, + error: { code: 'not_found', message: 'Session subscription was not found' }, + }); + await coordinator.settled(); + assert.equal(releases, 1); + await coordinator.close(); +}); + test('an in-flight transcript page cannot outlive its Guest observation grant', async () => { const message = assistantMessage('界'.repeat(20_000)); const continued = deferred(); diff --git a/packages/runtime-host/src/__tests__/session-subscription-client.test.ts b/packages/runtime-host/src/__tests__/session-subscription-client.test.ts index bca8eaf612..76be30c72d 100644 --- a/packages/runtime-host/src/__tests__/session-subscription-client.test.ts +++ b/packages/runtime-host/src/__tests__/session-subscription-client.test.ts @@ -544,6 +544,108 @@ test('decodes one bounded page without walking the remaining transcript', async assert.deepEqual(requests, []); }); +test('production subscription API queries semantic positions and publishes only a complete window', async () => { + const logical = { + snapshotToken: 'snapshot-1', + startOrdinal: 0, + endOrdinalExclusive: 1, + totalPositions: 1, + positions: [ + { + position: { ordinal: 0, key: { kind: 'turn' as const, id: 'turn-1' } }, + messages: [{ type: 'user', id: 'user-1', turnId: 'turn-1', ts: 1, text: 'hello' }], + }, + ], + hasOlder: false, + hasNewer: false, + }; + const bytes = Buffer.from(JSON.stringify(logical), 'utf8'); + const split = Math.floor(bytes.byteLength / 2); + const digest = `sha256:${createHash('sha256').update(bytes).digest('hex')}` as const; + let windowCalls = 0; + const subscription = new ClientSessionSubscription( + openResult('host-1', 'subscription-semantic', transcriptBootstrap(Buffer.from('{}'))), + async () => undefined, + async () => transcriptPage(), + async () => undefined, + { + queryPositions: async (input) => ({ + kind: 'page', + subscriptionId: input.subscriptionId, + snapshotToken: 'snapshot-1', + totalPositions: 1, + startOrdinal: 0, + positions: [{ ordinal: 0, key: { kind: 'turn', id: 'turn-1' } }], + olderCursor: null, + newerCursor: null, + }), + readTurnWindow: async (input) => { + windowCalls += 1; + const first = input.kind === 'open'; + return { + kind: 'page', + subscriptionId: input.subscriptionId, + snapshotToken: 'snapshot-1', + windowId: 'window-1', + byteOffset: first ? 0 : split, + totalBytes: bytes.byteLength, + payloadDigest: digest, + data: (first ? bytes.subarray(0, split) : bytes.subarray(split)).toString('base64'), + nextCursor: first ? 'window-cursor' : null, + }; + }, + }, + ); + + const positions = await subscription.queryTranscriptPositions({ + kind: 'acquire', + anchor: { kind: 'tail' }, + maxPositions: 1, + }); + assert.equal(positions.kind, 'page'); + const window = await subscription.loadTranscriptTurnWindow( + { + snapshotToken: 'snapshot-1', + startOrdinal: 0, + maxPositions: 1, + replaceCursor: null, + }, + decodeStoredMessage, + ); + assert.equal(windowCalls, 2); + assert.deepEqual(window, logical); + + const mismatched = Buffer.from( + JSON.stringify({ + ...logical, + positions: [ + { + position: { ordinal: 0, key: { kind: 'turn', id: 'turn-1' } }, + messages: [{ type: 'user', id: 'user-2', turnId: 'turn-2', ts: 2, text: 'wrong Turn' }], + }, + ], + }), + 'utf8', + ); + await assert.rejects( + subscription.decodeTranscriptTurnWindowPage( + { + kind: 'page', + subscriptionId: 'subscription-semantic', + snapshotToken: 'snapshot-1', + windowId: 'window-mismatched', + byteOffset: 0, + totalBytes: mismatched.byteLength, + payloadDigest: `sha256:${createHash('sha256').update(mismatched).digest('hex')}`, + data: mismatched.toString('base64'), + nextCursor: null, + }, + decodeStoredMessage, + ), + /position identity changed/, + ); +}); + test('assembles the complete edge Turn while paging newer transcript', async () => { const prompt = { type: 'user' as const, @@ -1018,6 +1120,66 @@ test('rejects a durable message that does not match its payload digest', async ( assert.deepEqual(assemblyDeltas, [message.byteLength, -message.byteLength]); }); +test('releases each completed legacy message assembly before decoding the next message', async () => { + const encoded = [0, 1].map((index) => + Buffer.from( + JSON.stringify({ + type: 'user', + id: `user-${index}`, + turnId: `turn-${index}`, + ts: index + 1, + text: `message-${index}`, + }), + 'utf8', + ), + ); + const subscription = new ClientSessionSubscription( + openResult('host-1', 'subscription-immediate-assembly-release', { + throughSequence: 1, + durableCoverage: 'complete', + overlayMessageCount: 0, + durable: { + ...transcriptPage({ + rawBytes: encoded[0]!.byteLength + encoded[1]!.byteLength, + fragments: encoded + .map((message, sequence) => ({ + kind: 'durable' as const, + sequence, + byteOffset: 0, + totalBytes: message.byteLength, + payloadDigest: null, + data: message.toString('base64'), + })) + .reverse(), + }), + throughSequence: 1, + }, + overlay: { ...transcriptPage({ source: 'overlay' }), throughSequence: 1 }, + }), + async () => undefined, + async () => { + throw new Error('unexpected page request'); + }, + ); + const assemblyDeltas: number[] = []; + const result = await subscription.decodeTranscriptPage( + subscription.transcriptBootstrap!.durable, + decodeStoredMessage, + undefined, + (deltaBytes) => assemblyDeltas.push(deltaBytes), + ); + assert.deepEqual( + result.messages.map(({ identity }) => identity), + [0, 1], + ); + assert.deepEqual(assemblyDeltas, [ + encoded[1]!.byteLength, + -encoded[1]!.byteLength, + encoded[0]!.byteLength, + -encoded[0]!.byteLength, + ]); +}); + test('rejects a transcript cursor that does not advance', async () => { const message = Buffer.from( JSON.stringify({ diff --git a/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts b/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts index fd35f9fb67..60faee1603 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts @@ -20,6 +20,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { + classifySharedSessionTranscriptVisibility, decodeStoredMessage as decodePersistedStoredMessage, type StoredMessage, } from '@maka/core/session'; @@ -35,8 +36,23 @@ import { } from '../server/session-transcript-pager.js'; import type { SessionTranscriptReader } from '../server/session-transcript-reader.js'; import { projectSharedSessionTranscriptMessage } from '../server/shared-session-transcript.js'; +import { + decodeTranscriptSignedToken, + encodeTranscriptSignedToken, +} from '../server/transcript-signed-token.js'; import { transcriptReader } from './fixtures/session-transcript-reader.js'; +test('signed transcript tokens are canonical and domain separated', () => { + const secret = Buffer.alloc(32, 0x5a); + const token = encodeTranscriptSignedToken('positions', { version: 1, ordinal: 4 }, secret); + assert.deepEqual(decodeTranscriptSignedToken('positions', token, secret), { + version: 1, + ordinal: 4, + }); + assert.throws(() => decodeTranscriptSignedToken('window', token, secret)); + assert.throws(() => decodeTranscriptSignedToken('positions', `${token.slice(0, -1)}x`, secret)); +}); + test('reads newly durable messages forward from an announced watermark', async () => { const durable = [userMessage(0), userMessage(1)]; const reader = transcriptReader(durable); @@ -223,6 +239,82 @@ test('projects durable and active transcript records before sharing them', async } }); +test('shares the Core visibility decision while retaining Host-only body sanitization', () => { + const messages: StoredMessage[] = [ + userMessage(0), + assistantMessage(1), + { + type: 'tool_call', + id: 'call', + turnId: 'turn-0', + ts: 2, + toolName: 'Read', + args: {}, + modelVisibility: 'hidden', + }, + { + type: 'tool_result', + id: 'result', + turnId: 'turn-0', + ts: 3, + toolUseId: 'call', + isError: false, + content: { kind: 'text', text: 'ok' }, + modelVisibility: 'hidden', + }, + { + type: 'permission_decision', + id: 'permission', + turnId: 'turn-0', + ts: 4, + toolUseId: 'call', + toolName: 'Read', + decision: 'allow', + }, + { type: 'token_usage', id: 'usage', turnId: 'turn-0', ts: 5, input: 1, output: 1 }, + { + type: 'turn_state', + id: 'state', + turnId: 'turn-0', + ts: 6, + status: 'completed', + partialOutputRetained: false, + }, + { + type: 'workhub_coordination', + id: 'coordination', + turnId: 'turn-0', + ts: 7, + schemaVersion: 1, + kind: 'delegation_assigned', + actionId: 'action', + actionFingerprint: `sha256:${'0'.repeat(64)}`, + coordinationTurnId: 'turn-0', + targetSessionId: 'target', + disposition: 'delegate_existing', + userText: 'work', + delegationId: 'delegation', + targetTurnId: 'target-turn', + targetMessageId: 'target-message', + targetSessionName: 'target', + }, + { type: 'system_note', id: 'visible-note', ts: 8, kind: 'step_limit' }, + { type: 'system_note', id: 'hidden-note', ts: 9, kind: 'mode_change' }, + ]; + for (const message of messages) { + const visibility = classifySharedSessionTranscriptVisibility( + message.type === 'system_note' + ? { type: message.type, kind: message.kind } + : { type: message.type }, + ); + assert.equal( + projectSharedSessionTranscriptMessage(message, 'session-1') === null, + visibility === 'hidden', + message.type, + ); + } +}); + test('rejects cursor tampering and cross-subscription replay', async () => { const reader = transcriptReader([userMessage(0, 'x'.repeat(2_000))]); const first = await createSessionTranscriptBootstrap({ diff --git a/packages/runtime-host/src/__tests__/session-transcript-protocol.test.ts b/packages/runtime-host/src/__tests__/session-transcript-protocol.test.ts index da33e1ce1f..9b1bd0bc88 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-protocol.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-protocol.test.ts @@ -21,12 +21,17 @@ import { RuntimeHostProtocolError } from '../protocol/errors.js'; import assert from 'node:assert/strict'; import test from 'node:test'; import { + decodeSessionTranscriptPositionsInput, + decodeSessionTranscriptPositionsResult, decodeSessionTranscriptBootstrap, decodeSessionTranscriptPage, decodeSessionTranscriptPageInput, + decodeSessionTranscriptTurnWindowInput, + decodeSessionTranscriptTurnWindowResult, encodeProtocolMessage, HOST_OPERATION_SPECS, RUNTIME_HOST_MAX_MESSAGE_BYTES, + SESSION_TRANSCRIPT_POSITION_PAGE_MAX_POSITIONS, SESSION_TRANSCRIPT_PAGE_MAX_BYTES, } from '../protocol/index.js'; @@ -216,6 +221,221 @@ test('Session transcript protocol rejects malformed and uncorrelated values', () ); }); +test('semantic transcript operations accept only opaque subscription-scoped requests', () => { + const acquire = { + kind: 'acquire' as const, + subscriptionId: 'subscription-1', + anchor: { kind: 'tail' as const }, + maxPositions: SESSION_TRANSCRIPT_POSITION_PAGE_MAX_POSITIONS, + }; + const page = { + kind: 'page' as const, + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + anchor: { kind: 'turn' as const, turnId: 'turn-1' }, + maxPositions: 1, + }; + const continuation = { + kind: 'continue' as const, + subscriptionId: 'subscription-1', + cursor: 'positions-cursor', + }; + const replace = { + kind: 'replace' as const, + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + anchor: { kind: 'ordinal' as const, ordinal: 4 }, + maxPositions: 8, + }; + const release = { + kind: 'release' as const, + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + }; + for (const input of [acquire, page, continuation, replace, release]) { + assert.deepEqual(decodeSessionTranscriptPositionsInput(input), input); + } + + assert.throws( + () => decodeSessionTranscriptPositionsInput({ ...acquire, sessionId: 'session-1' }), + isProtocolError, + ); + assert.throws( + () => decodeSessionTranscriptPositionsInput({ ...acquire, maxPositions: 0 }), + isProtocolError, + ); + assert.throws( + () => decodeSessionTranscriptPositionsInput({ ...acquire, maxPositions: 129 }), + isProtocolError, + ); + assert.throws( + () => decodeSessionTranscriptPositionsInput({ ...page, snapshotToken: 'x'.repeat(1_025) }), + isProtocolError, + ); + + const result = { + kind: 'page' as const, + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + totalPositions: 3, + startOrdinal: 1, + positions: [ + { ordinal: 1, key: { kind: 'note' as const, id: 'note-1' } }, + { ordinal: 2, key: { kind: 'turn' as const, id: 'turn-2' } }, + ], + olderCursor: 'older-cursor', + newerCursor: null, + }; + assert.deepEqual(decodeSessionTranscriptPositionsResult(result), result); + assert.throws( + () => + decodeSessionTranscriptPositionsResult({ + ...result, + positions: [{ ordinal: 1, key: { kind: 'turn', id: '' } }], + }), + isProtocolError, + ); + assert.throws( + () => decodeSessionTranscriptPositionsResult({ ...result, authorityRevision: 1 }), + isProtocolError, + ); + assert.throws( + () => + decodeSessionTranscriptPositionsResult({ + ...result, + totalPositions: 2, + startOrdinal: 0, + positions: [ + { ordinal: 0, key: { kind: 'empty' } }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-1' } }, + ], + }), + isProtocolError, + ); + for (const semanticResult of [ + { + kind: 'building', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + retryAfterMs: 25, + }, + { + kind: 'capacity', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + retryAfterMs: 1_000, + }, + { + kind: 'snapshot_stale', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + }, + { + kind: 'anchor_not_found', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + }, + { kind: 'released', subscriptionId: 'subscription-1' }, + ]) { + assert.deepEqual(decodeSessionTranscriptPositionsResult(semanticResult), semanticResult); + } +}); + +test('semantic Turn-window protocol keeps retained fragments opaque and bounded', () => { + const open = { + kind: 'open' as const, + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + startOrdinal: 4, + maxPositions: 10, + replaceCursor: null, + }; + const continuation = { + kind: 'continue' as const, + subscriptionId: 'subscription-1', + cursor: 'window-cursor', + }; + assert.deepEqual(decodeSessionTranscriptTurnWindowInput(open), open); + assert.deepEqual( + decodeSessionTranscriptTurnWindowInput({ + kind: 'open', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + startOrdinal: 4, + maxPositions: 10, + }), + { + kind: 'open', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + startOrdinal: 4, + maxPositions: 10, + }, + ); + assert.deepEqual(decodeSessionTranscriptTurnWindowInput(continuation), continuation); + assert.throws( + () => decodeSessionTranscriptTurnWindowInput({ ...open, maxPositions: 0 }), + isProtocolError, + ); + assert.throws( + () => decodeSessionTranscriptTurnWindowInput({ ...open, maxPositions: 11 }), + isProtocolError, + ); + + const bytes = Buffer.from('{"positions":[]}'); + const result = { + kind: 'page' as const, + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + windowId: 'window-1', + byteOffset: 0, + totalBytes: bytes.byteLength, + payloadDigest: `sha256:${'a'.repeat(64)}` as const, + data: bytes.toString('base64'), + nextCursor: null, + }; + assert.deepEqual(decodeSessionTranscriptTurnWindowResult(result), result); + assert.throws( + () => decodeSessionTranscriptTurnWindowResult({ ...result, data: 'not base64' }), + isProtocolError, + ); + assert.throws( + () => decodeSessionTranscriptTurnWindowResult({ ...result, snapshotGeneration: 3 }), + isProtocolError, + ); + for (const semanticResult of [ + { + kind: 'building', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + retryAfterMs: 25, + }, + { + kind: 'capacity', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + retryAfterMs: 1_000, + }, + { + kind: 'snapshot_stale', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + }, + { + kind: 'anchor_not_found', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + }, + { + kind: 'position_too_large', + subscriptionId: 'subscription-1', + snapshotToken: 'snapshot-token', + }, + ]) { + assert.deepEqual(decodeSessionTranscriptTurnWindowResult(semanticResult), semanticResult); + } +}); + function isProtocolError(error: unknown): boolean { return error instanceof RuntimeHostProtocolError && error.code === 'invalid_frame'; } diff --git a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts index c34c30db73..dd0e85cad4 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-reader.test.ts @@ -322,6 +322,65 @@ test('stops an oversized active projection before retaining the full RuntimeEven assert.equal(visited, 8_193); }); +test('forwards semantic position snapshot operations through the existing execution-store adapter', async () => { + const calls: string[] = []; + const stores = { + sessionStore: { + readTurnPositionPageSnapshot: async () => { + calls.push('positions'); + return { + kind: 'capacity' as const, + projection: 'owner' as const, + throughSequence: 3, + authorityRevision: 1, + retainedSnapshots: 2 as const, + }; + }, + readTranscriptRecordsByPositionKeysSnapshot: async () => { + calls.push('records'); + return { + projection: 'owner' as const, + snapshotKey: { throughSequence: 3, authorityRevision: 1, snapshotGeneration: 2 }, + records: [], + rawBytes: 0, + }; + }, + releaseTurnPositionSnapshot: async () => { + calls.push('release'); + }, + }, + } as unknown as ExecutionStoresWriter<'interactive'>; + const reader = createSessionTranscriptReader({ + stores, + canonicalPermissionOutcomes: { readPermissionOutcome: async () => undefined }, + }); + const snapshotKey = { throughSequence: 3, authorityRevision: 1, snapshotGeneration: 2 }; + await reader.readPositionPage({ + sessionId: 'session-1', + projection: 'owner', + snapshotLeaseId: 'lease-1', + throughSequence: 3, + anchor: { kind: 'tail' }, + maxPositions: 1, + }); + await reader.readPositionRecords({ + sessionId: 'session-1', + projection: 'owner', + snapshotLeaseId: 'lease-1', + snapshotKey, + positionKeys: [{ kind: 'turn', id: 'turn-1' }], + maxBytes: 1024, + maxRecords: 1, + }); + await reader.releasePositionSnapshot({ + sessionId: 'session-1', + projection: 'owner', + snapshotLeaseId: 'lease-1', + snapshotKey, + }); + assert.deepEqual(calls, ['positions', 'records', 'release']); +}); + function runHeader(sessionId: string): AgentRunHeader { return { runId: 'run-1', diff --git a/packages/runtime-host/src/client/connection.ts b/packages/runtime-host/src/client/connection.ts index 465c3cc567..f8bbb90553 100644 --- a/packages/runtime-host/src/client/connection.ts +++ b/packages/runtime-host/src/client/connection.ts @@ -586,6 +586,12 @@ class RuntimeHostConnectionImpl implements RuntimeHostConnection { throw error; } }, + { + queryPositions: (query) => + this.request('session.transcript.positions.query', query, timeoutMs), + readTurnWindow: (query) => + this.request('session.transcript.turn_window.page', query, timeoutMs), + }, ); this.#subscriptions.set(result.subscriptionId, subscription); return subscription; diff --git a/packages/runtime-host/src/client/session-subscription.ts b/packages/runtime-host/src/client/session-subscription.ts index a458dc9743..e3eabd614e 100644 --- a/packages/runtime-host/src/client/session-subscription.ts +++ b/packages/runtime-host/src/client/session-subscription.ts @@ -17,7 +17,6 @@ * under the License. */ -import { createHash } from 'node:crypto'; import { encodeProtocolMessage, type SessionAssistantStreamIdentity, @@ -31,7 +30,13 @@ import { type SessionTranscriptFragment, type SessionTranscriptPage, type SessionTranscriptPageInput, + type SessionTranscriptPositionsInput, + type SessionTranscriptPositionsResult, + type SessionTranscriptSemanticPosition, + type SessionTranscriptTurnWindowInput, + type SessionTranscriptTurnWindowResult, } from '../protocol/index.js'; +import { TranscriptFragmentAssembler } from './transcript-fragment-assembler.js'; const MAX_CLIENT_QUEUED_FRAMES = 32; const MAX_CLIENT_QUEUED_BYTES = 256 * 1024; @@ -60,6 +65,182 @@ function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } +function acceptSemanticWindowFragment( + assembler: TranscriptFragmentAssembler, + page: Extract, +): void { + const data = Buffer.from(page.data, 'base64'); + try { + assembler.accept(page.byteOffset, data); + } catch (cause) { + throw new RuntimeHostSubscriptionError('correlation_changed', errorMessage(cause), { cause }); + } +} + +function decodeSemanticTranscriptWindow( + bytes: Buffer, + expectedSnapshotToken: string, + decodeMessage: (value: unknown) => T, +): DecodedSessionTranscriptTurnWindow { + let value: unknown; + try { + value = JSON.parse(bytes.toString('utf8')) as unknown; + } catch (cause) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + `Semantic transcript window is not valid JSON: ${errorMessage(cause)}`, + ); + } + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Semantic transcript window is not an object', + ); + } + const record = value as Record; + const keys = [ + 'snapshotToken', + 'startOrdinal', + 'endOrdinalExclusive', + 'totalPositions', + 'positions', + 'hasOlder', + 'hasNewer', + ]; + if ( + Object.keys(record).length !== keys.length || + keys.some((key) => !Object.hasOwn(record, key)) || + record.snapshotToken !== expectedSnapshotToken || + !isClientCount(record.startOrdinal) || + !isClientCount(record.endOrdinalExclusive) || + !isClientCount(record.totalPositions) || + typeof record.hasOlder !== 'boolean' || + typeof record.hasNewer !== 'boolean' || + !Array.isArray(record.positions) || + record.positions.length < 1 || + record.positions.length > 10 || + record.endOrdinalExclusive !== record.startOrdinal + record.positions.length || + record.endOrdinalExclusive > record.totalPositions || + record.hasOlder !== record.startOrdinal > 0 || + record.hasNewer !== record.endOrdinalExclusive < record.totalPositions + ) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Semantic transcript window metadata changed', + ); + } + const startOrdinal = record.startOrdinal as number; + const endOrdinalExclusive = record.endOrdinalExclusive as number; + const totalPositions = record.totalPositions as number; + const hasOlder = record.hasOlder as boolean; + const hasNewer = record.hasNewer as boolean; + const messageIds = new Set(); + const positions = record.positions.map((entry, index) => { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Invalid semantic transcript position', + ); + } + const item = entry as Record; + if ( + Object.keys(item).length !== 2 || + !Object.hasOwn(item, 'position') || + !Object.hasOwn(item, 'messages') || + !item.position || + typeof item.position !== 'object' || + Array.isArray(item.position) || + !Array.isArray(item.messages) + ) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Invalid semantic transcript position', + ); + } + const position = item.position as Record; + if ( + Object.keys(position).length !== 2 || + position.ordinal !== startOrdinal + index || + !isSemanticPositionKey(position.key) + ) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Semantic transcript position order changed', + ); + } + const key = position.key as SessionTranscriptSemanticPosition['key']; + const messages = item.messages.map((message) => { + if (!semanticMessageMatchesPosition(message, key, messageIds)) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Semantic transcript position identity changed', + ); + } + return decodeMessage(message); + }); + return { + position: position as unknown as SessionTranscriptSemanticPosition, + messages, + }; + }); + if ( + positions.some((entry) => entry.position.key.kind === 'empty') && + (totalPositions !== 1 || startOrdinal !== 0 || positions.length !== 1) + ) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Semantic transcript empty position changed', + ); + } + return { + snapshotToken: expectedSnapshotToken, + startOrdinal, + endOrdinalExclusive, + totalPositions, + positions, + hasOlder, + hasNewer, + }; +} + +function semanticMessageMatchesPosition( + value: unknown, + key: SessionTranscriptSemanticPosition['key'], + messageIds: Set, +): boolean { + if (!value || typeof value !== 'object' || Array.isArray(value) || key.kind === 'empty') { + return false; + } + const message = value as Record; + if (typeof message.id !== 'string' || message.id.length === 0 || messageIds.has(message.id)) { + return false; + } + const matches = + key.kind === 'turn' + ? message.turnId === key.id + : message.type === 'system_note' && + message.id === key.id && + !Object.hasOwn(message, 'turnId'); + if (matches) messageIds.add(message.id); + return matches; +} + +function isSemanticPositionKey(value: unknown): boolean { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const key = value as Record; + if (key.kind === 'empty') return Object.keys(key).length === 1; + return ( + (key.kind === 'turn' || key.kind === 'note') && + Object.keys(key).length === 2 && + typeof key.id === 'string' && + key.id.length > 0 + ); +} + +function isClientCount(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} + export interface RuntimeHostSessionSubscription extends AsyncIterable { readonly hostEpoch: string; readonly subscriptionId: string; @@ -81,9 +262,54 @@ export interface RuntimeHostSessionSubscription extends AsyncIterable, ): Promise; + queryTranscriptPositions( + input: ClientTranscriptPositionsInput, + ): Promise; + loadTranscriptTurnWindow( + input: ClientTranscriptTurnWindowOpenInput, + decodeMessage: (value: unknown) => T, + ): Promise | NonPageTranscriptTurnWindowResult>; + decodeTranscriptTurnWindowPage( + page: Extract, + decodeMessage: (value: unknown) => T, + ): Promise>; close(): Promise; } +type WithoutSubscription = T extends { readonly subscriptionId: string } + ? Omit + : never; + +export type ClientTranscriptPositionsInput = WithoutSubscription; +export type ClientTranscriptTurnWindowOpenInput = Omit< + Extract, + 'kind' | 'subscriptionId' +>; +export type NonPageTranscriptTurnWindowResult = Exclude< + SessionTranscriptTurnWindowResult, + { readonly kind: 'page' } +>; + +export interface DecodedSessionTranscriptTurnWindow { + readonly snapshotToken: string; + readonly startOrdinal: number; + readonly endOrdinalExclusive: number; + readonly totalPositions: number; + readonly positions: readonly { + readonly position: SessionTranscriptSemanticPosition; + readonly messages: readonly T[]; + }[]; + readonly hasOlder: boolean; + readonly hasNewer: boolean; +} + +export interface SessionTranscriptSemanticClientPort { + queryPositions(input: SessionTranscriptPositionsInput): Promise; + readTurnWindow( + input: SessionTranscriptTurnWindowInput, + ): Promise; +} + export interface DecodedSessionTranscriptPage { readonly messages: readonly { readonly identity: number; @@ -110,6 +336,7 @@ export class ClientSessionSubscription input: SessionTranscriptPageInput, ) => Promise; readonly #releaseTranscriptOverlay: () => Promise; + readonly #semanticTranscript: SessionTranscriptSemanticClientPort | undefined; readonly #expectedSessionId: string; readonly #queue: QueuedFrame[] = []; #queuedBytes = 0; @@ -136,6 +363,7 @@ export class ClientSessionSubscription requestClose: () => Promise, readTranscriptPage: (input: SessionTranscriptPageInput) => Promise, releaseTranscriptOverlay: () => Promise = async () => undefined, + semanticTranscript?: SessionTranscriptSemanticClientPort, ) { this.hostEpoch = result.hostEpoch; this.subscriptionId = result.subscriptionId; @@ -149,6 +377,7 @@ export class ClientSessionSubscription this.#requestClose = requestClose; this.#readTranscriptPage = readTranscriptPage; this.#releaseTranscriptOverlay = releaseTranscriptOverlay; + this.#semanticTranscript = semanticTranscript; } [Symbol.asyncIterator](): AsyncIterator { @@ -228,7 +457,7 @@ export class ClientSessionSubscription throughSequence: page.throughSequence, maxBytes: Math.max(1, page.rawBytes), }); - const assembler = new TranscriptFragmentAssembler( + const assembler = new TranscriptMessageAssembler( page.source, page.direction, maxMessageBytes, @@ -334,6 +563,119 @@ export class ClientSessionSubscription }); } + queryTranscriptPositions( + input: ClientTranscriptPositionsInput, + ): Promise { + this.#assertTranscriptReadable(); + if (!this.transcriptBootstrap || !this.#semanticTranscript) { + return Promise.reject( + new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Session subscription was opened without semantic transcript access', + ), + ); + } + return this.#semanticTranscript + .queryPositions({ + ...input, + subscriptionId: this.subscriptionId, + } as SessionTranscriptPositionsInput) + .then((result) => { + this.#assertTranscriptReadable(); + if (result.subscriptionId !== this.subscriptionId) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Semantic transcript position subscription changed', + ); + } + return result; + }); + } + + async loadTranscriptTurnWindow( + input: ClientTranscriptTurnWindowOpenInput, + decodeMessage: (value: unknown) => T, + ): Promise | NonPageTranscriptTurnWindowResult> { + this.#assertTranscriptReadable(); + if (!this.transcriptBootstrap || !this.#semanticTranscript) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Session subscription was opened without semantic transcript access', + ); + } + const page = await this.#semanticTranscript.readTurnWindow({ + kind: 'open', + subscriptionId: this.subscriptionId, + ...input, + }); + if (page.subscriptionId !== this.subscriptionId) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Semantic transcript window subscription changed', + ); + } + return page.kind === 'page' ? this.decodeTranscriptTurnWindowPage(page, decodeMessage) : page; + } + + async decodeTranscriptTurnWindowPage( + page: Extract, + decodeMessage: (value: unknown) => T, + ): Promise> { + this.#assertTranscriptReadable(); + const semantic = this.#semanticTranscript; + if (!semantic || page.subscriptionId !== this.subscriptionId || page.byteOffset !== 0) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Invalid semantic transcript window start', + ); + } + const assembler = new TranscriptFragmentAssembler( + 'newer', + page.totalBytes, + page.payloadDigest, + SESSION_TRANSCRIPT_RANGE_MAX_BYTES, + ); + try { + acceptSemanticWindowFragment(assembler, page); + let cursor = page.nextCursor; + while (cursor !== null) { + const requestedCursor = cursor; + const continuation = await semantic.readTurnWindow({ + kind: 'continue', + subscriptionId: this.subscriptionId, + cursor, + }); + if ( + continuation.kind !== 'page' || + continuation.subscriptionId !== this.subscriptionId || + continuation.snapshotToken !== page.snapshotToken || + continuation.windowId !== page.windowId || + continuation.totalBytes !== page.totalBytes || + continuation.payloadDigest !== page.payloadDigest || + continuation.nextCursor === requestedCursor + ) { + throw new RuntimeHostSubscriptionError( + 'correlation_changed', + 'Semantic transcript window continuation changed', + ); + } + acceptSemanticWindowFragment(assembler, continuation); + cursor = continuation.nextCursor; + } + let bytes: Buffer; + try { + bytes = assembler.finish(); + } catch (cause) { + throw new RuntimeHostSubscriptionError('correlation_changed', errorMessage(cause), { + cause, + }); + } + return decodeSemanticTranscriptWindow(bytes, page.snapshotToken, decodeMessage); + } finally { + assembler.release(); + } + } + async #loadTranscript(): Promise { this.#assertTranscriptReadable(); const bootstrap = this.transcriptBootstrap; @@ -424,7 +766,7 @@ export class ClientSessionSubscription throughSequence: initial.throughSequence, maxBytes: Math.max(1, initial.rawBytes), }); - const assembler = new TranscriptFragmentAssembler( + const assembler = new TranscriptMessageAssembler( initial.source, initial.direction, maxMessageBytes, @@ -608,16 +950,14 @@ export class ClientSessionSubscription } } -class TranscriptFragmentAssembler { +class TranscriptMessageAssembler { readonly #messages: Array<{ identity: number; value: unknown }> = []; - #assemblyBytes = 0; #current: | { identity: number; totalBytes: number; payloadDigest: `sha256:${string}` | null; - data: Buffer; - edge: number; + assembly: TranscriptFragmentAssembler; } | undefined; #lastStartedIdentity: number | undefined; @@ -636,7 +976,7 @@ class TranscriptFragmentAssembler { get continuationBytes(): number | null { const current = this.#current; if (!current) return null; - return this.direction === 'older' ? current.edge : current.totalBytes - current.edge; + return current.assembly.continuationBytes; } finish(): Array<{ identity: number; value: unknown }> { @@ -651,9 +991,8 @@ class TranscriptFragmentAssembler { } release(): void { - if (this.#assemblyBytes === 0) return; - this.accountAssemblyBytes(-this.#assemblyBytes); - this.#assemblyBytes = 0; + this.#current?.assembly.release(); + this.#current = undefined; } #accept(fragment: SessionTranscriptFragment): void { @@ -677,29 +1016,12 @@ class TranscriptFragmentAssembler { 'Session transcript message identity changed between fragments', ); } - const expectedOffset = - this.direction === 'older' ? this.#current.edge - bytes.byteLength : this.#current.edge; - if (fragment.byteOffset !== expectedOffset) { - throw new RuntimeHostSubscriptionError( - 'correlation_changed', - 'Session transcript message has a fragment gap', - ); - } - if (fragment.byteOffset + bytes.byteLength > this.#current.totalBytes) { - throw new RuntimeHostSubscriptionError( - 'correlation_changed', - 'Session transcript fragment exceeds its declared message size', - ); - } - bytes.copy(this.#current.data, fragment.byteOffset); - this.#current.edge = - this.direction === 'older' ? fragment.byteOffset : fragment.byteOffset + bytes.byteLength; - if ( - (this.direction === 'older' && this.#current.edge === 0) || - (this.direction === 'newer' && this.#current.edge === fragment.totalBytes) - ) { - this.#completeCurrent(); + try { + this.#current.assembly.accept(fragment.byteOffset, bytes); + } catch (cause) { + throw new RuntimeHostSubscriptionError('correlation_changed', errorMessage(cause), { cause }); } + if (this.#current.assembly.complete) this.#completeCurrent(); } #start(identity: number, totalBytes: number, payloadDigest: `sha256:${string}` | null): void { @@ -718,43 +1040,32 @@ class TranscriptFragmentAssembler { ); } this.#lastStartedIdentity = identity; - this.accountAssemblyBytes(totalBytes); - try { - this.#current = { - identity, - totalBytes, - payloadDigest, - data: Buffer.allocUnsafe(totalBytes), - edge: this.direction === 'older' ? totalBytes : 0, - }; - this.#assemblyBytes += totalBytes; - } catch (error) { - this.accountAssemblyBytes(-totalBytes); - throw error; - } + const assembly = new TranscriptFragmentAssembler( + this.direction, + totalBytes, + payloadDigest, + this.maxMessageBytes, + this.accountAssemblyBytes, + ); + this.#current = { identity, totalBytes, payloadDigest, assembly }; } #completeCurrent(): void { const current = this.#current!; try { - if ( - current.payloadDigest !== null && - `sha256:${createHash('sha256').update(current.data).digest('hex')}` !== - current.payloadDigest - ) { - throw new Error('payload digest mismatch'); - } this.#messages.push({ identity: current.identity, - value: JSON.parse(current.data.toString('utf8')) as unknown, + value: JSON.parse(current.assembly.finish().toString('utf8')) as unknown, }); } catch (cause) { throw new RuntimeHostSubscriptionError( 'correlation_changed', `Session transcript message failed integrity validation: ${errorMessage(cause)}`, ); + } finally { + current.assembly.release(); + this.#current = undefined; } - this.#current = undefined; } } diff --git a/packages/runtime-host/src/client/transcript-fragment-assembler.ts b/packages/runtime-host/src/client/transcript-fragment-assembler.ts new file mode 100644 index 0000000000..14ddbc2a39 --- /dev/null +++ b/packages/runtime-host/src/client/transcript-fragment-assembler.ts @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash } from 'node:crypto'; + +export class TranscriptFragmentAssemblyError extends Error { + readonly name = 'TranscriptFragmentAssemblyError'; +} + +/** The single bounded integrity path for transcript message and semantic-window fragments. */ +export class TranscriptFragmentAssembler { + readonly #data: Buffer; + #edge: number; + #released = false; + + constructor( + readonly direction: 'older' | 'newer', + readonly totalBytes: number, + readonly payloadDigest: `sha256:${string}` | null, + maxBytes = Number.MAX_SAFE_INTEGER, + private readonly accountAssemblyBytes: (deltaBytes: number) => void = () => undefined, + ) { + if (!Number.isSafeInteger(totalBytes) || totalBytes < 1 || totalBytes > maxBytes) { + throw new RangeError('Session transcript payload exceeds the local byte limit'); + } + accountAssemblyBytes(totalBytes); + try { + this.#data = Buffer.allocUnsafe(totalBytes); + this.#edge = direction === 'older' ? totalBytes : 0; + } catch (error) { + accountAssemblyBytes(-totalBytes); + throw error; + } + } + + get continuationBytes(): number | null { + if (this.complete) return null; + return this.direction === 'older' ? this.#edge : this.totalBytes - this.#edge; + } + + get complete(): boolean { + return this.direction === 'older' ? this.#edge === 0 : this.#edge === this.totalBytes; + } + + accept(byteOffset: number, bytes: Buffer): void { + if (this.#released) + throw new TranscriptFragmentAssemblyError('Transcript assembly is released'); + const expectedOffset = this.direction === 'older' ? this.#edge - bytes.byteLength : this.#edge; + if ( + bytes.byteLength < 1 || + byteOffset !== expectedOffset || + byteOffset < 0 || + byteOffset + bytes.byteLength > this.totalBytes + ) { + throw new TranscriptFragmentAssemblyError('Session transcript payload has a fragment gap'); + } + bytes.copy(this.#data, byteOffset); + this.#edge = this.direction === 'older' ? byteOffset : byteOffset + bytes.byteLength; + } + + finish(): Buffer { + if (!this.complete || this.#released) { + throw new TranscriptFragmentAssemblyError( + 'Session transcript payload ended before every fragment arrived', + ); + } + if ( + this.payloadDigest !== null && + `sha256:${createHash('sha256').update(this.#data).digest('hex')}` !== this.payloadDigest + ) { + throw new TranscriptFragmentAssemblyError('Session transcript payload digest mismatch'); + } + return this.#data; + } + + release(): void { + if (this.#released) return; + this.#released = true; + this.#data.fill(0); + this.accountAssemblyBytes(-this.totalBytes); + } +} diff --git a/packages/runtime-host/src/protocol/operations.ts b/packages/runtime-host/src/protocol/operations.ts index 8476927962..78084fe470 100644 --- a/packages/runtime-host/src/protocol/operations.ts +++ b/packages/runtime-host/src/protocol/operations.ts @@ -316,6 +316,8 @@ export const REMOTE_OWNER_OPERATION_GRANTS = Object.freeze([ 'session.revision.create', 'session.transcript.page', 'session.transcript.overlay.release', + 'session.transcript.positions.query', + 'session.transcript.turn_window.page', 'session.turn_landmarks.query', 'session.turns.query', 'session.workspace.relocate', diff --git a/packages/runtime-host/src/protocol/session-transcript.ts b/packages/runtime-host/src/protocol/session-transcript.ts index e50b0d03e9..93583e7be0 100644 --- a/packages/runtime-host/src/protocol/session-transcript.ts +++ b/packages/runtime-host/src/protocol/session-transcript.ts @@ -24,6 +24,7 @@ import { requireExactRecord, requireId, requireRecord, + requireShapedRecord, requireUtf8String, } from './codec.js'; import { invalidProtocolFrame } from './errors.js'; @@ -37,6 +38,123 @@ export const SESSION_TRANSCRIPT_RANGE_MAX_MESSAGES = SESSION_TRANSCRIPT_PAGE_MAX export const SESSION_TRANSCRIPT_OVERLAY_MAX_MESSAGES = 4_096; export const SESSION_TRANSCRIPT_PAGE_RESULT_MAX_BYTES = 744 * 1024; export const SESSION_TRANSCRIPT_CURSOR_MAX_BYTES = 1024; +export const SESSION_TRANSCRIPT_POSITION_PAGE_MAX_POSITIONS = 128; +export const SESSION_TRANSCRIPT_POSITION_PAGE_MAX_BYTES = 64 * 1024; +export const SESSION_TRANSCRIPT_WINDOW_MAX_POSITIONS = 10; +export const SESSION_TRANSCRIPT_WINDOW_MAX_BYTES = 16 * 1024 * 1024; + +export type SessionTranscriptSemanticPositionKey = + | { readonly kind: 'turn'; readonly id: string } + | { readonly kind: 'note'; readonly id: string } + | { readonly kind: 'empty' }; + +export interface SessionTranscriptSemanticPosition { + readonly ordinal: number; + readonly key: SessionTranscriptSemanticPositionKey; +} + +export type SessionTranscriptVisiblePositionAnchor = + | { readonly kind: 'tail' } + | { readonly kind: 'ordinal'; readonly ordinal: number } + | { readonly kind: 'turn'; readonly turnId: string }; + +export type SessionTranscriptPositionsInput = + | { + readonly kind: 'acquire'; + readonly subscriptionId: string; + readonly anchor: SessionTranscriptVisiblePositionAnchor; + readonly maxPositions: number; + } + | { + readonly kind: 'page'; + readonly subscriptionId: string; + readonly snapshotToken: string; + readonly anchor: SessionTranscriptVisiblePositionAnchor; + readonly maxPositions: number; + } + | { + readonly kind: 'continue'; + readonly subscriptionId: string; + readonly cursor: string; + } + | { + readonly kind: 'replace'; + readonly subscriptionId: string; + readonly snapshotToken: string; + readonly anchor: SessionTranscriptVisiblePositionAnchor; + readonly maxPositions: number; + } + | { + readonly kind: 'release'; + readonly subscriptionId: string; + readonly snapshotToken: string; + }; + +export type SessionTranscriptPositionsResult = + | { + readonly kind: 'page'; + readonly subscriptionId: string; + readonly snapshotToken: string; + readonly totalPositions: number; + readonly startOrdinal: number; + readonly positions: readonly SessionTranscriptSemanticPosition[]; + readonly olderCursor: string | null; + readonly newerCursor: string | null; + } + | { + readonly kind: 'building' | 'capacity'; + readonly subscriptionId: string; + readonly snapshotToken: string; + readonly retryAfterMs: number; + } + | { + readonly kind: 'snapshot_stale' | 'anchor_not_found'; + readonly subscriptionId: string; + readonly snapshotToken: string; + } + | { + readonly kind: 'released'; + readonly subscriptionId: string; + }; + +export type SessionTranscriptTurnWindowInput = + | { + readonly kind: 'open'; + readonly subscriptionId: string; + readonly snapshotToken: string; + readonly startOrdinal: number; + readonly maxPositions: number; + readonly replaceCursor?: string | null; + } + | { + readonly kind: 'continue'; + readonly subscriptionId: string; + readonly cursor: string; + }; + +export type SessionTranscriptTurnWindowResult = + | { + readonly kind: 'page'; + readonly subscriptionId: string; + readonly snapshotToken: string; + readonly windowId: string; + readonly byteOffset: number; + readonly totalBytes: number; + readonly payloadDigest: `sha256:${string}`; + readonly data: string; + readonly nextCursor: string | null; + } + | { + readonly kind: 'building' | 'capacity'; + readonly subscriptionId: string; + readonly snapshotToken: string; + readonly retryAfterMs: number; + } + | { + readonly kind: 'snapshot_stale' | 'anchor_not_found' | 'position_too_large'; + readonly subscriptionId: string; + readonly snapshotToken: string; + }; export type SessionTranscriptPageSource = 'durable' | 'overlay'; export type SessionTranscriptPageDirection = 'older' | 'newer'; @@ -132,8 +250,438 @@ export const SESSION_TRANSCRIPT_OPERATION_SPECS = { } }, }), + 'session.transcript.positions.query': defineOperation< + SessionTranscriptPositionsInput, + SessionTranscriptPositionsResult, + (typeof QUERY_ERRORS)[number] + >({ + mode: 'query', + availability: 'ready', + errors: QUERY_ERRORS, + decodeInput: decodeSessionTranscriptPositionsInput, + decodeOutput: decodeSessionTranscriptPositionsResult, + assertOutputForInput: assertSemanticTranscriptPositionsOutput, + }), + 'session.transcript.turn_window.page': defineOperation< + SessionTranscriptTurnWindowInput, + SessionTranscriptTurnWindowResult, + (typeof QUERY_ERRORS)[number] + >({ + mode: 'query', + availability: 'ready', + errors: QUERY_ERRORS, + decodeInput: decodeSessionTranscriptTurnWindowInput, + decodeOutput: decodeSessionTranscriptTurnWindowResult, + assertOutputForInput: assertSemanticTranscriptTurnWindowOutput, + }), } as const; +export function decodeSessionTranscriptPositionsInput( + value: unknown, +): SessionTranscriptPositionsInput { + const record = requireRecord(value, 'Session transcript positions input'); + switch (record.kind) { + case 'acquire': { + const input = requireExactRecord(record, 'Session transcript positions acquire input', [ + 'kind', + 'subscriptionId', + 'anchor', + 'maxPositions', + ]); + return { + kind: 'acquire', + subscriptionId: requireId(input.subscriptionId, 'subscriptionId'), + anchor: decodeSemanticPositionAnchor(input.anchor), + maxPositions: requireSemanticPositionLimit(input.maxPositions), + }; + } + case 'page': + case 'replace': { + const input = requireExactRecord( + record, + `Session transcript positions ${record.kind} input`, + ['kind', 'subscriptionId', 'snapshotToken', 'anchor', 'maxPositions'], + ); + return { + kind: record.kind, + subscriptionId: requireId(input.subscriptionId, 'subscriptionId'), + snapshotToken: requireOpaqueTranscriptToken(input.snapshotToken, 'snapshot token'), + anchor: decodeSemanticPositionAnchor(input.anchor), + maxPositions: requireSemanticPositionLimit(input.maxPositions), + }; + } + case 'continue': { + const input = requireExactRecord(record, 'Session transcript positions continue input', [ + 'kind', + 'subscriptionId', + 'cursor', + ]); + return { + kind: 'continue', + subscriptionId: requireId(input.subscriptionId, 'subscriptionId'), + cursor: requireOpaqueTranscriptToken(input.cursor, 'positions cursor'), + }; + } + case 'release': { + const input = requireExactRecord(record, 'Session transcript positions release input', [ + 'kind', + 'subscriptionId', + 'snapshotToken', + ]); + return { + kind: 'release', + subscriptionId: requireId(input.subscriptionId, 'subscriptionId'), + snapshotToken: requireOpaqueTranscriptToken(input.snapshotToken, 'snapshot token'), + }; + } + default: + throw invalidProtocolFrame('Invalid Session transcript positions input kind'); + } +} + +export function decodeSessionTranscriptPositionsResult( + value: unknown, +): SessionTranscriptPositionsResult { + requireEncodedByteLimit( + value, + 'Session transcript positions result', + SESSION_TRANSCRIPT_POSITION_PAGE_MAX_BYTES, + ); + const record = requireRecord(value, 'Session transcript positions result'); + if (record.kind === 'page') { + const result = requireExactRecord(record, 'Session transcript positions page result', [ + 'kind', + 'subscriptionId', + 'snapshotToken', + 'totalPositions', + 'startOrdinal', + 'positions', + 'olderCursor', + 'newerCursor', + ]); + if ( + !Array.isArray(result.positions) || + result.positions.length > SESSION_TRANSCRIPT_POSITION_PAGE_MAX_POSITIONS + ) { + throw invalidProtocolFrame('Invalid Session transcript semantic positions'); + } + const startOrdinal = requireCount(result.startOrdinal, 'Session transcript start ordinal'); + const positions = result.positions.map((position, index) => + decodeSemanticPosition(position, startOrdinal + index), + ); + const totalPositions = requireCount( + result.totalPositions, + 'Session transcript total positions', + ); + if ( + positions.length === 0 || + startOrdinal >= totalPositions || + startOrdinal + positions.length > totalPositions || + (positions.some((position) => position.key.kind === 'empty') && + (totalPositions !== 1 || startOrdinal !== 0 || positions.length !== 1)) + ) { + throw invalidProtocolFrame('Invalid Session transcript position page bounds'); + } + return { + kind: 'page', + subscriptionId: requireId(result.subscriptionId, 'subscriptionId'), + snapshotToken: requireOpaqueTranscriptToken(result.snapshotToken, 'snapshot token'), + totalPositions, + startOrdinal, + positions, + olderCursor: decodeNullableTranscriptToken(result.olderCursor, 'older positions cursor'), + newerCursor: decodeNullableTranscriptToken(result.newerCursor, 'newer positions cursor'), + }; + } + if (record.kind === 'building' || record.kind === 'capacity') { + const result = requireExactRecord(record, `Session transcript ${record.kind} result`, [ + 'kind', + 'subscriptionId', + 'snapshotToken', + 'retryAfterMs', + ]); + const retryAfterMs = requireCount(result.retryAfterMs, 'Session transcript retry hint'); + if (retryAfterMs < 25 || retryAfterMs > 1_000) { + throw invalidProtocolFrame('Invalid Session transcript retry hint'); + } + return { + kind: record.kind, + subscriptionId: requireId(result.subscriptionId, 'subscriptionId'), + snapshotToken: requireOpaqueTranscriptToken(result.snapshotToken, 'snapshot token'), + retryAfterMs, + }; + } + if (record.kind === 'snapshot_stale' || record.kind === 'anchor_not_found') { + const result = requireExactRecord(record, `Session transcript ${record.kind} result`, [ + 'kind', + 'subscriptionId', + 'snapshotToken', + ]); + return { + kind: record.kind, + subscriptionId: requireId(result.subscriptionId, 'subscriptionId'), + snapshotToken: requireOpaqueTranscriptToken(result.snapshotToken, 'snapshot token'), + }; + } + if (record.kind === 'released') { + const result = requireExactRecord(record, 'Session transcript released result', [ + 'kind', + 'subscriptionId', + ]); + return { + kind: 'released', + subscriptionId: requireId(result.subscriptionId, 'subscriptionId'), + }; + } + throw invalidProtocolFrame('Invalid Session transcript positions result kind'); +} + +export function decodeSessionTranscriptTurnWindowInput( + value: unknown, +): SessionTranscriptTurnWindowInput { + const record = requireRecord(value, 'Session transcript Turn-window input'); + if (record.kind === 'open') { + const input = requireShapedRecord( + record, + 'Session transcript Turn-window open input', + ['kind', 'subscriptionId', 'snapshotToken', 'startOrdinal', 'maxPositions'], + ['replaceCursor'], + ); + const maxPositions = requireCount( + input.maxPositions, + 'Session transcript Turn-window position limit', + ); + if (maxPositions < 1 || maxPositions > SESSION_TRANSCRIPT_WINDOW_MAX_POSITIONS) { + throw invalidProtocolFrame('Invalid Session transcript Turn-window position limit'); + } + return { + kind: 'open', + subscriptionId: requireId(input.subscriptionId, 'subscriptionId'), + snapshotToken: requireOpaqueTranscriptToken(input.snapshotToken, 'snapshot token'), + startOrdinal: requireCount(input.startOrdinal, 'Session transcript start ordinal'), + maxPositions, + ...(Object.hasOwn(input, 'replaceCursor') + ? { + replaceCursor: + input.replaceCursor === null + ? null + : requireOpaqueTranscriptToken(input.replaceCursor, 'window replacement cursor'), + } + : {}), + }; + } + if (record.kind === 'continue') { + const input = requireExactRecord(record, 'Session transcript Turn-window continue input', [ + 'kind', + 'subscriptionId', + 'cursor', + ]); + return { + kind: 'continue', + subscriptionId: requireId(input.subscriptionId, 'subscriptionId'), + cursor: requireOpaqueTranscriptToken(input.cursor, 'window cursor'), + }; + } + throw invalidProtocolFrame('Invalid Session transcript Turn-window input kind'); +} + +export function decodeSessionTranscriptTurnWindowResult( + value: unknown, +): SessionTranscriptTurnWindowResult { + requireEncodedByteLimit( + value, + 'Session transcript Turn-window result', + SESSION_TRANSCRIPT_PAGE_RESULT_MAX_BYTES, + ); + const record = requireRecord(value, 'Session transcript Turn-window result'); + if (record.kind === 'page') { + const result = requireExactRecord(record, 'Session transcript Turn-window page result', [ + 'kind', + 'subscriptionId', + 'snapshotToken', + 'windowId', + 'byteOffset', + 'totalBytes', + 'payloadDigest', + 'data', + 'nextCursor', + ]); + const byteOffset = requireCount(result.byteOffset, 'Session transcript window byte offset'); + const totalBytes = requireCount(result.totalBytes, 'Session transcript window total bytes'); + const data = requireBase64Fragment(result.data); + const dataBytes = Buffer.from(data, 'base64').byteLength; + if ( + totalBytes < 1 || + totalBytes > SESSION_TRANSCRIPT_WINDOW_MAX_BYTES || + byteOffset >= totalBytes || + byteOffset + dataBytes > totalBytes + ) { + throw invalidProtocolFrame('Invalid Session transcript Turn-window fragment bounds'); + } + return { + kind: 'page', + subscriptionId: requireId(result.subscriptionId, 'subscriptionId'), + snapshotToken: requireOpaqueTranscriptToken(result.snapshotToken, 'snapshot token'), + windowId: requireId(result.windowId, 'windowId'), + byteOffset, + totalBytes, + payloadDigest: requirePayloadDigest(result.payloadDigest, 'Turn-window payload digest'), + data, + nextCursor: decodeNullableTranscriptToken(result.nextCursor, 'window cursor'), + }; + } + if (record.kind === 'building' || record.kind === 'capacity') { + const result = requireExactRecord(record, `Session transcript ${record.kind} result`, [ + 'kind', + 'subscriptionId', + 'snapshotToken', + 'retryAfterMs', + ]); + const retryAfterMs = requireCount(result.retryAfterMs, 'Session transcript retry hint'); + if (retryAfterMs < 25 || retryAfterMs > 1_000) { + throw invalidProtocolFrame('Invalid Session transcript retry hint'); + } + return { + kind: record.kind, + subscriptionId: requireId(result.subscriptionId, 'subscriptionId'), + snapshotToken: requireOpaqueTranscriptToken(result.snapshotToken, 'snapshot token'), + retryAfterMs, + }; + } + if ( + record.kind === 'snapshot_stale' || + record.kind === 'anchor_not_found' || + record.kind === 'position_too_large' + ) { + const result = requireExactRecord(record, `Session transcript ${record.kind} result`, [ + 'kind', + 'subscriptionId', + 'snapshotToken', + ]); + return { + kind: record.kind, + subscriptionId: requireId(result.subscriptionId, 'subscriptionId'), + snapshotToken: requireOpaqueTranscriptToken(result.snapshotToken, 'snapshot token'), + }; + } + throw invalidProtocolFrame('Invalid Session transcript Turn-window result kind'); +} + +function assertSemanticTranscriptSubscription( + input: { readonly subscriptionId: string }, + output: { readonly subscriptionId: string }, +): void { + if (input.subscriptionId !== output.subscriptionId) { + throw invalidProtocolFrame('Session semantic transcript subscription changed'); + } +} + +function assertSemanticTranscriptPositionsOutput( + input: SessionTranscriptPositionsInput, + output: SessionTranscriptPositionsResult, +): void { + assertSemanticTranscriptSubscription(input, output); + if ( + output.kind === 'page' && + input.kind !== 'continue' && + input.kind !== 'release' && + output.positions.length > input.maxPositions + ) { + throw invalidProtocolFrame('Session transcript position page exceeds request limit'); + } + if ( + input.kind === 'page' && + 'snapshotToken' in output && + output.snapshotToken !== input.snapshotToken + ) { + throw invalidProtocolFrame('Session transcript snapshot token changed'); + } +} + +function assertSemanticTranscriptTurnWindowOutput( + input: SessionTranscriptTurnWindowInput, + output: SessionTranscriptTurnWindowResult, +): void { + assertSemanticTranscriptSubscription(input, output); + if (input.kind === 'open' && output.snapshotToken !== input.snapshotToken) { + throw invalidProtocolFrame('Session transcript window snapshot token changed'); + } +} + +function decodeSemanticPositionAnchor(value: unknown): SessionTranscriptVisiblePositionAnchor { + const anchor = requireRecord(value, 'Session transcript semantic position anchor'); + if (anchor.kind === 'tail') { + requireExactRecord(anchor, 'Session transcript tail anchor', ['kind']); + return { kind: 'tail' }; + } + if (anchor.kind === 'ordinal') { + const exact = requireExactRecord(anchor, 'Session transcript ordinal anchor', [ + 'kind', + 'ordinal', + ]); + return { + kind: 'ordinal', + ordinal: requireCount(exact.ordinal, 'Session transcript position ordinal'), + }; + } + if (anchor.kind === 'turn') { + const exact = requireExactRecord(anchor, 'Session transcript Turn anchor', ['kind', 'turnId']); + return { kind: 'turn', turnId: requireId(exact.turnId, 'turnId') }; + } + throw invalidProtocolFrame('Invalid Session transcript semantic position anchor'); +} + +function decodeSemanticPosition( + value: unknown, + expectedOrdinal: number, +): SessionTranscriptSemanticPosition { + const position = requireExactRecord(value, 'Session transcript semantic position', [ + 'ordinal', + 'key', + ]); + const ordinal = requireCount(position.ordinal, 'Session transcript position ordinal'); + if (ordinal !== expectedOrdinal) { + throw invalidProtocolFrame('Session transcript semantic position ordinals are not contiguous'); + } + return { ordinal, key: decodeSemanticPositionKey(position.key) }; +} + +function decodeSemanticPositionKey(value: unknown): SessionTranscriptSemanticPositionKey { + const key = requireRecord(value, 'Session transcript semantic position key'); + if (key.kind === 'empty') { + requireExactRecord(key, 'Session transcript empty position key', ['kind']); + return { kind: 'empty' }; + } + if (key.kind === 'turn' || key.kind === 'note') { + const exact = requireExactRecord(key, 'Session transcript semantic position key', [ + 'kind', + 'id', + ]); + return { kind: key.kind, id: requireId(exact.id, 'Session transcript position identity') }; + } + throw invalidProtocolFrame('Invalid Session transcript semantic position key'); +} + +function requireSemanticPositionLimit(value: unknown): number { + const limit = requireCount(value, 'Session transcript semantic position limit'); + if (limit < 1 || limit > SESSION_TRANSCRIPT_POSITION_PAGE_MAX_POSITIONS) { + throw invalidProtocolFrame('Invalid Session transcript semantic position limit'); + } + return limit; +} + +function requireOpaqueTranscriptToken(value: unknown, label: string): string { + return requireUtf8String( + value, + `Session transcript ${label}`, + SESSION_TRANSCRIPT_CURSOR_MAX_BYTES, + ); +} + +function decodeNullableTranscriptToken(value: unknown, label: string): string | null { + return value === null ? null : requireOpaqueTranscriptToken(value, label); +} + function decodeSessionTranscriptOverlayReleaseInput( value: unknown, ): SessionTranscriptOverlayReleaseInput { diff --git a/packages/runtime-host/src/server/access-credential-store.ts b/packages/runtime-host/src/server/access-credential-store.ts index 292cede830..2d33018592 100644 --- a/packages/runtime-host/src/server/access-credential-store.ts +++ b/packages/runtime-host/src/server/access-credential-store.ts @@ -36,6 +36,8 @@ const LEGACY_TRANSCRIPT_QUERY_GRANT = 'session.transcript.query'; const TRANSCRIPT_QUERY_REPLACEMENT_GRANTS = [ 'session.transcript.page', 'session.transcript.overlay.release', + 'session.transcript.positions.query', + 'session.transcript.turn_window.page', ] as const satisfies readonly OperationKey[]; const TURN_QUERY_GRANT = 'session.turns.query'; const TURN_QUERY_REPLACEMENT_GRANTS = [ @@ -69,6 +71,8 @@ export const SESSION_GUEST_OPERATION_GRANTS = Object.freeze([ 'subscription.close', 'session.transcript.page', 'session.transcript.overlay.release', + 'session.transcript.positions.query', + 'session.transcript.turn_window.page', ] as const satisfies readonly OperationKey[]); export interface StoredAccessCredential { diff --git a/packages/runtime-host/src/server/connection-session.ts b/packages/runtime-host/src/server/connection-session.ts index 7e800e1755..f4cd28b66a 100644 --- a/packages/runtime-host/src/server/connection-session.ts +++ b/packages/runtime-host/src/server/connection-session.ts @@ -23,6 +23,7 @@ import { RUNTIME_HOST_MAX_IN_FLIGHT_DOMAIN_REQUESTS, type ClientCapabilityClientFrame, type HostOperationErrorCode, + type OperationKey, type RequestFrame, } from '../protocol/index.js'; import type { RuntimeHostMessageTransport } from '../transport/message-transport.js'; @@ -82,10 +83,11 @@ export class RuntimeHostConnectionSession { readonly #options: RuntimeHostConnectionSessionOptions; readonly #writer: BoundedSerialOutboundWriter; readonly #requests = new Map>(); - #transcriptPageTail: Promise = Promise.resolve(); + #transcriptDataPlaneTail: Promise = Promise.resolve(); #inFlightStatusRequests = 0; #continuityService: SessionContinuityService | undefined; #continuity: SessionContinuityConnection | undefined; + #continuityCloseTask: Promise | undefined; #clientCapabilityService: ClientCapabilityService | undefined; #clientCapabilities: ClientCapabilityConnection | undefined; #clientCapabilityCloseTask: Promise | undefined; @@ -116,6 +118,7 @@ export class RuntimeHostConnectionSession { this.#writer.settled(), this.#options.transport.closed, this.#clientCapabilityCloseTask?.catch(() => undefined), + this.#continuityCloseTask?.catch(() => undefined), ]); } } @@ -175,10 +178,9 @@ export class RuntimeHostConnectionSession { #dispatch(frame: RequestFrame): void { if (frame.operation === 'host.status') this.#inFlightStatusRequests += 1; - const handling = - frame.operation === 'session.transcript.page' - ? this.#transcriptPageTail.then(() => this.#handleRequest(frame)) - : this.#handleRequest(frame); + const handling = isTranscriptDataPlaneOperation(frame.operation) + ? this.#transcriptDataPlaneTail.then(() => this.#handleRequest(frame)) + : this.#handleRequest(frame); const task = handling .catch(() => this.#teardown()) .finally(() => { @@ -188,8 +190,8 @@ export class RuntimeHostConnectionSession { } }); this.#requests.set(frame.requestId, task); - if (frame.operation === 'session.transcript.page') { - this.#transcriptPageTail = task.catch(() => undefined); + if (isTranscriptDataPlaneOperation(frame.operation)) { + this.#transcriptDataPlaneTail = task.catch(() => undefined); } } @@ -221,7 +223,8 @@ export class RuntimeHostConnectionSession { const continuity = frame.operation === 'subscription.open' || frame.operation === 'subscription.close' || - frame.operation === 'session.transcript.page' + isTranscriptDataPlaneOperation(frame.operation) || + frame.operation === 'session.transcript.overlay.release' ? this.#ensureContinuity() : undefined; const response = await dispatchOperation(frame, this.#options.resolveHandlers(), { @@ -277,7 +280,8 @@ export class RuntimeHostConnectionSession { } #detachContinuity(): void { - this.#continuity?.close(); + const closing = this.#continuity?.close(); + if (closing) this.#continuityCloseTask = closing; this.#continuity = undefined; this.#continuityService = undefined; } @@ -386,6 +390,14 @@ export class RuntimeHostConnectionSession { } } +function isTranscriptDataPlaneOperation(operation: OperationKey): boolean { + return ( + operation === 'session.transcript.page' || + operation === 'session.transcript.positions.query' || + operation === 'session.transcript.turn_window.page' + ); +} + function isReadEof(error: unknown): boolean { return error instanceof RuntimeHostTransportError && error.code === 'read_eof'; } diff --git a/packages/runtime-host/src/server/operation-dispatcher.ts b/packages/runtime-host/src/server/operation-dispatcher.ts index b752189df3..86336c4350 100644 --- a/packages/runtime-host/src/server/operation-dispatcher.ts +++ b/packages/runtime-host/src/server/operation-dispatcher.ts @@ -117,6 +117,8 @@ export type SessionContinuityOperationKey = Extract< | 'subscription.close' | 'session.transcript.page' | 'session.transcript.overlay.release' + | 'session.transcript.positions.query' + | 'session.transcript.turn_window.page' >; export type SessionRevisionOperationKey = Extract< OperationKey, diff --git a/packages/runtime-host/src/server/semantic-session-transcript-pager.ts b/packages/runtime-host/src/server/semantic-session-transcript-pager.ts new file mode 100644 index 0000000000..2ef1a0bff1 --- /dev/null +++ b/packages/runtime-host/src/server/semantic-session-transcript-pager.ts @@ -0,0 +1,1015 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash, randomUUID } from 'node:crypto'; +import type { StoredMessage } from '@maka/core/session'; +import { + SessionTurnPositionAnchorNotFoundError, + SessionTurnPositionLimitError, + SessionTurnPositionSnapshotMismatchError, + SESSION_TURN_POSITION_BODY_MAX_BYTES, + SESSION_TURN_POSITION_BODY_MAX_RECORDS, + type SessionTranscriptProjection, + type SessionTranscriptPositionKey, + type SessionTurnPositionAnchor, + type SessionTurnPositionSnapshotKey, +} from '@maka/storage/execution-stores'; +import { + SESSION_TRANSCRIPT_PAGE_MAX_BYTES, + SESSION_TRANSCRIPT_WINDOW_MAX_POSITIONS, + type SessionTranscriptPositionsInput, + type SessionTranscriptPositionsResult, + type SessionTranscriptTurnWindowInput, + type SessionTranscriptTurnWindowResult, +} from '../protocol/index.js'; +import { projectSharedSessionTranscriptMessage } from './shared-session-transcript.js'; +import type { SessionTranscriptReader } from './session-transcript-reader.js'; +import { + decodeTranscriptSignedToken, + encodeTranscriptSignedToken, +} from './transcript-signed-token.js'; +import { selectTranscriptBuffer } from './transcript-buffer-slice.js'; + +const SEMANTIC_RETRY_AFTER_MS = 50; +export const SEMANTIC_TRANSCRIPT_WINDOW_IDLE_TTL_MS = 5 * 60 * 1_000; + +interface SemanticSnapshot { + readonly handleId: string; + readonly incarnation: string; + readonly token: string; + readonly leaseId: string; + key?: SessionTurnPositionSnapshotKey; +} + +interface RetainedSemanticWindow { + readonly id: string; + readonly incarnation: string; + readonly snapshotTokenDigest: string; + readonly bytes: Buffer; + readonly digest: `sha256:${string}`; + currentCursor: string | null; + retained: boolean; + expiry?: NodeJS.Timeout; +} + +export interface SubscriberSemanticTranscriptState { + readonly sessionId: string; + readonly subscriptionId: string; + readonly projection: SessionTranscriptProjection; + readonly cursorSecret: Buffer; + semanticEligibleThroughSequence: number | null | undefined; + currentSnapshot?: SemanticSnapshot; + pendingSnapshot?: SemanticSnapshot; + activeWindow?: RetainedSemanticWindow; + closed: boolean; + releaseTail: Promise; +} + +export interface SemanticTranscriptRetainedAccounting { + retain(bytes: number): boolean; + release(bytes: number): void; +} + +export class SemanticTranscriptRequestError extends Error { + readonly name = 'SemanticTranscriptRequestError'; +} + +export class SemanticTranscriptConflictError extends Error { + readonly name = 'SemanticTranscriptConflictError'; +} + +export function createSubscriberSemanticTranscriptState(input: { + sessionId: string; + subscriptionId: string; + projection: SessionTranscriptProjection; + cursorSecret: Buffer; +}): SubscriberSemanticTranscriptState { + return { + ...input, + semanticEligibleThroughSequence: undefined, + closed: false, + releaseTail: Promise.resolve(), + }; +} + +export function confirmSemanticTranscriptWatermark( + state: SubscriberSemanticTranscriptState, + throughSequence: number | null, +): void { + const current = state.semanticEligibleThroughSequence; + if ( + current !== undefined && + current !== null && + throughSequence !== null && + throughSequence < current + ) { + throw new Error('Session semantic transcript watermark moved backwards'); + } + state.semanticEligibleThroughSequence = throughSequence; +} + +export async function querySemanticTranscriptPositions(input: { + reader: SessionTranscriptReader; + state: SubscriberSemanticTranscriptState; + request: SessionTranscriptPositionsInput; + accounting?: SemanticTranscriptRetainedAccounting; +}): Promise { + const { reader, state, request } = input; + assertSubscription(state, request.subscriptionId); + if (state.closed) throw new SemanticTranscriptRequestError('Semantic transcript is closed'); + if (state.semanticEligibleThroughSequence === undefined) { + throw new SemanticTranscriptRequestError('Semantic transcript is not active'); + } + if (request.kind === 'release') { + if (state.currentSnapshot?.token !== request.snapshotToken) { + decodeExactToken('snapshot', request.snapshotToken, state); + return { kind: 'released', subscriptionId: state.subscriptionId }; + } + const snapshot = requireSnapshotToken(state, request.snapshotToken, 'current'); + await releaseSnapshot(reader, state, snapshot, input.accounting); + return { kind: 'released', subscriptionId: state.subscriptionId }; + } + if (request.kind === 'continue') { + const cursor = decodePositionsCursor(request.cursor, state); + const snapshot = state.currentSnapshot; + if (!snapshot || tokenDigest(snapshot.token) !== cursor.snapshotTokenDigest) { + throw new SemanticTranscriptRequestError('Positions cursor snapshot is stale'); + } + const result = await readPositionPage( + reader, + state, + snapshot, + { + kind: 'ordinal', + ordinal: cursor.nextOrdinal, + }, + cursor.maxPositions, + ); + clearWindowForStaleResult(state, result, input.accounting); + return result; + } + if (request.kind === 'replace') { + if (state.currentSnapshot?.token !== request.snapshotToken) { + decodeExactToken('snapshot', request.snapshotToken, state); + return { + kind: 'snapshot_stale', + subscriptionId: state.subscriptionId, + snapshotToken: request.snapshotToken, + }; + } + const current = requireSnapshotToken(state, request.snapshotToken, 'current'); + const pending = state.pendingSnapshot ?? createSnapshot(state); + state.pendingSnapshot = pending; + let result: SessionTranscriptPositionsResult; + try { + result = await readPositionPage(reader, state, pending, request.anchor, request.maxPositions); + } catch (error) { + await releaseSpecificSnapshotSerialized(reader, state, pending); + state.pendingSnapshot = undefined; + throw error; + } + if (result.kind === 'page') { + if (state.currentSnapshot !== current) { + await releaseSpecificSnapshotSerialized(reader, state, pending); + state.pendingSnapshot = undefined; + throw new SemanticTranscriptConflictError('Semantic snapshot replacement lost its race'); + } + await releaseSpecificSnapshotSerialized(reader, state, current); + if (state.closed) { + await releaseSubscriberSemanticTranscript(reader, state, input.accounting); + throw new SemanticTranscriptRequestError('Semantic transcript is closed'); + } + state.currentSnapshot = pending; + state.pendingSnapshot = undefined; + if (state.activeWindow) { + releaseWindow(state.activeWindow, input.accounting); + state.activeWindow = undefined; + } + } else if (result.kind === 'snapshot_stale' || result.kind === 'anchor_not_found') { + await releaseSpecificSnapshotSerialized(reader, state, pending); + state.pendingSnapshot = undefined; + } + return result; + } + let snapshot: SemanticSnapshot; + if (request.kind === 'acquire') { + snapshot = state.currentSnapshot ?? createSnapshot(state); + state.currentSnapshot ??= snapshot; + } else { + if (state.currentSnapshot?.token !== request.snapshotToken) { + decodeExactToken('snapshot', request.snapshotToken, state); + return { + kind: 'snapshot_stale', + subscriptionId: state.subscriptionId, + snapshotToken: request.snapshotToken, + }; + } + snapshot = requireSnapshotToken(state, request.snapshotToken, 'current'); + } + const result = await readPositionPage( + reader, + state, + snapshot, + request.anchor, + request.maxPositions, + ); + clearWindowForStaleResult(state, result, input.accounting); + return result; +} + +export async function readSemanticTranscriptTurnWindow(input: { + reader: SessionTranscriptReader; + state: SubscriberSemanticTranscriptState; + request: SessionTranscriptTurnWindowInput; + fragmentBytes?: number; + accounting?: SemanticTranscriptRetainedAccounting; +}): Promise { + const { state, request } = input; + assertSubscription(state, request.subscriptionId); + if (state.closed) throw new SemanticTranscriptRequestError('Semantic transcript is closed'); + const fragmentBytes = input.fragmentBytes ?? SESSION_TRANSCRIPT_PAGE_MAX_BYTES; + if (request.kind === 'continue') { + const cursor = decodeWindowCursor(request.cursor, state); + const window = state.activeWindow; + if ( + !window || + window.id !== cursor.windowId || + window.incarnation !== cursor.incarnation || + window.snapshotTokenDigest !== cursor.snapshotTokenDigest || + window.currentCursor !== request.cursor + ) { + throw new SemanticTranscriptRequestError('Window cursor is stale'); + } + return sliceWindow(state, window, cursor.offset, fragmentBytes, input.accounting); + } + if (state.currentSnapshot?.token !== request.snapshotToken) { + decodeExactToken('snapshot', request.snapshotToken, state); + return { + kind: 'snapshot_stale', + subscriptionId: state.subscriptionId, + snapshotToken: request.snapshotToken, + }; + } + const snapshot = requireSnapshotToken(state, request.snapshotToken, 'current'); + if (!snapshot.key) { + return { + kind: 'snapshot_stale', + subscriptionId: state.subscriptionId, + snapshotToken: snapshot.token, + }; + } + const oldWindow = state.activeWindow; + if (oldWindow) { + if ( + typeof request.replaceCursor !== 'string' || + oldWindow.currentCursor !== request.replaceCursor + ) { + throw new SemanticTranscriptRequestError( + 'Active semantic window requires its current cursor', + ); + } + decodeWindowCursor(request.replaceCursor, state); + } else if (request.replaceCursor != null) { + throw new SemanticTranscriptRequestError('Semantic window replacement cursor is stale'); + } + const prepared = await prepareWindow(input.reader, state, snapshot, request); + if (prepared.kind !== 'ready') { + if (prepared.result.kind === 'snapshot_stale' && state.activeWindow) { + releaseWindow(state.activeWindow, input.accounting); + state.activeWindow = undefined; + } + return prepared.result; + } + if (state.closed) throw new SemanticTranscriptRequestError('Semantic transcript is closed'); + if (input.accounting && !input.accounting.retain(prepared.window.bytes.byteLength)) { + return { + kind: 'capacity', + subscriptionId: state.subscriptionId, + snapshotToken: snapshot.token, + retryAfterMs: SEMANTIC_RETRY_AFTER_MS, + }; + } + prepared.window.retained = input.accounting !== undefined; + state.activeWindow = prepared.window; + if (oldWindow) releaseWindow(oldWindow, input.accounting); + return sliceWindow(state, prepared.window, 0, fragmentBytes, input.accounting); +} + +export function releaseSubscriberSemanticTranscript( + reader: SessionTranscriptReader, + state: SubscriberSemanticTranscriptState, + accounting?: SemanticTranscriptRetainedAccounting, +): Promise { + state.closed = true; + const release = async () => { + if (state.activeWindow) { + releaseWindow(state.activeWindow, accounting); + state.activeWindow = undefined; + } + const failures: unknown[] = []; + for (const [property, snapshot] of [ + ['pendingSnapshot', state.pendingSnapshot], + ['currentSnapshot', state.currentSnapshot], + ] as const) { + if (!snapshot?.key) continue; + try { + await releaseSpecificSnapshot(reader, state, snapshot); + if (state[property] === snapshot) state[property] = undefined; + } catch (error) { + failures.push(error); + } + } + if (failures.length > 0) { + throw new AggregateError(failures, 'Failed to release semantic transcript snapshots'); + } + }; + const task = state.releaseTail.then(release, release); + state.releaseTail = task.catch(() => undefined); + return task; +} + +async function readPositionPage( + reader: SessionTranscriptReader, + state: SubscriberSemanticTranscriptState, + snapshot: SemanticSnapshot, + anchor: SessionTurnPositionAnchor, + maxPositions: number, +): Promise { + try { + const result = await reader.readPositionPage({ + sessionId: state.sessionId, + projection: state.projection, + snapshotLeaseId: snapshot.leaseId, + ...(snapshot.key + ? { snapshotKey: snapshot.key } + : { throughSequence: state.semanticEligibleThroughSequence ?? null }), + anchor, + maxPositions, + }); + if (result.kind !== 'capacity') snapshot.key = result.snapshotKey; + if (state.closed) { + await releaseSubscriberSemanticTranscript(reader, state); + throw new SemanticTranscriptRequestError('Semantic transcript is closed'); + } + if (result.kind === 'building') { + return { + kind: 'building', + subscriptionId: state.subscriptionId, + snapshotToken: snapshot.token, + retryAfterMs: SEMANTIC_RETRY_AFTER_MS, + }; + } + if (result.kind === 'capacity') { + return { + kind: 'capacity', + subscriptionId: state.subscriptionId, + snapshotToken: snapshot.token, + retryAfterMs: SEMANTIC_RETRY_AFTER_MS, + }; + } + const positions = result.positions.map(({ ordinal, key }) => ({ ordinal, key })); + const digest = tokenDigest(snapshot.token); + const olderOrdinal = Math.max(0, result.startOrdinal - maxPositions); + const olderMaxPositions = result.startOrdinal - olderOrdinal; + const newerOrdinal = result.startOrdinal + positions.length; + return { + kind: 'page', + subscriptionId: state.subscriptionId, + snapshotToken: snapshot.token, + totalPositions: result.totalPositions, + startOrdinal: result.startOrdinal, + positions, + olderCursor: result.hasOlder + ? encodePositionsCursor(state, digest, 'older', olderOrdinal, olderMaxPositions) + : null, + newerCursor: result.hasNewer + ? encodePositionsCursor(state, digest, 'newer', newerOrdinal, maxPositions) + : null, + }; + } catch (error) { + if (error instanceof SessionTurnPositionSnapshotMismatchError) { + snapshot.key = undefined; + if (state.currentSnapshot === snapshot) state.currentSnapshot = undefined; + if (state.pendingSnapshot === snapshot) state.pendingSnapshot = undefined; + return { + kind: 'snapshot_stale', + subscriptionId: state.subscriptionId, + snapshotToken: snapshot.token, + }; + } + if (error instanceof SessionTurnPositionAnchorNotFoundError) { + return { + kind: 'anchor_not_found', + subscriptionId: state.subscriptionId, + snapshotToken: snapshot.token, + }; + } + throw error; + } +} + +async function prepareWindow( + reader: SessionTranscriptReader, + state: SubscriberSemanticTranscriptState, + snapshot: SemanticSnapshot, + request: Extract, +): Promise< + | { readonly kind: 'ready'; readonly window: RetainedSemanticWindow } + | { readonly kind: 'result'; readonly result: SessionTranscriptTurnWindowResult } +> { + let page; + try { + page = await reader.readPositionPage({ + sessionId: state.sessionId, + projection: state.projection, + snapshotLeaseId: snapshot.leaseId, + snapshotKey: snapshot.key!, + anchor: { kind: 'ordinal', ordinal: request.startOrdinal }, + maxPositions: Math.min(request.maxPositions, SESSION_TRANSCRIPT_WINDOW_MAX_POSITIONS), + }); + if (state.closed) throw new SemanticTranscriptRequestError('Semantic transcript is closed'); + } catch (error) { + if (error instanceof SessionTurnPositionSnapshotMismatchError) { + snapshot.key = undefined; + if (state.currentSnapshot === snapshot) state.currentSnapshot = undefined; + return { kind: 'result', result: simpleWindowResult('snapshot_stale', state, snapshot) }; + } + if (error instanceof SessionTurnPositionAnchorNotFoundError) { + return { kind: 'result', result: simpleWindowResult('anchor_not_found', state, snapshot) }; + } + throw error; + } + if (page.kind === 'building' || page.kind === 'capacity') { + return { + kind: 'result', + result: { + kind: page.kind, + subscriptionId: state.subscriptionId, + snapshotToken: snapshot.token, + retryAfterMs: SEMANTIC_RETRY_AFTER_MS, + }, + }; + } + const encodedPositions: Buffer[] = []; + let acceptedStartOrdinal: number | undefined; + let acceptedEndOrdinalExclusive: number | undefined; + let remainingRecords = SESSION_TURN_POSITION_BODY_MAX_RECORDS; + let remainingBytes = SESSION_TURN_POSITION_BODY_MAX_BYTES; + for (const position of page.positions) { + if (position.key.kind === 'empty') { + const encodedPosition = encodeSemanticPosition({ + position: { ordinal: position.ordinal, key: position.key }, + messages: [], + }); + if ( + !semanticWindowFits( + snapshot.token, + page.totalPositions, + encodedPositions, + encodedPosition, + acceptedStartOrdinal ?? position.ordinal, + position.ordinal + 1, + ) + ) { + if (encodedPositions.length === 0) { + return { + kind: 'result', + result: simpleWindowResult('position_too_large', state, snapshot), + }; + } + break; + } + encodedPositions.push(encodedPosition); + acceptedStartOrdinal ??= position.ordinal; + acceptedEndOrdinalExclusive = position.ordinal + 1; + continue; + } + if (remainingRecords < 1 || remainingBytes < 1) break; + try { + const body = await reader.readPositionRecords({ + sessionId: state.sessionId, + projection: state.projection, + snapshotLeaseId: snapshot.leaseId, + snapshotKey: snapshot.key!, + positionKeys: [position.key], + maxBytes: remainingBytes, + maxRecords: remainingRecords, + }); + if (state.closed) throw new SemanticTranscriptRequestError('Semantic transcript is closed'); + if ( + body.rawBytes < 0 || + body.rawBytes > remainingBytes || + body.records.length > remainingRecords + ) { + throw new Error('Semantic transcript position body exceeded its requested budget'); + } + let previousSequence = -1; + const messages: StoredMessage[] = []; + for (const record of body.records) { + if ( + !samePositionKey(record.positionKey, position.key) || + record.sequence <= previousSequence + ) { + throw new Error('Semantic transcript position body order changed'); + } + previousSequence = record.sequence; + const message = + state.projection === 'shared' + ? projectSharedSessionTranscriptMessage(record.message, state.sessionId) + : record.message; + if (message) messages.push(message); + } + const encodedPosition = encodeSemanticPosition({ + position: { ordinal: position.ordinal, key: position.key }, + messages, + }); + if ( + !semanticWindowFits( + snapshot.token, + page.totalPositions, + encodedPositions, + encodedPosition, + acceptedStartOrdinal ?? position.ordinal, + position.ordinal + 1, + ) + ) { + if (encodedPositions.length === 0) { + return { + kind: 'result', + result: simpleWindowResult('position_too_large', state, snapshot), + }; + } + break; + } + encodedPositions.push(encodedPosition); + acceptedStartOrdinal ??= position.ordinal; + acceptedEndOrdinalExclusive = position.ordinal + 1; + remainingRecords -= body.records.length; + remainingBytes -= body.rawBytes; + } catch (error) { + if ( + error instanceof SessionTurnPositionLimitError && + (error.reason === 'transcript_record_bytes' || error.reason === 'transcript_record_count') + ) { + if (encodedPositions.length === 0) { + return { + kind: 'result', + result: simpleWindowResult('position_too_large', state, snapshot), + }; + } + break; + } + if (error instanceof SessionTurnPositionSnapshotMismatchError) { + snapshot.key = undefined; + if (state.currentSnapshot === snapshot) state.currentSnapshot = undefined; + return { kind: 'result', result: simpleWindowResult('snapshot_stale', state, snapshot) }; + } + throw error; + } + } + if ( + encodedPositions.length === 0 || + acceptedStartOrdinal === undefined || + acceptedEndOrdinalExclusive === undefined + ) { + return { kind: 'result', result: simpleWindowResult('position_too_large', state, snapshot) }; + } + const encoded = encodeSemanticWindow( + snapshot.token, + page.totalPositions, + acceptedStartOrdinal, + acceptedEndOrdinalExclusive, + encodedPositions, + ); + const digest = `sha256:${createHash('sha256').update(encoded).digest('hex')}` as const; + return { + kind: 'ready', + window: { + id: randomUUID(), + incarnation: randomUUID(), + snapshotTokenDigest: tokenDigest(snapshot.token), + bytes: encoded, + digest, + currentCursor: null, + retained: false, + }, + }; +} + +function sliceWindow( + state: SubscriberSemanticTranscriptState, + window: RetainedSemanticWindow, + offset: number, + fragmentBytes: number, + accounting: SemanticTranscriptRetainedAccounting | undefined, +): SessionTranscriptTurnWindowResult { + if (offset < 0 || offset >= window.bytes.byteLength || fragmentBytes < 1) { + throw new SemanticTranscriptRequestError('Invalid semantic window offset'); + } + const selected = selectTranscriptBuffer( + window.bytes, + 'newer', + offset, + fragmentBytes, + () => new SemanticTranscriptRequestError('Invalid semantic window offset'), + ); + if (!selected) throw new SemanticTranscriptRequestError('Invalid semantic window byte budget'); + const end = selected.nextOffset; + const nextCursor = + end === window.bytes.byteLength ? null : encodeWindowCursor(state, window, end); + window.currentCursor = nextCursor; + const result: SessionTranscriptTurnWindowResult = { + kind: 'page', + subscriptionId: state.subscriptionId, + snapshotToken: state.currentSnapshot!.token, + windowId: window.id, + byteOffset: offset, + totalBytes: window.bytes.byteLength, + payloadDigest: window.digest, + data: selected.data.toString('base64'), + nextCursor, + }; + if (nextCursor === null) { + releaseWindow(window, accounting); + if (state.activeWindow === window) state.activeWindow = undefined; + } else { + armWindowExpiry(state, window, accounting); + } + return result; +} + +function armWindowExpiry( + state: SubscriberSemanticTranscriptState, + window: RetainedSemanticWindow, + accounting: SemanticTranscriptRetainedAccounting | undefined, +): void { + if (window.expiry) clearTimeout(window.expiry); + window.expiry = setTimeout(() => { + if (state.activeWindow !== window) return; + releaseWindow(window, accounting); + state.activeWindow = undefined; + }, SEMANTIC_TRANSCRIPT_WINDOW_IDLE_TTL_MS); + window.expiry.unref(); +} + +async function releaseSnapshot( + reader: SessionTranscriptReader, + state: SubscriberSemanticTranscriptState, + snapshot: SemanticSnapshot, + accounting: SemanticTranscriptRetainedAccounting | undefined, +): Promise { + const pending = state.pendingSnapshot; + if (pending && pending !== snapshot) { + await releaseSpecificSnapshotSerialized(reader, state, pending); + if (state.pendingSnapshot === pending) state.pendingSnapshot = undefined; + } + await releaseSpecificSnapshotSerialized(reader, state, snapshot); + if (state.activeWindow) { + releaseWindow(state.activeWindow, accounting); + state.activeWindow = undefined; + } + if (state.currentSnapshot === snapshot) state.currentSnapshot = undefined; + if (state.pendingSnapshot === snapshot) state.pendingSnapshot = undefined; +} + +async function releaseSpecificSnapshot( + reader: SessionTranscriptReader, + state: SubscriberSemanticTranscriptState, + snapshot: SemanticSnapshot, +): Promise { + if (!snapshot.key) return; + const key = snapshot.key; + await reader.releasePositionSnapshot({ + sessionId: state.sessionId, + projection: state.projection, + snapshotLeaseId: snapshot.leaseId, + snapshotKey: key, + }); + if (snapshot.key === key) snapshot.key = undefined; +} + +function releaseSpecificSnapshotSerialized( + reader: SessionTranscriptReader, + state: SubscriberSemanticTranscriptState, + snapshot: SemanticSnapshot, +): Promise { + const release = () => releaseSpecificSnapshot(reader, state, snapshot); + const task = state.releaseTail.then(release, release); + state.releaseTail = task.catch(() => undefined); + return task; +} + +function releaseWindow( + window: RetainedSemanticWindow, + accounting: SemanticTranscriptRetainedAccounting | undefined, +): void { + if (window.expiry) { + clearTimeout(window.expiry); + window.expiry = undefined; + } + if (!window.retained) return; + window.retained = false; + accounting?.release(window.bytes.byteLength); +} + +function clearWindowForStaleResult( + state: SubscriberSemanticTranscriptState, + result: SessionTranscriptPositionsResult, + accounting: SemanticTranscriptRetainedAccounting | undefined, +): void { + if (result.kind !== 'snapshot_stale' || !state.activeWindow) return; + releaseWindow(state.activeWindow, accounting); + state.activeWindow = undefined; +} + +function createSnapshot(state: SubscriberSemanticTranscriptState): SemanticSnapshot { + const handleId = randomUUID(); + const incarnation = randomUUID(); + return { + handleId, + incarnation, + leaseId: randomUUID(), + token: encodeTranscriptSignedToken( + 'snapshot', + { version: 1, handleId, incarnation }, + state.cursorSecret, + ), + }; +} + +function requireSnapshotToken( + state: SubscriberSemanticTranscriptState, + token: string, + expected: 'current' | 'pending', +): SemanticSnapshot { + const decoded = decodeExactToken('snapshot', token, state); + const snapshot = expected === 'current' ? state.currentSnapshot : state.pendingSnapshot; + if ( + typeof decoded.handleId !== 'string' || + typeof decoded.incarnation !== 'string' || + !snapshot || + decoded.handleId !== snapshot.handleId || + decoded.incarnation !== snapshot.incarnation || + token !== snapshot.token + ) { + throw new SemanticTranscriptRequestError('Semantic snapshot token is stale'); + } + return snapshot; +} + +function encodePositionsCursor( + state: SubscriberSemanticTranscriptState, + snapshotTokenDigest: string, + direction: 'older' | 'newer', + nextOrdinal: number, + maxPositions: number, +): string { + return encodeTranscriptSignedToken( + 'positions', + { + version: 1, + subscriptionId: state.subscriptionId, + snapshotTokenDigest, + direction, + nextOrdinal, + maxPositions, + }, + state.cursorSecret, + ); +} + +function decodePositionsCursor( + value: string, + state: SubscriberSemanticTranscriptState, +): { + snapshotTokenDigest: string; + direction: 'older' | 'newer'; + nextOrdinal: number; + maxPositions: number; +} { + const decoded = decodeExactToken('positions', value, state); + if ( + typeof decoded.snapshotTokenDigest !== 'string' || + (decoded.direction !== 'older' && decoded.direction !== 'newer') || + !isCount(decoded.nextOrdinal) || + !isCount(decoded.maxPositions) + ) { + throw new SemanticTranscriptRequestError('Invalid positions cursor'); + } + return decoded as { + snapshotTokenDigest: string; + direction: 'older' | 'newer'; + nextOrdinal: number; + maxPositions: number; + }; +} + +function encodeWindowCursor( + state: SubscriberSemanticTranscriptState, + window: RetainedSemanticWindow, + offset: number, +): string { + return encodeTranscriptSignedToken( + 'window', + { + version: 1, + subscriptionId: state.subscriptionId, + snapshotTokenDigest: window.snapshotTokenDigest, + windowId: window.id, + incarnation: window.incarnation, + offset, + }, + state.cursorSecret, + ); +} + +function decodeWindowCursor( + value: string, + state: SubscriberSemanticTranscriptState, +): { + snapshotTokenDigest: string; + windowId: string; + incarnation: string; + offset: number; +} { + const decoded = decodeExactToken('window', value, state); + if ( + typeof decoded.snapshotTokenDigest !== 'string' || + typeof decoded.windowId !== 'string' || + typeof decoded.incarnation !== 'string' || + !isCount(decoded.offset) + ) { + throw new SemanticTranscriptRequestError('Invalid semantic window cursor'); + } + return decoded as { + snapshotTokenDigest: string; + windowId: string; + incarnation: string; + offset: number; + }; +} + +function decodeExactToken( + domain: 'snapshot' | 'positions' | 'window', + value: string, + state: SubscriberSemanticTranscriptState, +): Record { + let decoded: unknown; + try { + decoded = decodeTranscriptSignedToken(domain, value, state.cursorSecret); + } catch (cause) { + throw new SemanticTranscriptRequestError('Invalid semantic transcript token', { cause }); + } + if (!decoded || typeof decoded !== 'object' || Array.isArray(decoded)) { + throw new SemanticTranscriptRequestError('Invalid semantic transcript token'); + } + const record = decoded as Record; + const keys = + domain === 'snapshot' + ? ['version', 'handleId', 'incarnation'] + : domain === 'positions' + ? [ + 'version', + 'subscriptionId', + 'snapshotTokenDigest', + 'direction', + 'nextOrdinal', + 'maxPositions', + ] + : ['version', 'subscriptionId', 'snapshotTokenDigest', 'windowId', 'incarnation', 'offset']; + if ( + record.version !== 1 || + (domain !== 'snapshot' && record.subscriptionId !== state.subscriptionId) || + (domain === 'snapshot' && + (typeof record.handleId !== 'string' || typeof record.incarnation !== 'string')) || + Object.keys(record).length !== keys.length || + keys.some((key) => !Object.hasOwn(record, key)) + ) { + throw new SemanticTranscriptRequestError( + 'Semantic transcript token does not match subscription', + ); + } + return record; +} + +function simpleWindowResult( + kind: 'snapshot_stale' | 'anchor_not_found' | 'position_too_large', + state: SubscriberSemanticTranscriptState, + snapshot: SemanticSnapshot, +): SessionTranscriptTurnWindowResult { + return { + kind, + subscriptionId: state.subscriptionId, + snapshotToken: snapshot.token, + }; +} + +function encodeSemanticPosition(value: { + readonly position: { readonly ordinal: number; readonly key: SessionTranscriptPositionKey }; + readonly messages: readonly StoredMessage[]; +}): Buffer { + return Buffer.from(JSON.stringify(value), 'utf8'); +} + +function semanticWindowFits( + snapshotToken: string, + totalPositions: number, + accepted: readonly Buffer[], + candidate: Buffer, + startOrdinal: number, + endOrdinalExclusive: number, +): boolean { + const { prefix, suffix } = semanticWindowEnvelope( + snapshotToken, + totalPositions, + startOrdinal, + endOrdinalExclusive, + ); + const positionsBytes = + accepted.reduce((total, position) => total + position.byteLength, 0) + candidate.byteLength; + const separators = accepted.length; + return ( + prefix.byteLength + positionsBytes + separators + suffix.byteLength <= + SESSION_TURN_POSITION_BODY_MAX_BYTES + ); +} + +function encodeSemanticWindow( + snapshotToken: string, + totalPositions: number, + startOrdinal: number, + endOrdinalExclusive: number, + positions: readonly Buffer[], +): Buffer { + const { prefix, suffix } = semanticWindowEnvelope( + snapshotToken, + totalPositions, + startOrdinal, + endOrdinalExclusive, + ); + const fragments: Buffer[] = [prefix]; + for (const [index, position] of positions.entries()) { + if (index > 0) fragments.push(Buffer.from(',')); + fragments.push(position); + } + fragments.push(suffix); + return Buffer.concat(fragments); +} + +function semanticWindowEnvelope( + snapshotToken: string, + totalPositions: number, + startOrdinal: number, + endOrdinalExclusive: number, +): { readonly prefix: Buffer; readonly suffix: Buffer } { + return { + prefix: Buffer.from( + `{"snapshotToken":${JSON.stringify(snapshotToken)},"startOrdinal":${startOrdinal},"endOrdinalExclusive":${endOrdinalExclusive},"totalPositions":${totalPositions},"positions":[`, + 'utf8', + ), + suffix: Buffer.from( + `],"hasOlder":${startOrdinal > 0},"hasNewer":${endOrdinalExclusive < totalPositions}}`, + 'utf8', + ), + }; +} + +function samePositionKey( + left: Exclude, + right: SessionTranscriptPositionKey, +): boolean { + return right.kind !== 'empty' && left.kind === right.kind && left.id === right.id; +} + +function tokenDigest(token: string): string { + return createHash('sha256').update(token, 'utf8').digest('base64url'); +} + +function assertSubscription( + state: SubscriberSemanticTranscriptState, + subscriptionId: string, +): void { + if (state.subscriptionId !== subscriptionId) { + throw new SemanticTranscriptRequestError('Semantic transcript subscription changed'); + } +} + +function isCount(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} diff --git a/packages/runtime-host/src/server/session-continuity-coordinator.ts b/packages/runtime-host/src/server/session-continuity-coordinator.ts index 2596e1a8af..7b6eee97c8 100644 --- a/packages/runtime-host/src/server/session-continuity-coordinator.ts +++ b/packages/runtime-host/src/server/session-continuity-coordinator.ts @@ -46,6 +46,8 @@ import { type SessionToolEvent, type SessionTranscriptAdvancedFrame, type SessionTranscriptPageInput, + type SessionTranscriptPositionsInput, + type SessionTranscriptTurnWindowInput, type OperationOutcome, type SubscriptionFrame, type SubscriptionOpenInput, @@ -82,6 +84,16 @@ import { type SessionTranscriptReader, } from './session-transcript-reader.js'; import { projectSharedSessionMessageContent } from './shared-session-transcript.js'; +import { + confirmSemanticTranscriptWatermark, + createSubscriberSemanticTranscriptState, + querySemanticTranscriptPositions, + readSemanticTranscriptTurnWindow, + releaseSubscriberSemanticTranscript, + SemanticTranscriptConflictError, + SemanticTranscriptRequestError, + type SubscriberSemanticTranscriptState, +} from './semantic-session-transcript-pager.js'; const MAX_CONNECTION_SUBSCRIPTIONS = 16; const MAX_SUBSCRIBER_QUEUED_FRAMES = 32; @@ -171,6 +183,7 @@ interface Subscriber { pumping: boolean; terminalQueued: boolean; transcript?: SubscriberTranscriptState; + semanticTranscript?: SubscriberSemanticTranscriptState; retainedTranscriptOverlay?: RetainedTranscriptOverlay; } @@ -253,6 +266,10 @@ export class SessionContinuityCoordinator implements SessionContinuityService { }, 'session.transcript.page': (input, context) => this.#readTranscriptPage(context.connectionId, input), + 'session.transcript.positions.query': (input, context) => + this.#queryTranscriptPositions(context.connectionId, input), + 'session.transcript.turn_window.page': (input, context) => + this.#readTranscriptTurnWindow(context.connectionId, input), 'session.transcript.overlay.release': async (input, context) => { const existing = this.#subscriptions.get(input.subscriptionId); if (!existing) { @@ -284,6 +301,8 @@ export class SessionContinuityCoordinator implements SessionContinuityService { #closed = false; #preparingTranscriptOverlayBytes = 0; #retainedTranscriptOverlayBytes = 0; + #retainedSemanticWindows = 0; + readonly #semanticReleaseTasks = new Set>(); readonly #sessionAccessAuthority: | Pick | undefined; @@ -345,9 +364,9 @@ export class SessionContinuityCoordinator implements SessionContinuityService { if (attached) this.#abortSubscription(connectionId, subscriptionId); }, close: () => { - if (!attached) return; + if (!attached) return this.settled(); attached = false; - this.#closeConnection(connectionId); + return this.#closeConnection(connectionId); }, }; } @@ -821,8 +840,8 @@ export class SessionContinuityCoordinator implements SessionContinuityService { } } - close(): void { - if (this.#closed) return; + close(): Promise { + if (this.#closed) return this.settled(); this.#closed = true; this.#unsubscribeGrantRevocations?.(); this.#cancelTranscriptOverlayPreparationWaiters(); @@ -833,6 +852,13 @@ export class SessionContinuityCoordinator implements SessionContinuityService { this.#pendingRefreshes.clear(); this.#pendingAgentGraphChanges.clear(); this.#pendingSessionDomainChanges.clear(); + return this.settled(); + } + + async settled(): Promise { + while (this.#semanticReleaseTasks.size > 0) { + await Promise.allSettled([...this.#semanticReleaseTasks]); + } } async #open( @@ -1011,6 +1037,16 @@ export class SessionContinuityCoordinator implements SessionContinuityService { pumping: false, terminalQueued: false, ...(transcript ? { transcript } : {}), + ...(transcript + ? { + semanticTranscript: createSubscriberSemanticTranscriptState({ + sessionId, + subscriptionId, + projection: transcript.projection, + cursorSecret: transcript.cursorSecret, + }), + } + : {}), ...(retainedTranscriptOverlay ? { retainedTranscriptOverlay } : {}), }; if (subscriber.retainedTranscriptOverlay) @@ -1150,6 +1186,180 @@ export class SessionContinuityCoordinator implements SessionContinuityService { }); } + async #queryTranscriptPositions( + connectionId: string, + input: SessionTranscriptPositionsInput, + ): Promise> { + const subscriber = this.#ownedSubscriber(connectionId, input.subscriptionId); + if (!subscriber) return transcriptSubscriptionNotFound(); + if (!this.#transcriptReader || !subscriber.semanticTranscript) { + return { + ok: false, + error: { code: 'operation_unavailable', message: 'Semantic transcript is unavailable' }, + }; + } + const semanticTranscript = subscriber.semanticTranscript; + const connection = this.#connections.get(connectionId); + if (!connection || !this.#canObserve(subscriber, subscriber.sessionId)) { + this.#closeSubscriber(subscriber, 'access_revoked'); + return transcriptSubscriptionNotFound(); + } + return this.sessionAdmission.run(subscriber.sessionId, async () => { + if (!this.#semanticRequestStillAuthorized(connectionId, connection, subscriber)) { + this.#closeSubscriber(subscriber, 'access_revoked'); + return transcriptSubscriptionNotFound(); + } + try { + const result = await querySemanticTranscriptPositions({ + reader: this.#transcriptReader!, + state: semanticTranscript, + request: input, + accounting: { + retain: (bytes) => this.#retainSemanticWindow(bytes), + release: (bytes) => this.#releaseSemanticWindow(bytes), + }, + }); + if (!this.#semanticRequestStillAuthorized(connectionId, connection, subscriber)) { + this.#closeSubscriber(subscriber, 'access_revoked'); + await this.#releaseDetachedSemanticTranscript(semanticTranscript); + return transcriptSubscriptionNotFound(); + } + return { ok: true, result }; + } catch (error) { + if (!this.#semanticRequestStillAuthorized(connectionId, connection, subscriber)) { + this.#closeSubscriber(subscriber, 'access_revoked'); + await this.#releaseDetachedSemanticTranscript(semanticTranscript); + return transcriptSubscriptionNotFound(); + } + if (error instanceof SemanticTranscriptRequestError) { + return { ok: false, error: { code: 'invalid_request', message: error.message } }; + } + if (error instanceof SemanticTranscriptConflictError) { + return { ok: false, error: { code: 'operation_conflict', message: error.message } }; + } + return { + ok: false, + error: { code: 'persistence_failed', message: 'Semantic transcript is unavailable' }, + }; + } + }); + } + + async #readTranscriptTurnWindow( + connectionId: string, + input: SessionTranscriptTurnWindowInput, + ): Promise> { + const subscriber = this.#ownedSubscriber(connectionId, input.subscriptionId); + if (!subscriber) return transcriptSubscriptionNotFound(); + if (!this.#transcriptReader || !subscriber.semanticTranscript) { + return { + ok: false, + error: { code: 'operation_unavailable', message: 'Semantic transcript is unavailable' }, + }; + } + const semanticTranscript = subscriber.semanticTranscript; + const connection = this.#connections.get(connectionId); + if (!connection || !this.#canObserve(subscriber, subscriber.sessionId)) { + this.#closeSubscriber(subscriber, 'access_revoked'); + return transcriptSubscriptionNotFound(); + } + let preparationPermit: TranscriptOverlayPreparationPermit | undefined; + let releasePreparation: (() => void) | undefined; + try { + if (input.kind === 'open') { + preparationPermit = await this.#acquireTranscriptOverlayPreparation(connection); + releasePreparation = preparationPermit.take(); + } + return await this.sessionAdmission.run(subscriber.sessionId, async () => { + if (!this.#semanticRequestStillAuthorized(connectionId, connection, subscriber)) { + this.#closeSubscriber(subscriber, 'access_revoked'); + return transcriptSubscriptionNotFound(); + } + try { + const result = await readSemanticTranscriptTurnWindow({ + reader: this.#transcriptReader!, + state: semanticTranscript, + request: input, + accounting: { + retain: (bytes) => this.#retainSemanticWindow(bytes), + release: (bytes) => this.#releaseSemanticWindow(bytes), + }, + }); + if (!this.#semanticRequestStillAuthorized(connectionId, connection, subscriber)) { + this.#closeSubscriber(subscriber, 'access_revoked'); + await this.#releaseDetachedSemanticTranscript(semanticTranscript); + return transcriptSubscriptionNotFound(); + } + return { ok: true, result }; + } catch (error) { + if (!this.#semanticRequestStillAuthorized(connectionId, connection, subscriber)) { + this.#closeSubscriber(subscriber, 'access_revoked'); + await this.#releaseDetachedSemanticTranscript(semanticTranscript); + return transcriptSubscriptionNotFound(); + } + if (error instanceof SemanticTranscriptRequestError) { + return { ok: false, error: { code: 'invalid_request', message: error.message } }; + } + if (error instanceof SemanticTranscriptConflictError) { + return { ok: false, error: { code: 'operation_conflict', message: error.message } }; + } + return { + ok: false, + error: { code: 'persistence_failed', message: 'Semantic transcript is unavailable' }, + }; + } + }); + } catch (error) { + return error instanceof TranscriptOverlayCapacityError && input.kind === 'open' + ? { + ok: true, + result: { + kind: 'capacity', + subscriptionId: subscriber.subscriptionId, + snapshotToken: input.snapshotToken, + retryAfterMs: 50, + }, + } + : transcriptSubscriptionNotFound(); + } finally { + releasePreparation?.(); + preparationPermit?.release(); + } + } + + #semanticRequestStillAuthorized( + connectionId: string, + connection: ConnectionState, + subscriber: Subscriber, + ): boolean { + return ( + this.#ownedSubscriber(connectionId, subscriber.subscriptionId) === subscriber && + this.#connections.get(connectionId) === connection && + this.#canObserve(subscriber, subscriber.sessionId) + ); + } + + #retainSemanticWindow(bytes: number): boolean { + if ( + this.#retainedSemanticWindows >= 64 || + this.#retainedTranscriptOverlayBytes + bytes > MAX_RETAINED_TRANSCRIPT_OVERLAY_BYTES + ) { + return false; + } + this.#retainedSemanticWindows += 1; + this.#retainedTranscriptOverlayBytes += bytes; + return true; + } + + #releaseSemanticWindow(bytes: number): void { + if (this.#retainedSemanticWindows < 1 || this.#retainedTranscriptOverlayBytes < bytes) { + throw new Error('Semantic transcript retained accounting underflow'); + } + this.#retainedSemanticWindows -= 1; + this.#retainedTranscriptOverlayBytes -= bytes; + this.#drainTranscriptOverlayPreparationWaiters(); + } + #prepareTranscriptOverlay( state: SessionProjectionState, sessionId: string, @@ -1412,6 +1622,12 @@ export class SessionContinuityCoordinator implements SessionContinuityService { const subscriber = this.#ownedSubscriber(connectionId, subscriptionId); if (!subscriber || subscriber.activated || subscriber.phase === 'closed') return; subscriber.activated = true; + if (subscriber.semanticTranscript && subscriber.transcript) { + confirmSemanticTranscriptWatermark( + subscriber.semanticTranscript, + subscriber.transcript.openedThroughSequence, + ); + } this.#pump(subscriber); } @@ -1435,15 +1651,16 @@ export class SessionContinuityCoordinator implements SessionContinuityService { return true; } - #closeConnection(connectionId: string): void { + #closeConnection(connectionId: string): Promise { const connection = this.#connections.get(connectionId); - if (!connection) return; + if (!connection) return this.settled(); connection.resolveClosed(); for (const subscriptionId of [...connection.subscriptionIds]) { const subscriber = this.#ownedSubscriber(connectionId, subscriptionId); if (subscriber) this.#removeSubscriber(subscriber); } this.#connections.delete(connectionId); + return this.settled(); } #enqueue(subscriber: Subscriber, frame: SubscriptionFrame): void { @@ -1511,6 +1728,7 @@ export class SessionContinuityCoordinator implements SessionContinuityService { #closeSubscriber(subscriber: Subscriber, reason: 'slow_consumer' | 'access_revoked'): void { if (subscriber.phase !== 'open') return; subscriber.phase = 'closing'; + this.#trackSemanticRelease(subscriber); const inFlight = subscriber.pumping ? subscriber.queue[0] : undefined; subscriber.queue = []; subscriber.queuedBytes = 0; @@ -1660,6 +1878,7 @@ export class SessionContinuityCoordinator implements SessionContinuityService { #enqueueSessionRemoved(subscriber: Subscriber): void { if (subscriber.phase !== 'open' || subscriber.terminalQueued) return; + this.#trackSemanticRelease(subscriber); const frame: SubscriptionFrame = { kind: 'subscription.closed', hostEpoch: this.#hostEpoch, @@ -1702,6 +1921,15 @@ export class SessionContinuityCoordinator implements SessionContinuityService { subscriber.queuedBytes -= queued.encodedBytes; } subscriber.lastFlushedSequence = queued.frame.sequence; + if ( + queued.frame.kind === 'subscription.transcript_advanced' && + subscriber.semanticTranscript + ) { + confirmSemanticTranscriptWatermark( + subscriber.semanticTranscript, + queued.frame.throughSequence, + ); + } if (queued.frame.kind === 'subscription.closed') { this.#removeSubscriber(subscriber); return; @@ -1723,6 +1951,7 @@ export class SessionContinuityCoordinator implements SessionContinuityService { this.#connections .get(subscriber.connectionId) ?.subscriptionIds.delete(subscriber.subscriptionId); + this.#trackSemanticRelease(subscriber); this.#releaseSubscriberTranscriptOverlay(subscriber); if (!this.#closed && state && removed) { if (!this.#hasTranscriptOverlayConsumer(state)) this.#invalidateTranscriptOverlay(state); @@ -1732,6 +1961,33 @@ export class SessionContinuityCoordinator implements SessionContinuityService { } } + #trackSemanticRelease(subscriber: Subscriber): void { + const semantic = subscriber.semanticTranscript; + if (!semantic || !this.#transcriptReader) return; + subscriber.semanticTranscript = undefined; + const task = releaseSubscriberSemanticTranscript(this.#transcriptReader, semantic, { + retain: (bytes) => this.#retainSemanticWindow(bytes), + release: (bytes) => this.#releaseSemanticWindow(bytes), + }) + .catch((error: unknown) => this.onPublicationFailure(error)) + .finally(() => this.#semanticReleaseTasks.delete(task)); + this.#semanticReleaseTasks.add(task); + } + + async #releaseDetachedSemanticTranscript( + semantic: SubscriberSemanticTranscriptState, + ): Promise { + if (!this.#transcriptReader) return; + try { + await releaseSubscriberSemanticTranscript(this.#transcriptReader, semantic, { + retain: (bytes) => this.#retainSemanticWindow(bytes), + release: (bytes) => this.#releaseSemanticWindow(bytes), + }); + } catch (error) { + this.onPublicationFailure(error); + } + } + #releaseSubscriberTranscriptOverlay(subscriber: Subscriber): void { if (subscriber.transcript) subscriber.transcript.overlayMessages = undefined; const retained = subscriber.retainedTranscriptOverlay; @@ -1868,11 +2124,16 @@ function assistantStreamKey(kind: SessionAssistantDelta['kind'], messageId: stri return `${kind}\0${messageId}`; } -function transcriptSubscriptionNotFound(): OperationOutcome<'session.transcript.page'> { +function transcriptSubscriptionNotFound< + K extends + | 'session.transcript.page' + | 'session.transcript.positions.query' + | 'session.transcript.turn_window.page', +>(): OperationOutcome { return { ok: false, error: { code: 'not_found', message: 'Session subscription was not found' }, - }; + } as OperationOutcome; } function terminalFrameByteBudget(subscriber: Subscriber, hostEpoch: string): number { diff --git a/packages/runtime-host/src/server/session-continuity-service.ts b/packages/runtime-host/src/server/session-continuity-service.ts index a830ff57ee..1d0c00421d 100644 --- a/packages/runtime-host/src/server/session-continuity-service.ts +++ b/packages/runtime-host/src/server/session-continuity-service.ts @@ -27,7 +27,7 @@ export interface SessionContinuityFrameSink { export interface SessionContinuityConnection { activate(subscriptionId: string): void; abort(subscriptionId: string): void; - close(): void; + close(): Promise; } export interface SessionContinuityService { diff --git a/packages/runtime-host/src/server/session-transcript-pager.ts b/packages/runtime-host/src/server/session-transcript-pager.ts index 09737024b0..3918e6e1ac 100644 --- a/packages/runtime-host/src/server/session-transcript-pager.ts +++ b/packages/runtime-host/src/server/session-transcript-pager.ts @@ -17,7 +17,7 @@ * under the License. */ -import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; +import { randomBytes } from 'node:crypto'; import type { StoredMessage } from '@maka/core/session'; import { SESSION_TRANSCRIPT_PAGE_MAX_MESSAGES, @@ -37,6 +37,11 @@ import { type SessionTranscriptReader, } from './session-transcript-reader.js'; import { projectSharedSessionTranscriptMessage } from './shared-session-transcript.js'; +import { + decodeTranscriptSignedToken, + encodeTranscriptSignedToken, +} from './transcript-signed-token.js'; +import { selectTranscriptBuffer } from './transcript-buffer-slice.js'; type SessionTranscriptProjection = 'owner' | 'shared'; @@ -490,11 +495,12 @@ async function readSharedDurablePage( record.sequence === position && request.byteOffset !== undefined ? request.byteOffset : null; - const selected = selectBuffer( + const selected = selectTranscriptBuffer( bytes, request.direction, continuationOffset, request.maxBytes - rawBytes, + () => new TranscriptPageRequestError('Invalid cursor byte offset'), ); if (!selected) { next = { position: record.sequence, byteOffset: null }; @@ -655,7 +661,13 @@ function selectOverlay( fragments.length < maxMessages ) { const message = messages[index]!; - const selected = selectBuffer(message, direction, offset, maxBytes - rawBytes); + const selected = selectTranscriptBuffer( + message, + direction, + offset, + maxBytes - rawBytes, + () => new TranscriptPageRequestError('Invalid cursor byte offset'), + ); if (!selected) break; fragments.push({ kind: 'overlay', @@ -682,43 +694,6 @@ function selectOverlay( }; } -function selectBuffer( - bytes: Buffer, - direction: SessionTranscriptPageDirection, - byteOffset: number | null, - budget: number, -): { - byteOffset: number; - data: Buffer; - complete: boolean; - nextOffset: number; -} | null { - if (budget < 1) return null; - if (direction === 'older') { - const end = byteOffset ?? bytes.byteLength; - if (end < 1 || end > bytes.byteLength) - throw new TranscriptPageRequestError('Invalid cursor byte offset'); - const start = Math.max(0, end - budget); - return { - byteOffset: start, - data: bytes.subarray(start, end), - complete: start === 0, - nextOffset: start, - }; - } - const start = byteOffset ?? 0; - if (start < 0 || start >= bytes.byteLength) { - throw new TranscriptPageRequestError('Invalid cursor byte offset'); - } - const end = Math.min(bytes.byteLength, start + budget); - return { - byteOffset: start, - data: bytes.subarray(start, end), - complete: end === bytes.byteLength, - nextOffset: end, - }; -} - function pageFromSelection( state: SubscriberTranscriptState, source: SessionTranscriptPageSource, @@ -783,28 +758,13 @@ function emptyPage( } function encodeCursor(cursor: TranscriptCursorState, secret: Buffer): string { - const payload = Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64url'); - return `${payload}.${signCursor(payload, secret).toString('base64url')}`; + return encodeTranscriptSignedToken('legacy-page', cursor, secret); } function decodeCursor(value: string, secret: Buffer): TranscriptCursorState { let decoded: unknown; try { - const parts = value.split('.'); - if (parts.length !== 2) throw new Error('invalid cursor envelope'); - const [payload, signatureValue] = parts as [string, string]; - const bytes = Buffer.from(payload, 'base64url'); - const signature = Buffer.from(signatureValue, 'base64url'); - const expected = signCursor(payload, secret); - if ( - bytes.toString('base64url') !== payload || - signature.toString('base64url') !== signatureValue || - signature.byteLength !== expected.byteLength || - !timingSafeEqual(signature, expected) - ) { - throw new Error('invalid cursor signature'); - } - decoded = JSON.parse(bytes.toString('utf8')) as unknown; + decoded = decodeTranscriptSignedToken('legacy-page', value, secret); } catch (cause) { throw new TranscriptPageRequestError('Invalid transcript cursor', { cause }); } @@ -845,10 +805,6 @@ function decodeCursor(value: string, secret: Buffer): TranscriptCursorState { return cursor as unknown as TranscriptCursorState; } -function signCursor(payload: string, secret: Buffer): Buffer { - return createHmac('sha256', secret).update(payload, 'utf8').digest(); -} - function mergeActiveAssistantStreams( overlay: readonly StoredMessage[], prefixes: Iterable, diff --git a/packages/runtime-host/src/server/session-transcript-reader.ts b/packages/runtime-host/src/server/session-transcript-reader.ts index 442d3b8851..9288126869 100644 --- a/packages/runtime-host/src/server/session-transcript-reader.ts +++ b/packages/runtime-host/src/server/session-transcript-reader.ts @@ -34,7 +34,12 @@ import type { SessionTranscriptPageRequest, SessionTranscriptRecordScanPage, SessionTranscriptRecordScanRequest, + SessionTranscriptRecordsByPositionKeysSnapshotRequest, + SessionTranscriptRecordsByPositionKeysSnapshotResult, SessionTranscriptStoragePage, + SessionTurnPositionPageSnapshotRequest, + SessionTurnPositionReadResult, + SessionTurnPositionSnapshotReleaseRequest, } from '@maka/storage/execution-stores'; import { SESSION_TRANSCRIPT_OVERLAY_MAX_MESSAGES, type TurnSnapshot } from '../protocol/index.js'; @@ -57,6 +62,11 @@ export function createSessionTranscriptReader(input: { input.stores.sessionStore.readTranscriptRecordsSnapshot(sessionId, request), readDurableMessagesById: (sessionId, request) => input.stores.sessionStore.readTranscriptMessagesSnapshot(sessionId, request), + readPositionPage: (request) => input.stores.sessionStore.readTurnPositionPageSnapshot(request), + readPositionRecords: (request) => + input.stores.sessionStore.readTranscriptRecordsByPositionKeysSnapshot(request), + releasePositionSnapshot: (request) => + input.stores.sessionStore.releaseTurnPositionSnapshot(request), readActiveOverlay: async (sessionId, rootTurn) => { if (!rootTurn || isTerminalTurn(rootTurn)) return []; @@ -93,6 +103,13 @@ export interface SessionTranscriptReader { sessionId: string, request: SessionTranscriptMessageLookupRequest, ): Promise; + readPositionPage( + request: SessionTurnPositionPageSnapshotRequest, + ): Promise; + readPositionRecords( + request: SessionTranscriptRecordsByPositionKeysSnapshotRequest, + ): Promise; + releasePositionSnapshot(request: SessionTurnPositionSnapshotReleaseRequest): Promise; readActiveOverlay( sessionId: string, rootTurn: TurnSnapshot | null, diff --git a/packages/runtime-host/src/server/shared-session-transcript.ts b/packages/runtime-host/src/server/shared-session-transcript.ts index 5c59dd8b0e..3d8bab879d 100644 --- a/packages/runtime-host/src/server/shared-session-transcript.ts +++ b/packages/runtime-host/src/server/shared-session-transcript.ts @@ -20,7 +20,7 @@ import type { AttachmentRef, MessageContent } from '@maka/core/events'; import { projectToolActivityArgs } from '@maka/core/tool-activity-args'; import { - isUserVisibleSessionSystemNote, + classifySharedSessionTranscriptVisibility, type AssistantThinking, type StoredMessage, userFacingText, @@ -49,6 +49,12 @@ export function projectSharedSessionTranscriptMessage( message: StoredMessage, sessionId: string, ): StoredMessage | null { + const visibility = classifySharedSessionTranscriptVisibility( + message.type === 'system_note' + ? { type: message.type, kind: message.kind } + : { type: message.type }, + ); + if (visibility === 'hidden') return null; switch (message.type) { case 'user': { return { @@ -153,15 +159,13 @@ export function projectSharedSessionTranscriptMessage( ...(message.costUsd === undefined ? {} : { costUsd: message.costUsd }), }; case 'system_note': - return isUserVisibleSessionSystemNote(message.kind) - ? { - type: message.type, - id: message.id, - ...(message.turnId === undefined ? {} : { turnId: message.turnId }), - ts: message.ts, - kind: message.kind, - } - : null; + return { + type: message.type, + id: message.id, + ...(message.turnId === undefined ? {} : { turnId: message.turnId }), + ts: message.ts, + kind: message.kind, + }; case 'permission_decision': case 'workhub_coordination': return null; diff --git a/packages/runtime-host/src/server/transcript-buffer-slice.ts b/packages/runtime-host/src/server/transcript-buffer-slice.ts new file mode 100644 index 0000000000..767712dc3f --- /dev/null +++ b/packages/runtime-host/src/server/transcript-buffer-slice.ts @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export function selectTranscriptBuffer( + bytes: Buffer, + direction: 'older' | 'newer', + byteOffset: number | null, + budget: number, + invalidOffset: () => Error = () => new RangeError('Invalid transcript byte offset'), +): { + byteOffset: number; + data: Buffer; + complete: boolean; + nextOffset: number; +} | null { + if (budget < 1) return null; + if (direction === 'older') { + const end = byteOffset ?? bytes.byteLength; + if (end < 1 || end > bytes.byteLength) throw invalidOffset(); + const start = Math.max(0, end - budget); + return { + byteOffset: start, + data: bytes.subarray(start, end), + complete: start === 0, + nextOffset: start, + }; + } + const start = byteOffset ?? 0; + if (start < 0 || start >= bytes.byteLength) throw invalidOffset(); + const end = Math.min(bytes.byteLength, start + budget); + return { + byteOffset: start, + data: bytes.subarray(start, end), + complete: end === bytes.byteLength, + nextOffset: end, + }; +} diff --git a/packages/runtime-host/src/server/transcript-signed-token.ts b/packages/runtime-host/src/server/transcript-signed-token.ts new file mode 100644 index 0000000000..1c1c8721dd --- /dev/null +++ b/packages/runtime-host/src/server/transcript-signed-token.ts @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHmac, timingSafeEqual } from 'node:crypto'; + +export type TranscriptSignedTokenDomain = 'legacy-page' | 'snapshot' | 'positions' | 'window'; + +export class TranscriptSignedTokenError extends Error { + readonly name = 'TranscriptSignedTokenError'; +} + +export function encodeTranscriptSignedToken( + domain: TranscriptSignedTokenDomain, + value: unknown, + secret: Buffer, +): string { + const payload = Buffer.from(JSON.stringify(value), 'utf8').toString('base64url'); + return `${payload}.${sign(domain, payload, secret).toString('base64url')}`; +} + +export function decodeTranscriptSignedToken( + domain: TranscriptSignedTokenDomain, + value: string, + secret: Buffer, +): unknown { + try { + const parts = value.split('.'); + if (parts.length !== 2) throw new Error('invalid token envelope'); + const [payload, signatureValue] = parts as [string, string]; + const bytes = Buffer.from(payload, 'base64url'); + const signature = Buffer.from(signatureValue, 'base64url'); + const expected = sign(domain, payload, secret); + if ( + bytes.toString('base64url') !== payload || + signature.toString('base64url') !== signatureValue || + signature.byteLength !== expected.byteLength || + !timingSafeEqual(signature, expected) + ) { + throw new Error('invalid token signature'); + } + return JSON.parse(bytes.toString('utf8')) as unknown; + } catch (cause) { + throw new TranscriptSignedTokenError('Invalid transcript token', { cause }); + } +} + +function sign(domain: TranscriptSignedTokenDomain, payload: string, secret: Buffer): Buffer { + return createHmac('sha256', secret) + .update(`maka:session-transcript:${domain}:v1\0`, 'utf8') + .update(payload, 'utf8') + .digest(); +} diff --git a/packages/storage/package.json b/packages/storage/package.json index cbcbaddc18..7323ed5446 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -68,6 +68,7 @@ }, "dependencies": { "@maka/core": "0.1.0", - "fs-native-extensions": "^1.5.0" + "fs-native-extensions": "^1.5.0", + "hash-wasm": "4.12.0" } } diff --git a/packages/storage/src/__tests__/session-turn-identity-scanner.test.ts b/packages/storage/src/__tests__/session-turn-identity-scanner.test.ts new file mode 100644 index 0000000000..b6952bf4a7 --- /dev/null +++ b/packages/storage/src/__tests__/session-turn-identity-scanner.test.ts @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { classifySharedSessionTranscriptVisibility, decodeStoredMessage } from '@maka/core/session'; +import { + advanceSessionTurnIdentityScanner, + completeSessionTurnIdentityScanner, + createSessionTurnIdentityScannerState, + restoreSessionTurnIdentityScannerState, + serializeSessionTurnIdentityScannerState, + SessionTurnIdentityScannerError, +} from '../session-turn-identity-scanner.js'; + +function scan( + json: string, + splits: readonly number[] = [], + sql = { messageId: 'message-1', messageType: 'user' }, +) { + let state = createSessionTurnIdentityScannerState(); + const bytes = Buffer.from(json); + let offset = 0; + for (const end of [...splits, bytes.length]) { + advanceSessionTurnIdentityScanner(state, bytes.subarray(offset, end)); + state = restoreSessionTurnIdentityScannerState(serializeSessionTurnIdentityScannerState(state)); + offset = end; + } + return completeSessionTurnIdentityScanner(state, sql); +} + +test('extracts only the top-level identity envelope across arbitrary fragment boundaries', () => { + const json = JSON.stringify({ + text: '正文 😀 \\ escaped', + nested: { id: 'fake-id', type: 'system_note', turnId: 'fake-turn', kind: 'step_limit' }, + turnId: 'turn-一', + type: 'user', + id: 'message-1', + ts: 1, + }); + const expected = { kind: 'turn', positionId: 'turn-一', sharedVisibility: true }; + assert.deepEqual(scan(json), expected); + for (let split = 1; split < Buffer.byteLength(json); split += 1) { + assert.deepEqual(scan(json, [split]), expected); + } +}); + +test('handles escaped identity values and visible/invisible turnless notes', () => { + assert.deepEqual( + scan( + '{"kind":"step_limit","id":"note\\u002d1","type":"system_note","text":"x"}', + [2, 7, 19, 31, 47], + { messageId: 'note-1', messageType: 'system_note' }, + ), + { kind: 'note', positionId: 'note-1', sharedVisibility: true }, + ); + assert.deepEqual( + scan('{"id":"hidden","type":"system_note","kind":"mode_change"}', [], { + messageId: 'hidden', + messageType: 'system_note', + }), + { kind: 'ignored' }, + ); +}); + +test('rejects duplicate, empty, missing, nested, mismatched, and non-string identity', () => { + const invalid = [ + '{"id":"message-1","id":"message-1","type":"user","turnId":"turn"}', + '{"id":"message-1","type":"user","turnId":""}', + '{"id":"message-1","type":"user"}', + '{"id":"message-1","type":"user","nested":{"turnId":"turn"}}', + '{"id":"message-1","type":"user","turnId":1}', + '{"id":"message-1","type":"user","turnId":{"value":"turn"}}', + '{"id":"message-1","type":"future_message","turnId":"turn"}', + ]; + for (const json of invalid) { + assert.throws(() => scan(json), SessionTurnIdentityScannerError); + } + assert.throws(() => scan('{"id":"other","type":"user","turnId":"turn"}'), /mismatch/u); + assert.throws( + () => + scan('{"id":"note","type":"system_note","kind":"future_note"}', [], { + messageId: 'note', + messageType: 'system_note', + }), + SessionTurnIdentityScannerError, + ); + assert.throws( + () => + scan('{"id":"note","type":"system_note","kind":"step_limit","turnId":""}', [], { + messageId: 'note', + messageType: 'system_note', + }), + SessionTurnIdentityScannerError, + ); +}); + +test('rejects invalid UTF-8, truncated JSON, excessive nesting, and corrupt persisted state', () => { + const state = createSessionTurnIdentityScannerState(); + assert.throws( + () => advanceSessionTurnIdentityScanner(state, Uint8Array.from([0x7b, 0x22, 0xc0])), + /UTF-8/u, + ); + assert.throws(() => scan('{"id":"message-1","type":"user","turnId":"turn"'), /truncated/u); + assert.throws(() => scan('{"id":"message-1","type":"user","turnId":"turn",}'), /JSON/u); + assert.throws( + () => scan('{"id":"message-1","type":"user","turnId":"turn","body":[1,]}'), + /JSON/u, + ); + assert.throws(() => scan('{"id":"message-1",\u00a0"type":"user","turnId":"turn"}'), /JSON/u); + assert.throws( + () => + scan( + `{"id":"message-1","type":"user","turnId":"turn","x":${'['.repeat(4_097)}0${']'.repeat(4_097)}}`, + ), + /nesting/u, + ); + assert.throws(() => restoreSessionTurnIdentityScannerState('{"version":999}'), /unsupported/u); + assert.throws( + () => + restoreSessionTurnIdentityScannerState( + JSON.stringify({ + version: 1, + rootStarted: true, + rootComplete: false, + stack: [{ kind: 'object', expectation: 'invented', pendingKey: null }], + lexical: { kind: 'invented' }, + utf8: { needed: 0, codePoint: 0, minimum: 0 }, + seenIdentityKeys: [], + identity: { turnIdPresent: false, kindPresent: false }, + capturedBytes: 0, + }), + ), + /unsupported/u, + ); +}); + +test('rejects captured identity larger than 32 KiB without retaining body fields', () => { + assert.throws( + () => + scan( + JSON.stringify({ + id: 'message-1', + type: 'user', + turnId: 'x'.repeat(32 * 1024 + 1), + text: 'body'.repeat(100_000), + }), + ), + /32 KiB/u, + ); + assert.deepEqual( + scan( + JSON.stringify({ + id: 'message-1', + type: 'user', + turnId: 'turn', + text: 'body'.repeat(100_000), + }), + [4 * 1024], + ), + { kind: 'turn', positionId: 'turn', sharedVisibility: true }, + ); + assert.deepEqual( + scan( + JSON.stringify({ + id: 'message-1', + type: 'user', + turnId: 'turn', + ['x'.repeat(100 * 1024)]: 'ignored body field', + }), + [4 * 1024], + ), + { kind: 'turn', positionId: 'turn', sharedVisibility: true }, + ); +}); + +test('differentially extracts every canonical StoredMessage identity under key reordering', () => { + const messages = [ + { type: 'user', id: 'message-1', turnId: 'turn-1', ts: 1, text: 'user' }, + { + type: 'assistant', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + text: 'assistant', + modelId: 'model', + }, + { + type: 'tool_call', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + toolName: 'tool', + args: { turnId: 'nested-fake', kind: 'step_limit' }, + }, + { + type: 'tool_result', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + toolUseId: 'call-1', + isError: false, + content: { kind: 'text', text: 'result' }, + }, + { + type: 'permission_decision', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + toolUseId: 'call-1', + toolName: 'tool', + decision: 'allow', + }, + { type: 'token_usage', id: 'message-1', turnId: 'turn-1', ts: 1, input: 1, output: 2 }, + { + type: 'turn_state', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + status: 'completed', + partialOutputRetained: false, + }, + { + type: 'workhub_coordination', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + schemaVersion: 1, + kind: 'delegation_assigned', + actionId: 'action-1', + actionFingerprint: `sha256:${'0'.repeat(64)}`, + coordinationTurnId: 'turn-1', + targetSessionId: 'target-session', + disposition: 'delegate_existing', + userText: 'work', + delegationId: 'delegation-1', + targetTurnId: 'target-turn', + targetMessageId: 'target-message', + targetSessionName: 'target', + }, + { + type: 'system_note', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + kind: 'error', + }, + ]; + for (const message of messages) { + const reordered = Object.fromEntries(Object.entries(message).reverse()); + const json = JSON.stringify(reordered); + const decoded = decodeStoredMessage(JSON.parse(json) as never); + assert.equal(decoded.id, 'message-1'); + assert.deepEqual( + scan(json, [1, 2, 7, Math.floor(Buffer.byteLength(json) / 2)], { + messageId: 'message-1', + messageType: message.type, + }), + { + kind: 'turn', + positionId: 'turn-1', + sharedVisibility: + classifySharedSessionTranscriptVisibility( + decoded.type === 'system_note' + ? { type: decoded.type, kind: decoded.kind } + : { type: decoded.type }, + ) === 'visible', + }, + ); + } +}); diff --git a/packages/storage/src/__tests__/session-turn-position-index.test.ts b/packages/storage/src/__tests__/session-turn-position-index.test.ts new file mode 100644 index 0000000000..3d420f3f32 --- /dev/null +++ b/packages/storage/src/__tests__/session-turn-position-index.test.ts @@ -0,0 +1,3812 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { constants as sqliteConstants, DatabaseSync } from 'node:sqlite'; +import { describe, test } from 'node:test'; +import { createSHA256 } from 'hash-wasm'; +import type { CreateSessionInput } from '@maka/core/runtime-inputs'; +import { createSqliteAgentRunStore } from '../agent-run-store.js'; +import { createConversationOperationalStateStore } from '../conversation-operational-state.js'; +import { OPERATIONAL_STATE_DATABASE_NAME } from '../operational-state-store.js'; +import { createSessionStore } from '../session-store.js'; +import { advanceSessionTurnIdentityRecovery } from '../session-turn-identity-recovery.js'; +import { invalidateSessionTurnPositionIndex } from '../session-turn-position-index.js'; + +describe('Session Turn position snapshots', () => { + test('materializes owner and shared tagged positions in one exact generation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-dual-turn-position-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages(session.id, [ + { + type: 'permission_decision', + id: 'hidden-first', + turnId: 'same-id', + ts: 1, + toolUseId: 'tool-use', + toolName: 'Bash', + decision: 'deny', + }, + { type: 'user', id: 'visible-second', turnId: 'same-id', ts: 2, text: 'visible' }, + { type: 'system_note', id: 'same-id', ts: 3, kind: 'step_limit' }, + ]); + + const owner = await readyPage(store, session.id, 'owner-lease', 'owner'); + const shared = await readyPage(store, session.id, 'shared-lease', 'shared'); + assert.equal(owner.snapshotKey.snapshotGeneration, shared.snapshotKey.snapshotGeneration); + assert.equal(owner.totalPositions, 2); + assert.equal(shared.totalPositions, 2); + assert.deepEqual(owner.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'same-id' }, firstSequence: 0 }, + { ordinal: 1, key: { kind: 'note', id: 'same-id' }, firstSequence: 2 }, + ]); + assert.deepEqual(shared.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'same-id' }, firstSequence: 1 }, + { ordinal: 1, key: { kind: 'note', id: 'same-id' }, firstSequence: 2 }, + ]); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('makes empty and all-hidden shared projections observably identical', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-shared-empty-position-')); + const store = createSessionStore(root); + try { + const emptySession = await store.create(makeInput({ name: 'Empty' })); + const hiddenSession = await store.create(makeInput({ name: 'Hidden' })); + await store.appendMessages(hiddenSession.id, [ + hiddenPermission('hidden-turn', 0), + { + type: 'system_note', + id: 'hidden-note', + ts: 1, + kind: 'mode_change', + }, + ]); + const empty = await readyPage(store, emptySession.id, 'empty-shared', 'shared'); + const hidden = await readyPage(store, hiddenSession.id, 'hidden-shared', 'shared'); + assert.equal(empty.totalPositions, 1); + assert.equal(hidden.totalPositions, 1); + assert.deepEqual(empty.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ]); + assert.deepEqual(hidden.positions, empty.positions); + const owner = await readyPage(store, hiddenSession.id, 'hidden-owner', 'owner'); + assert.deepEqual(owner.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'hidden-turn' }, firstSequence: 0 }, + ]); + assert.equal(owner.snapshotKey.snapshotGeneration, hidden.snapshotKey.snapshotGeneration); + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: hiddenSession.id, + projection: 'shared', + snapshotLeaseId: 'hidden-shared', + snapshotKey: hidden.snapshotKey, + positionKeys: bodyKeys('hidden-turn'), + maxRecords: 1, + maxBytes: 64 * 1024, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('binds leases to a projection while sharing one retained generation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-projection-lease-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, user('turn-a', 0)); + const owner = await readyPage(store, session.id, 'owner-only', 'owner'); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'owner-only', + snapshotKey: owner.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const shared = await readyPage(store, session.id, 'shared-only', 'shared'); + assert.deepEqual(shared.snapshotKey, owner.snapshotKey); + await store.releaseTurnPositionSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'owner-only', + snapshotKey: owner.snapshotKey, + }); + const retained = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'shared-only', + snapshotKey: shared.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(retained.kind, 'page'); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed and lazily rebuilds when persisted shared policy version drifts', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-shared-policy-version-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, user('turn-a', 0)); + const before = await readyPage(store, session.id, 'policy-before', 'shared'); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_turn_authority_revisions + SET visibility_policy_version = 999 WHERE session_id = ?`) + .run(session.id); + } finally { + database.close(); + } + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'policy-before', + snapshotKey: before.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const after = await readyPage(store, session.id, 'policy-after', 'shared'); + assert.equal(after.snapshotKey.authorityRevision, before.snapshotKey.authorityRevision + 1); + assert.ok(after.snapshotKey.snapshotGeneration > before.snapshotKey.snapshotGeneration); + assert.deepEqual(after.positions, before.positions); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('publishes one stable synthetic position for an empty Session', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-empty-turn-position-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + const page = await readyPage(store, session.id); + + assert.equal(page.kind, 'page'); + if (page.kind !== 'page') assert.fail('expected an atomically ready page'); + assert.deepEqual(page.snapshotKey, { + throughSequence: null, + authorityRevision: 0, + snapshotGeneration: 1, + }); + assert.equal(page.startOrdinal, 0); + assert.equal(page.totalPositions, 1); + assert.deepEqual(page.positions, [ + { + ordinal: 0, + key: { kind: 'empty' }, + firstSequence: null, + }, + ]); + assert.equal(page.hasOlder, false); + assert.equal(page.hasNewer, false); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('uses inclusive predecessor semantics for legacy ordinal, sequence, and turn anchors', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-legacy-turn-position-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages(session.id, [ + { type: 'user', id: 'user-a', turnId: 'turn-a', ts: 1, text: 'a' }, + { + type: 'assistant', + id: 'assistant-a', + turnId: 'turn-a', + ts: 2, + text: 'A', + modelId: 'test-model', + }, + { type: 'user', id: 'user-b', turnId: 'turn-b', ts: 3, text: 'b' }, + { + type: 'assistant', + id: 'assistant-b', + turnId: 'turn-b', + ts: 4, + text: 'B', + modelId: 'test-model', + }, + { type: 'system_note', id: 'note', ts: 5, kind: 'step_limit' }, + ]); + + const ready = await readyPage(store, session.id); + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 1 }, + maxPositions: 2, + }); + assert.equal(first.kind, 'page'); + if (first.kind !== 'page') assert.fail('expected a ready legacy page'); + assert.deepEqual(first.positions, [ + { ordinal: 1, key: { kind: 'turn', id: 'turn-b' }, firstSequence: 2 }, + { ordinal: 2, key: { kind: 'note', id: 'note' }, firstSequence: 4 }, + ]); + + for (const anchor of [ + { kind: 'sequence' as const, sequence: 3 }, + { kind: 'turn' as const, turnId: 'turn-b' }, + ]) { + const anchored = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: first.snapshotKey, + anchor, + maxPositions: 2, + }); + assert.equal(anchored.kind, 'page'); + if (anchored.kind !== 'page') assert.fail('expected an exact snapshot page'); + assert.equal(anchored.startOrdinal, 1); + assert.deepEqual(anchored.positions, first.positions); + } + const clamped = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 999 }, + maxPositions: 2, + }); + assert.equal(clamped.kind, 'page'); + if (clamped.kind !== 'page') assert.fail('expected a clamped inclusive ordinal'); + assert.deepEqual(clamped.positions, [ + { ordinal: 2, key: { kind: 'note', id: 'note' }, firstSequence: 4 }, + ]); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'turn', turnId: 'missing-turn' }, + maxPositions: 2, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_anchor_not_found', + ); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'turn', turnId: '' }, + maxPositions: 2, + }), + /invalid/iu, + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('orders modern Turns only by root admission and keeps steering in its existing position', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-modern-turn-position-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-b', 'user-b', 10)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-a', 'user-a', 10)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-c', 'user-c', 5)); + await store.appendMessages(session.id, [ + { type: 'user', id: 'user-b', turnId: 'turn-b', ts: 1, text: 'b' }, + { type: 'user', id: 'user-a', turnId: 'turn-a', ts: 2, text: 'a' }, + { type: 'user', id: 'user-c', turnId: 'turn-c', ts: 3, text: 'c' }, + { + type: 'user', + id: 'steering-b', + turnId: 'turn-b', + ts: 4, + text: 'steer', + steeringEventId: 'steering-b', + }, + ]); + + const page = await readyPage(store, session.id); + assert.equal(page.kind, 'page'); + if (page.kind !== 'page') assert.fail('expected modern positions'); + assert.deepEqual(page.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-c' }, firstSequence: 2 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-a' }, firstSequence: 1 }, + { ordinal: 2, key: { kind: 'turn', id: 'turn-b' }, firstSequence: 0 }, + ]); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('represents admission-before-body and applies the closed turnless-note visibility policy', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-before-body-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-bodyless', 'future-user', 10)); + const bodyless = await readyPage(store, session.id, 'lease-bodyless'); + assert.deepEqual(bodyless.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-bodyless' }, firstSequence: null }, + ]); + const bodylessRecords = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-bodyless', + snapshotKey: bodyless.snapshotKey, + positionKeys: bodyKeys('turn-bodyless'), + maxRecords: 1, + maxBytes: 1, + }); + assert.deepEqual(bodylessRecords.records, []); + assert.equal(bodylessRecords.rawBytes, 0); + const sharedBodyless = await readyPage(store, session.id, 'lease-bodyless-shared', 'shared'); + assert.equal( + sharedBodyless.snapshotKey.snapshotGeneration, + bodyless.snapshotKey.snapshotGeneration, + ); + assert.deepEqual(sharedBodyless.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ]); + + await store.appendMessages(session.id, [ + { type: 'system_note', id: 'hidden-note', ts: 1, kind: 'mode_change' }, + { + type: 'system_note', + id: 'associated-note', + turnId: 'turn-bodyless', + ts: 2, + kind: 'error', + }, + { type: 'user', id: 'future-user', turnId: 'turn-bodyless', ts: 3, text: 'body' }, + { type: 'system_note', id: 'visible-note', ts: 4, kind: 'step_limit' }, + ]); + const materialized = await readyPage(store, session.id, 'lease-materialized'); + assert.deepEqual(materialized.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-bodyless' }, firstSequence: 1 }, + { ordinal: 1, key: { kind: 'note', id: 'visible-note' }, firstSequence: 3 }, + ]); + const sharedMaterialized = await readyPage( + store, + session.id, + 'lease-materialized-shared', + 'shared', + ); + assert.deepEqual(sharedMaterialized.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-bodyless' }, firstSequence: 2 }, + { ordinal: 1, key: { kind: 'note', id: 'visible-note' }, firstSequence: 3 }, + ]); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed on an explicitly empty Turn identity before committing the append', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-empty-turn-identity-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await assert.rejects( + store.appendMessage(session.id, { + type: 'user', + id: 'empty-turn', + turnId: '', + ts: 1, + text: 'invalid', + }), + (error: unknown) => { + assert.equal((error as { reason?: unknown }).reason, 'incompatible_identity'); + return true; + }, + ); + assert.equal(await store.readTranscriptHighWaterSnapshot(session.id), null); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('keeps equal note and Turn strings distinct through composite identity', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-note-turn-identity-collision-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, { + type: 'system_note', + id: 'session-note:collision', + ts: 1, + kind: 'step_limit', + }); + await store.appendMessage(session.id, { + type: 'user', + id: 'colliding-turn', + turnId: 'session-note:collision', + ts: 2, + text: 'separate Turn namespace', + }); + await store.appendMessage(session.id, { + type: 'user', + id: 'prefixed-turn', + turnId: 'session:real-turn', + ts: 3, + text: 'prefix is opaque', + }); + const page = await readyPage(store, session.id, 'lease-collision'); + assert.deepEqual( + page.positions.map(({ key }) => key), + [ + { kind: 'note', id: 'session-note:collision' }, + { kind: 'turn', id: 'session-note:collision' }, + { kind: 'turn', id: 'session:real-turn' }, + ], + ); + const turnAnchor = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-collision', + snapshotKey: page.snapshotKey, + anchor: { kind: 'turn', turnId: 'session-note:collision' }, + maxPositions: 1, + }); + assert.equal(turnAnchor.kind, 'page'); + if (turnAnchor.kind !== 'page') assert.fail('expected exact Turn anchor'); + assert.equal(turnAnchor.startOrdinal, 1); + const bodies = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-collision', + snapshotKey: page.snapshotKey, + positionKeys: [ + { kind: 'turn', id: 'session-note:collision' }, + { kind: 'note', id: 'session-note:collision' }, + ], + maxRecords: 2, + maxBytes: 64 * 1024, + }); + assert.deepEqual( + bodies.records.map(({ positionKey, message }) => [positionKey, message.id]), + [ + [{ kind: 'note', id: 'session-note:collision' }, 'session-note:collision'], + [{ kind: 'turn', id: 'session-note:collision' }, 'colliding-turn'], + ], + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('purging admission authority removes bodyless placeholders and downgrades body Turns', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-purge-position-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + const conversation = createConversationOperationalStateStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-bodyless', 'future-user', 5)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-body', 'body-user', 10)); + await store.appendMessage(session.id, { + type: 'user', + id: 'body-user', + turnId: 'turn-body', + ts: 1, + text: 'body', + }); + const admitted = await readyPage(store, session.id, 'lease-admitted'); + assert.deepEqual( + admitted.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id)), + ['turn-bodyless', 'turn-body'], + ); + + await conversation.purge(session.id); + const retained = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-admitted', + snapshotKey: admitted.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(retained.kind, 'page'); + if (retained.kind !== 'page') assert.fail('expected leased ready snapshot to survive purge'); + assert.deepEqual(retained.positions, admitted.positions); + const legacy = await readyPage(store, session.id, 'lease-purged'); + assert.equal( + legacy.snapshotKey.authorityRevision, + admitted.snapshotKey.authorityRevision + 1, + ); + assert.deepEqual(legacy.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-body' }, firstSequence: 0 }, + ]); + await store.remove(session.id); + await conversation.purge(session.id); + } finally { + conversation.close(); + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('invalidates v34 transcript recovery when purge deletes an unreconciled admission', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-purge-v34-building-')); + let store = createSessionStore(root); + let conversation: ReturnType | undefined; + const runs = createSqliteAgentRunStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages( + session.id, + Array.from({ length: 1_025 }, (_, index) => user(`turn-${index}`, index)), + ); + await runs.admitRootTurn( + rootAdmission(session.id, 'turn-bodyless-purged', 'future-user-purged', 2_000), + ); + runs.close?.(); + await store.close?.(); + downgradeTurnProjectionToV34(root); + + store = createSessionStore(root); + conversation = createConversationOperationalStateStore(root); + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-v34-building-old', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(first.kind, 'building'); + if (first.kind !== 'building') assert.fail('expected bounded transcript recovery'); + assert.equal(first.progress.lastStepRecords, 1_024); + const beforePurge = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + beforePurge + .prepare(`SELECT COUNT(*) AS count FROM session_turn_metadata + WHERE session_id = ? AND order_source = 'admission'`) + .get(session.id) as { count: number } + ).count, + 0, + ); + assert.equal( + ( + beforePurge + .prepare(`SELECT COUNT(*) AS count FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, first.snapshotKey.snapshotGeneration) as { count: number } + ).count, + 0, + ); + } finally { + beforePurge.close(); + } + + await conversation.purge(session.id); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-v34-building-old', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const afterPurge = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + afterPurge + .prepare(`SELECT authority_revision FROM session_turn_authority_revisions + WHERE session_id = ?`) + .get(session.id) as { authority_revision: number } + ).authority_revision, + first.snapshotKey.authorityRevision + 1, + ); + assert.equal( + afterPurge + .prepare(`SELECT 1 FROM session_turn_position_snapshots + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, first.snapshotKey.snapshotGeneration), + undefined, + ); + } finally { + afterPurge.close(); + } + + let next = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-v34-building-new', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(next.kind, 'building'); + if (next.kind !== 'building') assert.fail('expected post-purge rebuild'); + const noPartial = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + noPartial + .prepare(`SELECT COUNT(*) AS count FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, next.snapshotKey.snapshotGeneration) as { count: number } + ).count, + 0, + ); + } finally { + noPartial.close(); + } + while (next.kind === 'building') { + next = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-v34-building-new', + snapshotKey: next.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(next.kind, 'page'); + if (next.kind !== 'page') assert.fail('expected post-purge ready page'); + assert.equal(next.totalPositions, 1_025); + assert.equal( + next.positions.some(({ key }) => key.kind === 'turn' && key.id === 'turn-bodyless-purged'), + false, + ); + } finally { + conversation?.close(); + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('resets partial admission recovery when purge deletes canonical authority', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-purge-partial-recovery-')); + let store = createSessionStore(root); + let conversation: ReturnType | undefined; + const session = await store.create(makeInput()); + try { + await store.close?.(); + const seeded = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const insert = seeded.prepare(`INSERT INTO core_root_turn_admissions( + session_id, turn_id, admitted_at, record_json + ) VALUES (?, ?, ?, '{}')`); + seeded.exec('BEGIN IMMEDIATE'); + for (let index = 0; index < 1_025; index += 1) { + insert.run(session.id, `turn-${index.toString().padStart(4, '0')}`, index); + } + seeded.exec('COMMIT'); + } finally { + seeded.close(); + } + downgradeTurnProjectionToV34(root); + + store = createSessionStore(root); + conversation = createConversationOperationalStateStore(root); + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-partial-old', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(first.kind, 'building'); + if (first.kind !== 'building') assert.fail('expected partial admission recovery'); + const partial = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.deepEqual( + { + ...(partial + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: 1_023, + admission_cursor_turn_id: 'turn-1023', + admission_recovery_complete: 0, + }, + ); + } finally { + partial.close(); + } + + await conversation.purge(session.id); + const purged = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.deepEqual( + { + ...(purged + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: null, + admission_cursor_turn_id: null, + admission_recovery_complete: 0, + }, + ); + assert.equal( + ( + purged + .prepare(`SELECT authority_revision FROM session_turn_authority_revisions + WHERE session_id = ?`) + .get(session.id) as { authority_revision: number } + ).authority_revision, + first.snapshotKey.authorityRevision + 1, + ); + } finally { + purged.close(); + } + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-partial-old', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const rebuilt = await readyPage(store, session.id, 'purge-partial-new', 'owner'); + assert.deepEqual(rebuilt.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ]); + } finally { + conversation?.close(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('keeps admission authority revision stable for an idempotent empty purge', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-purge-idempotent-')); + const store = createSessionStore(root); + const conversation = createConversationOperationalStateStore(root); + try { + const session = await store.create(makeInput()); + const before = await readyPage(store, session.id, 'purge-idempotent-ready', 'owner'); + await conversation.purge(session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + database + .prepare(`SELECT authority_revision FROM session_turn_authority_revisions + WHERE session_id = ?`) + .get(session.id) as { authority_revision: number } + ).authority_revision, + before.snapshotKey.authorityRevision, + ); + } finally { + database.close(); + } + } finally { + conversation.close(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('clears an admission-origin corrupt source after purge removes its cause', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-failure-purge-')); + const store = createSessionStore(root); + const conversation = createConversationOperationalStateStore(root); + try { + const session = await store.create(makeInput()); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`INSERT INTO core_root_turn_admissions( + session_id, turn_id, admitted_at, record_json + ) VALUES (?, '', 1, '{}')`) + .run(session.id); + } finally { + database.close(); + } + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'admission-failure-before-purge', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'corrupt_source', + ); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: 'admission', + failure_reason: 'corrupt_source', + failure_sequence: 0, + }); + + await conversation.purge(session.id); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: null, + failure_reason: null, + failure_sequence: null, + }); + const rebuilt = await readyPage(store, session.id, 'admission-failure-after-purge', 'owner'); + assert.deepEqual(rebuilt.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ]); + } finally { + conversation.close(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('retains transcript-origin corrupt source across an admission purge', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-transcript-failure-admission-purge-')); + let store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + let conversation: ReturnType | undefined; + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, user('turn-corrupt-transcript', 0)); + await runs.admitRootTurn( + rootAdmission(session.id, 'turn-bodyless-to-purge', 'future-purged-user', 5), + ); + runs.close?.(); + await store.close?.(); + resetProjection(root, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_messages SET record_json = '{' + WHERE session_id = ? AND sequence = 0`) + .run(session.id); + } finally { + database.close(); + } + + store = createSessionStore(root); + conversation = createConversationOperationalStateStore(root); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'transcript-failure-before-purge', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'corrupt_source', + ); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: 'transcript', + failure_reason: 'corrupt_source', + failure_sequence: 0, + }); + + await conversation.purge(session.id); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: 'transcript', + failure_reason: 'corrupt_source', + failure_sequence: 0, + }); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'transcript-failure-after-purge', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'corrupt_source', + ); + } finally { + conversation?.close(); + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('tags hybrid failure as admission and clears it only for resolving authority', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-hybrid-failure-origin-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + const conversation = createConversationOperationalStateStore(root); + try { + const resolvedSession = await store.create(makeInput({ name: 'Resolved hybrid' })); + await runs.admitRootTurn(rootAdmission(resolvedSession.id, 'turn-modern', 'user-modern', 10)); + await store.appendMessages(resolvedSession.id, [ + { type: 'user', id: 'user-modern', turnId: 'turn-modern', ts: 1, text: 'modern' }, + { type: 'user', id: 'user-missing', turnId: 'turn-missing', ts: 2, text: 'missing' }, + ]); + await assert.rejects(readyPage(store, resolvedSession.id), (error: unknown) => { + return (error as { reason?: unknown }).reason === 'hybrid_missing_admission'; + }); + assert.deepEqual(readTurnIndexFailure(root, resolvedSession.id), { + failure_origin: 'admission', + failure_reason: 'hybrid_missing_admission', + failure_sequence: 1, + }); + + await runs.admitRootTurn( + rootAdmission(resolvedSession.id, 'turn-unrelated', 'future-unrelated-user', 12), + ); + assert.deepEqual(readTurnIndexFailure(root, resolvedSession.id), { + failure_origin: 'admission', + failure_reason: 'hybrid_missing_admission', + failure_sequence: 1, + }); + await runs.admitRootTurn( + rootAdmission(resolvedSession.id, 'turn-missing', 'user-missing', 11), + ); + assert.deepEqual(readTurnIndexFailure(root, resolvedSession.id), { + failure_origin: null, + failure_reason: null, + failure_sequence: null, + }); + const resolved = await readyPage(store, resolvedSession.id, 'hybrid-resolved', 'owner'); + assert.deepEqual( + resolved.positions.map(({ key }) => key), + [ + { kind: 'turn', id: 'turn-modern' }, + { kind: 'turn', id: 'turn-missing' }, + { kind: 'turn', id: 'turn-unrelated' }, + ], + ); + + const purgedSession = await store.create(makeInput({ name: 'Purged hybrid' })); + await runs.admitRootTurn( + rootAdmission(purgedSession.id, 'turn-modern-purge', 'user-modern-purge', 20), + ); + await store.appendMessages(purgedSession.id, [ + { + type: 'user', + id: 'user-modern-purge', + turnId: 'turn-modern-purge', + ts: 1, + text: 'modern', + }, + { + type: 'user', + id: 'user-missing-purge', + turnId: 'turn-missing-purge', + ts: 2, + text: 'missing', + }, + ]); + await assert.rejects(readyPage(store, purgedSession.id), (error: unknown) => { + return (error as { reason?: unknown }).reason === 'hybrid_missing_admission'; + }); + assert.deepEqual(readTurnIndexFailure(root, purgedSession.id), { + failure_origin: 'admission', + failure_reason: 'hybrid_missing_admission', + failure_sequence: 1, + }); + await conversation.purge(purgedSession.id); + assert.deepEqual(readTurnIndexFailure(root, purgedSession.id), { + failure_origin: null, + failure_reason: null, + failure_sequence: null, + }); + const purged = await readyPage(store, purgedSession.id, 'hybrid-purged', 'owner'); + assert.equal(purged.totalPositions, 2); + } finally { + conversation.close(); + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('enforces the persisted failure provenance pair matrix', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-failure-provenance-check-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await readyPage(store, session.id, 'failure-provenance-schema', 'owner'); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + for (const statement of [ + `UPDATE session_turn_index_state SET failure_origin = 'transcript' + WHERE session_id = ?`, + `UPDATE session_turn_index_state + SET failure_reason = 'corrupt_source', failure_sequence = 0 WHERE session_id = ?`, + `UPDATE session_turn_index_state + SET failure_origin = 'admission', failure_reason = 'corrupt_source' + WHERE session_id = ?`, + `UPDATE session_turn_index_state + SET failure_origin = 'other', failure_reason = 'corrupt_source', failure_sequence = 0 + WHERE session_id = ?`, + ]) { + assert.throws( + () => database.prepare(statement).run(session.id), + /CHECK constraint failed/, + ); + } + for (const [origin, reason] of [ + ['transcript', 'corrupt_source'], + ['transcript', 'incompatible_identity'], + ['admission', 'corrupt_source'], + ['admission', 'hybrid_missing_admission'], + ] as const) { + database + .prepare(`UPDATE session_turn_index_state + SET failure_origin = ?, failure_reason = ?, failure_sequence = 0 + WHERE session_id = ?`) + .run(origin, reason, session.id); + } + for (const [origin, reason] of [ + ['transcript', 'hybrid_missing_admission'], + ['admission', 'incompatible_identity'], + ] as const) { + assert.throws( + () => + database + .prepare(`UPDATE session_turn_index_state + SET failure_origin = ?, failure_reason = ?, failure_sequence = 0 + WHERE session_id = ?`) + .run(origin, reason, session.id), + /CHECK constraint failed/, + ); + } + assert.throws( + () => + database + .prepare(`UPDATE session_turn_index_state SET failure_origin = NULL + WHERE session_id = ?`) + .run(session.id), + /CHECK constraint failed/, + ); + } finally { + database.close(); + } + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('shares one exact generation across leases and releases only after the last consumer', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-shared-position-lease-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, user('turn-a', 0)); + const first = await readyPage(store, session.id, 'lease-one'); + const secondFacade = createSessionStore(root); + const shared = await secondFacade.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-two', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(shared.kind, 'page'); + if (shared.kind !== 'page') assert.fail('expected shared ready snapshot'); + assert.deepEqual(shared.snapshotKey, first.snapshotKey); + await secondFacade.close?.(); + + await store.releaseTurnPositionSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-one', + snapshotKey: first.snapshotKey, + }); + const retained = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-two', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(retained.kind, 'page'); + await store.releaseTurnPositionSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-two', + snapshotKey: first.snapshotKey, + }); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-two', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const rebuilt = await readyPage(store, session.id, 'lease-three'); + assert.ok(rebuilt.snapshotKey.snapshotGeneration > first.snapshotKey.snapshotGeneration); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('keeps a legacy prefix by sequence before admission-ordered modern Turns', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-hybrid-prefix-turn-position-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-b', 'user-b', 20)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-a', 'user-a', 10)); + await store.appendMessages(session.id, [ + user('turn-legacy', 0), + user('turn-b', 1), + user('turn-a', 2), + { type: 'system_note', id: 'modern-note', ts: 4, kind: 'step_limit' }, + ]); + + const page = await readyPage(store, session.id); + assert.deepEqual(page.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-legacy' }, firstSequence: 0 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-a' }, firstSequence: 2 }, + { ordinal: 2, key: { kind: 'turn', id: 'turn-b' }, firstSequence: 1 }, + { ordinal: 3, key: { kind: 'note', id: 'modern-note' }, firstSequence: 3 }, + ]); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed after bounded recovery finds a hybrid Turn without admission', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-hybrid-turn-position-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-modern', 'user-modern', 10)); + await store.appendMessages(session.id, [ + { type: 'user', id: 'user-modern', turnId: 'turn-modern', ts: 1, text: 'modern' }, + { type: 'user', id: 'user-missing', turnId: 'turn-missing', ts: 2, text: 'missing' }, + ]); + + await assert.rejects(readyPage(store, session.id), (error: unknown) => { + assert.equal((error as { code?: unknown }).code, 'session_turn_position_recovery_failed'); + assert.equal((error as { reason?: unknown }).reason, 'hybrid_missing_admission'); + return true; + }); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('increments authority at a fixed watermark while preserving the old exact snapshot', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-authority-revision-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages(session.id, [user('turn-a', 0), user('turn-b', 1)]); + const legacy = await readyPage(store, session.id); + assert.deepEqual( + legacy.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id)), + ['turn-a', 'turn-b'], + ); + + await runs.admitRootTurn(rootAdmission(session.id, 'turn-b', 'user-1', 10)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-a', 'user-0', 20)); + + const modern = await readyPage(store, session.id, 'lease-modern'); + assert.deepEqual(modern.snapshotKey, { + throughSequence: legacy.snapshotKey.throughSequence, + authorityRevision: legacy.snapshotKey.authorityRevision + 2, + snapshotGeneration: legacy.snapshotKey.snapshotGeneration + 1, + }); + assert.deepEqual( + modern.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id)), + ['turn-b', 'turn-a'], + ); + + const stableLegacy = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: legacy.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(stableLegacy.kind, 'page'); + if (stableLegacy.kind !== 'page') assert.fail('expected retained exact snapshot'); + assert.deepEqual( + stableLegacy.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id)), + ['turn-a', 'turn-b'], + ); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('atomically invalidates an unfinished snapshot when admission authority changes', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-building-authority-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages( + session.id, + Array.from({ length: 1_025 }, (_, index) => user(`turn-${index}`, index)), + ); + const building = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(building.kind, 'building'); + if (building.kind !== 'building') assert.fail('expected an unfinished position build'); + + await runs.admitRootTurn(rootAdmission(session.id, 'turn-1024', 'user-1024', 10)); + + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: building.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('returns capacity for a third exact snapshot and never reuses a released generation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-capacity-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, user('turn-a', 0)); + const first = await readyPage(store, session.id, 'lease-first'); + await store.appendMessage(session.id, user('turn-b', 1)); + const second = await readyPage(store, session.id, 'lease-second', 'shared'); + await store.appendMessage(session.id, user('turn-c', 2)); + + const capacity = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-third', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.deepEqual(capacity, { + kind: 'capacity', + projection: 'owner', + throughSequence: 2, + authorityRevision: 0, + retainedSnapshots: 2, + }); + + const stableFirst = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-first', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(stableFirst.kind, 'page'); + if (stableFirst.kind !== 'page') assert.fail('expected retained first snapshot'); + assert.equal(stableFirst.totalPositions, 1); + assert.deepEqual( + stableFirst.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id)), + ['turn-a'], + ); + + await store.releaseTurnPositionSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-first', + snapshotKey: first.snapshotKey, + }); + const third = await readyPage(store, session.id, 'lease-third'); + assert.equal(first.snapshotKey.snapshotGeneration, 1); + assert.equal(second.snapshotKey.snapshotGeneration, 2); + assert.equal(third.snapshotKey.snapshotGeneration, 3); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-first', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('caps one recovery step at 1,024 source records and resumes after reopen', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-recovery-count-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages( + session.id, + Array.from({ length: 1_025 }, (_, index) => user(`turn-${index}`, index)), + ); + await store.close?.(); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec('PRAGMA foreign_keys = ON'); + database + .prepare('DELETE FROM session_turn_position_snapshots WHERE session_id = ?') + .run(session.id); + database.prepare('DELETE FROM session_turn_metadata WHERE session_id = ?').run(session.id); + database + .prepare(`UPDATE session_turn_index_state SET indexed_through_sequence = -1, + source_records = 0, source_bytes = 0, + failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ?`) + .run(session.id); + } finally { + database.close(); + } + + store = createSessionStore(root); + const building = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(building.kind, 'building'); + if (building.kind !== 'building') assert.fail('expected bounded recovery progress'); + assert.equal(building.progress.sourceRecords, 1_024); + assert.equal(building.progress.nextSequence, 1_024); + + await store.close?.(); + store = createSessionStore(root); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: building.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const ready = await readyPage(store, session.id, 'lease-resumed'); + assert.equal(ready.totalPositions, 1_025); + assert.ok(ready.snapshotKey.snapshotGeneration > building.snapshotKey.snapshotGeneration); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('caps one recovery step below 4 MiB of source payload', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-recovery-bytes-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages( + session.id, + Array.from({ length: 5 }, (_, index) => ({ + ...user(`turn-${index}`, index), + text: `${index}${'x'.repeat(1024 * 1024)}`, + })), + ); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + + const building = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(building.kind, 'building'); + if (building.kind !== 'building') assert.fail('expected byte-bounded recovery'); + assert.equal(building.progress.sourceRecords, 3); + assert.ok(building.progress.sourceBytes < 4 * 1024 * 1024); + assert.equal(building.progress.nextSequence, 3); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('resumes an oversized first source record without a size-only failure', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-oversized-source-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, { + ...user('turn-oversized', 0), + text: 'x'.repeat(4 * 1024 * 1024 + 1024), + }); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(first.kind, 'building'); + if (first.kind !== 'building') assert.fail('expected partial scalar recovery'); + assert.equal(first.progress.lastStepBytes, 4 * 1024 * 1024); + assert.equal(first.progress.lastStepRecords, 0); + assert.equal(first.progress.currentByteOffset, 4 * 1024 * 1024); + const ready = await readyPage(store, session.id); + assert.equal(ready.totalPositions, 1); + assert.equal( + ready.positions[0] ? positionId(ready.positions[0]) : undefined, + 'turn-oversized', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('resumes legacy inline records larger than one recovery step', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-legacy-inline-source-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + const message = { + ...user('turn-legacy-inline', 0), + text: 'x'.repeat(4 * 1024 * 1024 + 1024), + }; + try { + await store.appendMessage(session.id, message); + await store.close?.(); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec('PRAGMA foreign_keys = ON'); + database.exec('BEGIN IMMEDIATE'); + database + .prepare(`UPDATE session_messages SET record_json = ? + WHERE session_id = ? AND sequence = 0`) + .run(JSON.stringify(message), session.id); + database + .prepare('DELETE FROM session_message_payloads WHERE session_id = ? AND sequence = 0') + .run(session.id); + database.exec('COMMIT'); + } finally { + database.close(); + } + resetProjection(root, session.id); + store = createSessionStore(root); + + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-legacy-inline', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(first.kind, 'building'); + if (first.kind !== 'building') assert.fail('expected partial inline scalar recovery'); + assert.equal(first.progress.lastStepBytes, 4 * 1024 * 1024); + assert.equal(first.progress.lastStepRecords, 0); + assert.equal(first.progress.currentByteOffset, 4 * 1024 * 1024); + + const ready = await readyPage(store, session.id, 'lease-legacy-inline'); + assert.equal( + ready.positions[0] ? positionId(ready.positions[0]) : undefined, + 'turn-legacy-inline', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('accounts exact 4 MiB boundaries and resumes identity located after a huge body', async () => { + const targets = [4 * 1024 * 1024 - 1, 4 * 1024 * 1024, 4 * 1024 * 1024 + 1, 9 * 1024 * 1024]; + for (const [index, targetBytes] of targets.entries()) { + const root = await mkdtemp(join(tmpdir(), `maka-turn-position-exact-bytes-${index}-`)); + let store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + const message = exactSizeUserMessage(targetBytes, `exact-turn-${index}`, index % 2 === 1); + assert.equal(Buffer.byteLength(JSON.stringify(message), 'utf8'), targetBytes); + await store.appendMessage(session.id, message); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + + let result = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: `lease-exact-${index}`, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + const recoveryBytes: number[] = []; + while (result.kind === 'building') { + if (result.progress.phase === 'recovering') { + recoveryBytes.push(result.progress.lastStepBytes); + assert.ok(result.progress.lastStepBytes <= 4 * 1024 * 1024); + assert.ok(result.progress.lastStepRecords <= 1_024); + } + result = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: `lease-exact-${index}`, + snapshotKey: result.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(result.kind, 'page'); + if (result.kind !== 'page') assert.fail('expected exact-size recovered page'); + assert.equal( + result.positions[0] ? positionId(result.positions[0]) : undefined, + `exact-turn-${index}`, + ); + assert.equal( + recoveryBytes.reduce((total, bytes) => total + bytes, 0), + targetBytes, + ); + assert.equal(Math.max(...recoveryBytes), Math.min(targetBytes, 4 * 1024 * 1024)); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + } + }); + + test('discards corrupt derived state and restarts only the partial source record after reopen', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-derived-recovery-reset-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage( + session.id, + exactSizeUserMessage(9 * 1024 * 1024, 'turn-derived-reset', true), + ); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + const partial = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-derived-before', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(partial.kind, 'building'); + if (partial.kind !== 'building') assert.fail('expected partial record'); + assert.equal(partial.progress.currentByteOffset, 4 * 1024 * 1024); + await store.close?.(); + + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const row = database + .prepare(`SELECT scanner_state FROM session_turn_identity_recovery + WHERE session_id = ?`) + .get(session.id) as { scanner_state: string }; + const scannerState = JSON.parse(row.scanner_state) as { + identity: { id: string }; + }; + scannerState.identity.id = 'x'.repeat(scannerState.identity.id.length); + database + .prepare(`UPDATE session_turn_identity_recovery + SET scanner_state = ? WHERE session_id = ?`) + .run(JSON.stringify(scannerState), session.id); + } finally { + database.close(); + } + store = createSessionStore(root); + const restarted = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-derived-after', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(restarted.kind, 'building'); + if (restarted.kind !== 'building') assert.fail('expected reconstructed partial state'); + assert.equal(restarted.progress.currentByteOffset, 4 * 1024 * 1024); + assert.equal(restarted.progress.lastStepBytes, 4 * 1024 * 1024); + assert.equal(restarted.progress.sourceBytes, 8 * 1024 * 1024); + const ready = await readyPage(store, session.id, 'lease-derived-after'); + assert.equal( + ready.positions[0] ? positionId(ready.positions[0]) : undefined, + 'turn-derived-reset', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('publishes recovery cursor, counters, and membership atomically at commit boundaries', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-recovery-transaction-boundary-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, user('turn-atomic', 0)); + await store.close?.(); + resetProjection(root, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec('PRAGMA foreign_keys = ON; BEGIN IMMEDIATE'); + const rolledBack = advanceSessionTurnIdentityRecovery(database, { + sessionId: session.id, + throughSequence: 0, + maxSourceBytes: 4 * 1024 * 1024, + maxCompletedRecords: 1_024, + hasher: await createSHA256(), + }); + assert.equal('failure' in rolledBack, false); + assert.equal( + ( + database + .prepare(`SELECT COUNT(*) AS count FROM session_turn_memberships + WHERE session_id = ?`) + .get(session.id) as { count: number } + ).count, + 1, + ); + database.exec('ROLLBACK'); + assert.equal( + ( + database + .prepare(`SELECT COUNT(*) AS count FROM session_turn_memberships + WHERE session_id = ?`) + .get(session.id) as { count: number } + ).count, + 0, + ); + database.exec('BEGIN IMMEDIATE'); + const committed = advanceSessionTurnIdentityRecovery(database, { + sessionId: session.id, + throughSequence: 0, + maxSourceBytes: 4 * 1024 * 1024, + maxCompletedRecords: 1_024, + hasher: await createSHA256(), + }); + assert.equal('failure' in committed, false); + database.exec('COMMIT'); + const state = database + .prepare(`SELECT indexed_through_sequence, source_records + FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as { indexed_through_sequence: number; source_records: number }; + assert.deepEqual([state.indexed_through_sequence, state.source_records], [0, 1]); + } finally { + database.close(); + } + store = createSessionStore(root); + const ready = await readyPage(store, session.id, 'lease-atomic'); + assert.equal(ready.positions[0] ? positionId(ready.positions[0]) : undefined, 'turn-atomic'); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('persists a partial-source mutation failure until authoritative invalidation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-partial-source-mutation-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage( + session.id, + exactSizeUserMessage(9 * 1024 * 1024, 'turn-source-mutation', false), + ); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + const partial = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-source-before', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(partial.kind, 'building'); + await store.close?.(); + + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_messages SET message_id = 'mutated-id' + WHERE session_id = ? AND sequence = 0`) + .run(session.id); + } finally { + database.close(); + } + store = createSessionStore(root); + for (const lease of ['lease-source-failed', 'lease-source-failed']) { + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: lease, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => { + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + assert.equal((error as { sequence?: unknown }).sequence, 0); + return true; + }, + ); + } + const failed = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + const failure = failed + .prepare(`SELECT failure_origin, failure_reason, failure_sequence + FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as { + failure_origin: string; + failure_reason: string; + failure_sequence: number; + }; + assert.equal(failure.failure_origin, 'transcript'); + assert.equal(failure.failure_reason, 'corrupt_source'); + assert.equal(failure.failure_sequence, 0); + } finally { + failed.close(); + } + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('binds transcript record reads to the exact snapshot key and Turn membership', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-records-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages(session.id, [ + hiddenPermission('turn-a', 0), + { + type: 'assistant', + id: 'assistant-a', + turnId: 'turn-a', + ts: 2, + text: 'answer', + modelId: 'test-model', + }, + user('turn-b', 1), + ]); + const page = await readyPage(store, session.id); + const sharedPage = await readyPage(store, session.id, 'lease-shared-records', 'shared'); + assert.equal(sharedPage.positions[0]?.firstSequence, 1); + + const records = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + positionKeys: bodyKeys('turn-a'), + maxBytes: 64 * 1024, + maxRecords: 8, + }); + assert.deepEqual(records.snapshotKey, page.snapshotKey); + assert.deepEqual( + records.records.map(({ positionKey, sequence, message }) => [ + positionKey, + sequence, + message.id, + ]), + [ + [{ kind: 'turn', id: 'turn-a' }, 0, 'permission-0'], + [{ kind: 'turn', id: 'turn-a' }, 1, 'assistant-a'], + ], + ); + const sharedRecords = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'lease-shared-records', + snapshotKey: sharedPage.snapshotKey, + positionKeys: bodyKeys('turn-a'), + maxBytes: 64 * 1024, + maxRecords: 8, + }); + assert.deepEqual( + sharedRecords.records.map(({ positionKey, sequence, message }) => [ + positionKey, + sequence, + message.id, + ]), + [ + [{ kind: 'turn', id: 'turn-a' }, 0, 'permission-0'], + [{ kind: 'turn', id: 'turn-a' }, 1, 'assistant-a'], + ], + ); + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'lease-shared-records', + snapshotKey: sharedPage.snapshotKey, + positionKeys: bodyKeys('turn-a'), + maxBytes: 64 * 1024, + maxRecords: 1, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'transcript_record_count', + ); + for (const [limits, reason] of [ + [{ maxBytes: 64 * 1024, maxRecords: 1 }, 'transcript_record_count'], + [{ maxBytes: 1, maxRecords: 8 }, 'transcript_record_bytes'], + ] as const) { + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + positionKeys: bodyKeys('turn-a'), + ...limits, + }), + (error: unknown) => { + assert.equal( + (error as { code?: unknown }).code, + 'session_turn_position_limit_exceeded', + ); + assert.equal((error as { reason?: unknown }).reason, reason); + return true; + }, + ); + } + for (const snapshotKey of [ + { ...page.snapshotKey, authorityRevision: page.snapshotKey.authorityRevision + 1 }, + { ...page.snapshotKey, snapshotGeneration: 999 }, + ]) { + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey, + positionKeys: bodyKeys('turn-a'), + maxBytes: 64 * 1024, + maxRecords: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + } + for (const request of [ + { positionKeys: bodyKeys('turn-a', 'turn-a'), maxRecords: 8, maxBytes: 64 * 1024 }, + { positionKeys: bodyKeys(''), maxRecords: 8, maxBytes: 64 * 1024 }, + { + positionKeys: Array.from({ length: 129 }, (_, index) => `turn-${index}`).map((id) => ({ + kind: 'turn' as const, + id, + })), + maxRecords: 8, + maxBytes: 64 * 1024, + }, + { positionKeys: bodyKeys('turn-a'), maxRecords: 257, maxBytes: 64 * 1024 }, + { positionKeys: bodyKeys('turn-a'), maxRecords: 8, maxBytes: 16 * 1024 * 1024 + 1 }, + { positionKeys: [{ kind: 'empty' }] as never, maxRecords: 8, maxBytes: 64 * 1024 }, + ]) { + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + ...request, + }), + /invalid|unique|stale|mismatched/iu, + ); + } + for (const request of [ + { snapshotLeaseId: 'lease-unknown', positionKeys: bodyKeys('turn-a') }, + { snapshotLeaseId: 'lease-default', positionKeys: bodyKeys('turn-unknown') }, + { + snapshotLeaseId: 'lease-default', + positionKeys: [{ kind: 'note' as const, id: 'turn-a' }], + }, + ]) { + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotKey: page.snapshotKey, + maxRecords: 8, + maxBytes: 64 * 1024, + ...request, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + } + + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_messages SET record_json = '{' + WHERE session_id = ? AND sequence = 1`) + .run(session.id); + } finally { + database.close(); + } + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + positionKeys: bodyKeys('turn-a'), + maxRecords: 1, + maxBytes: 64 * 1024, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'transcript_record_count', + ); + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + positionKeys: bodyKeys('turn-a'), + maxRecords: 2, + maxBytes: 64 * 1024, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'stored_session_message_incompatible', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed with a typed recovery error for a corrupted durable source record', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-corrupt-source-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, user('turn-a', 0)); + await store.close?.(); + resetProjection(root, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare( + 'UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = 0', + ) + .run('{', session.id); + } finally { + database.close(); + } + store = createSessionStore(root); + + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => { + assert.equal((error as { code?: unknown }).code, 'session_turn_position_recovery_failed'); + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + return true; + }, + ); + const failed = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + failed + .prepare(`SELECT COUNT(*) AS count FROM session_turn_position_snapshots + WHERE session_id = ?`) + .get(session.id) as { count: number } + ).count, + 0, + ); + } finally { + failed.close(); + } + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-retry-failed-source', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'corrupt_source', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed when a valid stored body no longer resolves to its exact position key', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-body-identity-drift-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + const original = user('turn-before', 0); + await store.appendMessage(session.id, original); + const page = await readyPage(store, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_messages SET record_json = ? + WHERE session_id = ? AND sequence = 0`) + .run(JSON.stringify({ ...original, turnId: 'turn-after' }), session.id); + } finally { + database.close(); + } + + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + positionKeys: bodyKeys('turn-before'), + maxRecords: 1, + maxBytes: 64 * 1024, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'stored_session_message_incompatible', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('enforces exact shared raw record-count and stored-byte boundaries', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-shared-position-body-limits-')); + const store = createSessionStore(root); + try { + const countSession = await store.create(makeInput({ name: 'Count boundary' })); + await store.appendMessages(countSession.id, [ + user('turn-count-boundary', 0), + ...Array.from({ length: 255 }, (_, index) => + hiddenPermission('turn-count-boundary', index + 1), + ), + ]); + const countAtLimit = await readyPage(store, countSession.id, 'count-at-limit', 'shared'); + const exactCount = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: countSession.id, + projection: 'shared', + snapshotLeaseId: 'count-at-limit', + snapshotKey: countAtLimit.snapshotKey, + positionKeys: bodyKeys('turn-count-boundary'), + maxRecords: 256, + maxBytes: 16 * 1024 * 1024, + }); + assert.equal(exactCount.records.length, 256); + await store.appendMessage(countSession.id, hiddenPermission('turn-count-boundary', 256)); + const countOverLimit = await readyPage(store, countSession.id, 'count-over-limit', 'shared'); + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: countSession.id, + projection: 'shared', + snapshotLeaseId: 'count-over-limit', + snapshotKey: countOverLimit.snapshotKey, + positionKeys: bodyKeys('turn-count-boundary'), + maxRecords: 256, + maxBytes: 16 * 1024 * 1024, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'transcript_record_count', + ); + + const byteSession = await store.create(makeInput({ name: 'Byte boundary' })); + await store.appendMessage( + byteSession.id, + exactSizeUserMessage(16 * 1024 * 1024, 'turn-byte-boundary', false), + ); + const bytesAtLimit = await readyPage(store, byteSession.id, 'bytes-at-limit', 'shared'); + const exactBytes = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: byteSession.id, + projection: 'shared', + snapshotLeaseId: 'bytes-at-limit', + snapshotKey: bytesAtLimit.snapshotKey, + positionKeys: bodyKeys('turn-byte-boundary'), + maxRecords: 256, + maxBytes: 16 * 1024 * 1024, + }); + assert.equal(exactBytes.rawBytes, 16 * 1024 * 1024); + await store.appendMessage(byteSession.id, hiddenPermission('turn-byte-boundary', 1)); + const bytesOverLimit = await readyPage(store, byteSession.id, 'bytes-over-limit', 'shared'); + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: byteSession.id, + projection: 'shared', + snapshotLeaseId: 'bytes-over-limit', + snapshotKey: bytesOverLimit.snapshotKey, + positionKeys: bodyKeys('turn-byte-boundary'), + maxRecords: 256, + maxBytes: 16 * 1024 * 1024, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'transcript_record_bytes', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('persists chunk-integrity failure across repair/reopen until authoritative invalidation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-chunk-corrupt-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, { + ...user('turn-chunk-corrupt', 0), + text: 'x'.repeat(1024 * 1024), + }); + await store.close?.(); + resetProjection(root, session.id); + const corrupt = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + corrupt + .prepare(`UPDATE session_message_chunks SET sha256 = ? + WHERE session_id = ? AND sequence = 0 AND chunk_index = 0`) + .run('0'.repeat(64), session.id); + } finally { + corrupt.close(); + } + store = createSessionStore(root); + await assert.rejects(readyPage(store, session.id), (error: unknown) => { + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + return true; + }); + await store.close?.(); + + const repaired = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const chunk = repaired + .prepare(`SELECT data FROM session_message_chunks + WHERE session_id = ? AND sequence = 0 AND chunk_index = 0`) + .get(session.id) as { data: Uint8Array }; + repaired + .prepare(`UPDATE session_message_chunks SET sha256 = ? + WHERE session_id = ? AND sequence = 0 AND chunk_index = 0`) + .run(createHash('sha256').update(chunk.data).digest('hex'), session.id); + } finally { + repaired.close(); + } + store = createSessionStore(root); + await assert.rejects(readyPage(store, session.id, 'lease-still-failed'), (error: unknown) => { + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + return true; + }); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + const recovered = await readyPage(store, session.id, 'lease-after-reset'); + assert.equal( + recovered.positions[0] ? positionId(recovered.positions[0]) : undefined, + 'turn-chunk-corrupt', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('uses the shared physical decoder to reject noncanonical complete-record chunks', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-body-chunk-shape-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, { + ...user('turn-chunk-shape', 0), + text: 'x'.repeat(1024 * 1024), + }); + const page = await readyPage(store, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const tail = database + .prepare(`SELECT chunk_index, data FROM session_message_chunks + WHERE session_id = ? AND sequence = 0 + ORDER BY chunk_index DESC LIMIT 2`) + .all(session.id) as Array<{ chunk_index: number; data: Uint8Array }>; + assert.equal(tail.length, 2); + const last = tail[0]!; + const prior = tail[1]!; + assert.equal(prior.data.byteLength, 64 * 1024); + const joined = Buffer.concat([Buffer.from(prior.data), Buffer.from(last.data)]); + const shiftedPrior = joined.subarray(0, 64 * 1024 - 1); + const shiftedLast = joined.subarray(64 * 1024 - 1); + const update = database.prepare(`UPDATE session_message_chunks + SET data = ?, sha256 = ? + WHERE session_id = ? AND sequence = 0 AND chunk_index = ?`); + update.run( + shiftedPrior, + createHash('sha256').update(shiftedPrior).digest('hex'), + session.id, + prior.chunk_index, + ); + update.run( + shiftedLast, + createHash('sha256').update(shiftedLast).digest('hex'), + session.id, + last.chunk_index, + ); + } finally { + database.close(); + } + + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + positionKeys: bodyKeys('turn-chunk-shape'), + maxRecords: 1, + maxBytes: 2 * 1024 * 1024, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'stored_session_message_incompatible', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed when recovered Turn identity conflicts with persisted projection metadata', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-identity-conflict-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, user('turn-a', 0)); + await store.close?.(); + resetProjection(root, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`INSERT INTO session_turn_metadata( + session_id, position_kind, position_id, order_source, + owner_first_sequence, shared_first_sequence + ) VALUES (?, 'turn', 'turn-conflict', 'legacy', 0, 0)`) + .run(session.id); + } finally { + database.close(); + } + store = createSessionStore(root); + + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => { + assert.equal((error as { code?: unknown }).code, 'session_turn_position_recovery_failed'); + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + return true; + }, + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('shortens a page to keep serialized position metadata below 64 KiB', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-page-bytes-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages( + session.id, + Array.from({ length: 128 }, (_, index) => user(`turn-${index}-${'t'.repeat(600)}`, index)), + ); + const ready = await readyPage(store, session.id); + const page = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 0 }, + maxPositions: 128, + }); + assert.equal(page.kind, 'page'); + if (page.kind !== 'page') assert.fail('expected bounded page'); + assert.ok(page.positions.length < 128); + assert.ok(Buffer.byteLength(JSON.stringify(page), 'utf8') <= 64 * 1024); + assert.equal(page.hasNewer, true); + + const tail = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 128, + }); + assert.equal(tail.kind, 'page'); + if (tail.kind !== 'page') assert.fail('expected bounded tail page'); + assert.ok(tail.positions.length < 128); + assert.equal(tail.positions.at(-1)?.ordinal, 127); + assert.equal(tail.hasNewer, false); + assert.equal(tail.hasOlder, true); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('returns a typed failure when one position alone exceeds the metadata budget', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-single-page-limit-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, user(`turn-${'x'.repeat(70 * 1024)}`, 0)); + await assert.rejects(readyPage(store, session.id), (error: unknown) => { + assert.equal((error as { code?: unknown }).code, 'session_turn_position_limit_exceeded'); + assert.equal((error as { reason?: unknown }).reason, 'page_metadata_bytes'); + return true; + }); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('indexes imported transcripts without copying a 16 KiB-plus body into positions', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-imported-')); + const store = createSessionStore(root); + try { + const messages = [ + { ...user('turn-imported', 0), text: 'x'.repeat(20 * 1024) }, + user('turn-second', 1), + ]; + const session = await store.createImportedSession(makeInput(), messages, { + adapterId: 'test-adapter', + sourceSessionId: 'source-session', + }); + const page = await readyPage(store, session.id); + assert.deepEqual(page.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-imported' }, firstSequence: 0 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-second' }, firstSequence: 1 }, + ]); + assert.ok(Buffer.byteLength(JSON.stringify(page), 'utf8') < 4 * 1024); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + const state = database + .prepare(`SELECT indexed_through_sequence, source_records, source_bytes + FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as { + indexed_through_sequence: number; + source_records: number; + source_bytes: number; + }; + assert.equal(state.indexed_through_sequence, 1); + assert.equal(state.source_records, 2); + assert.equal( + state.source_bytes, + messages.reduce( + (total, message) => total + Buffer.byteLength(JSON.stringify(message), 'utf8'), + 0, + ), + ); + } finally { + database.close(); + } + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('reclaims ready generations across owner reopen and preserves monotonic generation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-ready-reopen-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages(session.id, [user('turn-a', 0), user('turn-b', 1)]); + const before = await readyPage(store, session.id); + await store.close?.(); + store = createSessionStore(root); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: before.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const after = await readyPage(store, session.id, 'lease-after-reopen'); + assert.ok(after.snapshotKey.snapshotGeneration > before.snapshotKey.snapshotGeneration); + assert.deepEqual(after.positions, before.positions); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('keeps live snapshot leases when a second facade shares the same database owner', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-same-owner-')); + const first = createSessionStore(root); + let second: ReturnType | undefined; + try { + const session = await first.create(makeInput()); + await first.appendMessages(session.id, [user('turn-a', 0), user('turn-b', 1)]); + const original = await readyPage(first, session.id, 'lease-first'); + + second = createSessionStore(root); + const shared = await second.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-second', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(shared.kind, 'page'); + if (shared.kind !== 'page') assert.fail('expected a shared ready generation'); + assert.deepEqual(shared.snapshotKey, original.snapshotKey); + + await first.releaseTurnPositionSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-first', + snapshotKey: original.snapshotKey, + }); + const retained = await second.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-second', + snapshotKey: shared.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(retained.kind, 'page'); + } finally { + await second?.close?.(); + await first.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('database guards reject mutation of ready position rows', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-ready-corrupt-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages(session.id, [ + user('turn-a', 0), + user('turn-b', 1), + user('turn-c', 2), + ]); + const ready = await readyPage(store, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + for (const mutate of [ + () => + database + .prepare(`INSERT INTO session_turn_snapshot_positions( + session_id, snapshot_generation, position_kind, position_id, + owner_ordinal, shared_ordinal, owner_first_sequence, shared_first_sequence + ) VALUES (?, ?, 'turn', 'turn-extra', 3, 3, 3, 3)`) + .run(session.id, ready.snapshotKey.snapshotGeneration), + () => + database + .prepare(`UPDATE session_turn_snapshot_positions SET position_id = 'mutated' + WHERE session_id = ? AND snapshot_generation = ? AND owner_ordinal = 2`) + .run(session.id, ready.snapshotKey.snapshotGeneration), + () => + database + .prepare(`DELETE FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND owner_ordinal = 2`) + .run(session.id, ready.snapshotKey.snapshotGeneration), + () => + database + .prepare(`UPDATE session_turn_position_snapshots SET ready_owner_total = 2 + WHERE session_id = ? AND snapshot_generation = ?`) + .run(session.id, ready.snapshotKey.snapshotGeneration), + ]) { + assert.throws(mutate, /immutable/u); + } + } finally { + database.close(); + } + const stable = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(stable.kind, 'page'); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('atomically expires exact snapshots when historical handoff shifts sequence anchors', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-historical-insert-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages(session.id, [ + { + type: 'assistant', + id: 'prior-output', + turnId: 'turn-prior', + ts: 10, + text: 'prior', + modelId: 'test-model', + }, + { + type: 'assistant', + id: 'target-output', + turnId: 'turn-target', + ts: 20, + text: 'target', + modelId: 'test-model', + }, + { type: 'user', id: 'newer-user', turnId: 'turn-newer', ts: 30, text: 'newer' }, + ]); + const before = await readyPage(store, session.id); + + await store.markMessagesHandedOff({ + sessionId: session.id, + messageIds: ['historical-user'], + turnId: 'turn-target', + provenRootMessages: [ + { messageId: 'historical-user', content: { text: 'historical' }, admittedAt: 17 }, + ], + }); + + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: before.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const after = await readyPage(store, session.id); + assert.ok(after.snapshotKey.snapshotGeneration > before.snapshotKey.snapshotGeneration); + assert.deepEqual( + after.positions.map((position) => [positionId(position), position.firstSequence]), + [ + ['turn-prior', 0], + ['turn-target', 1], + ['turn-newer', 3], + ], + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('additively migrates a current v34 database and boundedly rebuilds legacy metadata', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-v34-migration-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages(session.id, [user('turn-a', 0), user('turn-b', 1)]); + await store.close?.(); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec(` + PRAGMA foreign_keys = OFF; + DROP TABLE session_turn_snapshot_leases; + DROP TABLE session_turn_snapshot_positions; + DROP TABLE session_turn_position_snapshots; + DROP TABLE session_turn_identity_recovery; + DROP TABLE session_turn_memberships; + DROP TABLE session_turn_metadata; + DROP TABLE session_turn_index_state; + DROP TABLE session_turn_authority_revisions; + UPDATE session_metadata_schema SET version = 34 WHERE scope = 'session_metadata'; + `); + } finally { + database.close(); + } + + store = createSessionStore(root); + const lazy = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + lazy + .prepare(`SELECT COUNT(*) AS count FROM session_turn_metadata WHERE session_id = ?`) + .get(session.id) as { count: number } + ).count, + 0, + ); + } finally { + lazy.close(); + } + const page = await readyPage(store, session.id); + assert.deepEqual(page.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-a' }, firstSequence: 0 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-b' }, firstSequence: 1 }, + ]); + const migrated = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + migrated + .prepare( + `SELECT version FROM session_metadata_schema WHERE scope = 'session_metadata'`, + ) + .get() as { version: number } + ).version, + 35, + ); + const metadataColumns = new Set( + ( + migrated.prepare(`PRAGMA table_info(session_turn_metadata)`).all() as Array<{ + name: string; + }> + ).map(({ name }) => name), + ); + assert.deepEqual( + ['position_kind', 'position_id', 'owner_first_sequence', 'shared_first_sequence'].map( + (name) => metadataColumns.has(name), + ), + [true, true, true, true], + ); + assert.equal(metadataColumns.has('turn_id'), false); + const snapshotColumns = new Set( + ( + migrated.prepare(`PRAGMA table_info(session_turn_position_snapshots)`).all() as Array<{ + name: string; + }> + ).map(({ name }) => name), + ); + assert.equal(snapshotColumns.has('ready_owner_total'), true); + assert.equal(snapshotColumns.has('ready_shared_total'), true); + } finally { + migrated.close(); + } + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('recovers a v34 bodyless root admission before publishing owner and shared snapshots', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-v34-bodyless-admission-')); + let store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + const session = await store.create(makeInput()); + try { + await runs.admitRootTurn( + rootAdmission(session.id, 'turn-bodyless-v34', 'future-user-v34', 42), + ); + runs.close?.(); + await store.close?.(); + downgradeTurnProjectionToV34(root); + + store = createSessionStore(root); + const owner = await readyPage(store, session.id, 'v34-bodyless-owner', 'owner'); + const shared = await readyPage(store, session.id, 'v34-bodyless-shared', 'shared'); + assert.deepEqual(owner.positions, [ + { + ordinal: 0, + key: { kind: 'turn', id: 'turn-bodyless-v34' }, + firstSequence: null, + }, + ]); + assert.deepEqual(shared.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ]); + const migrated = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.deepEqual( + ( + migrated + .prepare(`SELECT position_kind, position_id, order_source, admitted_at, + owner_first_sequence, shared_first_sequence + FROM session_turn_metadata WHERE session_id = ?`) + .all(session.id) as Array> + ).map((row) => ({ ...row })), + [ + { + position_kind: 'turn', + position_id: 'turn-bodyless-v34', + order_source: 'admission', + admitted_at: 42, + owner_first_sequence: null, + shared_first_sequence: null, + }, + ], + ); + } finally { + migrated.close(); + } + + await store.close?.(); + store = createSessionStore(root); + const reopenedOwner = await readyPage(store, session.id, 'v34-bodyless-reopen', 'owner'); + assert.deepEqual(reopenedOwner.positions, owner.positions); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('recovers v34 admission order while preserving owner and shared body anchors', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-v34-admission-bodies-')); + let store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + const session = await store.create(makeInput()); + try { + await runs.admitRootTurn(rootAdmission(session.id, 'turn-late', 'late-user', 20)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-early', 'early-user', 10)); + await store.appendMessages(session.id, [ + hiddenPermission('turn-late', 0), + { ...user('turn-early', 1), id: 'early-user' }, + { ...user('turn-late', 2), id: 'late-user' }, + ]); + runs.close?.(); + await store.close?.(); + downgradeTurnProjectionToV34(root); + + store = createSessionStore(root); + const owner = await readyPage(store, session.id, 'v34-bodies-owner', 'owner'); + const shared = await readyPage(store, session.id, 'v34-bodies-shared', 'shared'); + assert.deepEqual(owner.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-early' }, firstSequence: 1 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-late' }, firstSequence: 0 }, + ]); + assert.deepEqual(shared.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-early' }, firstSequence: 1 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-late' }, firstSequence: 2 }, + ]); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('reuses normalized admission authority across later exact snapshots', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-admission-fixed-point-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-a', 'user-a', 1)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-b', 'user-b', 2)); + const first = await readyPage(store, session.id, 'admission-fixed-first', 'owner'); + assert.deepEqual( + first.positions.map((position) => position.key), + [ + { kind: 'turn', id: 'turn-a' }, + { kind: 'turn', id: 'turn-b' }, + ], + ); + + await runs.admitRootTurn(rootAdmission(session.id, 'turn-c', 'user-c', 3)); + const database = (store as unknown as { metadata: { db: DatabaseSync } }).metadata.db; + let admissionReads = 0; + database.setAuthorizer((action, table) => { + if (action === sqliteConstants.SQLITE_READ && table === 'core_root_turn_admissions') { + admissionReads += 1; + } + return sqliteConstants.SQLITE_OK; + }); + try { + const next = await readyPage(store, session.id, 'admission-fixed-next', 'owner'); + assert.ok(next.snapshotKey.authorityRevision > first.snapshotKey.authorityRevision); + assert.deepEqual( + next.positions.map((position) => position.key), + [ + { kind: 'turn', id: 'turn-a' }, + { kind: 'turn', id: 'turn-b' }, + { kind: 'turn', id: 'turn-c' }, + ], + ); + } finally { + database.setAuthorizer(null); + } + assert.equal(admissionReads, 0); + assert.deepEqual( + { + ...(database + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: 3, + admission_cursor_turn_id: 'turn-c', + admission_recovery_complete: 1, + }, + ); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('explicit invalidation resets and rebuilds persisted admission recovery state', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-admission-reset-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-reset', 'user-reset', 9)); + await readyPage(store, session.id, 'admission-reset-before', 'owner'); + const database = (store as unknown as { metadata: { db: DatabaseSync } }).metadata.db; + database + .prepare(`UPDATE session_turn_index_state + SET failure_origin = 'transcript', failure_reason = 'corrupt_source', + failure_sequence = 0 WHERE session_id = ?`) + .run(session.id); + database.exec('BEGIN IMMEDIATE'); + try { + invalidateSessionTurnPositionIndex(database, session.id); + database.exec('COMMIT'); + } catch (error) { + database.exec('ROLLBACK'); + throw error; + } + assert.deepEqual( + { + ...(database + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: null, + admission_cursor_turn_id: null, + admission_recovery_complete: 0, + }, + ); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: null, + failure_reason: null, + failure_sequence: null, + }); + assert.equal( + ( + database + .prepare(`SELECT COUNT(*) AS count FROM session_turn_metadata WHERE session_id = ?`) + .get(session.id) as { count: number } + ).count, + 0, + ); + + const rebuilt = await readyPage(store, session.id, 'admission-reset-after', 'owner'); + assert.deepEqual(rebuilt.positions, [ + { + ordinal: 0, + key: { kind: 'turn', id: 'turn-reset' }, + firstSequence: null, + }, + ]); + assert.deepEqual( + { + ...(database + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: 9, + admission_cursor_turn_id: 'turn-reset', + admission_recovery_complete: 1, + }, + ); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('bounds legacy admission reconciliation and atomically rebuilds after owner reopen', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-admission-recovery-bound-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.close?.(); + const seeded = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const insert = seeded.prepare(`INSERT INTO core_root_turn_admissions( + session_id, turn_id, admitted_at, record_json + ) VALUES (?, ?, ?, '{}')`); + seeded.exec('BEGIN IMMEDIATE'); + for (let index = 0; index < 1_025; index += 1) { + insert.run(session.id, `turn-${index.toString().padStart(4, '0')}`, index); + } + seeded.exec('COMMIT'); + } finally { + seeded.close(); + } + downgradeTurnProjectionToV34(root); + + store = createSessionStore(root); + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'admission-bound-first', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(first.kind, 'building'); + if (first.kind !== 'building') assert.fail('expected bounded admission reconciliation'); + const bounded = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + bounded + .prepare(`SELECT COUNT(*) AS count FROM session_turn_metadata + WHERE session_id = ? AND order_source = 'admission'`) + .get(session.id) as { count: number } + ).count, + 1_024, + ); + const snapshotState = bounded + .prepare(`SELECT state, build_phase FROM session_turn_position_snapshots + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, first.snapshotKey.snapshotGeneration) as { + state: string; + build_phase: string; + }; + assert.deepEqual( + { ...snapshotState }, + { + state: 'building', + build_phase: 'recovering', + }, + ); + assert.deepEqual( + { + ...(bounded + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state + WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: 1_023, + admission_cursor_turn_id: 'turn-1023', + admission_recovery_complete: 0, + }, + ); + assert.equal( + ( + bounded + .prepare(`SELECT COUNT(*) AS count FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, first.snapshotKey.snapshotGeneration) as { count: number } + ).count, + 0, + ); + const plan = bounded + .prepare(`EXPLAIN QUERY PLAN SELECT turn_id, admitted_at + FROM core_root_turn_admissions + WHERE session_id = ? AND (admitted_at, turn_id) > (?, ?) + ORDER BY admitted_at, turn_id LIMIT ?`) + .all(session.id, 1_023, 'turn-1023', 1_024) + .map((row) => String((row as { detail?: unknown }).detail ?? '')) + .join('\n'); + assert.match(plan, /core_root_turn_admissions_order/u); + assert.match(plan, /\(admitted_at,turn_id\)>\(\?,\?\)/u); + assert.doesNotMatch(plan, /TEMP B-TREE|OFFSET|ROW_NUMBER|COUNT DISTINCT/iu); + } finally { + bounded.close(); + } + + await store.close?.(); + store = createSessionStore(root); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'admission-bound-first', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + let resumed = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'admission-bound-reopen', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(resumed.kind, 'building'); + if (resumed.kind !== 'building') assert.fail('expected resumed admission reconciliation'); + const resumedState = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.deepEqual( + { + ...(resumedState + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state + WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: 1_024, + admission_cursor_turn_id: 'turn-1024', + admission_recovery_complete: 1, + }, + ); + assert.equal( + ( + resumedState + .prepare(`SELECT COUNT(*) AS count FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, resumed.snapshotKey.snapshotGeneration) as { count: number } + ).count, + 0, + ); + } finally { + resumedState.close(); + } + while (resumed.kind === 'building') { + resumed = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'admission-bound-reopen', + snapshotKey: resumed.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(resumed.kind, 'page'); + if (resumed.kind !== 'page') assert.fail('expected ready page after resumed recovery'); + const ready = resumed; + assert.equal(ready.totalPositions, 1_025); + assert.ok(ready.snapshotKey.snapshotGeneration > first.snapshotKey.snapshotGeneration); + const completed = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + const positions = completed + .prepare(`SELECT owner_ordinal, position_id FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND owner_ordinal IS NOT NULL + ORDER BY owner_ordinal`) + .all(session.id, ready.snapshotKey.snapshotGeneration) as Array<{ + owner_ordinal: number; + position_id: string; + }>; + assert.equal(positions.length, 1_025); + for (let index = 0; index < positions.length; index += 1) { + assert.deepEqual( + { ...positions[index] }, + { + owner_ordinal: index, + position_id: `turn-${index.toString().padStart(4, '0')}`, + }, + ); + } + } finally { + completed.close(); + } + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('rehydrates a bodyless admission after shared visibility policy reset', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-bodyless-policy-reset-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn( + rootAdmission(session.id, 'turn-bodyless-policy', 'future-policy-user', 7), + ); + const before = await readyPage(store, session.id, 'bodyless-policy-before', 'owner'); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_turn_index_state + SET failure_origin = 'admission', failure_reason = 'corrupt_source', + failure_sequence = 0 WHERE session_id = ?`) + .run(session.id); + database + .prepare(`UPDATE session_turn_authority_revisions + SET visibility_policy_version = 999 WHERE session_id = ?`) + .run(session.id); + } finally { + database.close(); + } + + const owner = await readyPage(store, session.id, 'bodyless-policy-after', 'owner'); + const shared = await readyPage(store, session.id, 'bodyless-policy-shared', 'shared'); + assert.ok(owner.snapshotKey.snapshotGeneration > before.snapshotKey.snapshotGeneration); + assert.deepEqual(owner.positions, [ + { + ordinal: 0, + key: { kind: 'turn', id: 'turn-bodyless-policy' }, + firstSequence: null, + }, + ]); + assert.deepEqual(shared.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ]); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: null, + failure_reason: null, + failure_sequence: null, + }); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed on invalid or contradictory canonical admission scalars', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-invalid-admission-source-')); + let store = createSessionStore(root); + try { + const invalidIdentity = await store.create(makeInput({ name: 'Invalid identity' })); + const invalidTime = await store.create(makeInput({ name: 'Invalid time' })); + const contradiction = await store.create(makeInput({ name: 'Contradiction' })); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const insert = database.prepare(`INSERT INTO core_root_turn_admissions( + session_id, turn_id, admitted_at, record_json + ) VALUES (?, ?, ?, '{}')`); + insert.run(invalidIdentity.id, '', 1); + insert.run(invalidTime.id, 'turn-negative-time', -1); + insert.run(contradiction.id, 'turn-contradiction', 1); + database + .prepare(`INSERT INTO session_turn_metadata( + session_id, position_kind, position_id, order_source, admitted_at, + owner_first_sequence, shared_first_sequence + ) VALUES (?, 'turn', 'turn-contradiction', 'admission', 2, NULL, NULL)`) + .run(contradiction.id); + } finally { + database.close(); + } + + for (const sessionId of [invalidIdentity.id, invalidTime.id, contradiction.id]) { + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId, + projection: 'owner', + snapshotLeaseId: 'invalid-admission', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => { + assert.equal( + (error as { code?: unknown }).code, + 'session_turn_position_recovery_failed', + ); + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + return true; + }, + ); + const failed = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.deepEqual( + { + ...(failed + .prepare(`SELECT failure_origin, failure_reason, failure_sequence + FROM session_turn_index_state WHERE session_id = ?`) + .get(sessionId) as Record), + }, + { + failure_origin: 'admission', + failure_reason: 'corrupt_source', + failure_sequence: 0, + }, + ); + } finally { + failed.close(); + } + } + + await store.close?.(); + const repaired = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + repaired + .prepare(`UPDATE core_root_turn_admissions SET turn_id = 'turn-repaired' + WHERE session_id = ? AND turn_id = ''`) + .run(invalidIdentity.id); + } finally { + repaired.close(); + } + store = createSessionStore(root); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: invalidIdentity.id, + projection: 'owner', + snapshotLeaseId: 'invalid-admission-reopen', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'corrupt_source', + ); + + const reset = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + reset + .prepare(`UPDATE session_turn_authority_revisions + SET visibility_policy_version = 999 WHERE session_id = ?`) + .run(invalidIdentity.id); + } finally { + reset.close(); + } + const recovered = await readyPage( + store, + invalidIdentity.id, + 'invalid-admission-policy-reset', + 'owner', + ); + assert.deepEqual(recovered.positions, [ + { + ordinal: 0, + key: { kind: 'turn', id: 'turn-repaired' }, + firstSequence: null, + }, + ]); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('uses indexed owner/shared keyset plans for 10,000 alternating-visibility Turns', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-plan-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages( + session.id, + Array.from({ length: 10_000 }, (_, index) => + index % 2 === 0 ? user(`turn-${index}`, index) : hiddenPermission(`turn-${index}`, index), + ), + ); + const firstStep = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(firstStep.kind, 'building'); + if (firstStep.kind !== 'building') assert.fail('expected bounded position publication'); + assert.equal(firstStep.progress.phase, 'recovering'); + assert.equal(firstStep.progress.sourceRecords, 10_000); + assert.ok(firstStep.progress.sourceBytes > 0); + assert.equal(firstStep.progress.builtPositions, 0); + assert.equal(firstStep.progress.lastStepPositions, 0); + const secondStep = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: firstStep.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(secondStep.kind, 'building'); + if (secondStep.kind !== 'building') assert.fail('expected legacy ordinal build'); + assert.equal(secondStep.progress.phase, 'legacy'); + assert.equal(secondStep.progress.builtPositions, 1_024); + const sharedProgress = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'lease-shared-progress', + throughSequence: secondStep.snapshotKey.throughSequence, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(sharedProgress.kind, 'building'); + if (sharedProgress.kind !== 'building') assert.fail('expected shared ordinal progress'); + assert.equal(sharedProgress.progress.builtPositions, 1_024); + assert.equal(sharedProgress.progress.lastStepPositions, 512); + const bounded = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + bounded + .prepare(`SELECT COUNT(*) AS count FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, secondStep.snapshotKey.snapshotGeneration) as { count: number } + ).count, + 2_048, + ); + const recoveryPlans = [ + bounded + .prepare(`EXPLAIN QUERY PLAN SELECT MIN(owner_first_sequence) AS boundary + FROM session_turn_metadata + WHERE session_id = ? AND order_source = 'admission' AND owner_first_sequence <= ?`) + .all(session.id, 9_999), + bounded + .prepare(`EXPLAIN QUERY PLAN SELECT position_kind, position_id, + owner_first_sequence, NULL AS admitted_at + FROM session_turn_metadata + WHERE session_id = ? AND owner_first_sequence <= ? AND order_source = 'legacy' + AND (? IS NULL OR owner_first_sequence < ?) AND owner_first_sequence > ? + ORDER BY owner_first_sequence, position_kind, position_id LIMIT ?`) + .all(session.id, 9_999, null, null, -1, 1_025), + ].flat() as Array<{ detail: string }>; + assert.equal( + recoveryPlans.some(({ detail }) => /\bSCAN\b|USE TEMP B-TREE/.test(detail)), + false, + ); + assert.equal( + recoveryPlans.every(({ detail }) => detail.includes('SEARCH session_turn_metadata')), + true, + ); + } finally { + bounded.close(); + } + let tail = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: secondStep.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + while (tail.kind === 'building') { + tail = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: tail.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(tail.kind, 'page'); + if (tail.kind !== 'page') assert.fail('expected steady page'); + assert.equal(tail.totalPositions, 10_000); + assert.equal(tail.positions.length, 8); + const sharedTail = await readyPage(store, session.id, 'lease-shared-10k', 'shared'); + assert.equal(sharedTail.snapshotKey.snapshotGeneration, tail.snapshotKey.snapshotGeneration); + assert.equal(sharedTail.totalPositions, 5_000); + assert.equal(sharedTail.positions.length, 8); + const sharedMiddle = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'lease-shared-10k', + snapshotKey: sharedTail.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 2_500 }, + maxPositions: 8, + }); + assert.equal(sharedMiddle.kind, 'page'); + if (sharedMiddle.kind !== 'page') assert.fail('expected indexed shared page'); + assert.deepEqual(sharedMiddle.positions[0]?.key, { kind: 'turn', id: 'turn-5000' }); + + const sparseRecords = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: tail.snapshotKey, + positionKeys: bodyKeys('turn-9999', 'turn-0'), + maxRecords: 2, + maxBytes: 64 * 1024, + }); + assert.deepEqual( + sparseRecords.records.map(({ sequence }) => sequence), + [0, 9_999], + ); + + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + const plans = [ + database + .prepare(`EXPLAIN QUERY PLAN SELECT owner_ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND owner_ordinal IS NOT NULL AND owner_first_sequence <= ? + ORDER BY owner_first_sequence DESC LIMIT 1`) + .all(session.id, tail.snapshotKey.snapshotGeneration, 5_000), + database + .prepare(`EXPLAIN QUERY PLAN SELECT owner_ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND position_kind = 'turn' AND position_id = ? AND owner_ordinal IS NOT NULL`) + .all(session.id, tail.snapshotKey.snapshotGeneration, 'turn-5000'), + database + .prepare(`EXPLAIN QUERY PLAN SELECT owner_ordinal, position_kind, position_id, + owner_first_sequence + FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND owner_ordinal IS NOT NULL AND owner_ordinal >= ? + ORDER BY owner_ordinal LIMIT ?`) + .all(session.id, tail.snapshotKey.snapshotGeneration, 5_000, 8), + database + .prepare(`EXPLAIN QUERY PLAN SELECT shared_ordinal, position_kind, position_id, + shared_first_sequence + FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND shared_ordinal IS NOT NULL AND shared_ordinal >= ? + ORDER BY shared_ordinal LIMIT ?`) + .all(session.id, tail.snapshotKey.snapshotGeneration, 2_500, 8), + ].flat() as Array<{ detail: string }>; + assert.equal( + plans.some(({ detail }) => /\bSCAN\b|USE TEMP B-TREE/.test(detail)), + false, + ); + assert.equal( + plans.every(({ detail }) => detail.includes('SEARCH session_turn_snapshot_positions')), + true, + ); + const membershipPlan = database + .prepare(`EXPLAIN QUERY PLAN + SELECT membership.sequence, + coalesce(payload.record_bytes, length(CAST(message.record_json AS BLOB))) + FROM session_turn_memberships AS membership + INNER JOIN session_messages AS message + ON message.session_id = membership.session_id + AND message.sequence = membership.sequence + LEFT JOIN session_message_payloads AS payload + ON payload.session_id = message.session_id AND payload.sequence = message.sequence + WHERE membership.session_id = ? AND membership.position_kind = 'turn' + AND membership.position_id = ? + AND membership.sequence <= ? + ORDER BY membership.sequence LIMIT ?`) + .all(session.id, 'turn-9999', 9_999, 2) as Array<{ detail: string }>; + assert.equal( + membershipPlan.some(({ detail }) => /\bSCAN\b|USE TEMP B-TREE/u.test(detail)), + false, + ); + assert.equal( + membershipPlan.some(({ detail }) => + detail.includes('session_turn_memberships_by_position'), + ), + true, + ); + } finally { + database.close(); + } + + for (const anchor of [ + { kind: 'ordinal' as const, ordinal: 5_000 }, + { kind: 'sequence' as const, sequence: 5_000 }, + { kind: 'turn' as const, turnId: 'turn-5000' }, + ]) { + const page = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: tail.snapshotKey, + anchor, + maxPositions: 8, + }); + assert.equal(page.kind, 'page'); + if (page.kind !== 'page') assert.fail('expected indexed anchored page'); + assert.equal(page.startOrdinal, 5_000); + assert.equal(page.positions.length, 8); + } + const adjacent = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: tail.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 5_008 }, + maxPositions: 8, + }); + assert.equal(adjacent.kind, 'page'); + if (adjacent.kind !== 'page') assert.fail('expected adjacent keyset page'); + assert.deepEqual( + adjacent.positions.map(({ ordinal }) => ordinal), + Array.from({ length: 8 }, (_, index) => 5_008 + index), + ); + assert.equal( + new Set(adjacent.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id))).size, + 8, + ); + const prior = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: tail.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 5_000 }, + maxPositions: 8, + }); + assert.equal(prior.kind, 'page'); + if (prior.kind !== 'page') assert.fail('expected prior keyset page'); + assert.equal( + prior.positions.some((position) => + adjacent.positions.some((candidate) => positionId(candidate) === positionId(position)), + ), + false, + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); +}); + +type TestStore = ReturnType; + +async function readyPage( + store: TestStore, + sessionId: string, + snapshotLeaseId = 'lease-default', + projection: 'owner' | 'shared' = 'owner', +) { + let result = await store.readTurnPositionPageSnapshot({ + sessionId, + projection, + snapshotLeaseId, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + while (result.kind === 'building') { + result = await store.readTurnPositionPageSnapshot({ + sessionId, + projection, + snapshotLeaseId, + snapshotKey: result.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(result.kind, 'page'); + if (result.kind !== 'page') assert.fail('expected a ready page'); + return result; +} + +function user(turnId: string, index: number) { + return { + type: 'user' as const, + id: `user-${index}`, + turnId, + ts: index, + text: turnId, + }; +} + +function hiddenPermission(turnId: string, index: number) { + return { + type: 'permission_decision' as const, + id: `permission-${index}`, + turnId, + ts: index, + toolUseId: `tool-${index}`, + toolName: 'Read', + decision: 'deny' as const, + }; +} + +function bodyKeys(...ids: string[]) { + return ids.map((id) => ({ kind: 'turn' as const, id })); +} + +function positionId(position: { key: { kind: 'turn' | 'note'; id: string } | { kind: 'empty' } }) { + return position.key.kind === 'empty' ? '' : position.key.id; +} + +function exactSizeUserMessage(targetBytes: number, turnId: string, turnIdAfterBody: boolean) { + const message = turnIdAfterBody + ? { type: 'user' as const, id: `user-${turnId}`, ts: 1, text: '', turnId } + : { type: 'user' as const, id: `user-${turnId}`, turnId, ts: 1, text: '' }; + const envelopeBytes = Buffer.byteLength(JSON.stringify(message), 'utf8'); + assert.ok(targetBytes >= envelopeBytes); + message.text = 'x'.repeat(targetBytes - envelopeBytes); + return message; +} + +function rootAdmission( + sessionId: string, + turnId: string, + userMessageId: string, + admittedAt: number, +) { + return { + sessionId, + turnId, + proposedRunId: `run-${turnId}`, + proposedUserMessageId: userMessageId, + execution: { kind: 'external_message' as const }, + previousRootTurnId: null, + normalizedInput: { text: turnId }, + sourceMessages: [], + admittedAt, + }; +} + +function resetProjection(root: string, sessionId: string): void { + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec('PRAGMA foreign_keys = ON'); + database + .prepare('DELETE FROM session_turn_position_snapshots WHERE session_id = ?') + .run(sessionId); + database.prepare('DELETE FROM session_turn_metadata WHERE session_id = ?').run(sessionId); + database + .prepare(`UPDATE session_turn_index_state SET indexed_through_sequence = -1, + source_records = 0, source_bytes = 0, + failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ?`) + .run(sessionId); + } finally { + database.close(); + } +} + +function readTurnIndexFailure(root: string, sessionId: string): Record { + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + return { + ...(database + .prepare(`SELECT failure_origin, failure_reason, failure_sequence + FROM session_turn_index_state WHERE session_id = ?`) + .get(sessionId) as Record), + }; + } finally { + database.close(); + } +} + +function downgradeTurnProjectionToV34(root: string): void { + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec(` + PRAGMA foreign_keys = OFF; + DROP TABLE session_turn_snapshot_leases; + DROP TABLE session_turn_snapshot_positions; + DROP TABLE session_turn_position_snapshots; + DROP TABLE session_turn_identity_recovery; + DROP TABLE session_turn_memberships; + DROP TABLE session_turn_metadata; + DROP TABLE session_turn_index_state; + DROP TABLE session_turn_authority_revisions; + UPDATE session_metadata_schema SET version = 34 WHERE scope = 'session_metadata'; + `); + } finally { + database.close(); + } +} + +function makeInput(overrides: Partial = {}): CreateSessionInput { + return { + cwd: '/tmp/cwd', + llmConnectionSlug: 'test-connection', + model: 'test-model', + permissionMode: 'ask', + name: 'Session', + labels: [], + ...overrides, + }; +} diff --git a/packages/storage/src/agent-run-store.ts b/packages/storage/src/agent-run-store.ts index 27481f3fec..fb652a375a 100644 --- a/packages/storage/src/agent-run-store.ts +++ b/packages/storage/src/agent-run-store.ts @@ -34,6 +34,7 @@ import { type SubmittedTurnIntent, } from './submitted-turn-intent.js'; import { assertNoReservedWorkspaceAuthorityAppend } from './runtime-event-authority.js'; +import { recordRootTurnAdmissionForPositionIndex } from './session-turn-position-index.js'; import { acquireOperationalStateDatabase, type OperationalStateDatabaseLease, @@ -738,6 +739,12 @@ class SqliteAgentRunStore implements DurableAgentRunStore { admission.admittedAt, JSON.stringify(admission), ); + recordRootTurnAdmissionForPositionIndex( + this.#lease.database, + admission.sessionId, + admission.turnId, + admission.admittedAt, + ); for (const source of admission.sourceMessages) { this.#lease.database .prepare(` diff --git a/packages/storage/src/conversation-operational-state.ts b/packages/storage/src/conversation-operational-state.ts index 2f87bd8921..fbd766ec09 100644 --- a/packages/storage/src/conversation-operational-state.ts +++ b/packages/storage/src/conversation-operational-state.ts @@ -23,6 +23,7 @@ import { type OperationalStateDatabaseLease, } from './operational-state-store.js'; import { isRuntimeStorageSafeId } from './runtime-event-invariants.js'; +import { recordRootTurnAdmissionsPurgedForPositionIndex } from './session-turn-position-index.js'; export interface ConversationOperationalStateStore { purge(sessionId: string): Promise; @@ -81,7 +82,15 @@ class SqliteConversationOperationalStateStore implements ConversationOperational database .prepare('DELETE FROM core_agent_run_projections WHERE session_id = ?') .run(sessionId); - database.prepare('DELETE FROM core_root_turn_admissions WHERE session_id = ?').run(sessionId); + const canonicalAdmissionsDeleted = + database + .prepare('DELETE FROM core_root_turn_admissions WHERE session_id = ?') + .run(sessionId).changes > 0; + recordRootTurnAdmissionsPurgedForPositionIndex( + database, + sessionId, + canonicalAdmissionsDeleted, + ); database .prepare('DELETE FROM core_root_turn_start_rejections WHERE session_id = ?') .run(sessionId); diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index 33bdf087e7..0441fa2d45 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -85,6 +85,10 @@ export { normalizeRootTurnAdmissionPayload } from './agent-run-store.js'; export { isSessionNotFoundError, SessionReadMarkerMessageNotFoundError, + SessionTurnPositionAnchorNotFoundError, + SessionTurnPositionLimitError, + SessionTurnPositionRecoveryError, + SessionTurnPositionSnapshotMismatchError, } from './session-store.js'; export { SessionMetadataConflictError, @@ -119,6 +123,13 @@ export type { ProvenSteeringMessageHandoff, } from './message-admission-store.js'; export { submittedTurnIntentsEqual } from './submitted-turn-intent.js'; +export { + SESSION_TURN_POSITION_BODY_MAX_BYTES, + SESSION_TURN_POSITION_BODY_MAX_KEYS, + SESSION_TURN_POSITION_BODY_MAX_RECORDS, + SESSION_TURN_POSITION_MAX_PAGE_BYTES, + SESSION_TURN_POSITION_MAX_PAGE_POSITIONS, +} from './session-turn-position-snapshots.js'; export type { SubmittedTurnIntent } from './submitted-turn-intent.js'; export type { ProbeSessionRemovalResult, @@ -133,6 +144,17 @@ export type { SessionTranscriptRecordScanRequest, SessionTranscriptStoragePage, SessionTranscriptStorageFragment, + SessionTranscriptBodyPositionKey, + SessionTranscriptPositionKey, + SessionTranscriptProjection, + SessionTranscriptRecordsByPositionKeysSnapshotRequest, + SessionTranscriptRecordsByPositionKeysSnapshotResult, + SessionTurnPosition, + SessionTurnPositionAnchor, + SessionTurnPositionPageSnapshotRequest, + SessionTurnPositionReadResult, + SessionTurnPositionSnapshotKey, + SessionTurnPositionSnapshotReleaseRequest, } from './session-store.js'; export type ExecutionSessionWriter = SessionAuthorityStore; @@ -412,6 +434,12 @@ async function createExecutionStoresForWrite run(() => sessionStore.readTurnLandmarksSnapshot(sessionId, maxLandmarks)), + readTurnPositionPageSnapshot: (request) => + run(() => sessionStore.readTurnPositionPageSnapshot(request)), + readTranscriptRecordsByPositionKeysSnapshot: (request) => + run(() => sessionStore.readTranscriptRecordsByPositionKeysSnapshot(request)), + releaseTurnPositionSnapshot: (request) => + run(() => sessionStore.releaseTurnPositionSnapshot(request)), readMessagesForRecovery: (sessionId) => run(() => sessionStore.readMessagesForRecovery(sessionId)), listTurnsSnapshot: (sessionId) => run(() => sessionStore.listTurnsSnapshot(sessionId)), diff --git a/packages/storage/src/operational-state-store.ts b/packages/storage/src/operational-state-store.ts index 5f2772a680..47b2029435 100644 --- a/packages/storage/src/operational-state-store.ts +++ b/packages/storage/src/operational-state-store.ts @@ -57,6 +57,7 @@ import { ensureOperationalSchemaRegistry, isCurrentOperationalTargetSchema, } from './operational-target-schema.js'; +import { reclaimSessionTurnPositionSnapshotsForNewOwner } from './session-turn-position-index.js'; export const OPERATIONAL_STATE_DATABASE_NAME = 'runtime.sqlite'; export const OPERATIONAL_STATE_SCHEMA_VERSION = 2; @@ -201,6 +202,14 @@ class OperationalStateDatabaseOwner { this.database.exec('PRAGMA foreign_keys = ON'); inspectAndMigrateOperationalState(this.database, options.now ?? Date.now); configureSqliteRuntimeDatabase(this.database); + this.database.exec('BEGIN IMMEDIATE'); + try { + reclaimSessionTurnPositionSnapshotsForNewOwner(this.database); + this.database.exec('COMMIT'); + } catch (error) { + rollback(this.database); + throw error; + } } catch (error) { this.database.close(); this.closed = true; diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index 55eed4d1d4..c23a910407 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -301,6 +301,162 @@ export interface SessionTurnLandmarkSnapshot { readonly landmarks: readonly SessionTurnLandmark[]; } +export interface SessionTurnPositionSnapshotKey { + readonly throughSequence: number | null; + readonly authorityRevision: number; + readonly snapshotGeneration: number; +} + +export type SessionTranscriptProjection = 'owner' | 'shared'; + +export type SessionTranscriptPositionKey = + | { readonly kind: 'turn'; readonly id: string } + | { readonly kind: 'note'; readonly id: string } + | { readonly kind: 'empty' }; + +export type SessionTranscriptBodyPositionKey = Exclude< + SessionTranscriptPositionKey, + { readonly kind: 'empty' } +>; + +export type SessionTurnPositionAnchor = + /** The final position, returned as the inclusive end of a tail-sized page. */ + | { readonly kind: 'tail' } + /** An inclusive ordinal; values beyond the end clamp to the final position. */ + | { readonly kind: 'ordinal'; readonly ordinal: number } + /** The position with the greatest firstSequence at or before this inclusive sequence. */ + | { readonly kind: 'sequence'; readonly sequence: number } + /** An exact non-empty Turn identity; a missing Turn fails with a typed anchor error. */ + | { readonly kind: 'turn'; readonly turnId: string }; + +export interface SessionTurnPosition { + readonly ordinal: number; + readonly key: SessionTranscriptPositionKey; + readonly firstSequence: number | null; +} + +export interface SessionTurnPositionPageSnapshotRequest { + readonly sessionId: string; + readonly projection: SessionTranscriptProjection; + readonly snapshotLeaseId: string; + /** Exact continuation identity. Omit only when allocating a new snapshot. */ + readonly snapshotKey?: SessionTurnPositionSnapshotKey; + /** Inclusive durable watermark. Omit to capture the current durable tail. */ + readonly throughSequence?: number | null; + readonly anchor: SessionTurnPositionAnchor; + readonly maxPositions: number; +} + +export type SessionTurnPositionReadResult = + | { + readonly kind: 'building'; + readonly snapshotKey: SessionTurnPositionSnapshotKey; + readonly projection: SessionTranscriptProjection; + readonly progress: { + readonly phase: 'recovering' | 'legacy' | 'admission' | 'notes'; + readonly nextSequence: number; + readonly currentByteOffset: number; + readonly sourceRecords: number; + readonly sourceBytes: number; + readonly builtPositions: number; + readonly lastStepRecords: number; + readonly lastStepBytes: number; + readonly lastStepPositions: number; + }; + } + | { + readonly kind: 'capacity'; + readonly projection: SessionTranscriptProjection; + readonly throughSequence: number | null; + readonly authorityRevision: number; + readonly retainedSnapshots: 2; + } + | { + readonly kind: 'page'; + readonly snapshotKey: SessionTurnPositionSnapshotKey; + readonly projection: SessionTranscriptProjection; + readonly startOrdinal: number; + readonly totalPositions: number; + readonly positions: readonly SessionTurnPosition[]; + readonly hasOlder: boolean; + readonly hasNewer: boolean; + }; + +export interface SessionTranscriptRecordsByPositionKeysSnapshotRequest { + readonly sessionId: string; + readonly projection: SessionTranscriptProjection; + readonly snapshotLeaseId: string; + readonly snapshotKey: SessionTurnPositionSnapshotKey; + readonly positionKeys: readonly SessionTranscriptBodyPositionKey[]; + readonly maxBytes: number; + readonly maxRecords: number; +} + +export interface SessionTranscriptRecordsByPositionKeysSnapshotResult { + readonly snapshotKey: SessionTurnPositionSnapshotKey; + readonly projection: SessionTranscriptProjection; + readonly records: readonly { + readonly positionKey: SessionTranscriptBodyPositionKey; + readonly sequence: number; + readonly message: StoredMessage; + }[]; + readonly rawBytes: number; +} + +export interface SessionTurnPositionSnapshotReleaseRequest { + readonly sessionId: string; + readonly projection: SessionTranscriptProjection; + readonly snapshotLeaseId: string; + readonly snapshotKey: SessionTurnPositionSnapshotKey; +} + +export class SessionTurnPositionSnapshotMismatchError extends Error { + readonly name = 'SessionTurnPositionSnapshotMismatchError'; + readonly code = 'session_turn_position_snapshot_mismatch'; + + constructor(readonly sessionId: string) { + super(`Session Turn-position snapshot is stale or mismatched: ${sessionId}`); + } +} + +export class SessionTurnPositionRecoveryError extends Error { + readonly name = 'SessionTurnPositionRecoveryError'; + readonly code = 'session_turn_position_recovery_failed'; + + constructor( + readonly sessionId: string, + readonly reason: 'corrupt_source' | 'incompatible_identity' | 'hybrid_missing_admission', + readonly sequence?: number, + options?: ErrorOptions, + ) { + super(`Session Turn-position recovery failed (${reason}): ${sessionId}`, options); + } +} + +export class SessionTurnPositionAnchorNotFoundError extends Error { + readonly name = 'SessionTurnPositionAnchorNotFoundError'; + readonly code = 'session_turn_position_anchor_not_found'; + + constructor( + readonly sessionId: string, + readonly turnId: string, + ) { + super(`Session Turn-position anchor does not exist: ${sessionId}/${turnId}`); + } +} + +export class SessionTurnPositionLimitError extends Error { + readonly name = 'SessionTurnPositionLimitError'; + readonly code = 'session_turn_position_limit_exceeded'; + + constructor( + readonly sessionId: string, + readonly reason: 'page_metadata_bytes' | 'transcript_record_count' | 'transcript_record_bytes', + ) { + super(`Session Turn-position limit exceeded (${reason}): ${sessionId}`); + } +} + export interface SessionStore { create(input: CreateSessionInput, initialBoundary?: ExecutionBoundary): Promise; list(filter?: SessionListFilter): Promise; @@ -327,6 +483,13 @@ export interface SessionStore { sessionId: string, maxLandmarks: number, ): Promise; + readTurnPositionPageSnapshot( + request: SessionTurnPositionPageSnapshotRequest, + ): Promise; + readTranscriptRecordsByPositionKeysSnapshot( + request: SessionTranscriptRecordsByPositionKeysSnapshotRequest, + ): Promise; + releaseTurnPositionSnapshot(request: SessionTurnPositionSnapshotReleaseRequest): Promise; /** Read durable messages for startup recovery. */ readMessagesForRecovery(sessionId: string): Promise; /** Derive durable turns without triggering connection-lock self-healing. */ @@ -946,6 +1109,27 @@ class SqliteSessionStore implements SessionAuthorityStore { return this.metadata.readTurnLandmarks(sessionId, maxLandmarks); } + async readTurnPositionPageSnapshot( + request: SessionTurnPositionPageSnapshotRequest, + ): Promise { + await this.ensureReady(); + return this.metadata.readTurnPositionPage(request); + } + + async readTranscriptRecordsByPositionKeysSnapshot( + request: SessionTranscriptRecordsByPositionKeysSnapshotRequest, + ): Promise { + await this.ensureReady(); + return this.metadata.readTranscriptRecordsByPositionKeys(request); + } + + async releaseTurnPositionSnapshot( + request: SessionTurnPositionSnapshotReleaseRequest, + ): Promise { + await this.ensureReady(); + await this.metadata.releaseTurnPositionSnapshot(request); + } + async readMessagesForRecovery(sessionId: string): Promise { await this.ensureReady(); return this.metadata.readMessagesForRecovery(sessionId); diff --git a/packages/storage/src/session-turn-admission-recovery.ts b/packages/storage/src/session-turn-admission-recovery.ts new file mode 100644 index 0000000000..136c7af466 --- /dev/null +++ b/packages/storage/src/session-turn-admission-recovery.ts @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; +import { SessionTurnPositionRecoveryError } from './session-store.js'; + +export const SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS = 1_024; + +export interface SessionTurnAdmissionRecoveryResult { + readonly complete: boolean; +} + +interface PersistedRecoveryFailure { + readonly failure_reason: 'corrupt_source' | 'incompatible_identity' | 'hybrid_missing_admission'; + readonly failure_sequence: number; +} + +interface AdmissionRecoveryState { + readonly admission_cursor_admitted_at: number | null; + readonly admission_cursor_turn_id: string | null; + readonly admission_recovery_complete: number; + readonly failure_reason: PersistedRecoveryFailure['failure_reason'] | null; + readonly failure_sequence: number | null; +} + +/** + * Reconciles only canonical admission identity/order scalars. Transcript + * membership and body decoding remain owned by their existing recovery path. + */ +export function advanceSessionTurnAdmissionRecovery( + db: DatabaseSync, + input: { + readonly sessionId: string; + readonly maxAdmissions: number; + }, +): SessionTurnAdmissionRecoveryResult { + if ( + !Number.isSafeInteger(input.maxAdmissions) || + input.maxAdmissions < 1 || + input.maxAdmissions > SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS + ) { + throw new SessionTurnPositionRecoveryError(input.sessionId, 'corrupt_source'); + } + const state = db + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete, failure_reason, failure_sequence + FROM session_turn_index_state WHERE session_id = ?`) + .get(input.sessionId) as unknown as AdmissionRecoveryState | undefined; + if (!state) throw new SessionTurnPositionRecoveryError(input.sessionId, 'corrupt_source'); + if (state.failure_reason !== null) { + throw new SessionTurnPositionRecoveryError( + input.sessionId, + state.failure_reason, + state.failure_sequence ?? 0, + ); + } + if (state.admission_recovery_complete === 1) return { complete: true }; + const rows = ( + state.admission_cursor_admitted_at === null + ? db + .prepare(` + SELECT turn_id, admitted_at + FROM core_root_turn_admissions + WHERE session_id = ? + ORDER BY admitted_at, turn_id + LIMIT ? + `) + .all(input.sessionId, input.maxAdmissions) + : db + .prepare(` + SELECT turn_id, admitted_at + FROM core_root_turn_admissions + WHERE session_id = ? AND (admitted_at, turn_id) > (?, ?) + ORDER BY admitted_at, turn_id + LIMIT ? + `) + .all( + input.sessionId, + state.admission_cursor_admitted_at, + state.admission_cursor_turn_id, + input.maxAdmissions, + ) + ) as Array<{ turn_id?: unknown; admitted_at?: unknown }>; + const readExisting = db.prepare(` + SELECT order_source, admitted_at + FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' AND position_id = ? + `); + const upsert = db.prepare(` + INSERT INTO session_turn_metadata( + session_id, position_kind, position_id, order_source, admitted_at, + owner_first_sequence, shared_first_sequence + ) VALUES (?, 'turn', ?, 'admission', ?, NULL, NULL) + ON CONFLICT(session_id, position_kind, position_id) DO UPDATE SET + order_source = 'admission', admitted_at = excluded.admitted_at + `); + let cursorAdmittedAt = state.admission_cursor_admitted_at; + let cursorTurnId = state.admission_cursor_turn_id; + for (const row of rows) { + if ( + typeof row.turn_id !== 'string' || + row.turn_id.length === 0 || + typeof row.admitted_at !== 'number' || + !Number.isSafeInteger(row.admitted_at) || + row.admitted_at < 0 + ) { + failAdmissionRecovery(db, input.sessionId); + } + const existing = readExisting.get(input.sessionId, row.turn_id) as + | { order_source: string; admitted_at: number | null } + | undefined; + if (existing?.order_source === 'admission' && existing.admitted_at !== row.admitted_at) { + failAdmissionRecovery(db, input.sessionId); + } + upsert.run(input.sessionId, row.turn_id, row.admitted_at); + cursorAdmittedAt = row.admitted_at; + cursorTurnId = row.turn_id; + } + const complete = rows.length < input.maxAdmissions; + db.prepare(`UPDATE session_turn_index_state + SET admission_cursor_admitted_at = ?, admission_cursor_turn_id = ?, + admission_recovery_complete = ? + WHERE session_id = ?`).run(cursorAdmittedAt, cursorTurnId, complete ? 1 : 0, input.sessionId); + return { complete }; +} + +function failAdmissionRecovery(db: DatabaseSync, sessionId: string): never { + db.prepare(`UPDATE session_turn_index_state + SET failure_origin = 'admission', failure_reason = 'corrupt_source', failure_sequence = 0 + WHERE session_id = ?`).run(sessionId); + throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source', 0); +} diff --git a/packages/storage/src/session-turn-identity-recovery.ts b/packages/storage/src/session-turn-identity-recovery.ts new file mode 100644 index 0000000000..fcb760cdac --- /dev/null +++ b/packages/storage/src/session-turn-identity-recovery.ts @@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash } from 'node:crypto'; +import type { DatabaseSync } from 'node:sqlite'; +import type { IHasher } from 'hash-wasm'; +import { + publishSessionTurnMembership, + SessionTurnMembershipPublicationError, +} from './session-turn-membership.js'; +import { + advanceSessionTurnIdentityScanner, + completeSessionTurnIdentityScanner, + createSessionTurnIdentityScannerState, + restoreSessionTurnIdentityScannerState, + serializeSessionTurnIdentityScannerState, + SessionTurnIdentityScannerError, + type SessionTurnIdentityScannerStateV1, + type SessionTurnRecoveredIdentity, +} from './session-turn-identity-scanner.js'; +import { + planForwardTranscriptSlice, + readTranscriptSlices, +} from './sqlite-session-transcript-slices.js'; +import { SQLITE_SESSION_MESSAGE_CHUNK_BYTES } from './sqlite-session-metadata-schema.js'; + +export const SESSION_TURN_IDENTITY_HASH_STATE_VERSION = 1; +export const SESSION_TURN_IDENTITY_HASH_ALGORITHM = 'sha256'; +export const SESSION_TURN_IDENTITY_HASH_IMPLEMENTATION = 'hash-wasm@4.12.0'; + +export type SessionTurnIdentityRecoveryFailureReason = + | 'corrupt_source' + | 'incompatible_identity' + | 'hybrid_missing_admission'; + +export interface SessionTurnIdentityRecoveryProgress { + readonly complete: boolean; + readonly nextSequence: number; + readonly currentByteOffset: number; + readonly sourceRecords: number; + readonly sourceBytes: number; + readonly lastStepRecords: number; + readonly lastStepBytes: number; +} + +export interface SessionTurnIdentityRecoveryFailureFact { + readonly complete: false; + readonly failure: SessionTurnIdentityRecoveryFailureReason; + readonly failureSequence: number; +} + +export type SessionTurnIdentityRecoveryResult = + | SessionTurnIdentityRecoveryProgress + | SessionTurnIdentityRecoveryFailureFact; + +interface IndexStateRow { + readonly indexed_through_sequence: number; + readonly source_records: number; + readonly source_bytes: number; + readonly failure_reason: SessionTurnIdentityRecoveryFailureReason | null; + readonly failure_sequence: number | null; +} + +interface PartialRow { + readonly sequence: number; + readonly byte_offset: number; + readonly record_bytes: number; + readonly chunked: number; + readonly expected_digest: string | null; + readonly message_id: string; + readonly message_type: string; + readonly hash_state_version: number; + readonly hash_algorithm: string; + readonly hash_implementation: string; + readonly hash_state: Uint8Array; + readonly scanner_state_version: number; + readonly scanner_state: string; + readonly derived_state_digest: string; +} + +interface SourceMetadata { + readonly sequence: number; + readonly message_id: string; + readonly message_type: string; + readonly record_bytes: number; + readonly chunked: number; + readonly expected_digest: string | null; +} + +/** + * Advance only the scalar identity index. This function never returns source + * bytes, JSON, fragments, or StoredMessage objects to its caller. + */ +export function advanceSessionTurnIdentityRecovery( + db: DatabaseSync, + input: { + readonly sessionId: string; + readonly throughSequence: number; + readonly maxSourceBytes: number; + readonly maxCompletedRecords: number; + readonly hasher: IHasher; + }, +): SessionTurnIdentityRecoveryResult { + const { sessionId, throughSequence, maxSourceBytes, maxCompletedRecords, hasher } = input; + const initial = readIndexState(db, sessionId); + if (initial.failure_reason !== null) { + return { + complete: false, + failure: initial.failure_reason, + failureSequence: initial.failure_sequence!, + }; + } + let nextSequence = initial.indexed_through_sequence + 1; + let stepBytes = 0; + let stepRecords = 0; + while ( + nextSequence <= throughSequence && + stepBytes < maxSourceBytes && + stepRecords < maxCompletedRecords + ) { + const source = readSourceMetadata(db, sessionId, nextSequence); + if (!source) return fail(db, sessionId, 'corrupt_source', nextSequence); + let partial = readPartial(db, sessionId); + let scanner: SessionTurnIdentityScannerStateV1; + let byteOffset: number; + if (partial && partial.sequence !== nextSequence) { + return fail(db, sessionId, 'corrupt_source', nextSequence); + } + if (partial && !sameSource(partial, source)) { + return fail(db, sessionId, 'corrupt_source', nextSequence); + } + try { + if (partial) { + validateDerivedEnvelope(partial); + hasher.load(Uint8Array.from(partial.hash_state)); + scanner = restoreSessionTurnIdentityScannerState(partial.scanner_state); + byteOffset = partial.byte_offset; + } else { + hasher.init(); + scanner = createSessionTurnIdentityScannerState(); + byteOffset = 0; + } + } catch { + // Derived scanner/hash state is reconstructible. Discard only the + // current partial record and restart it from the authoritative bytes. + db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); + partial = undefined; + hasher.init(); + scanner = createSessionTurnIdentityScannerState(); + byteOffset = 0; + } + + const available = maxSourceBytes - stepBytes; + const byteLength = planForwardTranscriptSlice( + source.record_bytes, + byteOffset, + available, + source.chunked === 1 ? SQLITE_SESSION_MESSAGE_CHUNK_BYTES : 1, + ); + if (byteLength === 0) break; + let data: Buffer; + try { + const bySequence = readTranscriptSlices(db, sessionId, [ + { + sequence: source.sequence, + byteOffset, + totalBytes: source.record_bytes, + byteLength, + chunked: source.chunked === 1, + payloadDigest: + source.expected_digest === null ? null : `sha256:${source.expected_digest}`, + }, + ]); + data = bySequence.get(source.sequence)!; + if (!data || data.byteLength !== byteLength) { + return fail(db, sessionId, 'corrupt_source', source.sequence); + } + hasher.update(data); + advanceSessionTurnIdentityScanner(scanner, data); + } catch (error) { + return fail( + db, + sessionId, + error instanceof SessionTurnIdentityScannerError ? error.reason : 'corrupt_source', + source.sequence, + error, + ); + } + byteOffset += byteLength; + stepBytes += byteLength; + db.prepare(` + UPDATE session_turn_index_state SET source_bytes = source_bytes + ? WHERE session_id = ? + `).run(byteLength, sessionId); + if (byteOffset < source.record_bytes) { + writePartial(db, sessionId, source, byteOffset, hasher.save(), scanner); + break; + } + + let identity: SessionTurnRecoveredIdentity; + try { + const digest = hasher.digest('hex'); + if (source.expected_digest !== null && digest !== source.expected_digest) { + return fail(db, sessionId, 'corrupt_source', source.sequence); + } + identity = completeSessionTurnIdentityScanner(scanner, { + messageId: source.message_id, + messageType: source.message_type, + }); + } catch (error) { + return fail( + db, + sessionId, + error instanceof SessionTurnIdentityScannerError ? error.reason : 'corrupt_source', + source.sequence, + error, + ); + } + try { + publishSessionTurnMembership(db, sessionId, source.sequence, identity); + } catch (error) { + return fail( + db, + sessionId, + error instanceof SessionTurnMembershipPublicationError ? error.reason : 'corrupt_source', + source.sequence, + error, + ); + } + db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); + nextSequence += 1; + stepRecords += 1; + db.prepare(` + UPDATE session_turn_index_state + SET indexed_through_sequence = ?, source_records = source_records + 1 + WHERE session_id = ? + `).run(source.sequence, sessionId); + } + const final = readIndexState(db, sessionId); + const partial = readPartial(db, sessionId); + return { + complete: final.indexed_through_sequence >= throughSequence, + nextSequence: final.indexed_through_sequence + 1, + currentByteOffset: partial?.byte_offset ?? 0, + sourceRecords: final.source_records, + sourceBytes: final.source_bytes, + lastStepRecords: stepRecords, + lastStepBytes: stepBytes, + }; +} + +function readIndexState(db: DatabaseSync, sessionId: string): IndexStateRow { + return db + .prepare(` + SELECT indexed_through_sequence, source_records, source_bytes, + failure_reason, failure_sequence + FROM session_turn_index_state WHERE session_id = ? + `) + .get(sessionId) as unknown as IndexStateRow; +} + +function readPartial(db: DatabaseSync, sessionId: string): PartialRow | undefined { + return db + .prepare('SELECT * FROM session_turn_identity_recovery WHERE session_id = ?') + .get(sessionId) as PartialRow | undefined; +} + +function readSourceMetadata( + db: DatabaseSync, + sessionId: string, + sequence: number, +): SourceMetadata | undefined { + const row = db + .prepare(` + SELECT message.sequence, message.message_id, message.message_type, + coalesce(payload.record_bytes, length(CAST(message.record_json AS BLOB))) AS record_bytes, + payload.record_bytes IS NOT NULL AS chunked, payload.sha256 AS expected_digest + FROM session_messages AS message + LEFT JOIN session_message_payloads AS payload + ON payload.session_id = message.session_id AND payload.sequence = message.sequence + WHERE message.session_id = ? AND message.sequence = ? + `) + .get(sessionId, sequence) as Partial | undefined; + if ( + !row || + row.sequence !== sequence || + typeof row.message_id !== 'string' || + row.message_id.length === 0 || + typeof row.message_type !== 'string' || + row.message_type.length === 0 || + !Number.isSafeInteger(row.record_bytes) || + (row.record_bytes ?? 0) < 1 || + (row.chunked !== 0 && row.chunked !== 1) || + (row.expected_digest !== null && + (typeof row.expected_digest !== 'string' || !/^[0-9a-f]{64}$/u.test(row.expected_digest))) + ) { + return undefined; + } + return row as SourceMetadata; +} + +function sameSource(partial: PartialRow, source: SourceMetadata): boolean { + return ( + partial.sequence === source.sequence && + partial.record_bytes === source.record_bytes && + partial.chunked === source.chunked && + partial.expected_digest === source.expected_digest && + partial.message_id === source.message_id && + partial.message_type === source.message_type && + partial.byte_offset >= 0 && + partial.byte_offset < source.record_bytes + ); +} + +function validateDerivedEnvelope(partial: PartialRow): void { + if ( + partial.hash_state_version !== SESSION_TURN_IDENTITY_HASH_STATE_VERSION || + partial.hash_algorithm !== SESSION_TURN_IDENTITY_HASH_ALGORITHM || + partial.hash_implementation !== SESSION_TURN_IDENTITY_HASH_IMPLEMENTATION || + partial.scanner_state_version !== 1 || + !(partial.hash_state instanceof Uint8Array) || + partial.hash_state.byteLength < 1 || + partial.hash_state.byteLength > 64 * 1024 || + partial.derived_state_digest !== derivedStateDigest(partial.hash_state, partial.scanner_state) + ) { + throw new Error('Unsupported derived recovery state'); + } +} + +function writePartial( + db: DatabaseSync, + sessionId: string, + source: SourceMetadata, + byteOffset: number, + hashState: Uint8Array, + scanner: SessionTurnIdentityScannerStateV1, +): void { + const scannerState = serializeSessionTurnIdentityScannerState(scanner); + const derivedDigest = derivedStateDigest(hashState, scannerState); + db.prepare(` + INSERT INTO session_turn_identity_recovery( + session_id, sequence, byte_offset, record_bytes, chunked, expected_digest, + message_id, message_type, hash_state_version, hash_algorithm, + hash_implementation, hash_state, scanner_state_version, scanner_state + , derived_state_digest + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, 'sha256', 'hash-wasm@4.12.0', ?, 1, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + sequence = excluded.sequence, byte_offset = excluded.byte_offset, + record_bytes = excluded.record_bytes, chunked = excluded.chunked, + expected_digest = excluded.expected_digest, message_id = excluded.message_id, + message_type = excluded.message_type, hash_state_version = excluded.hash_state_version, + hash_algorithm = excluded.hash_algorithm, + hash_implementation = excluded.hash_implementation, hash_state = excluded.hash_state, + scanner_state_version = excluded.scanner_state_version, + scanner_state = excluded.scanner_state, + derived_state_digest = excluded.derived_state_digest + `).run( + sessionId, + source.sequence, + byteOffset, + source.record_bytes, + source.chunked, + source.expected_digest, + source.message_id, + source.message_type, + hashState, + scannerState, + derivedDigest, + ); +} + +function derivedStateDigest(hashState: Uint8Array, scannerState: string): string { + return createHash('sha256') + .update('session-turn-identity-recovery-v1\0') + .update(SESSION_TURN_IDENTITY_HASH_ALGORITHM) + .update('\0') + .update(SESSION_TURN_IDENTITY_HASH_IMPLEMENTATION) + .update('\0') + .update(hashState) + .update('\0') + .update(scannerState, 'utf8') + .digest('hex'); +} + +function fail( + db: DatabaseSync, + sessionId: string, + reason: 'corrupt_source' | 'incompatible_identity', + sequence: number, + cause?: unknown, +): SessionTurnIdentityRecoveryFailureFact { + db.prepare(` + UPDATE session_turn_index_state + SET failure_origin = 'transcript', failure_reason = ?, failure_sequence = ? + WHERE session_id = ? + `).run(reason, sequence, sessionId); + db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); + void cause; + return { complete: false, failure: reason, failureSequence: sequence }; +} diff --git a/packages/storage/src/session-turn-identity-scanner.ts b/packages/storage/src/session-turn-identity-scanner.ts new file mode 100644 index 0000000000..87cb4ce045 --- /dev/null +++ b/packages/storage/src/session-turn-identity-scanner.ts @@ -0,0 +1,676 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + classifySessionTurnIdentity, + SessionTurnIdentityClassificationError, + type SessionTurnIdentity, +} from './session-turn-membership.js'; + +export const SESSION_TURN_IDENTITY_SCANNER_VERSION = 1; +export const SESSION_TURN_IDENTITY_SCANNER_MAX_STATE_BYTES = 64 * 1024; +export const SESSION_TURN_IDENTITY_SCANNER_MAX_CAPTURE_BYTES = 32 * 1024; +export const SESSION_TURN_IDENTITY_SCANNER_MAX_DEPTH = 4_096; + +type ObjectExpectation = 'key_or_end' | 'key' | 'colon' | 'value' | 'comma_or_end'; +type ArrayExpectation = 'value_or_end' | 'value' | 'comma_or_end'; +type ContainerState = + | { readonly kind: 'object'; expectation: ObjectExpectation; pendingKey: string | null } + | { readonly kind: 'array'; expectation: ArrayExpectation }; + +type LexicalState = + | { readonly kind: 'default' } + | { + readonly kind: 'string'; + readonly role: 'key' | 'value'; + capture: boolean; + value: string; + } + | { + readonly kind: 'escape'; + readonly role: 'key' | 'value'; + capture: boolean; + value: string; + } + | { + readonly kind: 'unicode'; + readonly role: 'key' | 'value'; + capture: boolean; + value: string; + digits: string; + } + | { readonly kind: 'number'; value: string } + | { readonly kind: 'literal'; readonly expected: 'true' | 'false' | 'null'; offset: number }; + +interface Utf8State { + needed: number; + codePoint: number; + minimum: number; +} + +export interface SessionTurnIdentityScannerStateV1 { + readonly version: 1; + rootStarted: boolean; + rootComplete: boolean; + stack: ContainerState[]; + lexical: LexicalState; + utf8: Utf8State; + seenIdentityKeys: string[]; + identity: { + id?: string; + type?: string; + turnId?: string; + turnIdPresent: boolean; + kind?: string; + kindPresent: boolean; + }; + capturedBytes: number; +} + +export type SessionTurnRecoveredIdentity = SessionTurnIdentity; + +export class SessionTurnIdentityScannerError extends Error { + readonly code = 'session_turn_identity_incompatible'; + + constructor( + readonly detail: string, + readonly reason: 'corrupt_source' | 'incompatible_identity' = 'corrupt_source', + ) { + super(`Session Turn identity envelope is incompatible: ${detail}`); + this.name = 'SessionTurnIdentityScannerError'; + } +} + +const IDENTITY_KEYS = new Set(['id', 'type', 'turnId', 'kind']); + +export function createSessionTurnIdentityScannerState(): SessionTurnIdentityScannerStateV1 { + return { + version: SESSION_TURN_IDENTITY_SCANNER_VERSION, + rootStarted: false, + rootComplete: false, + stack: [], + lexical: { kind: 'default' }, + utf8: { needed: 0, codePoint: 0, minimum: 0 }, + seenIdentityKeys: [], + identity: { turnIdPresent: false, kindPresent: false }, + capturedBytes: 0, + }; +} + +export function restoreSessionTurnIdentityScannerState( + encoded: string, +): SessionTurnIdentityScannerStateV1 { + if (Buffer.byteLength(encoded, 'utf8') > SESSION_TURN_IDENTITY_SCANNER_MAX_STATE_BYTES) { + throw new SessionTurnIdentityScannerError( + 'scanner state exceeds 64 KiB', + 'incompatible_identity', + ); + } + let value: unknown; + try { + value = JSON.parse(encoded); + } catch (error) { + throw new SessionTurnIdentityScannerError(`invalid persisted scanner state: ${String(error)}`); + } + if (!isScannerState(value)) { + throw new SessionTurnIdentityScannerError('unsupported persisted scanner state'); + } + return value; +} + +export function serializeSessionTurnIdentityScannerState( + state: SessionTurnIdentityScannerStateV1, +): string { + const encoded = JSON.stringify(state); + if (Buffer.byteLength(encoded, 'utf8') > SESSION_TURN_IDENTITY_SCANNER_MAX_STATE_BYTES) { + throw new SessionTurnIdentityScannerError( + 'scanner state exceeds 64 KiB', + 'incompatible_identity', + ); + } + return encoded; +} + +export function advanceSessionTurnIdentityScanner( + state: SessionTurnIdentityScannerStateV1, + fragment: Uint8Array, +): void { + for (const byte of fragment) { + const codePoint = decodeUtf8Byte(state.utf8, byte); + if (codePoint === null) continue; + consumeCodePoint(state, codePoint); + } + serializeSessionTurnIdentityScannerState(state); +} + +export function completeSessionTurnIdentityScanner( + state: SessionTurnIdentityScannerStateV1, + sqlIdentity: { readonly messageId: string; readonly messageType: string }, +): SessionTurnRecoveredIdentity { + if (state.utf8.needed !== 0) throw new SessionTurnIdentityScannerError('truncated UTF-8'); + if (state.lexical.kind === 'number') { + finishNumber(state); + } else if (state.lexical.kind !== 'default') { + throw new SessionTurnIdentityScannerError('truncated JSON token'); + } + if (!state.rootComplete || state.stack.length !== 0) { + throw new SessionTurnIdentityScannerError('truncated JSON structure'); + } + const { id, type, turnId, turnIdPresent, kind, kindPresent } = state.identity; + if (typeof id !== 'string' || id.length === 0 || typeof type !== 'string' || type.length === 0) { + throw new SessionTurnIdentityScannerError('missing id or type', 'incompatible_identity'); + } + if (id !== sqlIdentity.messageId || type !== sqlIdentity.messageType) { + throw new SessionTurnIdentityScannerError( + 'SQL/body id or type mismatch', + 'incompatible_identity', + ); + } + try { + return classifySessionTurnIdentity({ + id, + type, + turnIdPresent, + turnId, + kindPresent, + kind, + }); + } catch (error) { + if (error instanceof SessionTurnIdentityClassificationError) { + throw new SessionTurnIdentityScannerError(error.detail, 'incompatible_identity'); + } + throw error; + } +} + +function consumeCodePoint(state: SessionTurnIdentityScannerStateV1, codePoint: number): void { + const character = String.fromCodePoint(codePoint); + switch (state.lexical.kind) { + case 'string': + if (codePoint === 0x22) { + const token = state.lexical; + state.lexical = { kind: 'default' }; + acceptString(state, token.role, token.value); + } else if (codePoint === 0x5c) { + state.lexical = { ...state.lexical, kind: 'escape' }; + } else { + if (codePoint < 0x20) throw new SessionTurnIdentityScannerError('control byte in string'); + appendCaptured(state, character); + } + return; + case 'escape': { + const escaped = escapeCharacter(codePoint); + if (escaped === null) { + if (codePoint !== 0x75) throw new SessionTurnIdentityScannerError('invalid JSON escape'); + state.lexical = { ...state.lexical, kind: 'unicode', digits: '' }; + } else { + const previous = state.lexical; + state.lexical = { + kind: 'string', + role: previous.role, + capture: previous.capture, + value: previous.value, + }; + appendCaptured(state, escaped); + } + return; + } + case 'unicode': { + if (!/[0-9a-f]/iu.test(character)) { + throw new SessionTurnIdentityScannerError('invalid Unicode escape'); + } + state.lexical.digits += character; + if (state.lexical.digits.length === 4) { + const previous = state.lexical; + state.lexical = { + kind: 'string', + role: previous.role, + capture: previous.capture, + value: previous.value, + }; + appendCaptured(state, String.fromCharCode(Number.parseInt(previous.digits, 16))); + } + return; + } + case 'number': + if (/[0-9eE+.-]/u.test(character)) { + if (state.lexical.value.length >= 128) { + throw new SessionTurnIdentityScannerError( + 'number token is too long', + 'incompatible_identity', + ); + } + state.lexical.value += character; + return; + } + finishNumber(state); + consumeCodePoint(state, codePoint); + return; + case 'literal': + if (character !== state.lexical.expected[state.lexical.offset]) { + throw new SessionTurnIdentityScannerError('invalid JSON literal'); + } + state.lexical.offset += 1; + if (state.lexical.offset === state.lexical.expected.length) { + state.lexical = { kind: 'default' }; + acceptPrimitive(state, 'non-string'); + } + return; + case 'default': + break; + } + + if (codePoint === 0x20 || codePoint === 0x09 || codePoint === 0x0a || codePoint === 0x0d) return; + if (state.rootComplete) throw new SessionTurnIdentityScannerError('trailing JSON data'); + if (!state.rootStarted) { + if (character !== '{') { + throw new SessionTurnIdentityScannerError('root must be an object', 'incompatible_identity'); + } + state.rootStarted = true; + pushContainer(state, { kind: 'object', expectation: 'key_or_end', pendingKey: null }); + return; + } + const current = state.stack.at(-1); + if (!current) throw new SessionTurnIdentityScannerError('invalid JSON structure'); + if (character === '}' || character === ']') { + closeContainer(state, character); + return; + } + if (character === ',') { + if (current.expectation !== 'comma_or_end') { + throw new SessionTurnIdentityScannerError('unexpected comma'); + } + current.expectation = current.kind === 'object' ? 'key' : 'value'; + if (current.kind === 'object') current.pendingKey = null; + return; + } + if (character === ':') { + if (current.kind !== 'object' || current.expectation !== 'colon') { + throw new SessionTurnIdentityScannerError('unexpected colon'); + } + current.expectation = 'value'; + return; + } + if (character === '"') { + const role = + current.kind === 'object' && + (current.expectation === 'key_or_end' || current.expectation === 'key') + ? 'key' + : 'value'; + if (role === 'value') requireValueExpectation(current); + const capture = + (role === 'key' && state.stack.length === 1) || + (role === 'value' && + state.stack.length === 1 && + current.kind === 'object' && + current.pendingKey !== null && + IDENTITY_KEYS.has(current.pendingKey)); + state.lexical = { kind: 'string', role, capture, value: '' }; + return; + } + if (character === '{' || character === '[') { + requireValueExpectation(current); + acceptPrimitive(state, 'container'); + pushContainer( + state, + character === '{' + ? { kind: 'object', expectation: 'key_or_end', pendingKey: null } + : { kind: 'array', expectation: 'value_or_end' }, + ); + return; + } + requireValueExpectation(current); + if (character === '-' || /[0-9]/u.test(character)) { + state.lexical = { kind: 'number', value: character }; + return; + } + if (character === 't' || character === 'f' || character === 'n') { + state.lexical = { + kind: 'literal', + expected: character === 't' ? 'true' : character === 'f' ? 'false' : 'null', + offset: 1, + }; + return; + } + throw new SessionTurnIdentityScannerError('invalid JSON token'); +} + +function acceptString( + state: SessionTurnIdentityScannerStateV1, + role: 'key' | 'value', + value: string, +): void { + const current = state.stack.at(-1); + if (!current) throw new SessionTurnIdentityScannerError('string outside root'); + if (role === 'key') { + if ( + current.kind !== 'object' || + (current.expectation !== 'key_or_end' && current.expectation !== 'key') + ) { + throw new SessionTurnIdentityScannerError('unexpected object key'); + } + current.pendingKey = state.stack.length === 1 ? value : null; + current.expectation = 'colon'; + return; + } + if (current.kind === 'object' && state.stack.length === 1 && current.pendingKey) { + recordIdentityValue(state, current.pendingKey, value); + } + acceptPrimitive(state, 'string'); +} + +function recordIdentityValue( + state: SessionTurnIdentityScannerStateV1, + key: string, + value: string, +): void { + if (!IDENTITY_KEYS.has(key)) return; + if (state.seenIdentityKeys.includes(key)) { + throw new SessionTurnIdentityScannerError( + `duplicate top-level ${key}`, + 'incompatible_identity', + ); + } + state.seenIdentityKeys.push(key); + state.capturedBytes += Buffer.byteLength(value, 'utf8'); + if (state.capturedBytes > SESSION_TURN_IDENTITY_SCANNER_MAX_CAPTURE_BYTES) { + throw new SessionTurnIdentityScannerError( + 'captured identity exceeds 32 KiB', + 'incompatible_identity', + ); + } + if (key === 'turnId') state.identity.turnIdPresent = true; + if (key === 'kind') state.identity.kindPresent = true; + state.identity[key as 'id' | 'type' | 'turnId' | 'kind'] = value; +} + +function acceptPrimitive( + state: SessionTurnIdentityScannerStateV1, + token: 'string' | 'non-string' | 'container', +): void { + const current = state.stack.at(-1); + if (!current) throw new SessionTurnIdentityScannerError('value outside root'); + requireValueExpectation(current); + if ( + token !== 'string' && + current.kind === 'object' && + state.stack.length === 1 && + current.pendingKey !== null && + IDENTITY_KEYS.has(current.pendingKey) + ) { + throw new SessionTurnIdentityScannerError( + `${current.pendingKey} must be a string`, + 'incompatible_identity', + ); + } + current.expectation = 'comma_or_end'; +} + +function requireValueExpectation(current: ContainerState): void { + const valid = + (current.kind === 'object' && current.expectation === 'value') || + (current.kind === 'array' && + (current.expectation === 'value_or_end' || current.expectation === 'value')); + if (!valid) throw new SessionTurnIdentityScannerError('unexpected JSON value'); +} + +function pushContainer(state: SessionTurnIdentityScannerStateV1, container: ContainerState): void { + if (state.stack.length >= SESSION_TURN_IDENTITY_SCANNER_MAX_DEPTH) { + throw new SessionTurnIdentityScannerError('JSON nesting exceeds 4096', 'incompatible_identity'); + } + state.stack.push(container); +} + +function closeContainer(state: SessionTurnIdentityScannerStateV1, character: string): void { + const current = state.stack.at(-1); + if (!current || (character === '}' ? current.kind !== 'object' : current.kind !== 'array')) { + throw new SessionTurnIdentityScannerError('mismatched JSON container'); + } + const mayClose = + current.kind === 'object' + ? current.expectation === 'key_or_end' || current.expectation === 'comma_or_end' + : current.expectation === 'value_or_end' || current.expectation === 'comma_or_end'; + if (!mayClose) throw new SessionTurnIdentityScannerError('incomplete JSON container'); + state.stack.pop(); + if (state.stack.length === 0) state.rootComplete = true; +} + +function finishNumber(state: SessionTurnIdentityScannerStateV1): void { + if (state.lexical.kind !== 'number') return; + if (!/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/u.test(state.lexical.value)) { + throw new SessionTurnIdentityScannerError('invalid JSON number'); + } + state.lexical = { kind: 'default' }; + acceptPrimitive(state, 'non-string'); +} + +function appendCaptured(state: SessionTurnIdentityScannerStateV1, value: string): void { + const lexical = state.lexical; + if (lexical.kind !== 'string' && lexical.kind !== 'escape' && lexical.kind !== 'unicode') { + throw new SessionTurnIdentityScannerError('invalid string scanner state'); + } + if (!lexical.capture) return; + if (lexical.role === 'key' && state.stack.length === 1) { + const candidate = lexical.value + value; + if (![...IDENTITY_KEYS].some((key) => key.startsWith(candidate))) { + lexical.capture = false; + lexical.value = ''; + return; + } + } + lexical.value += value; + if (Buffer.byteLength(lexical.value, 'utf8') > SESSION_TURN_IDENTITY_SCANNER_MAX_CAPTURE_BYTES) { + throw new SessionTurnIdentityScannerError( + 'scanner capture exceeds 32 KiB', + 'incompatible_identity', + ); + } +} + +function escapeCharacter(codePoint: number): string | null { + switch (codePoint) { + case 0x22: + return '"'; + case 0x5c: + return '\\'; + case 0x2f: + return '/'; + case 0x62: + return '\b'; + case 0x66: + return '\f'; + case 0x6e: + return '\n'; + case 0x72: + return '\r'; + case 0x74: + return '\t'; + default: + return null; + } +} + +function decodeUtf8Byte(state: Utf8State, byte: number): number | null { + if (state.needed === 0) { + if (byte <= 0x7f) return byte; + if (byte >= 0xc2 && byte <= 0xdf) { + state.needed = 1; + state.codePoint = byte & 0x1f; + state.minimum = 0x80; + return null; + } + if (byte >= 0xe0 && byte <= 0xef) { + state.needed = 2; + state.codePoint = byte & 0x0f; + state.minimum = 0x800; + return null; + } + if (byte >= 0xf0 && byte <= 0xf4) { + state.needed = 3; + state.codePoint = byte & 0x07; + state.minimum = 0x10000; + return null; + } + throw new SessionTurnIdentityScannerError('invalid UTF-8 lead byte'); + } + if (byte < 0x80 || byte > 0xbf) { + throw new SessionTurnIdentityScannerError('invalid UTF-8 continuation byte'); + } + state.codePoint = (state.codePoint << 6) | (byte & 0x3f); + state.needed -= 1; + if (state.needed > 0) return null; + const codePoint = state.codePoint; + const minimum = state.minimum; + state.codePoint = 0; + state.minimum = 0; + if (codePoint < minimum || codePoint > 0x10ffff || (codePoint >= 0xd800 && codePoint <= 0xdfff)) { + throw new SessionTurnIdentityScannerError('invalid UTF-8 scalar'); + } + return codePoint; +} + +function isScannerState(value: unknown): value is SessionTurnIdentityScannerStateV1 { + if (typeof value !== 'object' || value === null) return false; + const candidate = value as Partial; + if ( + !( + candidate.version === SESSION_TURN_IDENTITY_SCANNER_VERSION && + typeof candidate.rootStarted === 'boolean' && + typeof candidate.rootComplete === 'boolean' && + Array.isArray(candidate.stack) && + candidate.stack.length <= SESSION_TURN_IDENTITY_SCANNER_MAX_DEPTH && + candidate.stack.every(isContainerState) && + isLexicalState(candidate.lexical) && + isUtf8State(candidate.utf8) && + Array.isArray(candidate.seenIdentityKeys) && + candidate.seenIdentityKeys.every( + (key): key is string => typeof key === 'string' && IDENTITY_KEYS.has(key), + ) && + new Set(candidate.seenIdentityKeys).size === candidate.seenIdentityKeys.length && + typeof candidate.identity === 'object' && + candidate.identity !== null && + isOptionalString(candidate.identity.id) && + isOptionalString(candidate.identity.type) && + isOptionalString(candidate.identity.turnId) && + typeof candidate.identity.turnIdPresent === 'boolean' && + isOptionalString(candidate.identity.kind) && + typeof candidate.identity.kindPresent === 'boolean' && + Number.isSafeInteger(candidate.capturedBytes) && + (candidate.capturedBytes ?? -1) >= 0 && + (candidate.capturedBytes ?? 0) <= SESSION_TURN_IDENTITY_SCANNER_MAX_CAPTURE_BYTES + ) + ) { + return false; + } + const state = candidate as SessionTurnIdentityScannerStateV1; + if ( + (!state.rootStarted && (state.rootComplete || state.stack.length !== 0)) || + (state.rootComplete && state.stack.length !== 0) || + (!state.rootComplete && state.rootStarted && state.stack.length === 0) || + state.identity.turnIdPresent !== (state.identity.turnId !== undefined) || + state.identity.kindPresent !== (state.identity.kind !== undefined) + ) { + return false; + } + const capturedBytes = ['id', 'type', 'turnId', 'kind'].reduce((total, key) => { + const captured = state.identity[key as keyof typeof state.identity]; + return total + (typeof captured === 'string' ? Buffer.byteLength(captured, 'utf8') : 0); + }, 0); + return capturedBytes === state.capturedBytes; +} + +function isContainerState(value: unknown): value is ContainerState { + if (typeof value !== 'object' || value === null) return false; + const candidate = value as Partial; + if (candidate.kind === 'array') { + return ( + candidate.expectation === 'value_or_end' || + candidate.expectation === 'value' || + candidate.expectation === 'comma_or_end' + ); + } + return ( + candidate.kind === 'object' && + (candidate.expectation === 'key_or_end' || + candidate.expectation === 'key' || + candidate.expectation === 'colon' || + candidate.expectation === 'value' || + candidate.expectation === 'comma_or_end') && + (candidate.pendingKey === null || typeof candidate.pendingKey === 'string') + ); +} + +function isLexicalState(value: unknown): value is LexicalState { + if (typeof value !== 'object' || value === null) return false; + const candidate = value as Partial & Record; + if (candidate.kind === 'default') return true; + if (candidate.kind === 'number') { + return typeof candidate.value === 'string' && candidate.value.length <= 128; + } + if (candidate.kind === 'literal') { + return ( + (candidate.expected === 'true' || + candidate.expected === 'false' || + candidate.expected === 'null') && + Number.isSafeInteger(candidate.offset) && + (candidate.offset as number) >= 1 && + (candidate.offset as number) < candidate.expected.length + ); + } + if (candidate.kind !== 'string' && candidate.kind !== 'escape' && candidate.kind !== 'unicode') { + return false; + } + if ( + (candidate.role !== 'key' && candidate.role !== 'value') || + typeof candidate.capture !== 'boolean' || + typeof candidate.value !== 'string' || + Buffer.byteLength(candidate.value, 'utf8') > SESSION_TURN_IDENTITY_SCANNER_MAX_CAPTURE_BYTES + ) { + return false; + } + return ( + candidate.kind !== 'unicode' || + (typeof candidate.digits === 'string' && + candidate.digits.length <= 3 && + /^[0-9a-f]*$/iu.test(candidate.digits)) + ); +} + +function isUtf8State(value: unknown): value is Utf8State { + if (typeof value !== 'object' || value === null) return false; + const candidate = value as Partial; + if ( + !Number.isSafeInteger(candidate.needed) || + (candidate.needed ?? -1) < 0 || + (candidate.needed ?? 4) > 3 || + !Number.isSafeInteger(candidate.codePoint) || + (candidate.codePoint ?? -1) < 0 || + !Number.isSafeInteger(candidate.minimum) || + (candidate.minimum ?? -1) < 0 + ) { + return false; + } + return candidate.needed === 0 + ? candidate.codePoint === 0 && candidate.minimum === 0 + : candidate.minimum === 0x80 || candidate.minimum === 0x800 || candidate.minimum === 0x10000; +} + +function isOptionalString(value: unknown): value is string | undefined { + return value === undefined || typeof value === 'string'; +} diff --git a/packages/storage/src/session-turn-membership.ts b/packages/storage/src/session-turn-membership.ts new file mode 100644 index 0000000000..64583aa50d --- /dev/null +++ b/packages/storage/src/session-turn-membership.ts @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; +import { + classifySharedSessionTranscriptVisibility, + isSessionSystemNoteKind, + isStoredMessageType, + type StoredMessage, +} from '@maka/core/session'; +import { sqliteTableExists } from './sqlite-schema-introspection.js'; + +export type SessionTurnIdentity = + | { + readonly kind: 'turn' | 'note'; + readonly positionId: string; + readonly sharedVisibility: boolean; + } + | { readonly kind: 'ignored' }; + +export class SessionTurnIdentityClassificationError extends Error { + constructor(readonly detail: string) { + super(`Session Turn identity is incompatible: ${detail}`); + this.name = 'SessionTurnIdentityClassificationError'; + } +} + +export class SessionTurnMembershipPublicationError extends Error { + constructor( + readonly reason: 'corrupt_source' | 'incompatible_identity', + readonly sequence: number, + ) { + super(`Session Turn membership publication failed (${reason}) at sequence ${sequence}`); + this.name = 'SessionTurnMembershipPublicationError'; + } +} + +export function classifySessionTurnIdentity(input: { + readonly id: string; + readonly type: string; + readonly turnIdPresent: boolean; + readonly turnId?: string; + readonly kindPresent: boolean; + readonly kind?: string; +}): SessionTurnIdentity { + if (input.id.length === 0) { + throw new SessionTurnIdentityClassificationError('message id must be a non-empty string'); + } + if (!isStoredMessageType(input.type)) { + throw new SessionTurnIdentityClassificationError('message type is unknown'); + } + const visibility = (() => { + if (input.type !== 'system_note') { + return classifySharedSessionTranscriptVisibility({ type: input.type }); + } + if (!input.kindPresent || !isSessionSystemNoteKind(input.kind)) { + throw new SessionTurnIdentityClassificationError('system note kind is unknown or missing'); + } + return classifySharedSessionTranscriptVisibility({ type: input.type, kind: input.kind }); + })(); + if (input.turnIdPresent) { + if (typeof input.turnId !== 'string' || input.turnId.length === 0) { + throw new SessionTurnIdentityClassificationError('turnId must be a non-empty string'); + } + return { + kind: 'turn', + positionId: input.turnId, + sharedVisibility: visibility === 'visible', + }; + } + if (input.type !== 'system_note') { + throw new SessionTurnIdentityClassificationError('non-system message is missing turnId'); + } + return visibility === 'visible' + ? { kind: 'note', positionId: input.id, sharedVisibility: true } + : { kind: 'ignored' }; +} + +export function classifyStoredMessageTurnIdentity(message: StoredMessage): SessionTurnIdentity { + const turnId = (message as { turnId?: unknown }).turnId; + return classifySessionTurnIdentity({ + id: message.id, + type: message.type, + turnIdPresent: typeof turnId === 'string', + turnId: typeof turnId === 'string' ? turnId : undefined, + kindPresent: message.type === 'system_note', + kind: message.type === 'system_note' ? message.kind : undefined, + }); +} + +export function publishSessionTurnMembership( + db: DatabaseSync, + sessionId: string, + sequence: number, + identity: SessionTurnIdentity, +): void { + if (identity.kind === 'ignored') return; + const admission = + identity.kind === 'turn' && sqliteTableExists(db, 'core_root_turn_admissions') + ? (db + .prepare(` + SELECT admitted_at FROM core_root_turn_admissions + WHERE session_id = ? AND turn_id = ? + `) + .get(sessionId, identity.positionId) as { admitted_at: number } | undefined) + : undefined; + db.prepare(` + INSERT INTO session_turn_metadata( + session_id, position_kind, position_id, order_source, admitted_at, + owner_first_sequence, shared_first_sequence + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id, position_kind, position_id) DO UPDATE SET + order_source = CASE WHEN excluded.order_source = 'admission' THEN 'admission' + ELSE session_turn_metadata.order_source END, + admitted_at = COALESCE(excluded.admitted_at, session_turn_metadata.admitted_at), + owner_first_sequence = CASE WHEN session_turn_metadata.owner_first_sequence IS NULL + THEN excluded.owner_first_sequence + ELSE MIN(session_turn_metadata.owner_first_sequence, excluded.owner_first_sequence) END, + shared_first_sequence = CASE + WHEN excluded.shared_first_sequence IS NULL THEN session_turn_metadata.shared_first_sequence + WHEN session_turn_metadata.shared_first_sequence IS NULL THEN excluded.shared_first_sequence + ELSE MIN(session_turn_metadata.shared_first_sequence, excluded.shared_first_sequence) END + `).run( + sessionId, + identity.kind, + identity.positionId, + admission ? 'admission' : 'legacy', + admission?.admitted_at ?? null, + sequence, + identity.sharedVisibility ? sequence : null, + ); + const inserted = db + .prepare(` + INSERT INTO session_turn_memberships( + session_id, sequence, position_kind, position_id, shared_visibility + ) VALUES (?, ?, ?, ?, ?) ON CONFLICT(session_id, sequence) DO NOTHING + `) + .run( + sessionId, + sequence, + identity.kind, + identity.positionId, + identity.sharedVisibility ? 1 : 0, + ); + if (inserted.changes === 1) return; + const existing = db + .prepare(` + SELECT position_kind, position_id, shared_visibility + FROM session_turn_memberships WHERE session_id = ? AND sequence = ? + `) + .get(sessionId, sequence) as + | { position_kind: string; position_id: string; shared_visibility: number } + | undefined; + if ( + existing?.position_kind !== identity.kind || + existing.position_id !== identity.positionId || + existing.shared_visibility !== (identity.sharedVisibility ? 1 : 0) + ) { + throw new SessionTurnMembershipPublicationError('corrupt_source', sequence); + } +} diff --git a/packages/storage/src/session-turn-position-authority.ts b/packages/storage/src/session-turn-position-authority.ts new file mode 100644 index 0000000000..d640c2bf39 --- /dev/null +++ b/packages/storage/src/session-turn-position-authority.ts @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; +import { + SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION, + type StoredMessage, +} from '@maka/core/session'; +import { SessionTurnPositionRecoveryError } from './session-store.js'; +import { + classifyStoredMessageTurnIdentity, + publishSessionTurnMembership, + SessionTurnIdentityClassificationError, + SessionTurnMembershipPublicationError, +} from './session-turn-membership.js'; +import { sqliteTableExists } from './sqlite-schema-introspection.js'; + +export function recordAppendedSessionTurnMetadata( + db: DatabaseSync, + sessionId: string, + firstSequence: number, + entries: readonly { + readonly message: StoredMessage; + readonly recordBytes: number; + }[], +): void { + if (!sqliteTableExists(db, 'session_turn_metadata') || entries.length === 0) return; + ensureTurnIndexRows(db, sessionId); + const fillsBodylessAdmission = entries.some(({ message }) => { + const turnId = (message as { turnId?: unknown }).turnId; + if (typeof turnId !== 'string' || turnId.length === 0) return false; + return Boolean( + db + .prepare(` + SELECT 1 FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' AND position_id = ? + AND order_source = 'admission' AND owner_first_sequence IS NULL + `) + .get(sessionId, turnId), + ); + }); + for (let index = 0; index < entries.length; index += 1) { + publishCanonicalMembership(db, sessionId, firstSequence + index, entries[index]!.message); + } + const state = db + .prepare('SELECT indexed_through_sequence FROM session_turn_index_state WHERE session_id = ?') + .get(sessionId) as { indexed_through_sequence: number }; + if (state.indexed_through_sequence === firstSequence - 1) { + const sourceBytes = entries.reduce((total, { recordBytes }) => total + recordBytes, 0); + if (!Number.isSafeInteger(sourceBytes) || sourceBytes < 0) { + throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source'); + } + db.prepare(` + UPDATE session_turn_index_state + SET indexed_through_sequence = ?, + source_records = source_records + ?, source_bytes = source_bytes + ? + WHERE session_id = ? + `).run(firstSequence + entries.length - 1, entries.length, sourceBytes, sessionId); + } + if (fillsBodylessAdmission) invalidateBuildingSnapshots(db, sessionId); +} + +export function recordRootTurnAdmissionForPositionIndex( + db: DatabaseSync, + sessionId: string, + turnId: string, + admittedAt: number, +): void { + if (!sqliteTableExists(db, 'session_turn_metadata')) return; + if (!db.prepare('SELECT 1 FROM session_metadata WHERE session_id = ?').get(sessionId)) return; + if (!turnId || !Number.isSafeInteger(admittedAt) || admittedAt < 0) { + throw new SessionTurnPositionRecoveryError(sessionId, 'incompatible_identity'); + } + ensureTurnIndexRows(db, sessionId); + const current = db + .prepare(` + SELECT order_source, admitted_at, owner_first_sequence + FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' AND position_id = ? + `) + .get(sessionId, turnId) as + | { + order_source: string; + admitted_at: number | null; + owner_first_sequence: number | null; + } + | undefined; + if (current?.order_source === 'admission' && current.admitted_at !== admittedAt) { + throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source'); + } + db.prepare(` + INSERT INTO session_turn_metadata( + session_id, position_kind, position_id, order_source, admitted_at, + owner_first_sequence, shared_first_sequence + ) VALUES (?, 'turn', ?, 'admission', ?, NULL, NULL) + ON CONFLICT(session_id, position_kind, position_id) DO UPDATE SET + order_source = 'admission', admitted_at = excluded.admitted_at + `).run(sessionId, turnId, admittedAt); + db.prepare(` + UPDATE session_turn_index_state + SET admission_cursor_admitted_at = ?, admission_cursor_turn_id = ? + WHERE session_id = ? AND admission_recovery_complete = 1 + AND (admission_cursor_admitted_at IS NULL + OR (admission_cursor_admitted_at, admission_cursor_turn_id) < (?, ?)) + `).run(admittedAt, turnId, sessionId, admittedAt, turnId); + db.prepare(` + UPDATE session_turn_index_state + SET failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ? AND failure_origin = 'admission' + AND failure_reason = 'hybrid_missing_admission' + AND EXISTS ( + SELECT 1 FROM session_turn_memberships AS membership + WHERE membership.session_id = session_turn_index_state.session_id + AND membership.sequence = session_turn_index_state.failure_sequence + AND membership.position_kind = 'turn' AND membership.position_id = ? + ) + `).run(sessionId, turnId); + invalidateBuildingSnapshots(db, sessionId); + advanceAuthorityRevision(db, sessionId); +} + +export function recordRootTurnAdmissionsPurgedForPositionIndex( + db: DatabaseSync, + sessionId: string, + canonicalAdmissionsDeleted: boolean, +): void { + if (!sqliteTableExists(db, 'session_turn_metadata')) return; + if (!db.prepare('SELECT 1 FROM session_metadata WHERE session_id = ?').get(sessionId)) return; + ensureTurnIndexRows(db, sessionId); + const removed = db + .prepare(` + DELETE FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' + AND order_source = 'admission' AND owner_first_sequence IS NULL + `) + .run(sessionId); + const downgraded = db + .prepare(` + UPDATE session_turn_metadata SET order_source = 'legacy', admitted_at = NULL + WHERE session_id = ? AND position_kind = 'turn' + AND order_source = 'admission' AND owner_first_sequence IS NOT NULL + `) + .run(sessionId); + if (!canonicalAdmissionsDeleted && removed.changes === 0 && downgraded.changes === 0) return; + resetSessionTurnAdmissionRecoveryState(db, sessionId); + db.prepare(` + UPDATE session_turn_index_state + SET failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ? AND failure_origin = 'admission' + `).run(sessionId); + invalidateBuildingSnapshots(db, sessionId); + advanceAuthorityRevision(db, sessionId); +} + +export function invalidateSessionTurnPositionIndex(db: DatabaseSync, sessionId: string): void { + if (!sqliteTableExists(db, 'session_turn_metadata')) return; + ensureTurnIndexRows(db, sessionId); + db.prepare('DELETE FROM session_turn_position_snapshots WHERE session_id = ?').run(sessionId); + db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); + db.prepare('DELETE FROM session_turn_metadata WHERE session_id = ?').run(sessionId); + resetSessionTurnAdmissionRecoveryState(db, sessionId); + db.prepare(` + UPDATE session_turn_index_state + SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, + failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ? + `).run(sessionId); + advanceAuthorityRevision(db, sessionId); +} + +export function ensureTurnIndexRows(db: DatabaseSync, sessionId: string): void { + db.prepare(` + INSERT INTO session_turn_authority_revisions(session_id, visibility_policy_version) + VALUES (?, ?) ON CONFLICT(session_id) DO NOTHING + `).run(sessionId, SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION); + db.prepare(` + INSERT INTO session_turn_index_state(session_id) + VALUES (?) ON CONFLICT(session_id) DO NOTHING + `).run(sessionId); + const authority = db + .prepare(` + SELECT visibility_policy_version FROM session_turn_authority_revisions + WHERE session_id = ? + `) + .get(sessionId) as { visibility_policy_version: number }; + if (authority.visibility_policy_version === SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION) { + return; + } + db.prepare('DELETE FROM session_turn_position_snapshots WHERE session_id = ?').run(sessionId); + db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); + db.prepare('DELETE FROM session_turn_metadata WHERE session_id = ?').run(sessionId); + resetSessionTurnAdmissionRecoveryState(db, sessionId); + db.prepare(` + UPDATE session_turn_index_state + SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, + failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ? + `).run(sessionId); + db.prepare(` + UPDATE session_turn_authority_revisions + SET visibility_policy_version = ?, authority_revision = authority_revision + 1 + WHERE session_id = ? + `).run(SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION, sessionId); +} + +function publishCanonicalMembership( + db: DatabaseSync, + sessionId: string, + sequence: number, + message: StoredMessage, +): void { + try { + publishSessionTurnMembership( + db, + sessionId, + sequence, + classifyStoredMessageTurnIdentity(message), + ); + } catch (error) { + if (error instanceof SessionTurnIdentityClassificationError) { + throw new SessionTurnPositionRecoveryError(sessionId, 'incompatible_identity', sequence, { + cause: error, + }); + } + if (error instanceof SessionTurnMembershipPublicationError) { + throw new SessionTurnPositionRecoveryError(sessionId, error.reason, sequence, { + cause: error, + }); + } + throw error; + } +} + +function invalidateBuildingSnapshots(db: DatabaseSync, sessionId: string): void { + db.prepare(` + DELETE FROM session_turn_position_snapshots WHERE session_id = ? AND state = 'building' + `).run(sessionId); +} + +function resetSessionTurnAdmissionRecoveryState(db: DatabaseSync, sessionId: string): void { + db.prepare(` + UPDATE session_turn_index_state + SET admission_cursor_admitted_at = NULL, admission_cursor_turn_id = NULL, + admission_recovery_complete = 0 + WHERE session_id = ? + `).run(sessionId); +} + +function advanceAuthorityRevision(db: DatabaseSync, sessionId: string): void { + db.prepare(` + UPDATE session_turn_authority_revisions + SET authority_revision = authority_revision + 1 WHERE session_id = ? + `).run(sessionId); +} diff --git a/packages/storage/src/session-turn-position-index.ts b/packages/storage/src/session-turn-position-index.ts new file mode 100644 index 0000000000..e729e33412 --- /dev/null +++ b/packages/storage/src/session-turn-position-index.ts @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Compatibility barrel for the package-private PR1 seam. Responsibilities +// live in separate authority, recovery, and immutable snapshot modules. +export { + ensureTurnIndexRows, + invalidateSessionTurnPositionIndex, + recordAppendedSessionTurnMetadata, + recordRootTurnAdmissionForPositionIndex, + recordRootTurnAdmissionsPurgedForPositionIndex, +} from './session-turn-position-authority.js'; +export { + SESSION_TURN_POSITION_BODY_MAX_BYTES, + SESSION_TURN_POSITION_BODY_MAX_RECORDS, + SESSION_TURN_POSITION_BODY_MAX_KEYS, + SESSION_TURN_POSITION_BUILD_MAX_POSITIONS, + SESSION_TURN_POSITION_MAX_PAGE_BYTES, + SESSION_TURN_POSITION_MAX_PAGE_POSITIONS, + advanceSessionTurnAdmissionRecoveryForSnapshot, + advanceSessionTurnPositionOrdinalBuild, + allocateOrRequireSessionTurnPositionSnapshot, + markSessionTurnRecoveryComplete, + pageReadySessionTurnPositionSnapshot, + readSessionTurnMembershipPreflight, + reclaimSessionTurnPositionSnapshotsForNewOwner, + releaseSessionTurnPositionSnapshot, + requireSnapshot, + snapshotKeyFromRow, + type SessionTurnPositionBuildPhase, + type SessionTurnPositionSnapshotRow, +} from './session-turn-position-snapshots.js'; diff --git a/packages/storage/src/session-turn-position-snapshots.ts b/packages/storage/src/session-turn-position-snapshots.ts new file mode 100644 index 0000000000..92dca0ef4f --- /dev/null +++ b/packages/storage/src/session-turn-position-snapshots.ts @@ -0,0 +1,963 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; +import { + SessionTurnPositionAnchorNotFoundError, + SessionTurnPositionLimitError, + SessionTurnPositionRecoveryError, + SessionTurnPositionSnapshotMismatchError, + type SessionTranscriptBodyPositionKey, + type SessionTranscriptProjection, + type SessionTranscriptRecordsByPositionKeysSnapshotRequest, + type SessionTurnPositionPageSnapshotRequest, + type SessionTurnPositionReadResult, + type SessionTurnPositionSnapshotKey, + type SessionTurnPositionSnapshotReleaseRequest, +} from './session-store.js'; +import { ensureTurnIndexRows } from './session-turn-position-authority.js'; +import { + advanceSessionTurnAdmissionRecovery, + SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS, +} from './session-turn-admission-recovery.js'; +import { sqliteTableExists } from './sqlite-schema-introspection.js'; + +export const SESSION_TURN_POSITION_MAX_PAGE_POSITIONS = 128; +export const SESSION_TURN_POSITION_MAX_PAGE_BYTES = 64 * 1024; +export const SESSION_TURN_POSITION_BUILD_MAX_POSITIONS = 1_024; +export const SESSION_TURN_POSITION_BODY_MAX_KEYS = 128; +export const SESSION_TURN_POSITION_BODY_MAX_RECORDS = 256; +export const SESSION_TURN_POSITION_BODY_MAX_BYTES = 16 * 1024 * 1024; + +export type SessionTurnPositionBuildPhase = + | 'recovering' + | 'legacy' + | 'admission' + | 'notes' + | 'ready'; + +export interface SessionTurnPositionSnapshotRow { + readonly slot: number; + readonly through_sequence: number | null; + readonly authority_revision: number; + readonly snapshot_generation: number; + readonly state: 'building' | 'ready'; + readonly build_phase: SessionTurnPositionBuildPhase; + readonly build_next_owner_ordinal: number; + readonly build_next_shared_ordinal: number; + readonly build_cursor_sequence: number; + readonly build_cursor_admitted_at: number | null; + readonly build_cursor_position_kind: 'turn' | 'note' | null; + readonly build_cursor_position_id: string | null; + readonly ready_owner_total: number | null; + readonly ready_shared_total: number | null; +} + +export type SessionTurnPositionAllocation = + | { readonly kind: 'snapshot'; readonly snapshot: SessionTurnPositionSnapshotRow } + | Extract; + +export interface SessionTurnPositionBuildStep { + readonly snapshot: SessionTurnPositionSnapshotRow; + readonly executedPhase: Exclude; +} + +export interface SessionTurnMembershipPreflight { + readonly records: readonly { + readonly positionKey: SessionTranscriptBodyPositionKey; + readonly sequence: number; + }[]; + readonly storedBytes: number; +} + +export function allocateOrRequireSessionTurnPositionSnapshot( + db: DatabaseSync, + request: SessionTurnPositionPageSnapshotRequest, +): SessionTurnPositionAllocation { + validatePageRequest(request); + ensureSessionExists(db, request.sessionId); + ensureTurnIndexRows(db, request.sessionId); + const actualThrough = readHighWater(db, request.sessionId); + if (request.snapshotKey) { + return { + kind: 'snapshot', + snapshot: requireLeasedSnapshot( + db, + request.sessionId, + request.projection, + request.snapshotLeaseId, + request.snapshotKey, + ), + }; + } + const throughSequence = + request.throughSequence === undefined ? actualThrough : request.throughSequence; + if ( + throughSequence !== null && + (!Number.isSafeInteger(throughSequence) || + throughSequence < 0 || + actualThrough === null || + throughSequence > actualThrough) + ) { + throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + } + const authorityRevision = readAuthorityRevision(db, request.sessionId); + const existing = findSnapshotByAuthority( + db, + request.sessionId, + throughSequence, + authorityRevision, + ); + if (existing) { + acquireLease( + db, + request.sessionId, + existing.snapshot_generation, + request.projection, + request.snapshotLeaseId, + ); + return { kind: 'snapshot', snapshot: existing }; + } + const occupied = db + .prepare('SELECT slot FROM session_turn_position_snapshots WHERE session_id = ? ORDER BY slot') + .all(request.sessionId) as Array<{ slot: number }>; + if (occupied.length >= 2) { + return { + kind: 'capacity', + projection: request.projection, + throughSequence, + authorityRevision, + retainedSnapshots: 2, + }; + } + const used = new Set(occupied.map(({ slot }) => slot)); + const slot = used.has(0) ? 1 : 0; + const generation = ( + db + .prepare(` + UPDATE session_turn_authority_revisions + SET next_snapshot_generation = next_snapshot_generation + 1 + WHERE session_id = ? + RETURNING next_snapshot_generation - 1 AS generation + `) + .get(request.sessionId) as { generation: number } + ).generation; + db.prepare(` + INSERT INTO session_turn_position_snapshots( + session_id, slot, through_sequence, authority_revision, snapshot_generation, + state, build_phase + ) VALUES (?, ?, ?, ?, ?, 'building', 'recovering') + `).run(request.sessionId, slot, throughSequence, authorityRevision, generation); + acquireLease(db, request.sessionId, generation, request.projection, request.snapshotLeaseId); + return { + kind: 'snapshot', + snapshot: requireSnapshot(db, request.sessionId, { + throughSequence, + authorityRevision, + snapshotGeneration: generation, + }), + }; +} + +export function markSessionTurnRecoveryComplete( + db: DatabaseSync, + sessionId: string, + snapshot: SessionTurnPositionSnapshotRow, +): SessionTurnPositionSnapshotRow { + if (snapshot.state !== 'building' || snapshot.build_phase !== 'recovering') return snapshot; + const indexState = db + .prepare(`SELECT indexed_through_sequence, admission_recovery_complete + FROM session_turn_index_state WHERE session_id = ?`) + .get(sessionId) as { + indexed_through_sequence: number; + admission_recovery_complete: number; + }; + if ( + indexState.admission_recovery_complete !== 1 || + (snapshot.through_sequence !== null && + indexState.indexed_through_sequence < snapshot.through_sequence) + ) { + return snapshot; + } + const updated = db + .prepare(` + UPDATE session_turn_position_snapshots SET build_phase = 'legacy' + WHERE session_id = ? AND slot = ? AND state = 'building' AND build_phase = 'recovering' + `) + .run(sessionId, snapshot.slot); + if (updated.changes !== 1) throw new SessionTurnPositionSnapshotMismatchError(sessionId); + return requireSnapshot(db, sessionId, snapshotKeyFromRow(snapshot)); +} + +export function advanceSessionTurnAdmissionRecoveryForSnapshot( + db: DatabaseSync, + sessionId: string, + snapshot: SessionTurnPositionSnapshotRow, +): SessionTurnPositionSnapshotRow { + if (snapshot.state !== 'building' || snapshot.build_phase !== 'recovering') { + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } + const recovery = advanceSessionTurnAdmissionRecovery(db, { + sessionId, + maxAdmissions: SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS, + }); + return recovery.complete ? markSessionTurnRecoveryComplete(db, sessionId, snapshot) : snapshot; +} + +export function advanceSessionTurnPositionOrdinalBuild( + db: DatabaseSync, + sessionId: string, + snapshot: SessionTurnPositionSnapshotRow, +): SessionTurnPositionBuildStep { + if (snapshot.state !== 'building' || snapshot.build_phase === 'recovering') { + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } + const executedPhase = snapshot.build_phase; + if (executedPhase === 'ready') throw new SessionTurnPositionSnapshotMismatchError(sessionId); + const boundary = validateHybridBoundary(db, sessionId, snapshot.through_sequence); + const rows = readBuildRows(db, sessionId, snapshot, boundary, executedPhase); + const insert = db.prepare(` + INSERT INTO session_turn_snapshot_positions( + session_id, snapshot_generation, position_kind, position_id, + owner_ordinal, shared_ordinal, owner_first_sequence, shared_first_sequence + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `); + let nextOwnerOrdinal = snapshot.build_next_owner_ordinal; + let nextSharedOrdinal = snapshot.build_next_shared_ordinal; + let sequenceCursor = snapshot.build_cursor_sequence; + let admittedAtCursor = snapshot.build_cursor_admitted_at; + let positionKindCursor = snapshot.build_cursor_position_kind; + let positionIdCursor = snapshot.build_cursor_position_id; + for (const row of rows) { + const sharedVisible = + row.shared_first_sequence !== null && + snapshot.through_sequence !== null && + row.shared_first_sequence <= snapshot.through_sequence; + insert.run( + sessionId, + snapshot.snapshot_generation, + row.position_kind, + row.position_id, + nextOwnerOrdinal, + sharedVisible ? nextSharedOrdinal : null, + row.owner_first_sequence, + sharedVisible ? row.shared_first_sequence : null, + ); + nextOwnerOrdinal += 1; + if (sharedVisible) nextSharedOrdinal += 1; + sequenceCursor = row.owner_first_sequence ?? sequenceCursor; + admittedAtCursor = row.admitted_at; + positionKindCursor = row.position_kind; + positionIdCursor = row.position_id; + } + const exhausted = rows.length < SESSION_TURN_POSITION_BUILD_MAX_POSITIONS; + const nextPhase = exhausted ? followingPhase(executedPhase) : executedPhase; + if (nextPhase === 'ready') { + if (nextOwnerOrdinal === 0) { + insert.run(sessionId, snapshot.snapshot_generation, 'empty', '', 0, 0, null, null); + nextOwnerOrdinal = 1; + nextSharedOrdinal = 1; + } else if (nextSharedOrdinal === 0) { + insert.run(sessionId, snapshot.snapshot_generation, 'empty', '', null, 0, null, null); + nextSharedOrdinal = 1; + } + const published = db + .prepare(` + UPDATE session_turn_position_snapshots + SET state = 'ready', build_phase = 'ready', + build_next_owner_ordinal = ?, build_next_shared_ordinal = ?, + ready_owner_total = ?, ready_shared_total = ? + WHERE session_id = ? AND slot = ? AND state = 'building' AND build_phase = ? + `) + .run( + nextOwnerOrdinal, + nextSharedOrdinal, + nextOwnerOrdinal, + nextSharedOrdinal, + sessionId, + snapshot.slot, + executedPhase, + ); + if (published.changes !== 1) throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } else { + const phaseChanged = nextPhase !== executedPhase; + const updated = db + .prepare(` + UPDATE session_turn_position_snapshots + SET build_phase = ?, build_next_owner_ordinal = ?, build_next_shared_ordinal = ?, + build_cursor_sequence = ?, build_cursor_admitted_at = ?, + build_cursor_position_kind = ?, build_cursor_position_id = ? + WHERE session_id = ? AND slot = ? AND state = 'building' AND build_phase = ? + `) + .run( + nextPhase, + nextOwnerOrdinal, + nextSharedOrdinal, + phaseChanged ? -1 : sequenceCursor, + phaseChanged ? null : admittedAtCursor, + phaseChanged ? null : positionKindCursor, + phaseChanged ? null : positionIdCursor, + sessionId, + snapshot.slot, + executedPhase, + ); + if (updated.changes !== 1) throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } + return { + snapshot: requireSnapshot(db, sessionId, snapshotKeyFromRow(snapshot)), + executedPhase, + }; +} + +export function pageReadySessionTurnPositionSnapshot( + db: DatabaseSync, + request: SessionTurnPositionPageSnapshotRequest, + snapshot: SessionTurnPositionSnapshotRow, +): Extract { + validatePageRequest(request); + requireLease( + db, + request.sessionId, + request.projection, + request.snapshotLeaseId, + snapshot.snapshot_generation, + ); + const columns = projectionColumns(request.projection); + const totalPositions = + request.projection === 'owner' ? snapshot.ready_owner_total : snapshot.ready_shared_total; + if (snapshot.state !== 'ready' || totalPositions === null || totalPositions < 1) { + throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + } + let selectedOrdinal: number; + switch (request.anchor.kind) { + case 'tail': + selectedOrdinal = totalPositions - 1; + break; + case 'ordinal': + selectedOrdinal = Math.min(request.anchor.ordinal, totalPositions - 1); + break; + case 'sequence': { + const row = db + .prepare(` + SELECT ${columns.ordinal} AS ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND ${columns.ordinal} IS NOT NULL AND ${columns.firstSequence} <= ? + ORDER BY ${columns.firstSequence} DESC LIMIT 1 + `) + .get(request.sessionId, snapshot.snapshot_generation, request.anchor.sequence) as + | { ordinal: number } + | undefined; + selectedOrdinal = row?.ordinal ?? 0; + break; + } + case 'turn': { + const row = db + .prepare(` + SELECT ${columns.ordinal} AS ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND position_kind = 'turn' AND position_id = ? + AND ${columns.ordinal} IS NOT NULL + `) + .get(request.sessionId, snapshot.snapshot_generation, request.anchor.turnId) as + | { ordinal: number } + | undefined; + if (!row) { + throw new SessionTurnPositionAnchorNotFoundError(request.sessionId, request.anchor.turnId); + } + selectedOrdinal = row.ordinal; + break; + } + } + let startOrdinal = + request.anchor.kind === 'tail' + ? Math.max(0, totalPositions - request.maxPositions) + : selectedOrdinal; + const rows = db + .prepare(` + SELECT ${columns.ordinal} AS ordinal, position_kind, position_id, + ${columns.firstSequence} AS first_sequence + FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND ${columns.ordinal} >= ? + ORDER BY ${columns.ordinal} LIMIT ? + `) + .all( + request.sessionId, + snapshot.snapshot_generation, + startOrdinal, + request.maxPositions, + ) as Array<{ + ordinal: number; + position_kind: 'turn' | 'note' | 'empty'; + position_id: string; + first_sequence: number | null; + }>; + const expectedRows = Math.min(request.maxPositions, totalPositions - startOrdinal); + if ( + rows.length !== expectedRows || + rows.some((row, index) => row.ordinal !== startOrdinal + index) + ) { + throw new SessionTurnPositionRecoveryError(request.sessionId, 'corrupt_source'); + } + let positions = rows.map((row) => ({ + ordinal: row.ordinal, + key: + row.position_kind === 'empty' + ? ({ kind: 'empty' } as const) + : ({ kind: row.position_kind, id: row.position_id } as const), + firstSequence: row.first_sequence, + })); + const key = snapshotKeyFromRow(snapshot); + while ( + positions.length > 0 && + Buffer.byteLength( + JSON.stringify({ + kind: 'page', + snapshotKey: key, + projection: request.projection, + startOrdinal, + totalPositions, + positions, + hasOlder: startOrdinal > 0, + hasNewer: startOrdinal + positions.length < totalPositions, + }), + 'utf8', + ) > SESSION_TURN_POSITION_MAX_PAGE_BYTES + ) { + if (request.anchor.kind === 'tail') { + positions = positions.slice(1); + startOrdinal += 1; + } else { + positions = positions.slice(0, -1); + } + } + if (positions.length === 0) { + throw new SessionTurnPositionLimitError(request.sessionId, 'page_metadata_bytes'); + } + return { + kind: 'page', + snapshotKey: key, + projection: request.projection, + startOrdinal, + totalPositions, + positions, + hasOlder: startOrdinal > 0, + hasNewer: startOrdinal + positions.length < totalPositions, + }; +} + +export function readSessionTurnMembershipPreflight( + db: DatabaseSync, + request: SessionTranscriptRecordsByPositionKeysSnapshotRequest, +): SessionTurnMembershipPreflight { + validateRecordRequest(request); + const snapshot = requireLeasedSnapshot( + db, + request.sessionId, + request.projection, + request.snapshotLeaseId, + request.snapshotKey, + ); + if (snapshot.state !== 'ready') { + throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + } + const unique = new Set(request.positionKeys.map(positionKeyIdentity)); + if (unique.size !== request.positionKeys.length) { + throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + } + const ordinalColumn = projectionColumns(request.projection).ordinal; + const selected = request.positionKeys.map((positionKey) => { + const row = db + .prepare(` + SELECT ${ordinalColumn} AS ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND position_kind = ? AND position_id = ? AND ${ordinalColumn} IS NOT NULL + `) + .get(request.sessionId, snapshot.snapshot_generation, positionKey.kind, positionKey.id) as + | { ordinal: number } + | undefined; + if (!row) throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + return { positionKey, ordinal: row.ordinal }; + }); + selected.sort((left, right) => left.ordinal - right.ordinal); + const records: Array<{ + positionKey: SessionTranscriptBodyPositionKey; + sequence: number; + }> = []; + let storedBytes = 0; + for (const selectedPosition of selected) { + if (request.snapshotKey.throughSequence === null) continue; + const remaining = request.maxRecords - records.length; + const rows = db + .prepare(` + SELECT membership.sequence, + coalesce(payload.record_bytes, length(CAST(message.record_json AS BLOB))) AS stored_bytes + FROM session_turn_memberships AS membership + INNER JOIN session_messages AS message + ON message.session_id = membership.session_id AND message.sequence = membership.sequence + LEFT JOIN session_message_payloads AS payload + ON payload.session_id = message.session_id AND payload.sequence = message.sequence + WHERE membership.session_id = ? AND membership.position_kind = ? + AND membership.position_id = ? AND membership.sequence <= ? + ORDER BY membership.sequence LIMIT ? + `) + .all( + request.sessionId, + selectedPosition.positionKey.kind, + selectedPosition.positionKey.id, + request.snapshotKey.throughSequence, + remaining + 1, + ) as Array<{ sequence: number; stored_bytes: number }>; + if (rows.length > remaining) { + throw new SessionTurnPositionLimitError(request.sessionId, 'transcript_record_count'); + } + for (const row of rows) { + if (!Number.isSafeInteger(row.stored_bytes) || row.stored_bytes < 1) { + throw new SessionTurnPositionRecoveryError( + request.sessionId, + 'corrupt_source', + row.sequence, + ); + } + storedBytes += row.stored_bytes; + if (storedBytes > request.maxBytes) { + throw new SessionTurnPositionLimitError(request.sessionId, 'transcript_record_bytes'); + } + records.push({ positionKey: selectedPosition.positionKey, sequence: row.sequence }); + } + } + if (new Set(records.map((record) => record.sequence)).size !== records.length) { + throw new SessionTurnPositionRecoveryError(request.sessionId, 'corrupt_source'); + } + return { records, storedBytes }; +} + +export function releaseSessionTurnPositionSnapshot( + db: DatabaseSync, + request: SessionTurnPositionSnapshotReleaseRequest, +): void { + validateProjection(request.projection); + validateLeaseId(request.snapshotLeaseId); + ensureTurnIndexRows(db, request.sessionId); + const snapshot = requireLeasedSnapshot( + db, + request.sessionId, + request.projection, + request.snapshotLeaseId, + request.snapshotKey, + ); + const released = db + .prepare(` + DELETE FROM session_turn_snapshot_leases + WHERE session_id = ? AND snapshot_generation = ? AND projection = ? AND lease_id = ? + `) + .run( + request.sessionId, + snapshot.snapshot_generation, + request.projection, + request.snapshotLeaseId, + ); + if (released.changes !== 1) { + throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + } + const retained = db + .prepare(` + SELECT 1 FROM session_turn_snapshot_leases + WHERE session_id = ? AND snapshot_generation = ? LIMIT 1 + `) + .get(request.sessionId, snapshot.snapshot_generation); + if (!retained) { + db.prepare(` + DELETE FROM session_turn_position_snapshots + WHERE session_id = ? AND snapshot_generation = ? + `).run(request.sessionId, snapshot.snapshot_generation); + } +} + +export function reclaimSessionTurnPositionSnapshotsForNewOwner(db: DatabaseSync): void { + if (!sqliteTableExists(db, 'session_turn_position_snapshots')) return; + db.prepare('DELETE FROM session_turn_position_snapshots').run(); +} + +export function snapshotKeyFromRow( + row: SessionTurnPositionSnapshotRow, +): SessionTurnPositionSnapshotKey { + return { + throughSequence: row.through_sequence, + authorityRevision: row.authority_revision, + snapshotGeneration: row.snapshot_generation, + }; +} + +export function requireSnapshot( + db: DatabaseSync, + sessionId: string, + key: SessionTurnPositionSnapshotKey, +): SessionTurnPositionSnapshotRow { + validateSnapshotKey(key); + const row = db + .prepare(` + SELECT slot, through_sequence, authority_revision, snapshot_generation, state, + build_phase, build_next_owner_ordinal, build_next_shared_ordinal, + build_cursor_sequence, build_cursor_admitted_at, build_cursor_position_kind, + build_cursor_position_id, ready_owner_total, ready_shared_total + FROM session_turn_position_snapshots + WHERE session_id = ? AND snapshot_generation = ? + `) + .get(sessionId, key.snapshotGeneration) as SessionTurnPositionSnapshotRow | undefined; + if ( + !row || + row.through_sequence !== key.throughSequence || + row.authority_revision !== key.authorityRevision + ) { + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } + return row; +} + +function requireLeasedSnapshot( + db: DatabaseSync, + sessionId: string, + projection: SessionTranscriptProjection, + leaseId: string, + key: SessionTurnPositionSnapshotKey, +): SessionTurnPositionSnapshotRow { + validateProjection(projection); + validateLeaseId(leaseId); + const snapshot = requireSnapshot(db, sessionId, key); + requireLease(db, sessionId, projection, leaseId, snapshot.snapshot_generation); + return snapshot; +} + +function requireLease( + db: DatabaseSync, + sessionId: string, + projection: SessionTranscriptProjection, + leaseId: string, + generation: number, +): void { + if ( + !db + .prepare(` + SELECT 1 FROM session_turn_snapshot_leases + WHERE session_id = ? AND snapshot_generation = ? AND projection = ? AND lease_id = ? + `) + .get(sessionId, generation, projection, leaseId) + ) { + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } +} + +function acquireLease( + db: DatabaseSync, + sessionId: string, + generation: number, + projection: SessionTranscriptProjection, + leaseId: string, +): void { + validateProjection(projection); + validateLeaseId(leaseId); + const inserted = db + .prepare(` + INSERT INTO session_turn_snapshot_leases( + session_id, snapshot_generation, projection, lease_id + ) VALUES (?, ?, ?, ?) + ON CONFLICT(session_id, projection, lease_id) DO NOTHING + `) + .run(sessionId, generation, projection, leaseId); + if (inserted.changes === 0) { + const existing = db + .prepare(` + SELECT snapshot_generation FROM session_turn_snapshot_leases + WHERE session_id = ? AND projection = ? AND lease_id = ? + `) + .get(sessionId, projection, leaseId) as { snapshot_generation: number } | undefined; + if (existing?.snapshot_generation === generation) return; + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } +} + +function findSnapshotByAuthority( + db: DatabaseSync, + sessionId: string, + throughSequence: number | null, + authorityRevision: number, +): SessionTurnPositionSnapshotRow | undefined { + return db + .prepare(` + SELECT slot, through_sequence, authority_revision, snapshot_generation, state, + build_phase, build_next_owner_ordinal, build_next_shared_ordinal, + build_cursor_sequence, build_cursor_admitted_at, build_cursor_position_kind, + build_cursor_position_id, ready_owner_total, ready_shared_total + FROM session_turn_position_snapshots + WHERE session_id = ? + AND ((through_sequence = ?) OR (through_sequence IS NULL AND ? IS NULL)) + AND authority_revision = ? + `) + .get(sessionId, throughSequence, throughSequence, authorityRevision) as + | SessionTurnPositionSnapshotRow + | undefined; +} + +interface BuildRow { + readonly position_kind: 'turn' | 'note'; + readonly position_id: string; + readonly owner_first_sequence: number | null; + readonly shared_first_sequence: number | null; + readonly admitted_at: number | null; +} + +function readBuildRows( + db: DatabaseSync, + sessionId: string, + snapshot: SessionTurnPositionSnapshotRow, + boundary: number | null, + phase: Exclude, +): BuildRow[] { + const through = snapshot.through_sequence; + if (phase === 'legacy') { + if (through === null) return []; + return db + .prepare(` + SELECT position_kind, position_id, owner_first_sequence, + shared_first_sequence, NULL AS admitted_at + FROM session_turn_metadata + WHERE session_id = ? AND owner_first_sequence <= ? AND order_source = 'legacy' + AND (? IS NULL OR owner_first_sequence < ?) + AND owner_first_sequence > ? + ORDER BY owner_first_sequence, position_kind, position_id LIMIT ? + `) + .all( + sessionId, + through, + boundary, + boundary, + snapshot.build_cursor_sequence, + SESSION_TURN_POSITION_BUILD_MAX_POSITIONS, + ) as unknown as BuildRow[]; + } + if (phase === 'admission') { + const watermarkSql = + through === null + ? 'owner_first_sequence IS NULL' + : '(owner_first_sequence IS NULL OR owner_first_sequence <= ?)'; + const parameters: Array = [sessionId]; + if (through !== null) parameters.push(through); + parameters.push( + snapshot.build_cursor_admitted_at, + snapshot.build_cursor_admitted_at, + snapshot.build_cursor_admitted_at, + snapshot.build_cursor_position_id, + SESSION_TURN_POSITION_BUILD_MAX_POSITIONS, + ); + return db + .prepare(` + SELECT position_kind, position_id, owner_first_sequence, + shared_first_sequence, admitted_at + FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' + AND order_source = 'admission' AND ${watermarkSql} + AND (? IS NULL OR admitted_at > ? OR (admitted_at = ? AND position_id > ?)) + ORDER BY admitted_at, position_id LIMIT ? + `) + .all(...parameters) as unknown as BuildRow[]; + } + if (through === null || boundary === null) return []; + return db + .prepare(` + SELECT position_kind, position_id, owner_first_sequence, + shared_first_sequence, NULL AS admitted_at + FROM session_turn_metadata + WHERE session_id = ? AND owner_first_sequence <= ? AND position_kind = 'note' + AND order_source = 'legacy' AND owner_first_sequence >= ? + AND owner_first_sequence > ? + ORDER BY owner_first_sequence, position_id LIMIT ? + `) + .all( + sessionId, + through, + boundary, + snapshot.build_cursor_sequence, + SESSION_TURN_POSITION_BUILD_MAX_POSITIONS, + ) as unknown as BuildRow[]; +} + +function validateHybridBoundary( + db: DatabaseSync, + sessionId: string, + throughSequence: number | null, +): number | null { + if (throughSequence === null) return null; + const boundary = ( + db + .prepare(` + SELECT MIN(owner_first_sequence) AS boundary FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' + AND order_source = 'admission' AND owner_first_sequence <= ? + `) + .get(sessionId, throughSequence) as { boundary: number | null } + ).boundary; + if (boundary !== null) { + const hybrid = db + .prepare(` + SELECT owner_first_sequence FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' AND order_source = 'legacy' + AND owner_first_sequence >= ? AND owner_first_sequence <= ? + ORDER BY owner_first_sequence LIMIT 1 + `) + .get(sessionId, boundary, throughSequence) as { owner_first_sequence: number } | undefined; + if (hybrid) { + db.prepare(` + UPDATE session_turn_index_state + SET failure_origin = 'admission', failure_reason = 'hybrid_missing_admission', + failure_sequence = ? + WHERE session_id = ? + `).run(hybrid.owner_first_sequence, sessionId); + throw new SessionTurnPositionRecoveryError( + sessionId, + 'hybrid_missing_admission', + hybrid.owner_first_sequence, + ); + } + } + return boundary; +} + +function followingPhase( + phase: Exclude, +): Exclude { + if (phase === 'legacy') return 'admission'; + if (phase === 'admission') return 'notes'; + return 'ready'; +} + +function validatePageRequest(request: SessionTurnPositionPageSnapshotRequest): void { + validateProjection(request.projection); + validateLeaseId(request.snapshotLeaseId); + if ( + !Number.isSafeInteger(request.maxPositions) || + request.maxPositions < 1 || + request.maxPositions > SESSION_TURN_POSITION_MAX_PAGE_POSITIONS || + (request.snapshotKey !== undefined && request.throughSequence !== undefined) + ) { + throw new Error('Invalid Session Turn-position page request'); + } + if ( + request.anchor.kind === 'ordinal' && + (!Number.isSafeInteger(request.anchor.ordinal) || request.anchor.ordinal < 0) + ) { + throw new Error('Invalid Session Turn-position ordinal anchor'); + } + if ( + request.anchor.kind === 'sequence' && + (!Number.isSafeInteger(request.anchor.sequence) || request.anchor.sequence < 0) + ) { + throw new Error('Invalid Session Turn-position sequence anchor'); + } + if (request.anchor.kind === 'turn' && request.anchor.turnId.length === 0) { + throw new Error('Invalid Session Turn-position Turn anchor'); + } +} + +function validateRecordRequest( + request: SessionTranscriptRecordsByPositionKeysSnapshotRequest, +): void { + validateProjection(request.projection); + validateLeaseId(request.snapshotLeaseId); + if ( + request.positionKeys.length < 1 || + request.positionKeys.length > SESSION_TURN_POSITION_BODY_MAX_KEYS || + request.positionKeys.some( + (positionKey) => + (positionKey.kind !== 'turn' && positionKey.kind !== 'note') || + typeof positionKey.id !== 'string' || + positionKey.id.length === 0, + ) || + !Number.isSafeInteger(request.maxRecords) || + request.maxRecords < 1 || + request.maxRecords > SESSION_TURN_POSITION_BODY_MAX_RECORDS || + !Number.isSafeInteger(request.maxBytes) || + request.maxBytes < 1 || + request.maxBytes > SESSION_TURN_POSITION_BODY_MAX_BYTES + ) { + throw new Error('Invalid Session Turn-position record request'); + } + validateSnapshotKey(request.snapshotKey); +} + +function projectionColumns(projection: SessionTranscriptProjection): { + readonly ordinal: 'owner_ordinal' | 'shared_ordinal'; + readonly firstSequence: 'owner_first_sequence' | 'shared_first_sequence'; +} { + return projection === 'owner' + ? { ordinal: 'owner_ordinal', firstSequence: 'owner_first_sequence' } + : { ordinal: 'shared_ordinal', firstSequence: 'shared_first_sequence' }; +} + +function positionKeyIdentity(positionKey: SessionTranscriptBodyPositionKey): string { + return `${positionKey.kind}\0${positionKey.id}`; +} + +function validateProjection(projection: string): asserts projection is SessionTranscriptProjection { + if (projection !== 'owner' && projection !== 'shared') { + throw new Error('Invalid Session transcript projection'); + } +} + +function validateLeaseId(leaseId: string): void { + if (!/^[A-Za-z0-9_-]{1,128}$/u.test(leaseId)) { + throw new Error('Invalid Session Turn-position snapshot lease id'); + } +} + +function validateSnapshotKey(key: SessionTurnPositionSnapshotKey): void { + if ( + (key.throughSequence !== null && + (!Number.isSafeInteger(key.throughSequence) || key.throughSequence < 0)) || + !Number.isSafeInteger(key.authorityRevision) || + key.authorityRevision < 0 || + !Number.isSafeInteger(key.snapshotGeneration) || + key.snapshotGeneration < 1 + ) { + throw new Error('Invalid Session Turn-position snapshot key'); + } +} + +function readHighWater(db: DatabaseSync, sessionId: string): number | null { + return ( + db + .prepare('SELECT MAX(sequence) AS high_water FROM session_messages WHERE session_id = ?') + .get(sessionId) as { high_water: number | null } + ).high_water; +} + +function readAuthorityRevision(db: DatabaseSync, sessionId: string): number { + return ( + db + .prepare(` + SELECT authority_revision FROM session_turn_authority_revisions WHERE session_id = ? + `) + .get(sessionId) as { authority_revision: number } + ).authority_revision; +} + +function ensureSessionExists(db: DatabaseSync, sessionId: string): void { + if (!db.prepare('SELECT 1 FROM session_metadata WHERE session_id = ?').get(sessionId)) { + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } +} diff --git a/packages/storage/src/sqlite-legacy-scheduling.ts b/packages/storage/src/sqlite-legacy-scheduling.ts index c24042aa06..4bf736cdba 100644 --- a/packages/storage/src/sqlite-legacy-scheduling.ts +++ b/packages/storage/src/sqlite-legacy-scheduling.ts @@ -26,6 +26,7 @@ import type { } from '@maka/core/scheduled-task'; import type { DatabaseSync } from 'node:sqlite'; import { canonicalizeLegacyPlanReminderCronExpression } from './legacy-cron-expression.js'; +import { sqliteTableExists } from './sqlite-schema-introspection.js'; const LEGACY_AUTOMATION_TABLES = [ 'automation_authority_state', @@ -40,7 +41,9 @@ export function assertLegacySchedulingSchema( versions: ReadonlyMap, ): void { const automationVersion = versions.get('automation'); - const automationTables = LEGACY_AUTOMATION_TABLES.filter((table) => hasTable(database, table)); + const automationTables = LEGACY_AUTOMATION_TABLES.filter((table) => + sqliteTableExists(database, table), + ); if (automationTables.length > 0 && automationVersion === undefined) { throw new Error('Legacy Automation schema registry is missing'); } @@ -55,14 +58,14 @@ export function assertLegacySchedulingSchema( if ( workflowVersion !== undefined && workflowVersion <= LAST_RELEASED_PLAN_REMINDER_WORKFLOW_VERSION && - !hasTable(database, 'workflow_plan_reminders') + !sqliteTableExists(database, 'workflow_plan_reminders') ) { throw new Error('The released Workflow schema is missing workflow_plan_reminders'); } if ( workflowVersion !== undefined && workflowVersion > LAST_RELEASED_PLAN_REMINDER_WORKFLOW_VERSION && - hasTable(database, 'workflow_plan_reminders') + sqliteTableExists(database, 'workflow_plan_reminders') ) { throw new Error('The current Workflow schema still contains released Plan Reminder state'); } @@ -113,7 +116,7 @@ function assertLegacyAutomationEmpty(database: DatabaseSync): void { } function readLegacyPlanReminders(database: DatabaseSync): ScheduledTask[] { - if (!hasTable(database, 'workflow_plan_reminders')) { + if (!sqliteTableExists(database, 'workflow_plan_reminders')) { return []; } return database @@ -286,13 +289,6 @@ function enumValue(value: unknown, values: T) return value as T[number]; } -function hasTable(database: DatabaseSync, name: string): boolean { - return ( - database.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?").get(name) !== - undefined - ); -} - function hasColumn(database: DatabaseSync, table: string, column: string): boolean { return ( database.prepare('SELECT 1 FROM pragma_table_info(?) WHERE name = ?').get(table, column) !== diff --git a/packages/storage/src/sqlite-schema-introspection.ts b/packages/storage/src/sqlite-schema-introspection.ts new file mode 100644 index 0000000000..d08015968a --- /dev/null +++ b/packages/storage/src/sqlite-schema-introspection.ts @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; + +export function sqliteTableExists(db: DatabaseSync, table: string): boolean { + return Boolean( + db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?").get(table), + ); +} diff --git a/packages/storage/src/sqlite-session-metadata-schema.ts b/packages/storage/src/sqlite-session-metadata-schema.ts index da49f651a7..20439fade4 100644 --- a/packages/storage/src/sqlite-session-metadata-schema.ts +++ b/packages/storage/src/sqlite-session-metadata-schema.ts @@ -19,7 +19,7 @@ import type { DatabaseSync } from 'node:sqlite'; -export const SQLITE_SESSION_METADATA_SCHEMA_VERSION = 34; +export const SQLITE_SESSION_METADATA_SCHEMA_VERSION = 35; export const SQLITE_SESSION_MESSAGE_CHUNK_BYTES = 64 * 1024; export const SQLITE_SESSION_MESSAGE_CHUNK_MARKER = '{"$maka":"session-message-chunks-v1"}'; @@ -1225,6 +1225,264 @@ const MIGRATIONS: ReadonlyMap = new Map([ SELECT 1; `, ], + [ + 35, + ` + CREATE TABLE IF NOT EXISTS session_turn_authority_revisions ( + session_id TEXT PRIMARY KEY, + authority_revision INTEGER NOT NULL DEFAULT 0 CHECK (authority_revision >= 0), + visibility_policy_version INTEGER NOT NULL DEFAULT 1 + CHECK (visibility_policy_version >= 1), + next_snapshot_generation INTEGER NOT NULL DEFAULT 1 + CHECK (next_snapshot_generation >= 1), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE TABLE IF NOT EXISTS session_turn_index_state ( + session_id TEXT PRIMARY KEY, + indexed_through_sequence INTEGER NOT NULL DEFAULT -1 + CHECK (indexed_through_sequence >= -1), + source_records INTEGER NOT NULL DEFAULT 0 CHECK (source_records >= 0), + source_bytes INTEGER NOT NULL DEFAULT 0 CHECK (source_bytes >= 0), + admission_cursor_admitted_at INTEGER CHECK (admission_cursor_admitted_at >= 0), + admission_cursor_turn_id TEXT + CHECK (admission_cursor_turn_id IS NULL OR length(CAST(admission_cursor_turn_id AS BLOB)) > 0), + admission_recovery_complete INTEGER NOT NULL DEFAULT 0 + CHECK (admission_recovery_complete IN (0, 1)), + failure_origin TEXT CHECK (failure_origin IN ('transcript', 'admission')), + failure_reason TEXT + CHECK (failure_reason IN ('corrupt_source', 'incompatible_identity', + 'hybrid_missing_admission')), + failure_sequence INTEGER CHECK (failure_sequence >= 0), + CHECK ((admission_cursor_admitted_at IS NULL) = (admission_cursor_turn_id IS NULL)), + CHECK ((failure_origin IS NULL) = (failure_reason IS NULL) + AND (failure_reason IS NULL) = (failure_sequence IS NULL)), + CHECK (failure_origin IS NULL + OR (failure_origin = 'transcript' + AND failure_reason IN ('corrupt_source', 'incompatible_identity')) + OR (failure_origin = 'admission' + AND failure_reason IN ('corrupt_source', 'hybrid_missing_admission'))), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE TABLE IF NOT EXISTS session_turn_identity_recovery ( + session_id TEXT PRIMARY KEY, + sequence INTEGER NOT NULL CHECK (sequence >= 0), + byte_offset INTEGER NOT NULL CHECK (byte_offset >= 0), + record_bytes INTEGER NOT NULL CHECK (record_bytes > 0), + chunked INTEGER NOT NULL CHECK (chunked IN (0, 1)), + expected_digest TEXT CHECK (expected_digest IS NULL OR length(expected_digest) = 64), + message_id TEXT NOT NULL, + message_type TEXT NOT NULL, + hash_state_version INTEGER NOT NULL CHECK (hash_state_version = 1), + hash_algorithm TEXT NOT NULL CHECK (hash_algorithm = 'sha256'), + hash_implementation TEXT NOT NULL CHECK (hash_implementation = 'hash-wasm@4.12.0'), + hash_state BLOB NOT NULL CHECK (length(hash_state) BETWEEN 1 AND 65536), + scanner_state_version INTEGER NOT NULL CHECK (scanner_state_version = 1), + scanner_state TEXT NOT NULL CHECK (length(CAST(scanner_state AS BLOB)) <= 65536), + derived_state_digest TEXT NOT NULL CHECK (length(derived_state_digest) = 64), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE, + FOREIGN KEY(session_id, sequence) + REFERENCES session_messages(session_id, sequence) ON DELETE CASCADE ON UPDATE CASCADE + ) WITHOUT ROWID; + + CREATE TABLE IF NOT EXISTS session_turn_metadata ( + session_id TEXT NOT NULL, + position_kind TEXT NOT NULL CHECK (position_kind IN ('turn', 'note')), + position_id TEXT NOT NULL CHECK (length(CAST(position_id AS BLOB)) > 0), + order_source TEXT NOT NULL CHECK (order_source IN ('legacy', 'admission')), + admitted_at INTEGER CHECK (admitted_at >= 0), + owner_first_sequence INTEGER CHECK (owner_first_sequence >= 0), + shared_first_sequence INTEGER CHECK (shared_first_sequence >= 0), + PRIMARY KEY(session_id, position_kind, position_id), + UNIQUE(session_id, owner_first_sequence), + UNIQUE(session_id, shared_first_sequence), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE INDEX IF NOT EXISTS session_turn_metadata_by_legacy_order + ON session_turn_metadata(session_id, owner_first_sequence, position_kind, position_id) + WHERE order_source = 'legacy'; + + CREATE INDEX IF NOT EXISTS session_turn_metadata_by_admission_order + ON session_turn_metadata(session_id, admitted_at, position_id) + WHERE order_source = 'admission'; + + CREATE INDEX IF NOT EXISTS session_turn_metadata_by_admission_sequence + ON session_turn_metadata(session_id, owner_first_sequence) + WHERE order_source = 'admission'; + + CREATE INDEX IF NOT EXISTS session_turn_metadata_by_shared_sequence + ON session_turn_metadata(session_id, shared_first_sequence, position_kind, position_id) + WHERE shared_first_sequence IS NOT NULL; + + CREATE TABLE IF NOT EXISTS session_turn_memberships ( + session_id TEXT NOT NULL, + sequence INTEGER NOT NULL CHECK (sequence >= 0), + position_kind TEXT NOT NULL CHECK (position_kind IN ('turn', 'note')), + position_id TEXT NOT NULL CHECK (length(CAST(position_id AS BLOB)) > 0), + shared_visibility INTEGER NOT NULL CHECK (shared_visibility IN (0, 1)), + PRIMARY KEY(session_id, sequence), + FOREIGN KEY(session_id, position_kind, position_id) + REFERENCES session_turn_metadata(session_id, position_kind, position_id) ON DELETE CASCADE, + FOREIGN KEY(session_id, sequence) + REFERENCES session_messages(session_id, sequence) ON DELETE CASCADE ON UPDATE CASCADE + ) WITHOUT ROWID; + + CREATE INDEX IF NOT EXISTS session_turn_memberships_by_position + ON session_turn_memberships(session_id, position_kind, position_id, sequence); + + CREATE TABLE IF NOT EXISTS session_turn_position_snapshots ( + session_id TEXT NOT NULL, + slot INTEGER NOT NULL CHECK (slot IN (0, 1)), + through_sequence INTEGER CHECK (through_sequence >= 0), + authority_revision INTEGER NOT NULL CHECK (authority_revision >= 0), + snapshot_generation INTEGER NOT NULL CHECK (snapshot_generation >= 1), + state TEXT NOT NULL CHECK (state IN ('building', 'ready')), + build_phase TEXT NOT NULL DEFAULT 'recovering' + CHECK (build_phase IN ('recovering', 'legacy', 'admission', 'notes', 'ready')), + build_next_owner_ordinal INTEGER NOT NULL DEFAULT 0 + CHECK (build_next_owner_ordinal >= 0), + build_next_shared_ordinal INTEGER NOT NULL DEFAULT 0 + CHECK (build_next_shared_ordinal >= 0), + build_cursor_sequence INTEGER NOT NULL DEFAULT -1 CHECK (build_cursor_sequence >= -1), + build_cursor_admitted_at INTEGER CHECK (build_cursor_admitted_at >= 0), + build_cursor_position_kind TEXT CHECK ( + build_cursor_position_kind IS NULL OR build_cursor_position_kind IN ('turn', 'note') + ), + build_cursor_position_id TEXT, + ready_owner_total INTEGER CHECK (ready_owner_total >= 0), + ready_shared_total INTEGER CHECK (ready_shared_total >= 0), + CHECK ( + (state = 'building' AND build_phase <> 'ready' + AND ready_owner_total IS NULL AND ready_shared_total IS NULL) + OR + (state = 'ready' AND build_phase = 'ready' + AND ready_owner_total >= 1 AND ready_shared_total >= 1) + ), + CHECK ((build_cursor_position_kind IS NULL) = (build_cursor_position_id IS NULL)), + PRIMARY KEY(session_id, slot), + UNIQUE(session_id, snapshot_generation), + UNIQUE(session_id, through_sequence, authority_revision), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE UNIQUE INDEX IF NOT EXISTS session_turn_position_snapshots_by_exact_authority + ON session_turn_position_snapshots( + session_id, + COALESCE(through_sequence, -1), + authority_revision + ); + + CREATE TABLE IF NOT EXISTS session_turn_snapshot_positions ( + session_id TEXT NOT NULL, + snapshot_generation INTEGER NOT NULL CHECK (snapshot_generation >= 1), + position_kind TEXT NOT NULL CHECK (position_kind IN ('turn', 'note', 'empty')), + position_id TEXT NOT NULL, + owner_ordinal INTEGER CHECK (owner_ordinal >= 0), + shared_ordinal INTEGER CHECK (shared_ordinal >= 0), + owner_first_sequence INTEGER CHECK (owner_first_sequence >= 0), + shared_first_sequence INTEGER CHECK (shared_first_sequence >= 0), + CHECK ((position_kind = 'empty' AND position_id = '') OR + (position_kind <> 'empty' AND length(CAST(position_id AS BLOB)) > 0)), + CHECK (owner_ordinal IS NOT NULL OR shared_ordinal IS NOT NULL), + CHECK (position_kind = 'empty' OR owner_ordinal IS NOT NULL), + CHECK (shared_ordinal IS NULL OR position_kind = 'empty' + OR shared_first_sequence IS NOT NULL), + CHECK (shared_ordinal IS NOT NULL OR shared_first_sequence IS NULL), + CHECK (position_kind <> 'empty' OR + (owner_first_sequence IS NULL AND shared_first_sequence IS NULL)), + PRIMARY KEY(session_id, snapshot_generation, position_kind, position_id), + FOREIGN KEY(session_id, snapshot_generation) + REFERENCES session_turn_position_snapshots(session_id, snapshot_generation) + ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE UNIQUE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_owner_ordinal + ON session_turn_snapshot_positions(session_id, snapshot_generation, owner_ordinal) + WHERE owner_ordinal IS NOT NULL; + + CREATE UNIQUE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_shared_ordinal + ON session_turn_snapshot_positions(session_id, snapshot_generation, shared_ordinal) + WHERE shared_ordinal IS NOT NULL; + + CREATE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_owner_sequence + ON session_turn_snapshot_positions( + session_id, + snapshot_generation, + owner_first_sequence, + owner_ordinal + ) WHERE owner_ordinal IS NOT NULL; + + CREATE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_shared_sequence + ON session_turn_snapshot_positions( + session_id, + snapshot_generation, + shared_first_sequence, + shared_ordinal + ) WHERE shared_ordinal IS NOT NULL; + + CREATE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_turn_anchor + ON session_turn_snapshot_positions( + session_id, snapshot_generation, position_kind, position_id + ) WHERE position_kind = 'turn'; + + CREATE TABLE IF NOT EXISTS session_turn_snapshot_leases ( + session_id TEXT NOT NULL, + snapshot_generation INTEGER NOT NULL CHECK (snapshot_generation >= 1), + projection TEXT NOT NULL CHECK (projection IN ('owner', 'shared')), + lease_id TEXT NOT NULL CHECK (length(CAST(lease_id AS BLOB)) BETWEEN 1 AND 128), + PRIMARY KEY(session_id, snapshot_generation, projection, lease_id), + UNIQUE(session_id, projection, lease_id), + FOREIGN KEY(session_id, snapshot_generation) + REFERENCES session_turn_position_snapshots(session_id, snapshot_generation) + ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE TRIGGER IF NOT EXISTS session_turn_ready_positions_no_insert + BEFORE INSERT ON session_turn_snapshot_positions + WHEN EXISTS ( + SELECT 1 FROM session_turn_position_snapshots + WHERE session_id = NEW.session_id + AND snapshot_generation = NEW.snapshot_generation + AND state = 'ready' + ) + BEGIN + SELECT RAISE(ABORT, 'ready Session Turn snapshot positions are immutable'); + END; + + CREATE TRIGGER IF NOT EXISTS session_turn_ready_positions_no_update + BEFORE UPDATE ON session_turn_snapshot_positions + WHEN EXISTS ( + SELECT 1 FROM session_turn_position_snapshots + WHERE session_id = OLD.session_id + AND snapshot_generation = OLD.snapshot_generation + AND state = 'ready' + ) + BEGIN + SELECT RAISE(ABORT, 'ready Session Turn snapshot positions are immutable'); + END; + + CREATE TRIGGER IF NOT EXISTS session_turn_ready_positions_no_delete + BEFORE DELETE ON session_turn_snapshot_positions + WHEN EXISTS ( + SELECT 1 FROM session_turn_position_snapshots + WHERE session_id = OLD.session_id + AND snapshot_generation = OLD.snapshot_generation + AND state = 'ready' + ) + BEGIN + SELECT RAISE(ABORT, 'ready Session Turn snapshot positions are immutable'); + END; + + CREATE TRIGGER IF NOT EXISTS session_turn_ready_snapshot_no_update + BEFORE UPDATE ON session_turn_position_snapshots + WHEN OLD.state = 'ready' + BEGIN + SELECT RAISE(ABORT, 'ready Session Turn snapshot is immutable'); + END; + `, + ], ]); if (MIGRATIONS.size !== SQLITE_SESSION_METADATA_SCHEMA_VERSION) { diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index aeaa008467..80331a76f0 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -19,6 +19,7 @@ import { createRequire } from 'node:module'; import { createHash } from 'node:crypto'; +import { createSHA256 } from 'hash-wasm'; import { dirname, resolve } from 'node:path'; import { tmpdir } from 'node:os'; import { existsSync, mkdirSync } from 'node:fs'; @@ -88,6 +89,7 @@ import { isSubagentSessionParent, isSubagentSessionRuntime, isSubagentSessionSpawn, + SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION, type SessionHeader, type SessionHeaderPatch, type StoredMessage, @@ -134,6 +136,8 @@ import { decodePersistedSessionHeader, normalizeSessionHeader, SessionNotFoundError, + SessionTurnPositionLimitError, + SessionTurnPositionRecoveryError, type ExternalSessionImportLookupResult, type SessionTranscriptMessageLookupRequest, type SessionTranscriptPageRequest, @@ -143,12 +147,19 @@ import { type SessionTurnContribution, type SessionTurnContributionPage, type SessionTurnLandmarkSnapshot, + type SessionTranscriptRecordsByPositionKeysSnapshotRequest, + type SessionTranscriptRecordsByPositionKeysSnapshotResult, + type SessionTurnPositionPageSnapshotRequest, + type SessionTurnPositionReadResult, + type SessionTurnPositionSnapshotKey, + type SessionTurnPositionSnapshotReleaseRequest, } from './session-store.js'; import { isDiscardableConversationCopy, isValidConversationCopyTransition, } from './session-conversation-copy.js'; import { catalogPreviewForUserMessage } from './session-message-projection.js'; +import { classifyStoredMessageTurnIdentity } from './session-turn-membership.js'; import { configureSqliteSessionMetadataDatabase, migrateSqliteSessionMetadataDatabase, @@ -157,6 +168,20 @@ import { SQLITE_SESSION_MESSAGE_CHUNK_BYTES, SQLITE_SESSION_MESSAGE_CHUNK_MARKER, } from './sqlite-session-metadata-schema.js'; +import { + advanceSessionTurnAdmissionRecoveryForSnapshot, + advanceSessionTurnPositionOrdinalBuild, + allocateOrRequireSessionTurnPositionSnapshot, + ensureTurnIndexRows, + invalidateSessionTurnPositionIndex, + pageReadySessionTurnPositionSnapshot, + readSessionTurnMembershipPreflight, + recordAppendedSessionTurnMetadata, + releaseSessionTurnPositionSnapshot, + requireSnapshot, + snapshotKeyFromRow, +} from './session-turn-position-index.js'; +import { advanceSessionTurnIdentityRecovery } from './session-turn-identity-recovery.js'; import type { OperationalStateDatabaseLease } from './operational-state-store.js'; import { buildSqliteSessionCatalogPageQuery, @@ -166,6 +191,15 @@ import { sqliteOrdinarySessionRolePredicate, sqliteRecoverableSessionRolePredicate, } from './sqlite-session-role-scope.js'; +import { + planForwardTranscriptSlice, + readTranscriptSlices, + requireStoredMessageSequence, + StoredSessionMessageIncompatibleError, + type TranscriptRecordSlice, +} from './sqlite-session-transcript-slices.js'; + +export { StoredSessionMessageIncompatibleError } from './sqlite-session-transcript-slices.js'; export { SQLITE_SESSION_METADATA_SCHEMA_VERSION } from './sqlite-session-metadata-schema.js'; @@ -405,19 +439,6 @@ export class SessionMetadataConflictError extends Error { readonly name: string = 'SessionMetadataConflictError'; } -export class StoredSessionMessageIncompatibleError extends Error { - readonly name = 'StoredSessionMessageIncompatibleError'; - readonly code = 'stored_session_message_incompatible'; - - constructor( - readonly sessionId: string, - readonly sequence: number, - options?: ErrorOptions, - ) { - super(`Stored Session message ${sequence} for ${sessionId} is incompatible`, options); - } -} - export class SessionMetadataVersionConflictError extends SessionMetadataConflictError { readonly name = 'SessionMetadataVersionConflictError'; @@ -1475,7 +1496,13 @@ export class SqliteSessionMetadataStore { const inserted = this.tryInsertHeader(normalized, 1, normalized.createdAt, true); if (!inserted) return 'existing'; if (encoded.length > 0) { - this.insertSessionMessagesSync(normalized.id, 0, encoded); + const recordBytes = this.insertSessionMessagesSync(normalized.id, 0, encoded); + recordAppendedSessionTurnMetadata( + this.db, + normalized.id, + 0, + encoded.map(({ message }, index) => ({ message, recordBytes: recordBytes[index]! })), + ); // Align with appendMessages' connection-lock semantics: a session // with any user message is treated as connection-locked, even when // the legacy header did not record it. @@ -1594,7 +1621,13 @@ export class SqliteSessionMetadataStore { throw new Error(`Invalid Session message sequence for ${sessionId}`); } const sequence = row.last_sequence + 1; - this.insertSessionMessagesSync(sessionId, sequence, encoded); + const recordBytes = this.insertSessionMessagesSync(sessionId, sequence, encoded); + recordAppendedSessionTurnMetadata( + this.db, + sessionId, + sequence, + encoded.map(({ message }, index) => ({ message, recordBytes: recordBytes[index]! })), + ); this.updateCatalogProjectionSync(sessionId, projection, false, lockConnection); }); } @@ -1818,11 +1851,17 @@ export class SqliteSessionMetadataStore { ) { throw new SessionMetadataConflictError('Invalid WorkHub transcript sequence'); } - this.insertSessionMessagesSync( + const recordBytes = this.insertSessionMessagesSync( WORKHUB_COORDINATION_SESSION_ID, sequenceRow.last_sequence + 1, [{ message: assignment, json: assignmentJson }], ); + recordAppendedSessionTurnMetadata( + this.db, + WORKHUB_COORDINATION_SESSION_ID, + sequenceRow.last_sequence + 1, + [{ message: assignment, recordBytes: recordBytes[0]! }], + ); this.updateCatalogProjectionSync(WORKHUB_COORDINATION_SESSION_ID, request.projection, false); return { kind: 'assigned' as const, targetCreated, assignment }; }); @@ -2182,6 +2221,13 @@ export class SqliteSessionMetadataStore { this.insertSessionMessagesSync(input.sessionId, currentLastSequence + 1, ordinaryEntries); tailLatest = ordinaryEntries.at(-1)?.message; } + if (historicalMissingMessages.size > 0 || ordinaryMissingMessages.size > 0) { + // Handoff recovery can insert into the historical prefix and shift + // durable sequence anchors. Expire every exact snapshot atomically + // with that mutation; an old key must fail closed rather than observe + // a position list whose body anchors have moved. + invalidateSessionTurnPositionIndex(this.db, input.sessionId); + } if (tailLatest?.type === 'user') { this.updateCatalogProjectionSync( input.sessionId, @@ -2430,7 +2476,7 @@ export class SqliteSessionMetadataStore { const byteLength = request.direction === 'older' ? edge - byteOffset - : Math.min(totalBytes - edge, available); + : planForwardTranscriptSlice(totalBytes, edge, available); const complete = request.direction === 'older' ? byteOffset === 0 : byteOffset + byteLength === totalBytes; slices.push({ @@ -2671,6 +2717,244 @@ export class SqliteSessionMetadataStore { return nullableStoredMessageSequence(row.high_water, sessionId); } + async readTurnPositionPage( + request: SessionTurnPositionPageSnapshotRequest, + ): Promise { + this.assertOpen(); + assertSafeSessionId(request.sessionId); + if (!isSessionTurnVisibilityPolicyCurrent(this.db, request.sessionId)) { + this.transaction(() => ensureTurnIndexRows(this.db, request.sessionId)); + } + if (request.snapshotKey) { + const ready = this.readTransaction(() => { + const snapshot = requireSnapshot(this.db, request.sessionId, request.snapshotKey!); + return snapshot.state === 'ready' + ? pageReadySessionTurnPositionSnapshot(this.db, request, snapshot) + : null; + }); + if (ready) return ready; + } + const hasher = await createSHA256(); + const outcome = this.transaction(() => { + const allocation = allocateOrRequireSessionTurnPositionSnapshot(this.db, request); + if (allocation.kind === 'capacity') return allocation; + let snapshot = allocation.snapshot; + if (snapshot.state === 'ready') { + return pageReadySessionTurnPositionSnapshot(this.db, request, snapshot); + } + if (snapshot.build_phase === 'recovering') { + let recovery: ReturnType | undefined; + let transcriptRecoveryAdvanced = false; + if (snapshot.through_sequence !== null) { + const indexed = this.db + .prepare(` + SELECT indexed_through_sequence FROM session_turn_index_state WHERE session_id = ? + `) + .get(request.sessionId) as { indexed_through_sequence: number }; + if (indexed.indexed_through_sequence < snapshot.through_sequence) { + transcriptRecoveryAdvanced = true; + recovery = advanceSessionTurnIdentityRecovery(this.db, { + sessionId: request.sessionId, + throughSequence: snapshot.through_sequence, + maxSourceBytes: 4 * 1024 * 1024, + maxCompletedRecords: 1_024, + hasher, + }); + if ('failure' in recovery) { + this.db + .prepare(`DELETE FROM session_turn_position_snapshots + WHERE session_id = ? AND slot = ? AND state = 'building'`) + .run(request.sessionId, snapshot.slot); + return recovery; + } + } + } + if (!transcriptRecoveryAdvanced) { + try { + snapshot = advanceSessionTurnAdmissionRecoveryForSnapshot( + this.db, + request.sessionId, + snapshot, + ); + } catch (error) { + if (error instanceof SessionTurnPositionRecoveryError) { + this.db + .prepare(`DELETE FROM session_turn_position_snapshots + WHERE session_id = ? AND slot = ? AND state = 'building'`) + .run(request.sessionId, snapshot.slot); + return { + complete: false as const, + failure: error.reason, + failureSequence: error.sequence ?? 0, + }; + } + throw error; + } + } + const state = this.db + .prepare(` + SELECT indexed_through_sequence, source_records, source_bytes + FROM session_turn_index_state WHERE session_id = ? + `) + .get(request.sessionId) as { + indexed_through_sequence: number; + source_records: number; + source_bytes: number; + }; + const partial = this.db + .prepare(` + SELECT byte_offset FROM session_turn_identity_recovery WHERE session_id = ? + `) + .get(request.sessionId) as { byte_offset: number } | undefined; + // Admission reconciliation owns scalar authority, not transcript source + // records/bytes. Its persisted cursor is intentionally package-private; + // callers observe progress when this phase advances to ordinal building. + return { + kind: 'building' as const, + snapshotKey: snapshotKeyFromRow(snapshot), + projection: request.projection, + progress: { + phase: 'recovering' as const, + nextSequence: state.indexed_through_sequence + 1, + currentByteOffset: partial?.byte_offset ?? 0, + sourceRecords: state.source_records, + sourceBytes: state.source_bytes, + builtPositions: + request.projection === 'owner' + ? snapshot.build_next_owner_ordinal + : snapshot.build_next_shared_ordinal, + lastStepRecords: recovery && !('failure' in recovery) ? recovery.lastStepRecords : 0, + lastStepBytes: recovery && !('failure' in recovery) ? recovery.lastStepBytes : 0, + lastStepPositions: 0, + }, + }; + } + let step: ReturnType; + try { + step = advanceSessionTurnPositionOrdinalBuild(this.db, request.sessionId, snapshot); + } catch (error) { + if (error instanceof SessionTurnPositionRecoveryError) { + this.db + .prepare(`DELETE FROM session_turn_position_snapshots + WHERE session_id = ? AND slot = ? AND state = 'building'`) + .run(request.sessionId, snapshot.slot); + return { + complete: false as const, + failure: error.reason, + failureSequence: error.sequence ?? 0, + }; + } + throw error; + } + const state = this.db + .prepare(` + SELECT indexed_through_sequence, source_records, source_bytes + FROM session_turn_index_state WHERE session_id = ? + `) + .get(request.sessionId) as { + indexed_through_sequence: number; + source_records: number; + source_bytes: number; + }; + return { + kind: 'building' as const, + snapshotKey: snapshotKeyFromRow(step.snapshot), + projection: request.projection, + progress: { + phase: step.executedPhase, + nextSequence: state.indexed_through_sequence + 1, + currentByteOffset: 0, + sourceRecords: state.source_records, + sourceBytes: state.source_bytes, + builtPositions: + request.projection === 'owner' + ? step.snapshot.build_next_owner_ordinal + : step.snapshot.build_next_shared_ordinal, + lastStepRecords: 0, + lastStepBytes: 0, + lastStepPositions: + request.projection === 'owner' + ? step.snapshot.build_next_owner_ordinal - snapshot.build_next_owner_ordinal + : step.snapshot.build_next_shared_ordinal - snapshot.build_next_shared_ordinal, + }, + }; + }); + if ('failure' in outcome) { + throw new SessionTurnPositionRecoveryError( + request.sessionId, + outcome.failure, + outcome.failureSequence, + ); + } + return outcome; + } + + async readTranscriptRecordsByPositionKeys( + request: SessionTranscriptRecordsByPositionKeysSnapshotRequest, + ): Promise { + this.assertOpen(); + assertSafeSessionId(request.sessionId); + if (!isSessionTurnVisibilityPolicyCurrent(this.db, request.sessionId)) { + this.transaction(() => ensureTurnIndexRows(this.db, request.sessionId)); + } + return this.readTransaction(() => { + const preflight = readSessionTurnMembershipPreflight(this.db, request); + const decodedBySequence = new Map(); + for (const row of readStoredMessageRows( + this.db, + request.sessionId, + preflight.records.map((record) => record.sequence), + )) { + try { + decodedBySequence.set( + row.sequence, + decodeStoredMessage(JSON.parse(row.recordJson) as unknown), + ); + } catch (error) { + throw new StoredSessionMessageIncompatibleError(request.sessionId, row.sequence, { + cause: error, + }); + } + } + const records = preflight.records.map(({ positionKey, sequence }) => { + const message = decodedBySequence.get(sequence); + if (!message) throw new StoredSessionMessageIncompatibleError(request.sessionId, sequence); + try { + const identity = classifyStoredMessageTurnIdentity(message); + if ( + identity.kind === 'ignored' || + identity.kind !== positionKey.kind || + identity.positionId !== positionKey.id + ) { + throw new Error('decoded transcript identity no longer matches the exact position'); + } + } catch (error) { + throw new StoredSessionMessageIncompatibleError(request.sessionId, sequence, { + cause: error, + }); + } + return { positionKey, sequence, message }; + }); + return { + snapshotKey: request.snapshotKey, + projection: request.projection, + records, + rawBytes: preflight.storedBytes, + }; + }); + } + + async releaseTurnPositionSnapshot( + request: SessionTurnPositionSnapshotReleaseRequest, + ): Promise { + this.assertOpen(); + assertSafeSessionId(request.sessionId); + if (!isSessionTurnVisibilityPolicyCurrent(this.db, request.sessionId)) { + this.transaction(() => ensureTurnIndexRows(this.db, request.sessionId)); + } + this.transaction(() => releaseSessionTurnPositionSnapshot(this.db, request)); + } + async readTurnContributions( sessionId: string, throughSequence: number | null, @@ -2708,21 +2992,13 @@ export class SqliteSessionMetadataStore { let sourceMessages = 0; let sourceBytes = 0; while (nextPosition <= fixedThrough) { - const rows = this.db - .prepare( - ` - SELECT message.sequence, message.record_json, payload.record_bytes, payload.sha256 - FROM session_messages AS message - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE message.session_id = ? - AND message.sequence >= ? - AND message.sequence <= ? - ORDER BY message.sequence ASC - LIMIT 128 - `, - ) - .all(sessionId, nextPosition, fixedThrough) as StoredSessionMessagePayloadRow[]; + const rows = readStoredMessagePayloadRange( + this.db, + sessionId, + nextPosition, + fixedThrough, + 128, + ); if (rows.length === 0) { throw new StoredSessionMessageIncompatibleError(sessionId, nextPosition); } @@ -5182,7 +5458,7 @@ export class SqliteSessionMetadataStore { readonly message: StoredMessage; readonly json: string; }[], - ): void { + ): readonly number[] { if ( !Number.isSafeInteger(firstSequence) || firstSequence < 0 || @@ -5203,10 +5479,12 @@ export class SqliteSessionMetadataStore { INSERT INTO session_message_chunks(session_id, sequence, chunk_index, data, sha256) VALUES (?, ?, ?, ?, ?) `); + const recordBytes: number[] = []; for (let index = 0; index < entries.length; index += 1) { const entry = entries[index]!; const sequence = firstSequence + index; const encoded = Buffer.from(entry.json, 'utf8'); + recordBytes.push(encoded.byteLength); const chunked = encoded.byteLength > SQLITE_SESSION_MESSAGE_CHUNK_BYTES; insertMessage.run( sessionId, @@ -5238,6 +5516,7 @@ export class SqliteSessionMetadataStore { ); } } + return recordBytes; } private replaceSessionMessageSync( @@ -6799,8 +7078,9 @@ function readStoredMessageRecordJson( ) { throw new StoredSessionMessageIncompatibleError(sessionId, sequence); } - const data = readChunkedTranscriptRecord(db, sessionId, sequence, recordBytes); - if (createHash('sha256').update(data).digest('hex') !== row.sha256) { + const payloadDigest = requireTranscriptPayloadDigest(row.sha256, sessionId, sequence); + const data = readChunkedTranscriptRecord(db, sessionId, sequence, recordBytes, payloadDigest); + if (`sha256:${createHash('sha256').update(data).digest('hex')}` !== payloadDigest) { throw new StoredSessionMessageIncompatibleError(sessionId, sequence); } recordJson = data.toString('utf8'); @@ -6868,6 +7148,28 @@ function foldTurnContribution( }; } +function readStoredMessagePayloadRange( + db: DatabaseSync, + sessionId: string, + startSequence: number, + throughSequence: number, + limit: number, +): StoredSessionMessagePayloadRow[] { + return db + .prepare(` + SELECT message.sequence, message.record_json, payload.record_bytes, payload.sha256 + FROM session_messages AS message + LEFT JOIN session_message_payloads AS payload + ON payload.session_id = message.session_id AND payload.sequence = message.sequence + WHERE message.session_id = ? + AND message.sequence >= ? + AND message.sequence <= ? + ORDER BY message.sequence ASC + LIMIT ? + `) + .all(sessionId, startSequence, throughSequence, limit) as StoredSessionMessagePayloadRow[]; +} + function readStoredMessageRows( db: DatabaseSync, sessionId: string, @@ -6904,65 +7206,38 @@ function readChunkedTranscriptRecord( sessionId: string, sequence: number, recordBytes: number, + payloadDigest: `sha256:${string}`, ): Buffer { - const rows = db - .prepare( - ` - SELECT chunk_index, data, sha256 - FROM session_message_chunks - WHERE session_id = ? AND sequence = ? - ORDER BY chunk_index - `, - ) - .all(sessionId, sequence) as Array<{ - chunk_index?: unknown; - data?: unknown; - sha256?: unknown; - }>; - const expectedChunks = Math.ceil(recordBytes / SQLITE_SESSION_MESSAGE_CHUNK_BYTES); - if (rows.length !== expectedChunks) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - const chunks = rows.map((row, index) => { - if ( - row.chunk_index !== index || - !(row.data instanceof Uint8Array) || - typeof row.sha256 !== 'string' - ) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - const chunk = Buffer.from(row.data); - if (createHash('sha256').update(chunk).digest('hex') !== row.sha256) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - return chunk; - }); - const data = Buffer.concat(chunks, recordBytes); - if (data.byteLength !== recordBytes) { + const data = readTranscriptSlices(db, sessionId, [ + { + sequence, + byteOffset: 0, + totalBytes: recordBytes, + byteLength: recordBytes, + chunked: true, + payloadDigest, + }, + ]).get(sequence); + if (!data || data.byteLength !== recordBytes) { throw new StoredSessionMessageIncompatibleError(sessionId, sequence); } return data; } -function requireStoredMessageSequence(value: unknown, sessionId: string): number { - if (!Number.isSafeInteger(value) || (value as number) < 0) { - throw new StoredSessionMessageIncompatibleError(sessionId, -1); - } - return value as number; -} - function nullableStoredMessageSequence(value: unknown, sessionId: string): number | null { if (value === null || value === undefined) return null; return requireStoredMessageSequence(value, sessionId); } -interface TranscriptRecordSlice { - readonly sequence: number; - readonly byteOffset: number; - readonly totalBytes: number; - readonly byteLength: number; - readonly chunked: boolean; - readonly payloadDigest: `sha256:${string}` | null; +function isSessionTurnVisibilityPolicyCurrent(db: DatabaseSync, sessionId: string): boolean { + const row = db + .prepare(`SELECT visibility_policy_version FROM session_turn_authority_revisions + WHERE session_id = ?`) + .get(sessionId) as { visibility_policy_version: number } | undefined; + return ( + row === undefined || + row.visibility_policy_version === SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION + ); } function requireTranscriptPayloadDigest( @@ -6987,118 +7262,6 @@ function requireTranscriptRecordByteLength( return value as number; } -function readTranscriptSlices( - db: DatabaseSync, - sessionId: string, - slices: readonly TranscriptRecordSlice[], -): Map { - if (slices.length === 0) return new Map(); - const chunkedSlices = slices.filter((slice) => slice.chunked); - const values = chunkedSlices.map(() => '(?, ?, ?)').join(', '); - const parameters = chunkedSlices.flatMap((slice) => [ - slice.sequence, - Math.floor(slice.byteOffset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES), - Math.floor((slice.byteOffset + slice.byteLength - 1) / SQLITE_SESSION_MESSAGE_CHUNK_BYTES), - ]); - const rows = - chunkedSlices.length === 0 - ? [] - : (db - .prepare( - ` - WITH requested(sequence, first_chunk, last_chunk) AS (VALUES ${values}) - SELECT requested.sequence, chunk.chunk_index, chunk.data, chunk.sha256 - FROM requested - INNER JOIN session_message_chunks AS chunk - ON chunk.session_id = ? - AND chunk.sequence = requested.sequence - AND chunk.chunk_index BETWEEN requested.first_chunk AND requested.last_chunk - ORDER BY requested.sequence, chunk.chunk_index - `, - ) - .all(...parameters, sessionId) as Array<{ - sequence?: unknown; - chunk_index?: unknown; - data?: unknown; - sha256?: unknown; - }>); - const rowsBySequence = new Map(); - for (const row of rows) { - const sequence = requireStoredMessageSequence(row.sequence, sessionId); - const grouped = rowsBySequence.get(sequence); - if (grouped) grouped.push(row); - else rowsBySequence.set(sequence, [row]); - } - const result = new Map(); - for (const slice of slices) { - if (!slice.chunked) continue; - const selected = rowsBySequence.get(slice.sequence) ?? []; - const firstChunk = Math.floor(slice.byteOffset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES); - const lastChunk = Math.floor( - (slice.byteOffset + slice.byteLength - 1) / SQLITE_SESSION_MESSAGE_CHUNK_BYTES, - ); - if (selected.length !== lastChunk - firstChunk + 1) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - const chunks: Buffer[] = []; - for (let index = 0; index < selected.length; index += 1) { - const row = selected[index]!; - if ( - row.chunk_index !== firstChunk + index || - !(row.data instanceof Uint8Array) || - typeof row.sha256 !== 'string' - ) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - const chunk = Buffer.from(row.data); - if (createHash('sha256').update(chunk).digest('hex') !== row.sha256) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - chunks.push(chunk); - } - const joined = Buffer.concat(chunks); - const start = slice.byteOffset - firstChunk * SQLITE_SESSION_MESSAGE_CHUNK_BYTES; - const data = joined.subarray(start, start + slice.byteLength); - if (data.byteLength !== slice.byteLength) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - result.set(slice.sequence, data); - } - const inlineSlices = slices.filter((slice) => !slice.chunked); - if (inlineSlices.length > 0) { - const inlineValues = inlineSlices.map(() => '(?, ?, ?)').join(', '); - const inlineParameters = inlineSlices.flatMap((slice) => [ - slice.sequence, - slice.byteOffset + 1, - slice.byteLength, - ]); - const inlineRows = db - .prepare( - ` - WITH requested(sequence, byte_start, byte_length) AS (VALUES ${inlineValues}) - SELECT requested.sequence, - substr(CAST(message.record_json AS BLOB), requested.byte_start, requested.byte_length) - AS data - FROM requested - INNER JOIN session_messages AS message - ON message.session_id = ? AND message.sequence = requested.sequence - `, - ) - .all(...inlineParameters, sessionId) as Array<{ - sequence?: unknown; - data?: unknown; - }>; - for (const row of inlineRows) { - const sequence = requireStoredMessageSequence(row.sequence, sessionId); - if (!(row.data instanceof Uint8Array)) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - result.set(sequence, Buffer.from(row.data)); - } - } - return result; -} - function validateTranscriptRecord( data: string | Buffer, sessionId: string, diff --git a/packages/storage/src/sqlite-session-transcript-slices.ts b/packages/storage/src/sqlite-session-transcript-slices.ts new file mode 100644 index 0000000000..ca0bff84ed --- /dev/null +++ b/packages/storage/src/sqlite-session-transcript-slices.ts @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash } from 'node:crypto'; +import type { DatabaseSync } from 'node:sqlite'; +import { SQLITE_SESSION_MESSAGE_CHUNK_BYTES } from './sqlite-session-metadata-schema.js'; + +export class StoredSessionMessageIncompatibleError extends Error { + readonly name = 'StoredSessionMessageIncompatibleError'; + readonly code = 'stored_session_message_incompatible'; + + constructor( + readonly sessionId: string, + readonly sequence: number, + options?: ErrorOptions, + ) { + super(`Stored Session message ${sequence} for ${sessionId} is incompatible`, options); + } +} + +export interface TranscriptRecordSlice { + readonly sequence: number; + readonly byteOffset: number; + readonly totalBytes: number; + readonly byteLength: number; + readonly chunked: boolean; + readonly payloadDigest: `sha256:${string}` | null; +} + +export function planForwardTranscriptSlice( + totalBytes: number, + byteOffset: number, + availableBytes: number, + alignmentBytes = 1, +): number { + const remainingBytes = totalBytes - byteOffset; + if (remainingBytes <= availableBytes) return remainingBytes; + return Math.floor(availableBytes / alignmentBytes) * alignmentBytes; +} + +/** + * The single physical byte-slice primitive shared by the public transcript + * pager and the package-private scalar identity recovery scanner. + */ +export function readTranscriptSlices( + db: DatabaseSync, + sessionId: string, + slices: readonly TranscriptRecordSlice[], +): Map { + if (slices.length === 0) return new Map(); + const chunkedSlices = slices.filter((slice) => slice.chunked); + const values = chunkedSlices.map(() => '(?, ?, ?)').join(', '); + const parameters = chunkedSlices.flatMap((slice) => [ + slice.sequence, + Math.floor(slice.byteOffset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES), + Math.floor((slice.byteOffset + slice.byteLength - 1) / SQLITE_SESSION_MESSAGE_CHUNK_BYTES), + ]); + const rows = + chunkedSlices.length === 0 + ? [] + : (db + .prepare( + ` + WITH requested(sequence, first_chunk, last_chunk) AS (VALUES ${values}) + SELECT requested.sequence, chunk.chunk_index, chunk.data, chunk.sha256 + FROM requested + INNER JOIN session_message_chunks AS chunk + ON chunk.session_id = ? + AND chunk.sequence = requested.sequence + AND chunk.chunk_index BETWEEN requested.first_chunk AND requested.last_chunk + ORDER BY requested.sequence, chunk.chunk_index + `, + ) + .all(...parameters, sessionId) as Array<{ + sequence?: unknown; + chunk_index?: unknown; + data?: unknown; + sha256?: unknown; + }>); + const rowsBySequence = new Map(); + for (const row of rows) { + const sequence = requireStoredMessageSequence(row.sequence, sessionId); + const grouped = rowsBySequence.get(sequence); + if (grouped) grouped.push(row); + else rowsBySequence.set(sequence, [row]); + } + const result = new Map(); + for (const slice of slices) { + if (!slice.chunked) continue; + const selected = rowsBySequence.get(slice.sequence) ?? []; + const firstChunk = Math.floor(slice.byteOffset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES); + const lastChunk = Math.floor( + (slice.byteOffset + slice.byteLength - 1) / SQLITE_SESSION_MESSAGE_CHUNK_BYTES, + ); + if (selected.length !== lastChunk - firstChunk + 1) { + throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); + } + const chunks: Buffer[] = []; + for (let index = 0; index < selected.length; index += 1) { + const row = selected[index]!; + if ( + row.chunk_index !== firstChunk + index || + !(row.data instanceof Uint8Array) || + typeof row.sha256 !== 'string' || + !/^[0-9a-f]{64}$/u.test(row.sha256) + ) { + throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); + } + const chunk = Buffer.from(row.data); + const expectedLength = + row.chunk_index === Math.ceil(slice.totalBytes / SQLITE_SESSION_MESSAGE_CHUNK_BYTES) - 1 + ? slice.totalBytes - row.chunk_index * SQLITE_SESSION_MESSAGE_CHUNK_BYTES + : SQLITE_SESSION_MESSAGE_CHUNK_BYTES; + if ( + chunk.byteLength !== expectedLength || + createHash('sha256').update(chunk).digest('hex') !== row.sha256 + ) { + throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); + } + chunks.push(chunk); + } + const joined = Buffer.concat(chunks); + const start = slice.byteOffset - firstChunk * SQLITE_SESSION_MESSAGE_CHUNK_BYTES; + const data = joined.subarray(start, start + slice.byteLength); + if (data.byteLength !== slice.byteLength) { + throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); + } + result.set(slice.sequence, data); + } + const inlineSlices = slices.filter((slice) => !slice.chunked); + if (inlineSlices.length > 0) { + const inlineValues = inlineSlices.map(() => '(?, ?, ?)').join(', '); + const inlineParameters = inlineSlices.flatMap((slice) => [ + slice.sequence, + slice.byteOffset + 1, + slice.byteLength, + ]); + const inlineRows = db + .prepare( + ` + WITH requested(sequence, byte_start, byte_length) AS (VALUES ${inlineValues}) + SELECT requested.sequence, + substr(CAST(message.record_json AS BLOB), requested.byte_start, requested.byte_length) + AS data + FROM requested + INNER JOIN session_messages AS message + ON message.session_id = ? AND message.sequence = requested.sequence + `, + ) + .all(...inlineParameters, sessionId) as Array<{ + sequence?: unknown; + data?: unknown; + }>; + for (const row of inlineRows) { + const sequence = requireStoredMessageSequence(row.sequence, sessionId); + if (!(row.data instanceof Uint8Array)) { + throw new StoredSessionMessageIncompatibleError(sessionId, sequence); + } + result.set(sequence, Buffer.from(row.data)); + } + } + return result; +} + +export function requireStoredMessageSequence(value: unknown, sessionId: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new StoredSessionMessageIncompatibleError(sessionId, -1); + } + return value as number; +} diff --git a/scripts/generate-third-party-notices.mjs b/scripts/generate-third-party-notices.mjs index bd4d713427..b8da5b8c00 100644 --- a/scripts/generate-third-party-notices.mjs +++ b/scripts/generate-third-party-notices.mjs @@ -109,6 +109,42 @@ const APACHE_TEXT_OVERRIDE_KEYS = new Set([ '@ai-sdk/provider-utils@5.0.32', '@sigstore/verify@4.1.2', ]); +const BRUMME_ZLIB_TEXT = `Copyright (c) Stephan Brumme + +This software is provided 'as-is', without any express or implied warranty. In no event will the author be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. +2. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. +3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.`; +const GO_BSD3_TEXT = `Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.`; const EMBEDDED_COMPONENT_LICENSES = new Map([ [ '@ai-sdk/code-mode', @@ -131,6 +167,64 @@ const EMBEDDED_COMPONENT_LICENSES = new Map([ ], }, ], + [ + 'hash-wasm', + { + version: '4.12.0', + components: [ + { + name: 'Go crypto Argon2 implementation basis', + repository: 'https://go.googlesource.com/crypto', + license: 'BSD-3-Clause', + text: GO_BSD3_TEXT, + }, + { + name: 'Stephan Brumme CRC32/CRC64/xxHash32/xxHash64 implementations', + repository: 'https://create.stephan-brumme.com', + license: 'Zlib-like', + text: BRUMME_ZLIB_TEXT, + }, + { + name: 'Yanbo Li SM3 implementation', + repository: 'https://github.com/Daninet/hash-wasm', + copyright: 'Copyright 2016 Yanbo Li dreamfly281@gmail.com, goldboar@163.com', + }, + ], + }, + ], +]); +const EMBEDDED_SOURCE_NOTICES = new Map([ + [ + 'hash-wasm', + { + version: '4.12.0', + sourceDirectory: 'src', + files: [ + 'adler32.c', + 'argon2.c', + 'bcrypt.c', + 'blake2b.c', + 'blake2s.c', + 'blake3.c', + 'crc32.c', + 'crc64.c', + 'md4.c', + 'md5.c', + 'ripemd160.c', + 'scrypt.c', + 'sha1.c', + 'sha256.c', + 'sha3.c', + 'sha512.c', + 'sm3.c', + 'whirlpool.c', + 'xxhash128.c', + 'xxhash3.c', + 'xxhash32.c', + 'xxhash64.c', + ], + }, + ], ]); const MIT_COPYRIGHT_OVERRIDES = new Map([ // The published tarball omits the monorepo-root LICENSE. @@ -270,6 +364,24 @@ function readLicenseFiles(directory) { .sort((left, right) => left.name.localeCompare(right.name)); } +function readLeadingSourceNotice(path) { + const source = readFileSync(path, 'utf8').replace(/\r\n?/g, '\n'); + if (source.startsWith('/*')) { + const end = source.indexOf('*/'); + if (end < 0) throw new Error(`${path}: unterminated embedded source notice`); + return normalizeText(source.slice(0, end + 2)); + } + if (source.startsWith('//')) { + const lines = source.split('\n'); + let end = 0; + while (end < lines.length && (lines[end].startsWith('//') || lines[end].trim() === '')) { + end += 1; + } + return normalizeText(lines.slice(0, end).join('\n')); + } + throw new Error(`${path}: embedded source has no leading license/attribution notice`); +} + function overrideLicenseText(packageKey, selectedLicense) { if (selectedLicense === 'Apache-2.0' && APACHE_TEXT_OVERRIDE_KEYS.has(packageKey)) { const rootLicense = readFileSync(join(repoRoot, 'LICENSE'), 'utf8'); @@ -436,15 +548,48 @@ function renderNotice() { throw new Error(`${owner}: embedded component licenses require exact-version review`); } for (const component of inventory.components) { + const selectedLicense = component.license ?? 'MIT'; + const licenseText = component.text ?? MIT_TEXT(component.copyright); sections.push( [ `Embedded component: ${component.name}`, `Embedded by: ${owner}`, - 'Selected license: MIT', + `Selected license: ${selectedLicense}`, `Repository: ${component.repository}`, '', '--- VERSION-PINNED EMBEDDED LICENSE TEXT ---', - MIT_TEXT(component.copyright), + licenseText, + ].join('\n'), + ); + } + } + } + for (const [packageName, inventory] of EMBEDDED_SOURCE_NOTICES) { + const matchingDependencies = dependencies.filter((candidate) => candidate.name === packageName); + for (const dependency of matchingDependencies) { + const owner = `${dependency.name}@${dependency.version}`; + if (dependency.version !== inventory.version) { + throw new Error(`${owner}: embedded source notices require exact-version review`); + } + const candidates = lockIndex.get(owner); + const directory = packageDirectory(owner, candidates); + const sourceDirectory = join(directory, inventory.sourceDirectory); + const actualFiles = readdirSync(sourceDirectory) + .filter((name) => name.endsWith('.c')) + .sort(); + const expectedFiles = [...inventory.files].sort(); + if (actualFiles.join('\n') !== expectedFiles.join('\n')) { + throw new Error(`${owner}: embedded source inventory changed and requires review`); + } + for (const file of inventory.files) { + sections.push( + [ + `Embedded source notice: ${file}`, + `Embedded by: ${owner}`, + `Source: ${inventory.sourceDirectory}/${file}`, + '', + '--- VERSION-PINNED UPSTREAM SOURCE NOTICE ---', + readLeadingSourceNotice(join(sourceDirectory, file)), ].join('\n'), ); }